Skip to content

Commit

Permalink
WIP: initial concentrator import
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajgajg1134
ajgajg1134 committed Jul 30, 2024
1 parent f8a22c3 commit e04d3e5
Show file tree
Hide file tree
Showing 8 changed files with 582 additions and 171 deletions.
42 changes: 36 additions & 6 deletions ddtrace/tracer/span.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ import (
"sync/atomic"
"time"

pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace"
"github.com/DataDog/datadog-agent/pkg/trace/stats"
"github.com/DataDog/datadog-agent/pkg/trace/traceutil"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal"
Expand All @@ -33,9 +36,10 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames"
"gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof"

"github.com/DataDog/datadog-agent/pkg/obfuscate"
"github.com/tinylib/msgp/msgp"
"golang.org/x/xerrors"

"github.com/DataDog/datadog-agent/pkg/obfuscate"
)

type (
Expand Down Expand Up @@ -549,12 +553,38 @@ func (s *span) finish(finishTime int64) {
// we have an active tracer
if t.config.canComputeStats() && shouldComputeStats(s) {
// the agent supports computed stats
select {
case t.stats.In <- newAggregableSpan(s, t.obfuscator):
// ok
default:
log.Error("Stats channel full, disregarding span.")
statsInput := stats.Input{
Traces: []traceutil.ProcessedTrace{
{
TraceChunk: &pb.TraceChunk{
Priority: 0,
Origin: "",
Spans: []*pb.Span{
{
Service: s.Service,
Name: s.Name,
Resource: s.Resource,
TraceID: s.TraceID,
SpanID: s.SpanID,
ParentID: s.ParentID,
Start: s.Start,
Duration: s.Duration,
Error: s.Error,
Meta: s.Meta,
Metrics: s.Metrics,
Type: s.Type,
MetaStruct: nil,
SpanLinks: nil,
},
},
},
AppVersion: t.config.version,
GitCommitSha: "", // TODO: Do we need these?
ImageTag: "",
},
},
}
t.stats.Add(statsInput)
}
if t.config.canDropP0s() {
// the agent supports dropping p0's in the client
Expand Down
16 changes: 13 additions & 3 deletions ddtrace/tracer/tracer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof"

"github.com/DataDog/datadog-agent/pkg/obfuscate"
agentconfig "github.com/DataDog/datadog-agent/pkg/trace/config"
"github.com/DataDog/datadog-agent/pkg/trace/stats"
)

var _ ddtrace.Tracer = (*tracer)(nil)
Expand All @@ -49,7 +51,7 @@ type tracer struct {

// stats specifies the concentrator used to compute statistics, when client-side
// stats are enabled.
stats *concentrator
stats *stats.Concentrator

// traceWriter is responsible for sending finished traces to their
// destination, such as the Trace Agent or Datadog Forwarder.
Expand Down Expand Up @@ -267,6 +269,13 @@ func newUnstartedTracer(opts ...StartOption) *tracer {
if c.dataStreamsMonitoringEnabled {
dataStreamsProcessor = datastreams.NewProcessor(statsd, c.env, c.serviceName, c.version, c.agentURL, c.httpClient)
}
concentratorConfig := &agentconfig.AgentConfig{
BucketInterval: 10 * time.Second,
DefaultEnv: c.env,
Hostname: c.hostname,
AgentVersion: c.version, // Yes this is the tracer version going into a field named AgentVersion
// TODO: peer tags and compute stats by span kind
}
t := &tracer{
config: c,
traceWriter: writer,
Expand All @@ -276,7 +285,8 @@ func newUnstartedTracer(opts ...StartOption) *tracer {
rulesSampling: rulesSampler,
prioritySampling: sampler,
pid: os.Getpid(),
stats: newConcentrator(c, defaultStatsBucketSize),
// todo: provide a writer, provide a statsd client
stats: stats.NewConcentrator(concentratorConfig, nil, time.Now(), nil),
obfuscator: obfuscate.NewObfuscator(obfuscate.Config{
SQL: obfuscate.SQLConfig{
TableNames: c.agent.HasFlag("table_names"),
Expand Down Expand Up @@ -378,7 +388,7 @@ func (t *tracer) worker(tick <-chan time.Time) {
t.statsd.Incr("datadog.tracer.flush_triggered", []string{"reason:invoked"}, 1)
t.traceWriter.flush()
t.statsd.Flush()
t.stats.flushAndSend(time.Now(), withCurrentBucket)
t.stats.Flush(true)
// TODO(x): In reality, the traceWriter.flush() call is not synchronous
// when using the agent traceWriter. However, this functionality is used
// in Lambda so for that purpose this mechanism should suffice.
Expand Down
111 changes: 67 additions & 44 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
module gopkg.in/DataDog/dd-trace-go.v1

go 1.21
go 1.21.0

require (
cloud.google.com/go/pubsub v1.33.0
cloud.google.com/go/pubsub v1.36.1
github.com/99designs/gqlgen v0.17.36
github.com/DataDog/appsec-internal-go v1.7.0
github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1
github.com/DataDog/datadog-go/v5 v5.3.0
github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3
github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3
github.com/DataDog/datadog-go/v5 v5.5.0
github.com/DataDog/go-libddwaf/v3 v3.3.0
github.com/DataDog/gostackparse v0.7.0
github.com/DataDog/sketches-go v1.4.5
Expand Down Expand Up @@ -50,10 +51,10 @@ require (
github.com/go-sql-driver/mysql v1.6.0
github.com/gocql/gocql v0.0.0-20220224095938-0eacd3183625
github.com/gofiber/fiber/v2 v2.52.5
github.com/golang/protobuf v1.5.3
github.com/golang/protobuf v1.5.4
github.com/gomodule/redigo v1.8.9
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b
github.com/google/uuid v1.5.0
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.0
github.com/graph-gophers/graphql-go v1.5.0
github.com/graphql-go/graphql v0.8.1
Expand All @@ -73,7 +74,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.18
github.com/microsoft/go-mssqldb v0.21.0
github.com/miekg/dns v1.1.55
github.com/mitchellh/mapstructure v1.5.0
github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c
github.com/opentracing/opentracing-go v1.2.0
github.com/redis/go-redis/v9 v9.1.0
github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3
Expand All @@ -92,37 +93,40 @@ require (
github.com/vektah/gqlparser/v2 v2.5.16
github.com/zenazn/goji v1.0.1
go.mongodb.org/mongo-driver v1.12.1
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0
go.opentelemetry.io/otel v1.20.0
go.opentelemetry.io/otel/trace v1.20.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
go.opentelemetry.io/otel v1.27.0
go.opentelemetry.io/otel/trace v1.27.0
go.uber.org/atomic v1.11.0
golang.org/x/mod v0.14.0
golang.org/x/net v0.23.0
golang.org/x/oauth2 v0.9.0
golang.org/x/sys v0.20.0
golang.org/x/time v0.3.0
golang.org/x/mod v0.17.0
golang.org/x/net v0.26.0
golang.org/x/oauth2 v0.18.0
golang.org/x/sys v0.21.0
golang.org/x/time v0.5.0
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
google.golang.org/api v0.128.0
google.golang.org/grpc v1.57.1
google.golang.org/protobuf v1.33.0
google.golang.org/api v0.169.0
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.2
gopkg.in/jinzhu/gorm.v1 v1.9.2
gopkg.in/olivere/elastic.v3 v3.0.75
gopkg.in/olivere/elastic.v5 v5.0.84
gorm.io/driver/mysql v1.0.1
gorm.io/driver/postgres v1.4.6
gorm.io/driver/sqlserver v1.4.2
gorm.io/gorm v1.25.3
k8s.io/apimachinery v0.23.17
k8s.io/apimachinery v0.25.5
k8s.io/client-go v0.23.17
modernc.org/sqlite v1.28.0
)

require (
cloud.google.com/go v0.110.7 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute v1.25.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.2 // indirect
github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.3 // indirect
github.com/DataDog/go-sqllexer v0.0.12 // indirect
github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
Expand All @@ -144,7 +148,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 // indirect
github.com/bytedance/sonic v1.10.0 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
Expand All @@ -156,12 +160,13 @@ require (
github.com/ebitengine/purego v0.6.0-alpha.5 // indirect
github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-pg/zerochecker v0.2.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
Expand All @@ -173,11 +178,12 @@ require (
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.5 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
Expand Down Expand Up @@ -213,6 +219,7 @@ require (
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
Expand All @@ -227,11 +234,14 @@ require (
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
github.com/shirou/gopsutil/v3 v3.24.4 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tidwall/btree v1.6.0 // indirect
Expand All @@ -241,6 +251,8 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
Expand All @@ -255,26 +267,37 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.einride.tech/aip v0.66.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/metric v1.20.0 // indirect
go.opentelemetry.io/collector/component v0.104.0 // indirect
go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect
go.opentelemetry.io/collector/pdata v1.11.0 // indirect
go.opentelemetry.io/collector/semconv v0.104.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.4.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.23.17 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
lukechampine.com/uint128 v1.3.0 // indirect
mellium.im/sasl v0.3.1 // indirect
modernc.org/cc/v3 v3.41.0 // indirect
Expand All @@ -285,7 +308,7 @@ require (
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.2.0 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Loading

0 comments on commit e04d3e5

Please sign in to comment.