From f8c6f582cfed411eb18f451111269dde99af4157 Mon Sep 17 00:00:00 2001 From: Matt Johnstone Date: Tue, 15 Oct 2024 17:50:04 +0200 Subject: [PATCH 1/3] added block size monitoring --- cmd/solana_exporter/config.go | 15 +++++++++++- cmd/solana_exporter/exporter.go | 8 ++++--- cmd/solana_exporter/exporter_test.go | 8 +++++-- cmd/solana_exporter/slots.go | 34 ++++++++++++++++++++++++---- cmd/solana_exporter/slots_test.go | 4 ++-- pkg/rpc/client.go | 22 +++++++++++++----- pkg/rpc/responses.go | 13 ++++++----- 7 files changed, 80 insertions(+), 24 deletions(-) diff --git a/cmd/solana_exporter/config.go b/cmd/solana_exporter/config.go index d3af254..928f5e9 100644 --- a/cmd/solana_exporter/config.go +++ b/cmd/solana_exporter/config.go @@ -16,6 +16,7 @@ type ( NodeKeys []string BalanceAddresses []string ComprehensiveSlotTracking bool + MonitorBlockSizes bool } ) @@ -35,6 +36,7 @@ func NewExporterConfig( nodeKeys []string, balanceAddresses []string, comprehensiveSlotTracking bool, + monitorBlockSizes bool, ) *ExporterConfig { return &ExporterConfig{ HttpTimeout: time.Duration(httpTimeout) * time.Second, @@ -43,6 +45,7 @@ func NewExporterConfig( NodeKeys: nodeKeys, BalanceAddresses: balanceAddresses, ComprehensiveSlotTracking: comprehensiveSlotTracking, + MonitorBlockSizes: monitorBlockSizes, } } @@ -54,6 +57,7 @@ func NewExporterConfigFromCLI() *ExporterConfig { nodekeys arrayFlags balanceAddresses arrayFlags comprehensiveSlotTracking bool + monitorBlockSizes bool ) flag.IntVar( &httpTimeout, @@ -92,7 +96,16 @@ func NewExporterConfigFromCLI() *ExporterConfig { "Set this flag to track solana_leader_slots_by_epoch for ALL validators. "+ "Warning: this will lead to potentially thousands of new Prometheus metrics being created every epoch.", ) + flag.BoolVar( + &monitorBlockSizes, + "monitor-block-sizes", + false, + "Set this flag to track block sizes (number of transactions) for the configured validators. "+ + "Warning: this might grind the RPC node.", + ) flag.Parse() - return NewExporterConfig(httpTimeout, rpcUrl, listenAddress, nodekeys, balanceAddresses, comprehensiveSlotTracking) + return NewExporterConfig( + httpTimeout, rpcUrl, listenAddress, nodekeys, balanceAddresses, comprehensiveSlotTracking, monitorBlockSizes, + ) } diff --git a/cmd/solana_exporter/exporter.go b/cmd/solana_exporter/exporter.go index 01d3228..40781a0 100644 --- a/cmd/solana_exporter/exporter.go +++ b/cmd/solana_exporter/exporter.go @@ -57,13 +57,13 @@ func NewSolanaCollector( balanceAddresses: CombineUnique(balanceAddresses, nodekeys, votekeys), totalValidatorsDesc: prometheus.NewDesc( "solana_active_validators", - "Total number of active validators by state", + "Total number of active validators grouped by state (i.e., 'current' vs 'delinquent')", []string{StateLabel}, nil, ), validatorActivatedStake: prometheus.NewDesc( "solana_validator_activated_stake", - "Activated stake per validator", + "Activated stake per validator, in SOL.", []string{VotekeyLabel, NodekeyLabel}, nil, ), @@ -215,7 +215,9 @@ func main() { } collector := NewSolanaCollector(client, slotPacerSchedule, config.BalanceAddresses, config.NodeKeys, votekeys) - slotWatcher := NewSlotWatcher(client, config.NodeKeys, votekeys, config.ComprehensiveSlotTracking) + slotWatcher := NewSlotWatcher( + client, config.NodeKeys, votekeys, config.ComprehensiveSlotTracking, config.MonitorBlockSizes, + ) ctx, cancel := context.WithCancel(ctx) defer cancel() go slotWatcher.WatchSlots(ctx, collector.slotPace) diff --git a/cmd/solana_exporter/exporter_test.go b/cmd/solana_exporter/exporter_test.go index ecda661..a63edaf 100644 --- a/cmd/solana_exporter/exporter_test.go +++ b/cmd/solana_exporter/exporter_test.go @@ -172,7 +172,9 @@ func (c *staticRPCClient) GetLeaderSchedule( } //goland:noinspection GoUnusedParameter -func (c *staticRPCClient) GetBlock(ctx context.Context, commitment rpc.Commitment, slot int64) (*rpc.Block, error) { +func (c *staticRPCClient) GetBlock( + ctx context.Context, commitment rpc.Commitment, slot int64, transactionDetails string, +) (*rpc.Block, error) { return nil, nil } @@ -365,7 +367,9 @@ func (c *dynamicRPCClient) GetLeaderSchedule( } //goland:noinspection GoUnusedParameter -func (c *dynamicRPCClient) GetBlock(ctx context.Context, commitment rpc.Commitment, slot int64) (*rpc.Block, error) { +func (c *dynamicRPCClient) GetBlock( + ctx context.Context, commitment rpc.Commitment, slot int64, transactionDetails string, +) (*rpc.Block, error) { return nil, nil } diff --git a/cmd/solana_exporter/slots.go b/cmd/solana_exporter/slots.go index 5b2c7e6..5626b8a 100644 --- a/cmd/solana_exporter/slots.go +++ b/cmd/solana_exporter/slots.go @@ -24,6 +24,7 @@ type SlotWatcher struct { nodekeys []string votekeys []string comprehensiveSlotTracking bool + monitorBlockSizes bool // currentEpoch is the current epoch we are watching currentEpoch int64 @@ -46,16 +47,22 @@ type SlotWatcher struct { LeaderSlotsByEpochMetric *prometheus.CounterVec InflationRewardsMetric *prometheus.GaugeVec FeeRewardsMetric *prometheus.CounterVec + BlockSizeMetric *prometheus.GaugeVec } func NewSlotWatcher( - client rpc.Provider, nodekeys []string, votekeys []string, comprehensiveSlotTracking bool, + client rpc.Provider, + nodekeys []string, + votekeys []string, + comprehensiveSlotTracking bool, + monitorBlockSizes bool, ) *SlotWatcher { watcher := SlotWatcher{ client: client, nodekeys: nodekeys, votekeys: votekeys, comprehensiveSlotTracking: comprehensiveSlotTracking, + monitorBlockSizes: monitorBlockSizes, // metrics: TotalTransactionsMetric: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "solana_confirmed_transactions_total", @@ -105,6 +112,13 @@ func NewSlotWatcher( }, []string{NodekeyLabel, EpochLabel}, ), + BlockSizeMetric: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "solana_block_size", + Help: "Number of transactions per block, grouped by validator nodekey (identity)", + }, + []string{NodekeyLabel}, + ), } // register: for _, collector := range []prometheus.Collector{ @@ -117,6 +131,7 @@ func NewSlotWatcher( watcher.LeaderSlotsByEpochMetric, watcher.InflationRewardsMetric, watcher.FeeRewardsMetric, + watcher.BlockSizeMetric, } { if err := prometheus.Register(collector); err != nil { var ( @@ -306,7 +321,7 @@ func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot i klog.Infof("Fetched block production in [%v -> %v]", startSlot, endSlot) } -// fetchAndEmitFeeRewards fetches and emits all the fee rewards for the tracked addresses between the +// fetchAndEmitFeeRewards fetches and emits all the fee rewards (+ block sizes) for the tracked addresses between the // slotWatermark and endSlot func (c *SlotWatcher) fetchAndEmitFeeRewards(ctx context.Context, endSlot int64) { startSlot := c.slotWatermark + 1 @@ -333,11 +348,17 @@ func (c *SlotWatcher) fetchAndEmitFeeRewards(ctx context.Context, endSlot int64) klog.Infof("Fetched fee rewards in [%v -> %v]", startSlot, endSlot) } -// fetchAndEmitSingleFeeReward fetches and emits the fee reward for a single block. +// fetchAndEmitSingleFeeReward fetches and emits the fee reward + block size for a single block. func (c *SlotWatcher) fetchAndEmitSingleFeeReward( ctx context.Context, identity string, epoch int64, slot int64, ) error { - block, err := c.client.GetBlock(ctx, rpc.CommitmentConfirmed, slot) + var transactionDetails string + if c.monitorBlockSizes { + transactionDetails = "accounts" + } else { + transactionDetails = "none" + } + block, err := c.client.GetBlock(ctx, rpc.CommitmentConfirmed, slot, transactionDetails) if err != nil { var rpcError *rpc.RPCError if errors.As(err, &rpcError) { @@ -364,6 +385,11 @@ func (c *SlotWatcher) fetchAndEmitSingleFeeReward( } } + // track block size: + if c.monitorBlockSizes { + c.BlockSizeMetric.WithLabelValues(identity).Set(float64(len(block.Transactions))) + } + return nil } diff --git a/cmd/solana_exporter/slots_test.go b/cmd/solana_exporter/slots_test.go index c9bf505..bf07fbc 100644 --- a/cmd/solana_exporter/slots_test.go +++ b/cmd/solana_exporter/slots_test.go @@ -91,7 +91,7 @@ func assertSlotMetricsChangeCorrectly(t *testing.T, initial slotMetricValues, fi func TestSolanaCollector_WatchSlots_Static(t *testing.T) { client := staticRPCClient{} collector := NewSolanaCollector(&client, 100*time.Millisecond, nil, identities, votekeys) - watcher := NewSlotWatcher(&client, identities, votekeys, false) + watcher := NewSlotWatcher(&client, identities, votekeys, false, false) // reset metrics before running tests: watcher.LeaderSlotsTotalMetric.Reset() watcher.LeaderSlotsByEpochMetric.Reset() @@ -160,7 +160,7 @@ func TestSolanaCollector_WatchSlots_Dynamic(t *testing.T) { // create clients: client := newDynamicRPCClient() collector := NewSolanaCollector(client, 300*time.Millisecond, nil, identities, votekeys) - watcher := NewSlotWatcher(client, identities, votekeys, false) + watcher := NewSlotWatcher(client, identities, votekeys, false, false) // reset metrics before running tests: watcher.LeaderSlotsTotalMetric.Reset() watcher.LeaderSlotsByEpochMetric.Reset() diff --git a/pkg/rpc/client.go b/pkg/rpc/client.go index 27572af..bc77155 100644 --- a/pkg/rpc/client.go +++ b/pkg/rpc/client.go @@ -8,6 +8,7 @@ import ( "io" "k8s.io/klog/v2" "net/http" + "slices" "time" ) @@ -71,7 +72,7 @@ type Provider interface { GetLeaderSchedule(ctx context.Context, commitment Commitment, slot int64) (map[string][]int64, error) - GetBlock(ctx context.Context, commitment Commitment, slot int64) (*Block, error) + GetBlock(ctx context.Context, commitment Commitment, slot int64, transactionDetails string) (*Block, error) } func (c Commitment) MarshalJSON() ([]byte, error) { @@ -274,16 +275,25 @@ func (c *Client) GetLeaderSchedule(ctx context.Context, commitment Commitment, s // GetBlock returns identity and transaction information about a confirmed block in the ledger. // See API docs: https://solana.com/docs/rpc/http/getblock -func (c *Client) GetBlock(ctx context.Context, commitment Commitment, slot int64) (*Block, error) { +func (c *Client) GetBlock( + ctx context.Context, commitment Commitment, slot int64, transactionDetails string, +) (*Block, error) { + detailsOptions := []string{"full", "accounts", "none"} + if !slices.Contains(detailsOptions, transactionDetails) { + klog.Fatalf( + "%s is not a valid transaction-details option, must be one of %v", transactionDetails, detailsOptions, + ) + } if commitment == CommitmentProcessed { // as per https://solana.com/docs/rpc/http/getblock klog.Fatalf("commitment '%v' is not supported for GetBlock", CommitmentProcessed) } config := map[string]any{ - "commitment": commitment, - "encoding": "json", // this is default, but no harm in specifying it - "transactionDetails": "none", // for now, can hard-code this out, as we don't need it - "rewards": true, // what we here for! + "commitment": commitment, + "encoding": "json", // this is default, but no harm in specifying it + "transactionDetails": transactionDetails, + "rewards": true, // what we here for! + "maxSupportedTransactionVersion": 0, } var resp response[Block] if err := getResponse(ctx, c, "getBlock", []any{slot, config}, &resp); err != nil { diff --git a/pkg/rpc/responses.go b/pkg/rpc/responses.go index f6cc3dd..26ec720 100644 --- a/pkg/rpc/responses.go +++ b/pkg/rpc/responses.go @@ -79,12 +79,13 @@ type ( } Block struct { - BlockHeight int64 `json:"blockHeight"` - BlockTime int64 `json:"blockTime,omitempty"` - Blockhash string `json:"blockhash"` - ParentSlot int64 `json:"parentSlot"` - PreviousBlockhash string `json:"previousBlockhash"` - Rewards []BlockReward `json:"rewards"` + BlockHeight int64 `json:"blockHeight"` + BlockTime int64 `json:"blockTime,omitempty"` + Blockhash string `json:"blockhash"` + ParentSlot int64 `json:"parentSlot"` + PreviousBlockhash string `json:"previousBlockhash"` + Rewards []BlockReward `json:"rewards"` + Transactions []map[string]any `json:"transactions"` } BlockReward struct { From 048a229ad9cfdd2240006c1d5cf943c7c0172f0c Mon Sep 17 00:00:00 2001 From: Matt Johnstone Date: Tue, 15 Oct 2024 17:58:39 +0200 Subject: [PATCH 2/3] renamed fetchAndEmitFeeRewards -> fetchAndEmitBlockInfos --- cmd/solana_exporter/slots.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/solana_exporter/slots.go b/cmd/solana_exporter/slots.go index 5626b8a..f4ec113 100644 --- a/cmd/solana_exporter/slots.go +++ b/cmd/solana_exporter/slots.go @@ -280,7 +280,7 @@ func (c *SlotWatcher) checkValidSlotRange(from, to int64) error { // moveSlotWatermark performs all the slot-watching tasks required to move the slotWatermark to the provided 'to' slot. func (c *SlotWatcher) moveSlotWatermark(ctx context.Context, to int64) { c.fetchAndEmitBlockProduction(ctx, to) - c.fetchAndEmitFeeRewards(ctx, to) + c.fetchAndEmitBlockInfos(ctx, to) c.slotWatermark = to } @@ -321,9 +321,9 @@ func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot i klog.Infof("Fetched block production in [%v -> %v]", startSlot, endSlot) } -// fetchAndEmitFeeRewards fetches and emits all the fee rewards (+ block sizes) for the tracked addresses between the +// fetchAndEmitBlockInfos fetches and emits all the fee rewards (+ block sizes) for the tracked addresses between the // slotWatermark and endSlot -func (c *SlotWatcher) fetchAndEmitFeeRewards(ctx context.Context, endSlot int64) { +func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, endSlot int64) { startSlot := c.slotWatermark + 1 klog.Infof("Fetching fee rewards in [%v -> %v]", startSlot, endSlot) @@ -338,7 +338,7 @@ func (c *SlotWatcher) fetchAndEmitFeeRewards(ctx context.Context, endSlot int64) klog.Infof("Fetching fee rewards for %v in [%v -> %v]: %v ...", identity, startSlot, endSlot, leaderSlots) for _, slot := range leaderSlots { - err := c.fetchAndEmitSingleFeeReward(ctx, identity, c.currentEpoch, slot) + err := c.fetchAndEmitSingleBlockInfo(ctx, identity, c.currentEpoch, slot) if err != nil { klog.Errorf("Failed to fetch fee rewards for %v at %v: %v", identity, slot, err) } @@ -348,8 +348,8 @@ func (c *SlotWatcher) fetchAndEmitFeeRewards(ctx context.Context, endSlot int64) klog.Infof("Fetched fee rewards in [%v -> %v]", startSlot, endSlot) } -// fetchAndEmitSingleFeeReward fetches and emits the fee reward + block size for a single block. -func (c *SlotWatcher) fetchAndEmitSingleFeeReward( +// fetchAndEmitSingleBlockInfo fetches and emits the fee reward + block size for a single block. +func (c *SlotWatcher) fetchAndEmitSingleBlockInfo( ctx context.Context, identity string, epoch int64, slot int64, ) error { var transactionDetails string From f3cb194171ac8f2350d16ac95a354831b26e0812 Mon Sep 17 00:00:00 2001 From: Matt Johnstone Date: Tue, 15 Oct 2024 20:27:53 +0200 Subject: [PATCH 3/3] fixed tests --- cmd/solana_exporter/exporter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/solana_exporter/exporter.go b/cmd/solana_exporter/exporter.go index 40781a0..eb7cc2e 100644 --- a/cmd/solana_exporter/exporter.go +++ b/cmd/solana_exporter/exporter.go @@ -57,13 +57,13 @@ func NewSolanaCollector( balanceAddresses: CombineUnique(balanceAddresses, nodekeys, votekeys), totalValidatorsDesc: prometheus.NewDesc( "solana_active_validators", - "Total number of active validators grouped by state (i.e., 'current' vs 'delinquent')", + "Total number of active validators by state", []string{StateLabel}, nil, ), validatorActivatedStake: prometheus.NewDesc( "solana_validator_activated_stake", - "Activated stake per validator, in SOL.", + "Activated stake per validator", []string{VotekeyLabel, NodekeyLabel}, nil, ),