diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 7e289857a..a2c60cf1d 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -2,6 +2,7 @@ package indexer import ( "bytes" + "context" "fmt" "math/big" "sync" @@ -14,6 +15,7 @@ import ( "go.vocdoni.io/dvote/test/testcommon/testutil" "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" @@ -197,3 +199,84 @@ func BenchmarkNewProcess(b *testing.B) { log.Infof("indexed %d new processes, took %s", numProcesses, time.Since(startTime)) } + +func BenchmarkBlockList(b *testing.B) { + app := vochain.TestBaseApplication(b) + + idx, err := New(app, Options{DataDir: b.TempDir()}) + qt.Assert(b, err, qt.IsNil) + + count := 100000 + + createDummyBlocks(b, idx, count) + + b.ReportAllocs() + b.ResetTimer() + + benchmarkBlockList := func(b *testing.B, + limit int, offset int, chainID string, hash string, proposerAddress string, + ) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + blocks, total, err := idx.BlockList(limit, offset, chainID, hash, proposerAddress) + qt.Assert(b, err, qt.IsNil) + qt.Assert(b, blocks, qt.HasLen, limit, qt.Commentf("%+v", blocks)) + qt.Assert(b, blocks[0].TxCount, qt.Equals, int64(0)) + qt.Assert(b, total, qt.Equals, uint64(count)) + } + } + + // Run sub-benchmarks with different limits and filters + b.Run("BlockListLimit1", func(b *testing.B) { + benchmarkBlockList(b, 1, 0, "", "", "") + }) + + b.Run("BlockListLimit10", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "", "") + }) + + b.Run("BlockListLimit100", func(b *testing.B) { + benchmarkBlockList(b, 100, 0, "", "", "") + }) + + b.Run("BlockListOffset", func(b *testing.B) { + benchmarkBlockList(b, 10, count/2, "", "", "") + }) + + b.Run("BlockListWithChainID", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "test", "", "") + }) + + b.Run("BlockListWithHashSubstr", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "cafe", "") + }) + b.Run("BlockListWithHashExact", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe", "") + }) +} + +func createDummyBlocks(b *testing.B, idx *Indexer, n int) { + idx.blockMu.Lock() + defer idx.blockMu.Unlock() + + queries := idx.blockTxQueries() + for h := 1; h <= n; h++ { + _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: "test", + Height: int64(h), + Time: time.Now(), + Hash: nonNullBytes([]byte{ + 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, + 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, + }), + ProposerAddress: nonNullBytes([]byte{0xfe, 0xde}), + LastBlockHash: nonNullBytes([]byte{0xca, 0xfe}), + }, + ) + qt.Assert(b, err, qt.IsNil) + } + err := idx.blockTx.Commit() + qt.Assert(b, err, qt.IsNil) +} diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index c6021e530..b6528e890 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -71,22 +71,30 @@ func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress for _, row := range results { list = append(list, indexertypes.BlockFromDBRow(&row)) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.CountBlocks(chainID, hash, proposerAddress) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, count, nil } // CountBlocks returns how many blocks are indexed. -func (idx *Indexer) CountBlocks() (uint64, error) { - results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ - Limit: 1, +// If all args passed are empty ("") it will return the last block height, as an optimization. +func (idx *Indexer) CountBlocks(chainID, hash, proposerAddress string) (uint64, error) { + if chainID == "" && hash == "" && proposerAddress == "" { + count, err := idx.readOnlyQuery.LastBlockHeight(context.TODO()) + if err != nil { + return 0, err + } + return uint64(count), nil + } + count, err := idx.readOnlyQuery.CountBlocks(context.TODO(), indexerdb.CountBlocksParams{ + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, }) if err != nil { return 0, err } - if len(results) == 0 { - return 0, nil - } - return uint64(results[0].TotalCount), nil + return uint64(count), nil } diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index 669f59602..a57f31c6b 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -11,6 +11,35 @@ import ( "time" ) +const countBlocks = `-- name: CountBlocks :one +SELECT COUNT(*) +FROM blocks AS b +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +` + +type CountBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} +} + +func (q *Queries) CountBlocks(ctx context.Context, arg CountBlocksParams) (int64, error) { + row := q.queryRow(ctx, q.countBlocksStmt, countBlocks, arg.ChainID, arg.HashSubstr, arg.ProposerAddress) + var count int64 + err := row.Scan(&count) + return count, err +} + const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( chain_id, height, time, hash, proposer_address, last_block_hash @@ -85,11 +114,23 @@ func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, er return i, err } +const lastBlockHeight = `-- name: LastBlockHeight :one +SELECT height FROM blocks +ORDER BY height DESC +LIMIT 1 +` + +func (q *Queries) LastBlockHeight(ctx context.Context) (int64, error) { + row := q.queryRow(ctx, q.lastBlockHeightStmt, lastBlockHeight) + var height int64 + err := row.Scan(&height) + return height, err +} + const searchBlocks = `-- name: SearchBlocks :many SELECT b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, - COUNT(t.block_index) AS tx_count, - COUNT(*) OVER() AS total_count + COUNT(t.block_index) AS tx_count FROM blocks AS b LEFT JOIN transactions AS t ON b.height = t.block_height @@ -126,7 +167,6 @@ type SearchBlocksRow struct { ProposerAddress []byte LastBlockHash []byte TxCount int64 - TotalCount int64 } func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { @@ -152,7 +192,6 @@ func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]S &i.ProposerAddress, &i.LastBlockHash, &i.TxCount, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 47bf88d46..064c63538 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -30,6 +30,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countAccountsStmt, err = db.PrepareContext(ctx, countAccounts); err != nil { return nil, fmt.Errorf("error preparing query CountAccounts: %w", err) } + if q.countBlocksStmt, err = db.PrepareContext(ctx, countBlocks); err != nil { + return nil, fmt.Errorf("error preparing query CountBlocks: %w", err) + } if q.countTokenTransfersByAccountStmt, err = db.PrepareContext(ctx, countTokenTransfersByAccount); err != nil { return nil, fmt.Errorf("error preparing query CountTokenTransfersByAccount: %w", err) } @@ -96,6 +99,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) } + if q.lastBlockHeightStmt, err = db.PrepareContext(ctx, lastBlockHeight); err != nil { + return nil, fmt.Errorf("error preparing query LastBlockHeight: %w", err) + } if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } @@ -153,6 +159,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countAccountsStmt: %w", cerr) } } + if q.countBlocksStmt != nil { + if cerr := q.countBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countBlocksStmt: %w", cerr) + } + } if q.countTokenTransfersByAccountStmt != nil { if cerr := q.countTokenTransfersByAccountStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countTokenTransfersByAccountStmt: %w", cerr) @@ -263,6 +274,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getVoteStmt: %w", cerr) } } + if q.lastBlockHeightStmt != nil { + if cerr := q.lastBlockHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing lastBlockHeightStmt: %w", cerr) + } + } if q.searchAccountsStmt != nil { if cerr := q.searchAccountsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) @@ -374,6 +390,7 @@ type Queries struct { tx *sql.Tx computeProcessVoteCountStmt *sql.Stmt countAccountsStmt *sql.Stmt + countBlocksStmt *sql.Stmt countTokenTransfersByAccountStmt *sql.Stmt countTransactionsStmt *sql.Stmt countTransactionsByHeightStmt *sql.Stmt @@ -396,6 +413,7 @@ type Queries struct { getTransactionByHashStmt *sql.Stmt getTransactionByHeightAndIndexStmt *sql.Stmt getVoteStmt *sql.Stmt + lastBlockHeightStmt *sql.Stmt searchAccountsStmt *sql.Stmt searchBlocksStmt *sql.Stmt searchEntitiesStmt *sql.Stmt @@ -418,6 +436,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { tx: tx, computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, countAccountsStmt: q.countAccountsStmt, + countBlocksStmt: q.countBlocksStmt, countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, countTransactionsStmt: q.countTransactionsStmt, countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, @@ -440,6 +459,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTransactionByHashStmt: q.getTransactionByHashStmt, getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, getVoteStmt: q.getVoteStmt, + lastBlockHeightStmt: q.lastBlockHeightStmt, searchAccountsStmt: q.searchAccountsStmt, searchBlocksStmt: q.searchBlocksStmt, searchEntitiesStmt: q.searchEntitiesStmt, diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index b764c8812..4d6710560 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -444,7 +444,7 @@ func (idx *Indexer) ReindexBlocks(inTest bool) { return } - idxBlockCount, err := idx.CountBlocks() + idxBlockCount, err := idx.CountBlocks("", "", "") if err != nil { log.Warnf("indexer CountBlocks returned error: %s", err) } diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index d15332cdf..98533e60c 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -21,11 +21,15 @@ SELECT * FROM blocks WHERE hash = ? LIMIT 1; +-- name: LastBlockHeight :one +SELECT height FROM blocks +ORDER BY height DESC +LIMIT 1; + -- name: SearchBlocks :many SELECT b.*, - COUNT(t.block_index) AS tx_count, - COUNT(*) OVER() AS total_count + COUNT(t.block_index) AS tx_count FROM blocks AS b LEFT JOIN transactions AS t ON b.height = t.block_height @@ -44,3 +48,18 @@ GROUP BY b.height ORDER BY b.height DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); + +-- name: CountBlocks :one +SELECT COUNT(*) +FROM blocks AS b +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +);