From 3a59d5326dc133fd475d55fce0034ccac33e26b9 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 12:04:26 +0200 Subject: [PATCH] indexer: drop concept of transaction id, hash is now the primary key * api: remove endpoint /chain/transactions/reference/index/{index} --- api/chain.go | 38 ------------ vochain/indexer/bench_test.go | 4 +- vochain/indexer/db/blocks.sql.go | 10 +-- vochain/indexer/db/db.go | 10 --- vochain/indexer/db/models.go | 3 +- vochain/indexer/db/transactions.sql.go | 45 +++----------- vochain/indexer/indexer_test.go | 5 +- vochain/indexer/indexertypes/types.go | 3 - .../0015_recreate_table_transactions.sql | 62 +++++++++++++++++++ vochain/indexer/queries/blocks.sql | 10 +-- vochain/indexer/queries/transactions.sql | 7 +-- vochain/indexer/transaction.go | 12 ---- 12 files changed, 86 insertions(+), 123 deletions(-) create mode 100644 vochain/indexer/migrations/0015_recreate_table_transactions.sql diff --git a/api/chain.go b/api/chain.go index b41862cf6..6901d2417 100644 --- a/api/chain.go +++ b/api/chain.go @@ -104,14 +104,6 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.Endpoint.RegisterMethod( - "/chain/transactions/reference/index/{index}", - "GET", - apirest.MethodAccessTypePublic, - a.chainTxRefByIndexHandler, - ); err != nil { - return err - } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -705,36 +697,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxRefByIndexHandler -// -// @Summary Transaction metadata (by db index) -// @Description Get transaction by its internal index. This is not the transaction hash, and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). -// @Tags Chain -// @Accept json -// @Produce json -// @Param index path int true "Index of the transaction" -// @Success 200 {object} indexertypes.Transaction -// @Success 204 "See [errors](vocdoni-api#errors) section" -// @Router /chain/transactions/reference/index/{index} [get] -func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) - if err != nil { - return err - } - ref, err := a.indexer.GetTxMetadataByID(index) - if err != nil { - if errors.Is(err, indexer.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - data, err := json.Marshal(ref) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTxListHandler // // @Summary List transactions diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index f73645bd9..7e289857a 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -152,10 +152,10 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxMetadataByID(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by index, took %s", + log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index dcb19bf6c..c45e0bf6a 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -18,11 +18,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash) +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash ` type CreateBlockParams struct { diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 786977651..47bf88d46 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -87,9 +87,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { - return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) - } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } @@ -251,11 +248,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTransactionStmt != nil { - if cerr := q.getTransactionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) - } - } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) @@ -401,7 +393,6 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTransactionByHeightAndIndexStmt *sql.Stmt getVoteStmt *sql.Stmt @@ -446,7 +437,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, getTransactionByHashStmt: q.getTransactionByHashStmt, getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, getVoteStmt: q.getVoteStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index fab3d35ad..d69facf2f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -60,13 +60,12 @@ type TokenTransfer struct { } type Transaction struct { - ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 Type string + Subtype string RawTx []byte Signature []byte - Subtype string Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index c8628aebc..e6c5efa2d 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -67,31 +67,8 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa ) } -const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions -WHERE id = ? -LIMIT 1 -` - -func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { - row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) - var i Transaction - err := row.Scan( - &i.ID, - &i.Hash, - &i.BlockHeight, - &i.BlockIndex, - &i.Type, - &i.RawTx, - &i.Signature, - &i.Subtype, - &i.Signer, - ) - return i, err -} - const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -100,21 +77,20 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, &i.RawTx, &i.Signature, - &i.Subtype, &i.Signer, ) return i, err } const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -128,14 +104,13 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, &i.RawTx, &i.Signature, - &i.Subtype, &i.Signer, ) return i, err @@ -143,7 +118,7 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer + SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) @@ -152,9 +127,9 @@ WITH results AS ( AND (?6 = '' OR LOWER(HEX(signer)) = LOWER(?6)) ) ) -SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT ?2 OFFSET ?1 ` @@ -169,14 +144,13 @@ type SearchTransactionsParams struct { } type SearchTransactionsRow struct { - ID int64 Hash []byte BlockHeight int64 BlockIndex int64 Type string + Subtype string RawTx []byte Signature []byte - Subtype string Signer []byte TotalCount int64 } @@ -198,14 +172,13 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, &i.RawTx, &i.Signature, - &i.Subtype, &i.Signer, &i.TotalCount, ); err != nil { diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 0b348cf13..8f08bd398 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1405,7 +1405,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTxMetadataByID(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1423,8 +1423,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err := idx.SearchTransactions(15, 0, 0, "", "", "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { - // Index is between 1 and totalCount. - qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) @@ -1434,7 +1432,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err = idx.SearchTransactions(1, 5, 0, "", "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) - qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 263ab5b43..06eb8d0ef 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -178,7 +178,6 @@ type TxPackage struct { // TransactionMetadata contains tx information for the TransactionList api type TransactionMetadata struct { - Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` @@ -189,7 +188,6 @@ type TransactionMetadata struct { func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { return &TransactionMetadata{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), @@ -201,7 +199,6 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { return &TransactionMetadata{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), diff --git a/vochain/indexer/migrations/0015_recreate_table_transactions.sql b/vochain/indexer/migrations/0015_recreate_table_transactions.sql new file mode 100644 index 000000000..f92e8ddd8 --- /dev/null +++ b/vochain/indexer/migrations/0015_recreate_table_transactions.sql @@ -0,0 +1,62 @@ +-- +goose Up +PRAGMA foreign_keys = OFF; + +-- Create a new table with hash as primary key +CREATE TABLE transactions_new ( + hash BLOB NOT NULL PRIMARY KEY, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data from the old table to the new table +INSERT INTO transactions_new (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions; + +-- Drop the old table +DROP TABLE transactions; + +-- Rename the new table to the old table name +ALTER TABLE transactions_new RENAME TO transactions; + +-- Recreate necessary indexes +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +-- Add new columns +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +PRAGMA foreign_keys = ON; + +-- +goose Down +PRAGMA foreign_keys = OFF; + +-- Recreate the old table structure +CREATE TABLE transactions ( + id INTEGER NOT NULL PRIMARY KEY, + hash BLOB NOT NULL, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data back from the new table to the old table +INSERT INTO transactions (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions_new; + +-- Drop the new table +DROP TABLE transactions_new; + +-- Recreate the old indexes +CREATE INDEX transactions_hash +ON transactions(hash); + +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index f52ef3b4b..a00af5ca7 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -5,11 +5,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash); +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash; -- name: GetBlockByHeight :one SELECT * FROM blocks diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 4466a5387..fe32c46f6 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -5,11 +5,6 @@ INSERT INTO transactions ( ?, ?, ?, ?, ?, ?, ?, ? ); --- name: GetTransaction :one -SELECT * FROM transactions -WHERE id = ? -LIMIT 1; - -- name: GetTransactionByHash :one SELECT * FROM transactions WHERE hash = ? @@ -40,6 +35,6 @@ WITH results AS ( ) SELECT *, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 159d1b823..09ce738ab 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -30,18 +30,6 @@ func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) } -// GetTxMetadataByID fetches the tx metadata for the given tx height -func (idx *Indexer) GetTxMetadataByID(id uint64) (*indexertypes.TransactionMetadata, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrTransactionNotFound - } - return nil, fmt.Errorf("tx with id %d not found: %v", id, err) - } - return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil -} - // GetTxMetadataByHash fetches the tx metadata for the given tx hash func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash)