From 0f35bd61c1eef5c2df589e54fdf3a1c1d4db9c6c Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 12:04:26 +0200 Subject: [PATCH] indexer: drop concept of transaction id, hash is now the primary key * api: remove endpoint /chain/transactions/reference/index/{index} --- api/chain.go | 38 ------------- vochain/indexer/bench_test.go | 4 +- vochain/indexer/db/db.go | 10 ---- vochain/indexer/db/models.go | 1 - vochain/indexer/db/transactions.sql.go | 33 ++--------- vochain/indexer/indexer_test.go | 5 +- vochain/indexer/indexertypes/types.go | 10 ---- .../0015_recreate_table_transactions.sql | 56 +++++++++++++++++++ vochain/indexer/queries/transactions.sql | 7 +-- vochain/indexer/transaction.go | 13 ----- 10 files changed, 65 insertions(+), 112 deletions(-) create mode 100644 vochain/indexer/migrations/0015_recreate_table_transactions.sql diff --git a/api/chain.go b/api/chain.go index 7907d5d99..143c73c91 100644 --- a/api/chain.go +++ b/api/chain.go @@ -107,14 +107,6 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.Endpoint.RegisterMethod( - "/chain/transactions/reference/index/{index}", - "GET", - apirest.MethodAccessTypePublic, - a.chainTxRefByIndexHandler, - ); err != nil { - return err - } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -717,36 +709,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxRefByIndexHandler -// -// @Summary Transaction by index -// @Description Get transaction by its index. This is not transaction reference (hash), and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). -// @Tags Chain -// @Accept json -// @Produce json -// @Param index path int true "Index of the transaction" -// @Success 200 {object} indexertypes.Transaction -// @Success 204 "See [errors](vocdoni-api#errors) section" -// @Router /chain/transactions/reference/index/{index} [get] -func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) - if err != nil { - return err - } - ref, err := a.indexer.GetTransaction(index) - if err != nil { - if errors.Is(err, indexer.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - data, err := json.Marshal(ref) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTxListHandler // // @Summary List transactions diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 631e83205..9e8cb1210 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -147,10 +147,10 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTransaction(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by index, took %s", + log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 3695e8b73..dd22f9d0f 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -81,9 +81,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { - return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) - } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } @@ -232,11 +229,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTransactionStmt != nil { - if cerr := q.getTransactionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) - } - } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) @@ -375,7 +367,6 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt getVoteStmt *sql.Stmt @@ -417,7 +408,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, getTransactionByHashStmt: q.getTransactionByHashStmt, getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, getVoteStmt: q.getVoteStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 08a6e0e2b..b64fc72c8 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -57,7 +57,6 @@ type TokenTransfer struct { } type Transaction struct { - ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index 2d06f135a..a477ebad0 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -47,27 +47,8 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa ) } -const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type FROM transactions -WHERE id = ? -LIMIT 1 -` - -func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { - row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) - var i Transaction - err := row.Scan( - &i.ID, - &i.Hash, - &i.BlockHeight, - &i.BlockIndex, - &i.Type, - ) - return i, err -} - const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type FROM transactions WHERE hash = ? LIMIT 1 ` @@ -76,7 +57,6 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -86,7 +66,7 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr } const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -100,7 +80,6 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -111,16 +90,16 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type + SELECT hash, block_height, block_index, type FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) ) ) -SELECT id, hash, block_height, block_index, type, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT ?2 OFFSET ?1 ` @@ -133,7 +112,6 @@ type SearchTransactionsParams struct { } type SearchTransactionsRow struct { - ID int64 Hash []byte BlockHeight int64 BlockIndex int64 @@ -156,7 +134,6 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index fad19d248..76c306910 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1404,7 +1404,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTransaction(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1422,8 +1422,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err := idx.SearchTransactions(15, 0, 0, "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { - // Index is between 1 and totalCount. - qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) @@ -1433,7 +1431,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err = idx.SearchTransactions(1, 5, 0, "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) - qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index d0d2b29aa..b083b2e95 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,17 +176,8 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// TxMetadata contains tx information for the TransactionList api -type TxMetadata struct { - Type string `json:"type"` - BlockHeight uint32 `json:"blockHeight,omitempty"` - Index int32 `json:"index"` - Hash types.HexBytes `json:"hash"` -} - // Transaction holds the db reference for a single transaction type Transaction struct { - Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` @@ -195,7 +186,6 @@ type Transaction struct { func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { return &Transaction{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), diff --git a/vochain/indexer/migrations/0015_recreate_table_transactions.sql b/vochain/indexer/migrations/0015_recreate_table_transactions.sql new file mode 100644 index 000000000..bf091d7c9 --- /dev/null +++ b/vochain/indexer/migrations/0015_recreate_table_transactions.sql @@ -0,0 +1,56 @@ +-- +goose Up +PRAGMA foreign_keys = OFF; + +-- Create a new table with hash as primary key +CREATE TABLE transactions_new ( + hash BLOB NOT NULL PRIMARY KEY, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data from the old table to the new table +INSERT INTO transactions_new (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions; + +-- Drop the old table +DROP TABLE transactions; + +-- Rename the new table to the old table name +ALTER TABLE transactions_new RENAME TO transactions; + +-- Recreate necessary indexes +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; + +-- +goose Down +PRAGMA foreign_keys = OFF; + +-- Recreate the old table structure +CREATE TABLE transactions ( + id INTEGER NOT NULL PRIMARY KEY, + hash BLOB NOT NULL, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data back from the new table to the old table +INSERT INTO transactions (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions_new; + +-- Drop the new table +DROP TABLE transactions_new; + +-- Recreate the old indexes +CREATE INDEX transactions_hash +ON transactions(hash); + +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 0e625a197..eb8b2b617 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -5,11 +5,6 @@ INSERT INTO transactions ( ?, ?, ?, ? ); --- name: GetTransaction :one -SELECT * FROM transactions -WHERE id = ? -LIMIT 1; - -- name: GetTransactionByHash :one SELECT * FROM transactions WHERE hash = ? @@ -34,6 +29,6 @@ WITH results AS ( ) SELECT *, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8c9fe21..16e8cad81 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -22,18 +22,6 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTransaction fetches the txReference for the given tx height -func (idx *Indexer) GetTransaction(id uint64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrTransactionNotFound - } - return nil, fmt.Errorf("tx with id %d not found: %v", id, err) - } - return indexertypes.TransactionFromDB(&sqlTxRef), nil -} - // GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ @@ -83,7 +71,6 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx list := []*indexertypes.Transaction{} for _, row := range results { list = append(list, &indexertypes.Transaction{ - Index: uint64(row.ID), Hash: row.Hash, BlockHeight: uint32(row.BlockHeight), TxBlockIndex: int32(row.BlockIndex),