From 0c0284735df250245da2c066b2b1ac45c6b71733 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Tue, 6 Aug 2024 12:06:00 +0200 Subject: [PATCH 1/3] api: new endpoint /chain/transfers implements pagination and query params add a new endpoint, that includes `pagination` field in reply, and accepts QueryParams: * GET /chain/transfers * page * limit * accountId * accountIdFrom * accountIdTo * mark all of these endpoints as deprecated on swagger docs: * /accounts/{accountId}/transfers/page/{page} * /accounts/{accountId}/fees/page/{page} * introduced a breaking change in the endpoint `/accounts/{accountId}/transfers/page/{page}` since now the returned json structure changed. now it's a plain list of transfers instead of "received" and "sent" also fixed the encoding of the txHash field, now is base16 instead of base64 * api: add structs TransfersList, TransfersParams * indexer: replace GetTokenTransfersBy*Account methods with a new TokenTransfersList --- api/accounts.go | 45 +++------ api/api.go | 2 + api/api_types.go | 14 +++ api/chain.go | 80 +++++++++++++++ apiclient/account.go | 16 ++- cmd/end2endtest/account.go | 6 +- test/api_test.go | 15 +-- test/apierror_test.go | 4 - vochain/indexer/db/db.go | 78 +++++++-------- vochain/indexer/db/token_transfers.sql.go | 102 ++++++++------------ vochain/indexer/indexer.go | 90 ++++++----------- vochain/indexer/indexer_test.go | 13 ++- vochain/indexer/indexertypes/types.go | 2 +- vochain/indexer/queries/token_transfers.sql | 30 +++--- 14 files changed, 257 insertions(+), 240 deletions(-) diff --git a/api/accounts.go b/api/accounts.go index 15964a106..db79a7998 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -15,7 +15,6 @@ import ( "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" - "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/proto/build/go/models" "google.golang.org/protobuf/proto" @@ -433,56 +432,42 @@ func (a *API) accountElectionsCountHandler(_ *apirest.APIdata, ctx *httprouter.H // // @Summary List account received and sent token transfers // @Description Returns the token transfers for an account. A transfer is a token transference from one account to other (excepting the burn address). +// @Deprecated +// @Description (deprecated, in favor of /chain/transfers?accountId=xxx&page=xxx) // @Tags Accounts // @Accept json // @Produce json -// @Param accountId path string true "Specific accountId" +// @Param accountId path string true "Specific accountId that sent or received the tokens" // @Param page path number true "Page" -// @Success 200 {object} object{transfers=indexertypes.TokenTransfersAccount} +// @Success 200 {object} TransfersList // @Router /accounts/{accountId}/transfers/page/{page} [get] func (a *API) tokenTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - accountID, err := hex.DecodeString(util.TrimHex(ctx.URLParam(ParamAccountId))) - if err != nil || accountID == nil { - return ErrCantParseAccountID.Withf("%q", ctx.URLParam(ParamAccountId)) - } - acc, err := a.vocapp.State.GetAccount(common.BytesToAddress(accountID), true) - if acc == nil { - return ErrAccountNotFound - } - if err != nil { - return err - } - - page, err := parsePage(ctx.URLParam(ParamPage)) + params, err := parseTransfersParams( + ctx.URLParam(ParamPage), + "", + ctx.URLParam(ParamAccountId), + "", + "", + ) if err != nil { return err } - transfers, err := a.indexer.GetTokenTransfersByAccount(accountID, int32(page*DefaultItemsPerPage), DefaultItemsPerPage) - if err != nil { - return ErrCantFetchTokenTransfers.WithErr(err) - } - data, err := json.Marshal( - struct { - Transfers indexertypes.TokenTransfersAccount `json:"transfers"` - }{Transfers: transfers}, - ) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) + return a.sendTransfersList(ctx, params) } // tokenFeesHandler // // @Summary List account token fees // @Description Returns the token fees for an account. A spending is an amount of tokens burnt from one account for executing transactions. +// @Deprecated +// @Description (deprecated, in favor of /chain/transfers?accountId=xxx&page=xxx) // @Tags Accounts // @Accept json // @Produce json // @Param accountId path string true "Specific accountId" // @Param page path number true "Page" -// @Success 200 {object} object{fees=[]indexertypes.TokenFeeMeta} +// @Success 200 {object} FeesList // @Router /accounts/{accountId}/fees/page/{page} [get] func (a *API) tokenFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { params, err := parseFeesParams( diff --git a/api/api.go b/api/api.go index 64a5afe1c..773b0ff8b 100644 --- a/api/api.go +++ b/api/api.go @@ -74,6 +74,8 @@ const ( ParamHeight = "height" ParamReference = "reference" ParamType = "type" + ParamAccountIdFrom = "accountIdFrom" + ParamAccountIdTo = "accountIdTo" ) var ( diff --git a/api/api_types.go b/api/api_types.go index b018c2070..220bb1cdb 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -57,6 +57,14 @@ type FeesParams struct { AccountID string `json:"accountId,omitempty"` } +// TransfersParams allows the client to filter transfers +type TransfersParams struct { + PaginationParams + AccountID string `json:"accountId,omitempty"` + AccountIDFrom string `json:"accountIdFrom,omitempty"` + AccountIDTo string `json:"accountIdTo,omitempty"` +} + // VoteParams allows the client to filter votes type VoteParams struct { PaginationParams @@ -264,6 +272,12 @@ type FeesList struct { Pagination *Pagination `json:"pagination"` } +// TransfersList is used to return a paginated list to the client +type TransfersList struct { + Transfers []*indexertypes.TokenTransferMeta `json:"transfers"` + Pagination *Pagination `json:"pagination"` +} + type GenericTransactionWithInfo struct { TxContent json.RawMessage `json:"tx"` TxInfo indexertypes.Transaction `json:"txInfo"` diff --git a/api/chain.go b/api/chain.go index baa3f653f..e029efbbe 100644 --- a/api/chain.go +++ b/api/chain.go @@ -225,6 +225,14 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/chain/transfers", + "GET", + apirest.MethodAccessTypePublic, + a.chainTransfersListHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/chain/export/indexer", "GET", @@ -1076,6 +1084,63 @@ func (a *API) sendFeesList(ctx *httprouter.HTTPContext, params *FeesParams) erro return marshalAndSend(ctx, list) } +// chainTransfersListHandler +// +// @Summary List all token transfers +// @Description Returns the token transfers list ordered by date. +// @Tags Chain +// @Accept json +// @Produce json +// @Param page query number false "Page" +// @Param limit query number false "Items per page" +// @Param accountId query string false "Specific accountId that sent or received the tokens" +// @Param accountIdFrom query string false "Specific accountId that sent the tokens" +// @Param accountIdTo query string false "Specific accountId that received the tokens" +// @Success 200 {object} TransfersList +// @Router /chain/transfers [get] +func (a *API) chainTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseTransfersParams( + ctx.QueryParam(ParamPage), + ctx.QueryParam(ParamLimit), + ctx.QueryParam(ParamAccountId), + ctx.QueryParam(ParamAccountIdFrom), + ctx.QueryParam(ParamAccountIdTo), + ) + if err != nil { + return err + } + + return a.sendTransfersList(ctx, params) +} + +// sendTransfersList produces a filtered, paginated TokenTransfersList, +// and sends it marshalled over ctx.Send +// +// Errors returned are always of type APIerror. +func (a *API) sendTransfersList(ctx *httprouter.HTTPContext, params *TransfersParams) error { + transfers, total, err := a.indexer.TokenTransfersList( + params.Limit, + params.Page*params.Limit, + params.AccountID, + params.AccountIDFrom, + params.AccountIDTo, + ) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) + } + + pagination, err := calculatePagination(params.Page, params.Limit, total) + if err != nil { + return err + } + + list := &TransfersList{ + Transfers: transfers, + Pagination: pagination, + } + return marshalAndSend(ctx, list) +} + // chainIndexerExportHandler // // @Summary Exports the indexer database @@ -1122,6 +1187,21 @@ func parseFeesParams(paramPage, paramLimit, paramReference, paramType, paramAcco }, nil } +// parseTransfersParams returns an TransfersParams filled with the passed params +func parseTransfersParams(paramPage, paramLimit, paramAccountId, paramAccountIdFrom, paramAccountIdTo string) (*TransfersParams, error) { + pagination, err := parsePaginationParams(paramPage, paramLimit) + if err != nil { + return nil, err + } + + return &TransfersParams{ + PaginationParams: pagination, + AccountID: util.TrimHex(paramAccountId), + AccountIDFrom: util.TrimHex(paramAccountIdFrom), + AccountIDTo: util.TrimHex(paramAccountIdTo), + }, nil +} + // parseTransactionParams returns an TransactionParams filled with the passed params func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string) (*TransactionParams, error) { pagination, err := parsePaginationParams(paramPage, paramLimit) diff --git a/apiclient/account.go b/apiclient/account.go index 2beaed289..b0e974b45 100644 --- a/apiclient/account.go +++ b/apiclient/account.go @@ -290,21 +290,19 @@ func (c *HTTPclient) AccountSetMetadata(metadata *api.AccountMetadata) (types.He } // ListTokenTransfers returns the list of sent and received token transfers associated with an account -func (c *HTTPclient) ListTokenTransfers(account common.Address, page int) (indexertypes.TokenTransfersAccount, error) { +func (c *HTTPclient) ListTokenTransfers(account common.Address, page int) (*api.TransfersList, error) { resp, code, err := c.Request(HTTPGET, nil, "accounts", account.Hex(), "transfers", "page", strconv.Itoa(page)) if err != nil { - return indexertypes.TokenTransfersAccount{}, err + return nil, err } if code != apirest.HTTPstatusOK { - return indexertypes.TokenTransfersAccount{}, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp) + return nil, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp) } - tokenTxs := new(struct { - Transfers indexertypes.TokenTransfersAccount `json:"transfers"` - }) - if err := json.Unmarshal(resp, &tokenTxs); err != nil { - return indexertypes.TokenTransfersAccount{}, err + tokenTxs := &api.TransfersList{} + if err := json.Unmarshal(resp, tokenTxs); err != nil { + return nil, err } - return tokenTxs.Transfers, nil + return tokenTxs, nil } // SetSIK function allows to update the Secret Identity Key for the current diff --git a/cmd/end2endtest/account.go b/cmd/end2endtest/account.go index d96e6d22c..af0ab6c86 100644 --- a/cmd/end2endtest/account.go +++ b/cmd/end2endtest/account.go @@ -329,14 +329,12 @@ func checkTokenTransfersCount(api *apiclient.HTTPclient, address common.Address) if err != nil { return err } - countTokenTxs := uint64(len(tokenTxs.Received) + len(tokenTxs.Sent)) - count, err := api.CountTokenTransfers(address) if err != nil { return err } - if count != countTokenTxs { - return fmt.Errorf("expected %s to match transfers count %d and %d", address, count, countTokenTxs) + if count != tokenTxs.Pagination.TotalItems { + return fmt.Errorf("expected %s to match transfers count %d and %d", address, count, tokenTxs.Pagination.TotalItems) } log.Infow("current transfers count", "account", address.String(), "count", count) diff --git a/test/api_test.go b/test/api_test.go index dbc4e033e..80e4f3699 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -391,9 +391,7 @@ func TestAPIAccountTokentxs(t *testing.T) { resp, code = c.Request("GET", nil, "accounts", signer.Address().Hex(), "transfers", "page", "0") qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) - tokenTxs := new(struct { - Transfers indexertypes.TokenTransfersAccount `json:"transfers"` - }) + tokenTxs := &api.TransfersList{} err := json.Unmarshal(resp, tokenTxs) qt.Assert(t, err, qt.IsNil) @@ -407,18 +405,14 @@ func TestAPIAccountTokentxs(t *testing.T) { err = json.Unmarshal(resp, &countTnsAcc) qt.Assert(t, err, qt.IsNil) - totalTokenTxs := uint64(len(tokenTxs.Transfers.Received) + len(tokenTxs.Transfers.Sent)) - // compare count of total token transfers for the account 1 using the two response - qt.Assert(t, totalTokenTxs, qt.Equals, countTnsAcc.Count) + qt.Assert(t, tokenTxs.Pagination.TotalItems, qt.Equals, countTnsAcc.Count) // get the token transfers received and sent for account 2 resp, code = c.Request("GET", nil, "accounts", signer2.Address().Hex(), "transfers", "page", "0") qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) - tokenTxs2 := new(struct { - Transfers indexertypes.TokenTransfersAccount `json:"transfers"` - }) + tokenTxs2 := &api.TransfersList{} err = json.Unmarshal(resp, tokenTxs2) qt.Assert(t, err, qt.IsNil) @@ -432,9 +426,8 @@ func TestAPIAccountTokentxs(t *testing.T) { err = json.Unmarshal(resp, &countTnsAcc2) qt.Assert(t, err, qt.IsNil) - totalTokenTxs2 := uint64(len(tokenTxs2.Transfers.Received) + len(tokenTxs2.Transfers.Sent)) // compare count of total token transfers for the account 2 using the two response - qt.Assert(t, totalTokenTxs2, qt.Equals, countTnsAcc2.Count) + qt.Assert(t, tokenTxs2.Pagination.TotalItems, qt.Equals, countTnsAcc2.Count) resp, code = c.Request("GET", nil, "accounts", "page", "0") qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) diff --git a/test/apierror_test.go b/test/apierror_test.go index 912295bc4..1a413d5c9 100644 --- a/test/apierror_test.go +++ b/test/apierror_test.go @@ -63,10 +63,6 @@ func TestAPIerror(t *testing.T) { args: args{"POST", hugeFile, []string{"files", "cid"}}, want: api.ErrFileSizeTooBig, }, - { - args: args{"GET", nil, []string{"accounts", "totallyWrong!@#$", "transfers", "page", "0"}}, - want: api.ErrCantParseAccountID, - }, { args: args{"GET", nil, []string{ "votes", "verify", diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index f84c11994..acd06f7ed 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -84,12 +84,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTokenTransfersByFromAccountStmt, err = db.PrepareContext(ctx, getTokenTransfersByFromAccount); err != nil { - return nil, fmt.Errorf("error preparing query GetTokenTransfersByFromAccount: %w", err) - } - if q.getTokenTransfersByToAccountStmt, err = db.PrepareContext(ctx, getTokenTransfersByToAccount); err != nil { - return nil, fmt.Errorf("error preparing query GetTokenTransfersByToAccount: %w", err) - } if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) } @@ -111,6 +105,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.searchTokenFeesStmt, err = db.PrepareContext(ctx, searchTokenFees); err != nil { return nil, fmt.Errorf("error preparing query SearchTokenFees: %w", err) } + if q.searchTokenTransfersStmt, err = db.PrepareContext(ctx, searchTokenTransfers); err != nil { + return nil, fmt.Errorf("error preparing query SearchTokenTransfers: %w", err) + } if q.searchTransactionsStmt, err = db.PrepareContext(ctx, searchTransactions); err != nil { return nil, fmt.Errorf("error preparing query SearchTransactions: %w", err) } @@ -240,16 +237,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTokenTransfersByFromAccountStmt != nil { - if cerr := q.getTokenTransfersByFromAccountStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTokenTransfersByFromAccountStmt: %w", cerr) - } - } - if q.getTokenTransfersByToAccountStmt != nil { - if cerr := q.getTokenTransfersByToAccountStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTokenTransfersByToAccountStmt: %w", cerr) - } - } if q.getTransactionStmt != nil { if cerr := q.getTransactionStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) @@ -285,6 +272,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing searchTokenFeesStmt: %w", cerr) } } + if q.searchTokenTransfersStmt != nil { + if cerr := q.searchTokenTransfersStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchTokenTransfersStmt: %w", cerr) + } + } if q.searchTransactionsStmt != nil { if cerr := q.searchTransactionsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchTransactionsStmt: %w", cerr) @@ -384,8 +376,6 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt - getTokenTransfersByFromAccountStmt *sql.Stmt - getTokenTransfersByToAccountStmt *sql.Stmt getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt @@ -393,6 +383,7 @@ type Queries struct { searchEntitiesStmt *sql.Stmt searchProcessesStmt *sql.Stmt searchTokenFeesStmt *sql.Stmt + searchTokenTransfersStmt *sql.Stmt searchTransactionsStmt *sql.Stmt searchVotesStmt *sql.Stmt setProcessResultsCancelledStmt *sql.Stmt @@ -405,37 +396,36 @@ type Queries struct { func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getListAccountsStmt: q.getListAccountsStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTokenTransfersByFromAccountStmt: q.getTokenTransfersByFromAccountStmt, - getTokenTransfersByToAccountStmt: q.getTokenTransfersByToAccountStmt, - getTransactionStmt: q.getTransactionStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockStmt: q.getBlockStmt, + getEntityCountStmt: q.getEntityCountStmt, + getListAccountsStmt: q.getListAccountsStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionStmt: q.getTransactionStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, getVoteStmt: q.getVoteStmt, searchEntitiesStmt: q.searchEntitiesStmt, searchProcessesStmt: q.searchProcessesStmt, searchTokenFeesStmt: q.searchTokenFeesStmt, + searchTokenTransfersStmt: q.searchTokenTransfersStmt, searchTransactionsStmt: q.searchTransactionsStmt, searchVotesStmt: q.searchVotesStmt, setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, diff --git a/vochain/indexer/db/token_transfers.sql.go b/vochain/indexer/db/token_transfers.sql.go index 2d00e15e0..7e46f74da 100644 --- a/vochain/indexer/db/token_transfers.sql.go +++ b/vochain/indexer/db/token_transfers.sql.go @@ -14,9 +14,6 @@ import ( ) const countTokenTransfersByAccount = `-- name: CountTokenTransfersByAccount :one -; - - SELECT COUNT(*) FROM token_transfers WHERE to_account = ?1 OR from_account = ?1 @@ -79,75 +76,59 @@ func (q *Queries) GetTokenTransfer(ctx context.Context, txHash types.Hash) (Toke return i, err } -const getTokenTransfersByFromAccount = `-- name: GetTokenTransfersByFromAccount :many -SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time FROM token_transfers -WHERE from_account = ?1 +const searchTokenTransfers = `-- name: SearchTokenTransfers :many +WITH results AS ( + SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time + FROM token_transfers + WHERE ( + (?3 = '' OR ( + LOWER(HEX(from_account)) = LOWER(?3) + OR LOWER(HEX(to_account)) = LOWER(?3) + )) + AND (?4 = '' OR LOWER(HEX(from_account)) = LOWER(?4)) + AND (?5 = '' OR LOWER(HEX(to_account)) = LOWER(?5)) + ) +) +SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time, COUNT(*) OVER() AS total_count +FROM results ORDER BY transfer_time DESC -LIMIT ?3 -OFFSET ?2 +LIMIT ?2 +OFFSET ?1 ` -type GetTokenTransfersByFromAccountParams struct { - FromAccount types.AccountID - Offset int64 - Limit int64 +type SearchTokenTransfersParams struct { + Offset int64 + Limit int64 + FromOrToAccount interface{} + FromAccount interface{} + ToAccount interface{} } -func (q *Queries) GetTokenTransfersByFromAccount(ctx context.Context, arg GetTokenTransfersByFromAccountParams) ([]TokenTransfer, error) { - rows, err := q.query(ctx, q.getTokenTransfersByFromAccountStmt, getTokenTransfersByFromAccount, arg.FromAccount, arg.Offset, arg.Limit) - if err != nil { - return nil, err - } - defer rows.Close() - var items []TokenTransfer - for rows.Next() { - var i TokenTransfer - if err := rows.Scan( - &i.TxHash, - &i.BlockHeight, - &i.FromAccount, - &i.ToAccount, - &i.Amount, - &i.TransferTime, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const getTokenTransfersByToAccount = `-- name: GetTokenTransfersByToAccount :many -; - -SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time FROM token_transfers -WHERE to_account = ?1 -ORDER BY transfer_time DESC -LIMIT ?3 -OFFSET ?2 -` - -type GetTokenTransfersByToAccountParams struct { - ToAccount types.AccountID - Offset int64 - Limit int64 +type SearchTokenTransfersRow struct { + TxHash []byte + BlockHeight int64 + FromAccount []byte + ToAccount []byte + Amount int64 + TransferTime time.Time + TotalCount int64 } -func (q *Queries) GetTokenTransfersByToAccount(ctx context.Context, arg GetTokenTransfersByToAccountParams) ([]TokenTransfer, error) { - rows, err := q.query(ctx, q.getTokenTransfersByToAccountStmt, getTokenTransfersByToAccount, arg.ToAccount, arg.Offset, arg.Limit) +func (q *Queries) SearchTokenTransfers(ctx context.Context, arg SearchTokenTransfersParams) ([]SearchTokenTransfersRow, error) { + rows, err := q.query(ctx, q.searchTokenTransfersStmt, searchTokenTransfers, + arg.Offset, + arg.Limit, + arg.FromOrToAccount, + arg.FromAccount, + arg.ToAccount, + ) if err != nil { return nil, err } defer rows.Close() - var items []TokenTransfer + var items []SearchTokenTransfersRow for rows.Next() { - var i TokenTransfer + var i SearchTokenTransfersRow if err := rows.Scan( &i.TxHash, &i.BlockHeight, @@ -155,6 +136,7 @@ func (q *Queries) GetTokenTransfersByToAccount(ctx context.Context, arg GetToken &i.ToAccount, &i.Amount, &i.TransferTime, + &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index d431d4cb6..c84b78c27 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -709,31 +709,6 @@ func (idx *Indexer) OnCensusUpdate(pid, _ []byte, _ string, _ uint64) { idx.blockUpdateProcs[string(pid)] = true } -// GetTokenTransfersByFromAccount returns all the token transfers made from a given account -// from the database, ordered by timestamp and paginated by maxItems and offset -func (idx *Indexer) GetTokenTransfersByFromAccount(from []byte, offset, maxItems int32) ([]*indexertypes.TokenTransferMeta, error) { - ttFromDB, err := idx.readOnlyQuery.GetTokenTransfersByFromAccount(context.TODO(), indexerdb.GetTokenTransfersByFromAccountParams{ - FromAccount: from, - Limit: int64(maxItems), - Offset: int64(offset), - }) - if err != nil { - return nil, err - } - tt := []*indexertypes.TokenTransferMeta{} - for _, t := range ttFromDB { - tt = append(tt, &indexertypes.TokenTransferMeta{ - Amount: uint64(t.Amount), - From: t.FromAccount, - To: t.ToAccount, - Height: uint64(t.BlockHeight), - TxHash: t.TxHash, - Timestamp: t.TransferTime, - }) - } - return tt, nil -} - // OnSpendTokens indexes a token spending event. func (idx *Indexer) OnSpendTokens(address []byte, txType models.TxType, cost uint64, reference string) { idx.blockMu.Lock() @@ -789,47 +764,42 @@ func (idx *Indexer) TokenFeesList(limit, offset int, txType, reference, fromAcco return list, uint64(results[0].TotalCount), nil } -// GetTokenTransfersByToAccount returns all the token transfers made to a given account -// from the database, ordered by timestamp and paginated by maxItems and offset -func (idx *Indexer) GetTokenTransfersByToAccount(to []byte, offset, maxItems int32) ([]*indexertypes.TokenTransferMeta, error) { - ttFromDB, err := idx.readOnlyQuery.GetTokenTransfersByToAccount(context.TODO(), indexerdb.GetTokenTransfersByToAccountParams{ - ToAccount: to, - Limit: int64(maxItems), - Offset: int64(offset), +// TokenTransfersList returns all the token transfers, made to and/or from a given account +// (all optional filters), ordered by timestamp and paginated by limit and offset +func (idx *Indexer) TokenTransfersList(limit, offset int, fromOrToAccount, fromAccount, toAccount string) ( + []*indexertypes.TokenTransferMeta, uint64, error, +) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + results, err := idx.readOnlyQuery.SearchTokenTransfers(context.TODO(), indexerdb.SearchTokenTransfersParams{ + Limit: int64(limit), + Offset: int64(offset), + FromOrToAccount: fromOrToAccount, + FromAccount: fromAccount, + ToAccount: toAccount, }) if err != nil { - return nil, err + return nil, 0, err } - tt := []*indexertypes.TokenTransferMeta{} - for _, t := range ttFromDB { - tt = append(tt, &indexertypes.TokenTransferMeta{ - Amount: uint64(t.Amount), - From: t.FromAccount, - To: t.ToAccount, - Height: uint64(t.BlockHeight), - TxHash: t.TxHash, - Timestamp: t.TransferTime, + list := []*indexertypes.TokenTransferMeta{} + for _, row := range results { + list = append(list, &indexertypes.TokenTransferMeta{ + Amount: uint64(row.Amount), + From: row.FromAccount, + To: row.ToAccount, + Height: uint64(row.BlockHeight), + TxHash: row.TxHash, + Timestamp: row.TransferTime, }) } - return tt, nil -} - -// GetTokenTransfersByAccount returns all the token transfers made to and from a given account -// from the database, ordered by timestamp and paginated by maxItems and offset -func (idx *Indexer) GetTokenTransfersByAccount(acc []byte, offset, maxItems int32) (indexertypes.TokenTransfersAccount, error) { - transfersTo, err := idx.GetTokenTransfersByToAccount(acc, offset, maxItems) - if err != nil { - return indexertypes.TokenTransfersAccount{}, err - } - transfersFrom, err := idx.GetTokenTransfersByFromAccount(acc, offset, maxItems) - if err != nil { - return indexertypes.TokenTransfersAccount{}, err + if len(results) == 0 { + return list, 0, nil } - - return indexertypes.TokenTransfersAccount{ - Received: transfersTo, - Sent: transfersFrom, - }, nil + return list, uint64(results[0].TotalCount), nil } // CountTokenTransfersByAccount returns the count all the token transfers made from a given account diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 5db21ae30..851c16ad0 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1650,22 +1650,29 @@ func TestTokenTransfers(t *testing.T) { app.AdvanceTestBlock() // acct 1 must have only one token transfer received - acc1Tokentx, err := idx.GetTokenTransfersByToAccount(keys[1].Address().Bytes(), 0, 10) + acc1Tokentx, _, err := idx.TokenTransfersList(10, 0, "", "", hex.EncodeToString(keys[1].Address().Bytes())) qt.Assert(t, err, qt.IsNil) qt.Assert(t, len(acc1Tokentx), qt.Equals, 1) qt.Assert(t, acc1Tokentx[0].Amount, qt.Equals, uint64(5)) // acct 2 must two token transfers received - acc2Tokentx, err := idx.GetTokenTransfersByToAccount(keys[2].Address().Bytes(), 0, 10) + acc2Tokentx, _, err := idx.TokenTransfersList(10, 0, "", "", hex.EncodeToString(keys[2].Address().Bytes())) qt.Assert(t, err, qt.IsNil) qt.Assert(t, len(acc2Tokentx), qt.Equals, 2) qt.Assert(t, acc2Tokentx[0].Amount, qt.Equals, uint64(95)) qt.Assert(t, acc2Tokentx[1].Amount, qt.Equals, uint64(18)) // acct 0 must zero token transfers received - acc0Tokentx, err := idx.GetTokenTransfersByToAccount(keys[0].Address().Bytes(), 0, 10) + acc0Tokentx, _, err := idx.TokenTransfersList(10, 0, "", "", hex.EncodeToString(keys[0].Address().Bytes())) qt.Assert(t, err, qt.IsNil) qt.Assert(t, len(acc0Tokentx), qt.Equals, 0) + + // acct 1 must have two token transfer received or sent + acc1TokentxFromOrTo, _, err := idx.TokenTransfersList(10, 0, hex.EncodeToString(keys[1].Address().Bytes()), "", "") + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, len(acc1TokentxFromOrTo), qt.Equals, 2) + qt.Assert(t, acc1TokentxFromOrTo[0].Amount, qt.Equals, uint64(5)) + qt.Assert(t, acc1TokentxFromOrTo[1].Amount, qt.Equals, uint64(95)) } // friendlyResults translates votes into a matrix of strings diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 1ce755c16..d0d2b29aa 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -209,7 +209,7 @@ type TokenTransferMeta struct { Amount uint64 `json:"amount"` From types.AccountID `json:"from"` Height uint64 `json:"height"` - TxHash types.Hash `json:"txHash"` + TxHash types.HexBytes `json:"txHash"` Timestamp time.Time `json:"timestamp"` To types.AccountID `json:"to"` } diff --git a/vochain/indexer/queries/token_transfers.sql b/vochain/indexer/queries/token_transfers.sql index 345e82dc7..fdc6cee08 100644 --- a/vochain/indexer/queries/token_transfers.sql +++ b/vochain/indexer/queries/token_transfers.sql @@ -12,22 +12,24 @@ SELECT * FROM token_transfers WHERE tx_hash = ? LIMIT 1; --- name: GetTokenTransfersByFromAccount :many -SELECT * FROM token_transfers -WHERE from_account = sqlc.arg(from_account) -ORDER BY transfer_time DESC -LIMIT sqlc.arg(limit) -OFFSET sqlc.arg(offset) -; - --- name: GetTokenTransfersByToAccount :many -SELECT * FROM token_transfers -WHERE to_account = sqlc.arg(to_account) +-- name: SearchTokenTransfers :many +WITH results AS ( + SELECT * + FROM token_transfers + WHERE ( + (sqlc.arg(from_or_to_account) = '' OR ( + LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account)) + OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(from_or_to_account)) + )) + AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) + AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account))) + ) +) +SELECT *, COUNT(*) OVER() AS total_count +FROM results ORDER BY transfer_time DESC LIMIT sqlc.arg(limit) -OFFSET sqlc.arg(offset) -; - +OFFSET sqlc.arg(offset); -- name: CountTokenTransfersByAccount :one SELECT COUNT(*) FROM token_transfers From 03b3863e46e469b383008be1dc7c11e02af737d0 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 7 Aug 2024 11:33:20 +0200 Subject: [PATCH 2/3] api: endpoint /accounts now supports filtering by partial accountId --- api/accounts.go | 13 ++++++--- api/api_types.go | 1 + vochain/indexer/db/account.sql.go | 41 +++++++++++++++++++---------- vochain/indexer/db/db.go | 20 +++++++------- vochain/indexer/indexer.go | 12 +++++---- vochain/indexer/indexer_test.go | 4 +-- vochain/indexer/queries/account.sql | 22 ++++++++++++---- 7 files changed, 73 insertions(+), 40 deletions(-) diff --git a/api/accounts.go b/api/accounts.go index db79a7998..baab86251 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -534,6 +534,7 @@ func (a *API) accountListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC params, err := parseAccountParams( ctx.URLParam(ParamPage), "", + "", ) if err != nil { return err @@ -548,14 +549,16 @@ func (a *API) accountListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC // @Tags Accounts // @Accept json // @Produce json -// @Param page query number false "Page" -// @Param limit query number false "Items per page" -// @Success 200 {object} AccountsList +// @Param page query number false "Page" +// @Param limit query number false "Items per page" +// @Param accountId query string false "Filter by partial accountId" +// @Success 200 {object} AccountsList // @Router /accounts [get] func (a *API) accountListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { params, err := parseAccountParams( ctx.QueryParam(ParamPage), ctx.QueryParam(ParamLimit), + ctx.QueryParam(ParamAccountId), ) if err != nil { return err @@ -571,6 +574,7 @@ func (a *API) sendAccountList(ctx *httprouter.HTTPContext, params *AccountParams accounts, total, err := a.indexer.AccountList( params.Limit, params.Page*params.Limit, + params.AccountID, ) if err != nil { return ErrIndexerQueryFailed.WithErr(err) @@ -589,7 +593,7 @@ func (a *API) sendAccountList(ctx *httprouter.HTTPContext, params *AccountParams } // parseAccountParams returns an AccountParams filled with the passed params -func parseAccountParams(paramPage, paramLimit string) (*AccountParams, error) { +func parseAccountParams(paramPage, paramLimit, paramAccountID string) (*AccountParams, error) { pagination, err := parsePaginationParams(paramPage, paramLimit) if err != nil { return nil, err @@ -597,5 +601,6 @@ func parseAccountParams(paramPage, paramLimit string) (*AccountParams, error) { return &AccountParams{ PaginationParams: pagination, + AccountID: util.TrimHex(paramAccountID), }, nil } diff --git a/api/api_types.go b/api/api_types.go index 220bb1cdb..778b8a9cc 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -40,6 +40,7 @@ type OrganizationParams struct { // AccountParams allows the client to filter accounts type AccountParams struct { PaginationParams + AccountID string `json:"accountId,omitempty"` } // TransactionParams allows the client to filter transactions diff --git a/vochain/indexer/db/account.sql.go b/vochain/indexer/db/account.sql.go index 6e1691e5d..9280ea315 100644 --- a/vochain/indexer/db/account.sql.go +++ b/vochain/indexer/db/account.sql.go @@ -39,35 +39,48 @@ func (q *Queries) CreateAccount(ctx context.Context, arg CreateAccountParams) (s return q.exec(ctx, q.createAccountStmt, createAccount, arg.Account, arg.Balance, arg.Nonce) } -const getListAccounts = `-- name: GetListAccounts :many -SELECT account, balance, nonce, - COUNT(*) OVER() AS total_count -FROM accounts +const searchAccounts = `-- name: SearchAccounts :many +WITH results AS ( + SELECT account, balance, nonce + FROM accounts + WHERE ( + ( + ?3 = '' + OR (LENGTH(?3) = 40 AND LOWER(HEX(account)) = LOWER(?3)) + OR (LENGTH(?3) < 40 AND INSTR(LOWER(HEX(account)), LOWER(?3)) > 0) + -- TODO: consider keeping an account_hex column for faster searches + ) + ) +) +SELECT account, balance, nonce, COUNT(*) OVER() AS total_count +FROM results ORDER BY balance DESC -LIMIT ? OFFSET ? +LIMIT ?2 +OFFSET ?1 ` -type GetListAccountsParams struct { - Limit int64 - Offset int64 +type SearchAccountsParams struct { + Offset int64 + Limit int64 + AccountIDSubstr interface{} } -type GetListAccountsRow struct { - Account types.AccountID +type SearchAccountsRow struct { + Account []byte Balance int64 Nonce int64 TotalCount int64 } -func (q *Queries) GetListAccounts(ctx context.Context, arg GetListAccountsParams) ([]GetListAccountsRow, error) { - rows, err := q.query(ctx, q.getListAccountsStmt, getListAccounts, arg.Limit, arg.Offset) +func (q *Queries) SearchAccounts(ctx context.Context, arg SearchAccountsParams) ([]SearchAccountsRow, error) { + rows, err := q.query(ctx, q.searchAccountsStmt, searchAccounts, arg.Offset, arg.Limit, arg.AccountIDSubstr) if err != nil { return nil, err } defer rows.Close() - var items []GetListAccountsRow + var items []SearchAccountsRow for rows.Next() { - var i GetListAccountsRow + var i SearchAccountsRow if err := rows.Scan( &i.Account, &i.Balance, diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index acd06f7ed..3695e8b73 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -66,9 +66,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) } - if q.getListAccountsStmt, err = db.PrepareContext(ctx, getListAccounts); err != nil { - return nil, fmt.Errorf("error preparing query GetListAccounts: %w", err) - } if q.getProcessStmt, err = db.PrepareContext(ctx, getProcess); err != nil { return nil, fmt.Errorf("error preparing query GetProcess: %w", err) } @@ -96,6 +93,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) } + if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { + return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) + } if q.searchEntitiesStmt, err = db.PrepareContext(ctx, searchEntities); err != nil { return nil, fmt.Errorf("error preparing query SearchEntities: %w", err) } @@ -207,11 +207,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getEntityCountStmt: %w", cerr) } } - if q.getListAccountsStmt != nil { - if cerr := q.getListAccountsStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getListAccountsStmt: %w", cerr) - } - } if q.getProcessStmt != nil { if cerr := q.getProcessStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getProcessStmt: %w", cerr) @@ -257,6 +252,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getVoteStmt: %w", cerr) } } + if q.searchAccountsStmt != nil { + if cerr := q.searchAccountsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) + } + } if q.searchEntitiesStmt != nil { if cerr := q.searchEntitiesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchEntitiesStmt: %w", cerr) @@ -370,7 +370,6 @@ type Queries struct { createVoteStmt *sql.Stmt getBlockStmt *sql.Stmt getEntityCountStmt *sql.Stmt - getListAccountsStmt *sql.Stmt getProcessStmt *sql.Stmt getProcessCountStmt *sql.Stmt getProcessIDsByFinalResultsStmt *sql.Stmt @@ -380,6 +379,7 @@ type Queries struct { getTransactionByHashStmt *sql.Stmt getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt getVoteStmt *sql.Stmt + searchAccountsStmt *sql.Stmt searchEntitiesStmt *sql.Stmt searchProcessesStmt *sql.Stmt searchTokenFeesStmt *sql.Stmt @@ -412,7 +412,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { createVoteStmt: q.createVoteStmt, getBlockStmt: q.getBlockStmt, getEntityCountStmt: q.getEntityCountStmt, - getListAccountsStmt: q.getListAccountsStmt, getProcessStmt: q.getProcessStmt, getProcessCountStmt: q.getProcessCountStmt, getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, @@ -422,6 +421,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTransactionByHashStmt: q.getTransactionByHashStmt, getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, getVoteStmt: q.getVoteStmt, + searchAccountsStmt: q.searchAccountsStmt, searchEntitiesStmt: q.searchEntitiesStmt, searchProcessesStmt: q.searchProcessesStmt, searchTokenFeesStmt: q.searchTokenFeesStmt, diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index c84b78c27..46d77f0a3 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -814,17 +814,19 @@ func (idx *Indexer) CountTotalAccounts() (uint64, error) { return uint64(count), err } -// AccountList retrieves all accounts. -func (idx *Indexer) AccountList(limit, offset int) ([]*indexertypes.Account, uint64, error) { +// AccountList returns a list of accounts, accountID is a partial or full hex string, +// and is optional (declared as zero-value will be ignored). +func (idx *Indexer) AccountList(limit, offset int, accountID string) ([]*indexertypes.Account, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } if limit <= 0 { return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) } - results, err := idx.readOnlyQuery.GetListAccounts(context.TODO(), indexerdb.GetListAccountsParams{ - Limit: int64(limit), - Offset: int64(offset), + results, err := idx.readOnlyQuery.SearchAccounts(context.TODO(), indexerdb.SearchAccountsParams{ + Limit: int64(limit), + Offset: int64(offset), + AccountIDSubstr: accountID, }) if err != nil { return nil, 0, err diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 851c16ad0..cdf425388 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1564,7 +1564,7 @@ func TestAccountsList(t *testing.T) { last := 0 for i := 0; i < int(totalAccs); i++ { - accts, _, err := idx.AccountList(10, last) + accts, _, err := idx.AccountList(10, last, "") qt.Assert(t, err, qt.IsNil) for j, acc := range accts { @@ -1587,7 +1587,7 @@ func TestAccountsList(t *testing.T) { app.AdvanceTestBlock() // verify the updated balance and nonce - accts, _, err := idx.AccountList(5, 0) + accts, _, err := idx.AccountList(5, 0, "") qt.Assert(t, err, qt.IsNil) // the account in the position 0 must be the updated account balance due it has the major balance // indexer query has order BY balance DESC diff --git a/vochain/indexer/queries/account.sql b/vochain/indexer/queries/account.sql index 630e14657..c4e570cea 100644 --- a/vochain/indexer/queries/account.sql +++ b/vochain/indexer/queries/account.sql @@ -3,12 +3,24 @@ REPLACE INTO accounts ( account, balance, nonce ) VALUES (?, ?, ?); --- name: GetListAccounts :many -SELECT *, - COUNT(*) OVER() AS total_count -FROM accounts +-- name: SearchAccounts :many +WITH results AS ( + SELECT * + FROM accounts + WHERE ( + ( + sqlc.arg(account_id_substr) = '' + OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr))) + OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0) + -- TODO: consider keeping an account_hex column for faster searches + ) + ) +) +SELECT *, COUNT(*) OVER() AS total_count +FROM results ORDER BY balance DESC -LIMIT ? OFFSET ?; +LIMIT sqlc.arg(limit) +OFFSET sqlc.arg(offset); -- name: CountAccounts :one SELECT COUNT(*) FROM accounts; \ No newline at end of file From ca3f73844ef44637a0f2a3e33403dbd3976a616a Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 7 Aug 2024 10:55:27 +0200 Subject: [PATCH 3/3] api: paginated endpoints now return ErrAccountNotFound, ErrElectionNotFound, ErrOrgNotFound * add tests to avoid regressions --- api/accounts.go | 4 ++ api/chain.go | 14 ++++ api/elections.go | 8 +++ api/vote.go | 4 ++ test/apierror_test.go | 144 +++++++++++++++++++++++++++++++++++++ vochain/indexer/indexer.go | 10 +++ vochain/indexer/process.go | 20 ++++++ 7 files changed, 204 insertions(+) diff --git a/api/accounts.go b/api/accounts.go index baab86251..70f4fa11a 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -571,6 +571,10 @@ func (a *API) accountListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext // // Errors returned are always of type APIerror. func (a *API) sendAccountList(ctx *httprouter.HTTPContext, params *AccountParams) error { + if params.AccountID != "" && !a.indexer.AccountExists(params.AccountID) { + return ErrAccountNotFound + } + accounts, total, err := a.indexer.AccountList( params.Limit, params.Page*params.Limit, diff --git a/api/chain.go b/api/chain.go index e029efbbe..b0ac84669 100644 --- a/api/chain.go +++ b/api/chain.go @@ -332,6 +332,10 @@ func (a *API) organizationListByFilterAndPageHandler(msg *apirest.APIdata, ctx * // // Errors returned are always of type APIerror. func (a *API) sendOrganizationList(ctx *httprouter.HTTPContext, params *OrganizationParams) error { + if params.OrganizationID != "" && !a.indexer.EntityExists(params.OrganizationID) { + return ErrOrgNotFound + } + orgs, total, err := a.indexer.EntityList( params.Limit, params.Page*params.Limit, @@ -1061,6 +1065,10 @@ func (a *API) chainFeesListByTypeAndPageHandler(_ *apirest.APIdata, ctx *httprou // // Errors returned are always of type APIerror. func (a *API) sendFeesList(ctx *httprouter.HTTPContext, params *FeesParams) error { + if params.AccountID != "" && !a.indexer.AccountExists(params.AccountID) { + return ErrAccountNotFound + } + fees, total, err := a.indexer.TokenFeesList( params.Limit, params.Page*params.Limit, @@ -1118,6 +1126,12 @@ func (a *API) chainTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTP // // Errors returned are always of type APIerror. func (a *API) sendTransfersList(ctx *httprouter.HTTPContext, params *TransfersParams) error { + for _, param := range []string{params.AccountID, params.AccountIDFrom, params.AccountIDTo} { + if param != "" && !a.indexer.AccountExists(param) { + return ErrAccountNotFound.With(param) + } + } + transfers, total, err := a.indexer.TokenTransfersList( params.Limit, params.Page*params.Limit, diff --git a/api/elections.go b/api/elections.go index bbf99ad84..88ae90acc 100644 --- a/api/elections.go +++ b/api/elections.go @@ -262,6 +262,14 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex // // Errors returned are always of type APIerror. func (a *API) sendElectionList(ctx *httprouter.HTTPContext, params *ElectionParams) error { + if params.ElectionID != "" && !a.indexer.ProcessExists(params.ElectionID) { + return ErrElectionNotFound + } + + if params.OrganizationID != "" && !a.indexer.EntityExists(params.OrganizationID) { + return ErrOrgNotFound + } + status, err := parseStatus(params.Status) if err != nil { return err diff --git a/api/vote.go b/api/vote.go index b1cdce257..33322b103 100644 --- a/api/vote.go +++ b/api/vote.go @@ -226,6 +226,10 @@ func (a *API) votesListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) // // Errors returned are always of type APIerror. func (a *API) sendVotesList(ctx *httprouter.HTTPContext, params *VoteParams) error { + if params.ElectionID != "" && !a.indexer.ProcessExists(params.ElectionID) { + return ErrElectionNotFound + } + votes, total, err := a.indexer.VoteList( params.Limit, params.Page*params.Limit, diff --git a/test/apierror_test.go b/test/apierror_test.go index 1a413d5c9..e1f967d8b 100644 --- a/test/apierror_test.go +++ b/test/apierror_test.go @@ -55,6 +55,18 @@ func TestAPIerror(t *testing.T) { args: args{"GET", nil, []string{"accounts", "0123456789012345678901234567890123456789", "elections", "count"}}, want: api.ErrOrgNotFound, }, + { + args: args{"GET", nil, []string{"accounts", "0123456789012345678901234567890123456789", "elections", "page", "0"}}, + want: api.ErrOrgNotFound, + }, + { + args: args{"GET", nil, []string{"accounts", "0123456789012345678901234567890123456789", "transfers", "page", "0"}}, + want: api.ErrAccountNotFound, + }, + { + args: args{"GET", nil, []string{"accounts", "0123456789012345678901234567890123456789", "fees", "page", "0"}}, + want: api.ErrAccountNotFound, + }, { args: args{"GET", nil, []string{"chain", "blocks", "1234"}}, want: api.ErrBlockNotFound, @@ -116,6 +128,10 @@ func TestAPIerror(t *testing.T) { args: args{"GET", nil, []string{"elections", "page", "-1"}}, want: api.ErrPageNotFound, }, + { + args: args{"GET", nil, []string{"elections", "0123456789012345678901234567890123456789", "votes", "page", "0"}}, + want: api.ErrElectionNotFound, + }, } for _, tt := range tests { t.Run(tt.want.Error(), func(t *testing.T) { @@ -128,3 +144,131 @@ func TestAPIerror(t *testing.T) { }) } } + +func TestAPIerrorWithQuery(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + // Block 1 + server.VochainAPP.AdvanceTestBlock() + + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + type args struct { + method string + jsonBody any + urlPath []string + query string + } + tests := []struct { + name string + args args + want apirest.APIerror + }{ + { + args: args{"GET", nil, []string{"accounts"}, "page=1234"}, + want: api.ErrPageNotFound, + }, + { + args: args{"GET", nil, []string{"accounts"}, "accountId=0123456789"}, + want: api.ErrAccountNotFound, + }, + { + args: args{"GET", nil, []string{"accounts"}, "accountId=0123456789&page=1234"}, + want: api.ErrAccountNotFound, + }, + { + args: args{"GET", nil, []string{"elections"}, "electionId=0123456789"}, + want: api.ErrElectionNotFound, + }, + { + args: args{"GET", nil, []string{"elections"}, "electionId=0123456789&page=1234"}, + want: api.ErrElectionNotFound, + }, + { + args: args{"GET", nil, []string{"elections"}, "organizationId=0123456789"}, + want: api.ErrOrgNotFound, + }, + { + args: args{"GET", nil, []string{"elections"}, "organizationId=0123456789&page=1234"}, + want: api.ErrOrgNotFound, + }, + { + args: args{"GET", nil, []string{"elections"}, "status=FOOBAR"}, + want: api.ErrParamStatusInvalid, + }, + { + args: args{"GET", nil, []string{"elections"}, "manuallyEnded=FOOBAR"}, + want: api.ErrCantParseBoolean, + }, + { + args: args{"GET", nil, []string{"chain", "transactions"}, "page=1234"}, + want: api.ErrPageNotFound, + }, + // TODO: not yet implemented + // { + // args: args{"GET", nil, []string{"chain", "transactions"}, "height=1234"}, + // want: api.ErrBlockNotFound, + // }, + { + args: args{"GET", nil, []string{"chain", "transactions"}, "height=FOOBAR"}, + want: api.ErrCantParseNumber, + }, + // TODO: should this endpoint check `type` is a sane value? + // { + // args: args{"GET", nil, []string{"chain", "transactions"}, "type=FOOBAR"}, + // want: api.ErrParamTypeInvalid, + // }, + { + args: args{"GET", nil, []string{"chain", "organizations"}, "organizationId=0123456789"}, + want: api.ErrOrgNotFound, + }, + { + args: args{"GET", nil, []string{"chain", "fees"}, "accountId=0123456789"}, + want: api.ErrAccountNotFound, + }, + // TODO: should this endpoint check `reference` matches something? + // { + // args: args{"GET", nil, []string{"chain", "fees"}, "reference=0123456789"}, + // want: api.ErrTransactionNotFound, + // }, + // TODO: should this endpoint check `type` is a sane value? + // { + // args: args{"GET", nil, []string{"chain", "fees"}, "type=FOOBAR"}, + // want: api.ErrParamTypeInvalid, + // }, + { + args: args{"GET", nil, []string{"votes"}, "electionId=0123456789"}, + want: api.ErrElectionNotFound, + }, + { + args: args{"GET", nil, []string{"chain", "transfers"}, "accountId=0123456789"}, + want: api.ErrAccountNotFound, + }, + { + args: args{"GET", nil, []string{"chain", "transfers"}, "accountIdFrom=0123456789"}, + want: api.ErrAccountNotFound, + }, + { + args: args{"GET", nil, []string{"chain", "transfers"}, "accountIdTo=0123456789"}, + want: api.ErrAccountNotFound, + }, + } + for _, tt := range tests { + t.Run(tt.want.Error(), func(t *testing.T) { + resp, code := c.RequestWithQuery(tt.args.method, tt.args.jsonBody, tt.args.query, tt.args.urlPath...) + t.Logf("httpstatus=%d body=%s", code, resp) + qt.Assert(t, code, qt.Equals, tt.want.HTTPstatus) + apierr := &apirest.APIerror{} + qt.Assert(t, json.Unmarshal(resp, apierr), qt.IsNil) + qt.Assert(t, apierr.Code, qt.Equals, tt.want.Code) + }) + } +} diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 46d77f0a3..0981ba438 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -844,3 +844,13 @@ func (idx *Indexer) AccountList(limit, offset int, accountID string) ([]*indexer } return list, uint64(results[0].TotalCount), nil } + +// AccountExists returns whether the passed accountID matches at least one row in the db. +// accountID is a partial or full hex string. +func (idx *Indexer) AccountExists(accountID string) bool { + _, count, err := idx.AccountList(1, 0, accountID) + if err != nil { + log.Errorw(err, "indexer query failed") + } + return count > 0 +} diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index ee02f5ce8..5a875c05b 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -87,6 +87,16 @@ func (idx *Indexer) ProcessList(limit, offset int, entityID string, processID st return list, uint64(results[0].TotalCount), nil } +// ProcessExists returns whether the passed processID matches at least one row in the db. +// processID is a partial or full hex string. +func (idx *Indexer) ProcessExists(processID string) bool { + _, count, err := idx.ProcessList(1, 0, "", processID, 0, 0, 0, nil, nil, nil) + if err != nil { + log.Errorw(err, "indexer query failed") + } + return count > 0 +} + // CountTotalProcesses returns the total number of processes indexed. func (idx *Indexer) CountTotalProcesses() uint64 { count, err := idx.readOnlyQuery.GetProcessCount(context.TODO()) @@ -128,6 +138,16 @@ func (idx *Indexer) EntityList(limit, offset int, entityID string) ([]indexertyp return list, uint64(results[0].TotalCount), nil } +// EntityExists returns whether the passed entityID matches at least one row in the db. +// entityID is a partial or full hex string. +func (idx *Indexer) EntityExists(entityID string) bool { + _, count, err := idx.EntityList(1, 0, entityID) + if err != nil { + log.Errorw(err, "indexer query failed") + } + return count > 0 +} + // CountTotalEntities return the total number of entities indexed by the indexer func (idx *Indexer) CountTotalEntities() uint64 { count, err := idx.readOnlyQuery.GetEntityCount(context.TODO())