Skip to content

Commit

Permalink
ingest: asset_types and memo_type columns added (#29)
Browse files Browse the repository at this point in the history 
What
Adds three columns to the ingest_payments table, to be persisted when running ingestion.
Columns: src_asset_type, dest_asset_type and memo_type.

Why
These are relevant columns to the Vibrant wallet and potentially to other wallets as well.
  • Loading branch information
@daniel-burghardt
daniel-burghardt authored Jul 1, 2024
1 parent 8a5f525 commit 39df5f3
Show file tree
Hide file tree
Showing 13 changed files with 249 additions and 89 deletions.
2 changes: 1 addition & 1 deletion internal/data/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
func InsertTestPayments(t *testing.T, ctx context.Context, payments []Payment, connectionPool db.ConnectionPool) {
t.Helper()

const query = `INSERT INTO ingest_payments (operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo) VALUES (:operation_id, :operation_type, :transaction_id, :transaction_hash, :from_address, :to_address, :src_asset_code, :src_asset_issuer, :src_amount, :dest_asset_code, :dest_asset_issuer, :dest_amount, :created_at, :memo);`
const query = `INSERT INTO ingest_payments (operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_asset_type, src_amount, dest_asset_code, dest_asset_issuer, dest_asset_type, dest_amount, created_at, memo, memo_type) VALUES (:operation_id, :operation_type, :transaction_id, :transaction_hash, :from_address, :to_address, :src_asset_code, :src_asset_issuer, :src_asset_type, :src_amount, :dest_asset_code, :dest_asset_issuer, :dest_asset_type, :dest_amount, :created_at, :memo, :memo_type);`
_, err := connectionPool.NamedExecContext(ctx, query, payments)
require.NoError(t, err)
}
18 changes: 12 additions & 6 deletions internal/data/payments.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@ type Payment struct {
ToAddress string `db:"to_address" json:"toAddress"`
SrcAssetCode string `db:"src_asset_code" json:"srcAssetCode"`
SrcAssetIssuer string `db:"src_asset_issuer" json:"srcAssetIssuer"`
SrcAssetType string `db:"src_asset_type" json:"srcAssetType"`
SrcAmount int64 `db:"src_amount" json:"srcAmount"`
DestAssetCode string `db:"dest_asset_code" json:"destAssetCode"`
DestAssetIssuer string `db:"dest_asset_issuer" json:"destAssetIssuer"`
DestAssetType string `db:"dest_asset_type" json:"destAssetType"`
DestAmount int64 `db:"dest_amount" json:"destAmount"`
CreatedAt time.Time `db:"created_at" json:"createdAt"`
Memo *string `db:"memo" json:"memo"`
MemoType string `db:"memo_type" json:"memoType"`
}

func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) {
Expand Down Expand Up @@ -61,10 +64,10 @@ func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName
func (m *PaymentModel) AddPayment(ctx context.Context, tx db.Transaction, payment Payment) error {
const query = `
INSERT INTO ingest_payments (
operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount,
dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo
operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_asset_type, src_amount,
dest_asset_code, dest_asset_issuer, dest_asset_type, dest_amount, created_at, memo, memo_type
)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17
WHERE EXISTS (
SELECT 1 FROM accounts WHERE stellar_address IN ($5, $6)
)
Expand All @@ -76,16 +79,19 @@ func (m *PaymentModel) AddPayment(ctx context.Context, tx db.Transaction, paymen
to_address = EXCLUDED.to_address,
src_asset_code = EXCLUDED.src_asset_code,
src_asset_issuer = EXCLUDED.src_asset_issuer,
src_asset_type = EXCLUDED.src_asset_type,
src_amount = EXCLUDED.src_amount,
dest_asset_code = EXCLUDED.dest_asset_code,
dest_asset_issuer = EXCLUDED.dest_asset_issuer,
dest_asset_type = EXCLUDED.dest_asset_type,
dest_amount = EXCLUDED.dest_amount,
created_at = EXCLUDED.created_at,
memo = EXCLUDED.memo
memo = EXCLUDED.memo,
memo_type = EXCLUDED.memo_type
;
`
_, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.FromAddress, payment.ToAddress, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount,
payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAmount, payment.CreatedAt, payment.Memo)
_, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.FromAddress, payment.ToAddress, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAssetType, payment.SrcAmount,
payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAssetType, payment.DestAmount, payment.CreatedAt, payment.Memo, payment.MemoType)
if err != nil {
return fmt.Errorf("inserting payment: %w", err)
}
Expand Down
88 changes: 64 additions & 24 deletions internal/data/payments_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ package data

import (
"context"
"database/sql"
"testing"
"time"

"github.com/stellar/go/xdr"
"github.com/stellar/wallet-backend/internal/db"
"github.com/stellar/wallet-backend/internal/db/dbtest"
"github.com/stellar/wallet-backend/internal/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand All @@ -30,34 +33,35 @@ func TestPaymentModelAddPayment(t *testing.T) {
)
payment := Payment{
OperationID: 2120562792996865,
OperationType: "OperationTypePayment",
OperationType: xdr.OperationTypePayment.String(),
TransactionID: 2120562792996864,
TransactionHash: "a3daffa64dc46db84888b1206dc8014a480042e7fe8b19fd5d05465709f4e887",
FromAddress: fromAddress,
ToAddress: toAddress,
SrcAssetCode: "USDC",
SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5",
SrcAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(),
SrcAmount: 500000000,
DestAssetCode: "USDC",
DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5",
DestAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(),
DestAmount: 500000000,
CreatedAt: time.Date(2023, 12, 15, 1, 0, 0, 0, time.UTC),
Memo: nil,
MemoType: xdr.MemoTypeMemoNone.String(),
}

addPayment := func() {
addPayment := func(p Payment) {
err := db.RunInTransaction(ctx, m.DB, nil, func(dbTx db.Transaction) error {
return m.AddPayment(ctx, dbTx, payment)
return m.AddPayment(ctx, dbTx, p)
})
require.NoError(t, err)
}

fetchPaymentInserted := func() bool {
var inserted bool
err := dbConnectionPool.QueryRowxContext(ctx, "SELECT EXISTS(SELECT 1 FROM ingest_payments)").Scan(&inserted)
require.NoError(t, err)

return inserted
fetchPayment := func() (Payment, error) {
var dbPayment Payment
err := dbConnectionPool.GetContext(ctx, &dbPayment, "SELECT * FROM ingest_payments")
return dbPayment, err
}

cleanUpDB := func() {
Expand All @@ -66,10 +70,10 @@ func TestPaymentModelAddPayment(t *testing.T) {
}

t.Run("unkown_address", func(t *testing.T) {
addPayment()
addPayment(payment)

inserted := fetchPaymentInserted()
assert.False(t, inserted)
_, err := fetchPayment()
assert.ErrorIs(t, err, sql.ErrNoRows)

cleanUpDB()
})
Expand All @@ -78,10 +82,11 @@ func TestPaymentModelAddPayment(t *testing.T) {
_, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, fromAddress)
require.NoError(t, err)

addPayment()
addPayment(payment)

inserted := fetchPaymentInserted()
assert.True(t, inserted)
dbPayment, err := fetchPayment()
require.NoError(t, err)
assert.Equal(t, payment, dbPayment)

cleanUpDB()
})
Expand All @@ -90,10 +95,45 @@ func TestPaymentModelAddPayment(t *testing.T) {
_, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, toAddress)
require.NoError(t, err)

addPayment()
addPayment(payment)

dbPayment, err := fetchPayment()
require.NoError(t, err)
assert.Equal(t, payment, dbPayment)

cleanUpDB()
})

t.Run("to_known_address_update_on_reingestion", func(t *testing.T) {
updatedPayment := Payment{
OperationID: payment.OperationID, // Same OperationID
OperationType: xdr.OperationTypePathPaymentStrictSend.String(),
TransactionID: 2120562792996865,
TransactionHash: "a3daffa64dc46db84888b1206dc8014a480042e7fe8b19fd5d05465709f4e888",
FromAddress: fromAddress,
ToAddress: toAddress,
SrcAssetCode: "XLM",
SrcAssetIssuer: "",
SrcAssetType: xdr.AssetTypeAssetTypeCreditAlphanum12.String(),
SrcAmount: 300000000,
DestAssetCode: "ARST",
DestAssetIssuer: "GB7TAYRUZGE6TVT7NHP5SMIZRNQA6PLM423EYISAOAP3MKYIQMVYP2JO",
DestAssetType: xdr.AssetTypeAssetTypeCreditAlphanum12.String(),
DestAmount: 700000000,
CreatedAt: time.Date(2023, 12, 16, 1, 0, 0, 0, time.UTC),
Memo: utils.PointOf("diff"),
MemoType: xdr.MemoTypeMemoText.String(),
}

_, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, toAddress)
require.NoError(t, err)

addPayment(payment)
addPayment(updatedPayment)

inserted := fetchPaymentInserted()
assert.True(t, inserted)
dbPayment, err := fetchPayment()
require.NoError(t, err)
assert.Equal(t, updatedPayment, dbPayment)

cleanUpDB()
})
Expand Down Expand Up @@ -146,7 +186,7 @@ func TestPaymentModelUpdateLatestLedgerSynced(t *testing.T) {
assert.Equal(t, uint32(123), lastSyncedLedger)
}

func TestPaymentModelGetPayments(t *testing.T) {
func TestPaymentModelGetPaymentsPaginated(t *testing.T) {
dbt := dbtest.Open(t)
defer dbt.Close()
dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
Expand All @@ -159,11 +199,11 @@ func TestPaymentModelGetPayments(t *testing.T) {
}

dbPayments := []Payment{
{OperationID: 1, OperationType: "OperationTypePayment", TransactionID: 11, TransactionHash: "c370ff20144e4c96b17432b8d14664c1", FromAddress: "GAZ37ZO4TU3H", ToAddress: "GDD2HQO6IOFT", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAmount: 10, DestAssetCode: "XLM", DestAssetIssuer: "", DestAmount: 10, CreatedAt: time.Date(2024, 6, 21, 0, 0, 0, 0, time.UTC), Memo: nil},
{OperationID: 2, OperationType: "OperationTypePayment", TransactionID: 22, TransactionHash: "30850d8fc7d1439782885103390cd975", FromAddress: "GBZ5Q56JKHJQ", ToAddress: "GASV72SENBSY", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAmount: 20, DestAssetCode: "XLM", DestAssetIssuer: "", DestAmount: 20, CreatedAt: time.Date(2024, 6, 22, 0, 0, 0, 0, time.UTC), Memo: nil},
{OperationID: 3, OperationType: "OperationTypePayment", TransactionID: 33, TransactionHash: "d9521ed7057d4d1e9b9dd22ab515cbf1", FromAddress: "GAYFAYPOECBT", ToAddress: "GDWDPNMALNIT", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAmount: 30, DestAssetCode: "XLM", DestAssetIssuer: "", DestAmount: 30, CreatedAt: time.Date(2024, 6, 23, 0, 0, 0, 0, time.UTC), Memo: nil},
{OperationID: 4, OperationType: "OperationTypePayment", TransactionID: 44, TransactionHash: "2af98496a86741c6a6814200e06027fd", FromAddress: "GACKTNR2QQXU", ToAddress: "GBZ5KUZHAAVI", SrcAssetCode: "USDC", SrcAssetIssuer: "GAHLU7PDIQMZ", SrcAmount: 40, DestAssetCode: "USDC", DestAssetIssuer: "GAHLU7PDIQMZ", DestAmount: 40, CreatedAt: time.Date(2024, 6, 24, 0, 0, 0, 0, time.UTC), Memo: nil},
{OperationID: 5, OperationType: "OperationTypePayment", TransactionID: 55, TransactionHash: "edfab36f9f104c4fb74b549de44cfbcc", FromAddress: "GA4CMYJEC5W5", ToAddress: "GAZ37ZO4TU3H", SrcAssetCode: "USDC", SrcAssetIssuer: "GAHLU7PDIQMZ", SrcAmount: 50, DestAssetCode: "USDC", DestAssetIssuer: "GAHLU7PDIQMZ", DestAmount: 50, CreatedAt: time.Date(2024, 6, 25, 0, 0, 0, 0, time.UTC), Memo: nil},
{OperationID: 1, OperationType: xdr.OperationTypePayment.String(), TransactionID: 11, TransactionHash: "c370ff20144e4c96b17432b8d14664c1", FromAddress: "GAZ37ZO4TU3H", ToAddress: "GDD2HQO6IOFT", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAssetType: xdr.AssetTypeAssetTypeNative.String(), SrcAmount: 10, DestAssetCode: "XLM", DestAssetIssuer: "", DestAssetType: xdr.AssetTypeAssetTypeNative.String(), DestAmount: 10, CreatedAt: time.Date(2024, 6, 21, 0, 0, 0, 0, time.UTC), Memo: nil, MemoType: xdr.MemoTypeMemoNone.String()},
{OperationID: 2, OperationType: xdr.OperationTypePayment.String(), TransactionID: 22, TransactionHash: "30850d8fc7d1439782885103390cd975", FromAddress: "GBZ5Q56JKHJQ", ToAddress: "GASV72SENBSY", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAssetType: xdr.AssetTypeAssetTypeNative.String(), SrcAmount: 20, DestAssetCode: "XLM", DestAssetIssuer: "", DestAssetType: xdr.AssetTypeAssetTypeNative.String(), DestAmount: 20, CreatedAt: time.Date(2024, 6, 22, 0, 0, 0, 0, time.UTC), Memo: nil, MemoType: xdr.MemoTypeMemoNone.String()},
{OperationID: 3, OperationType: xdr.OperationTypePayment.String(), TransactionID: 33, TransactionHash: "d9521ed7057d4d1e9b9dd22ab515cbf1", FromAddress: "GAYFAYPOECBT", ToAddress: "GDWDPNMALNIT", SrcAssetCode: "XLM", SrcAssetIssuer: "", SrcAssetType: xdr.AssetTypeAssetTypeNative.String(), SrcAmount: 30, DestAssetCode: "XLM", DestAssetIssuer: "", DestAssetType: xdr.AssetTypeAssetTypeNative.String(), DestAmount: 30, CreatedAt: time.Date(2024, 6, 23, 0, 0, 0, 0, time.UTC), Memo: nil, MemoType: xdr.MemoTypeMemoNone.String()},
{OperationID: 4, OperationType: xdr.OperationTypePayment.String(), TransactionID: 44, TransactionHash: "2af98496a86741c6a6814200e06027fd", FromAddress: "GACKTNR2QQXU", ToAddress: "GBZ5KUZHAAVI", SrcAssetCode: "USDC", SrcAssetIssuer: "GAHLU7PDIQMZ", SrcAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(), SrcAmount: 40, DestAssetCode: "USDC", DestAssetIssuer: "GAHLU7PDIQMZ", DestAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(), DestAmount: 40, CreatedAt: time.Date(2024, 6, 24, 0, 0, 0, 0, time.UTC), Memo: nil, MemoType: xdr.MemoTypeMemoNone.String()},
{OperationID: 5, OperationType: xdr.OperationTypePayment.String(), TransactionID: 55, TransactionHash: "edfab36f9f104c4fb74b549de44cfbcc", FromAddress: "GA4CMYJEC5W5", ToAddress: "GAZ37ZO4TU3H", SrcAssetCode: "USDC", SrcAssetIssuer: "GAHLU7PDIQMZ", SrcAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(), SrcAmount: 50, DestAssetCode: "USDC", DestAssetIssuer: "GAHLU7PDIQMZ", DestAssetType: xdr.AssetTypeAssetTypeCreditAlphanum4.String(), DestAmount: 50, CreatedAt: time.Date(2024, 6, 25, 0, 0, 0, 0, time.UTC), Memo: nil, MemoType: xdr.MemoTypeMemoNone.String()},
}
InsertTestPayments(t, ctx, dbPayments, dbConnectionPool)

Expand Down
2 changes: 1 addition & 1 deletion internal/db/migrations/2024-05-22.0-ingest_payments.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ CREATE TABLE ingest_payments (
dest_asset_issuer text NOT NULL,
dest_amount bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
memo text,
memo text NULL,
PRIMARY KEY (operation_id)
);

Expand Down
13 changes: 13 additions & 0 deletions internal/db/migrations/2024-06-27.0-ingest_payments_2.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- +migrate Up

ALTER TABLE ingest_payments
ADD COLUMN src_asset_type text NOT NULL,
ADD COLUMN dest_asset_type text NOT NULL,
ADD COLUMN memo_type text NULL;

-- +migrate Down

ALTER TABLE ingest_payments
DROP COLUMN src_asset_type,
DROP COLUMN dest_asset_type,
DROP COLUMN memo_type;
Loading

0 comments on commit 39df5f3

Please sign in to comment.