From 0e99e9de941066b7f3094c0094e056d69b4e6976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 4 Nov 2024 22:27:07 -0600 Subject: [PATCH 01/37] wip(cat-gateway): add select and delete queries for purging volatile data --- .../bin/src/db/index/block/mod.rs | 1 + .../cql/delete_chain_root_for_role0_key.cql | 5 + .../cql/delete_chain_root_for_stake_addr.cql | 3 + .../cql/delete_chain_root_for_txn_id.cql | 5 + .../cql/delete_cip36_registration.cql | 6 + ...delete_cip36_registration_for_vote_key.cql | 7 + .../cql/delete_cip36_registration_invalid.cql | 5 + .../cql/delete_rbac509_registration.cql | 7 + .../cql/delete_stake_registration.cql | 6 + .../cql/delete_txi_by_txn_hashes.cql | 5 + .../cql/delete_txo_assets_by_stake_addr.cql | 8 + .../cql/delete_txo_by_stake_address.cql | 6 + ...delete_unstaked_txo_assets_by_txn_hash.cql | 6 + .../cql/delete_unstaked_txo_by_txn_hash.cql | 4 + .../cql/get_chain_root_for_role0_key.cql | 6 + .../cql/get_chain_root_for_stake_addr.cql | 3 + .../cql/get_chain_root_for_txn_id.cql | 6 + .../cql/get_cip36_registration.cql | 7 + .../get_cip36_registration_for_vote_key.cql | 8 + .../cql/get_rbac509_registration.cql | 8 + .../cql/get_stake_registration.cql | 7 + .../cql/get_txi_by_txn_hashes.cql | 5 + .../cql/get_txo_assets_by_stake_addr.cql | 9 + .../cql/get_txo_by_stake_address.cql | 7 + .../get_unstaked_txo_assets_by_txn_hash.cql | 7 + .../cql/get_unstaked_txo_by_txn_hash.cql | 5 + .../src/db/index/block/roll_forward/mod.rs | 131 +++++++++++ .../bin/src/db/index/queries/mod.rs | 210 +++++++++++++++++- 28 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql create mode 100644 catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs diff --git a/catalyst-gateway/bin/src/db/index/block/mod.rs b/catalyst-gateway/bin/src/db/index/block/mod.rs index 6f62abf0ff9..64a40853711 100644 --- a/catalyst-gateway/bin/src/db/index/block/mod.rs +++ b/catalyst-gateway/bin/src/db/index/block/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod certs; pub(crate) mod cip36; pub(crate) mod rbac509; +pub(crate) mod roll_forward; pub(crate) mod txi; pub(crate) mod txo; diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql new file mode 100644 index 00000000000..63ef2aedba8 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql @@ -0,0 +1,5 @@ +-- Delete all the chain roots for a role0 key +DELETE FROM chain_root_for_role0_key +WHERE role0_key = :role0_key + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql new file mode 100644 index 00000000000..41a776e4ef4 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql @@ -0,0 +1,3 @@ +-- Delete Chain Root For TX ID. RBAC 509 registrations. +DELETE FROM chain_root_for_txn_id +WHERE transaction_id = :transaction_id diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql new file mode 100644 index 00000000000..2a992be601a --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql @@ -0,0 +1,5 @@ +-- Delete all the chain roots for a stake address +DELETE FROM chain_root_for_stake_addr +WHERE stake_addr = :stake_addr + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql new file mode 100644 index 00000000000..2ab7b51b7cc --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql @@ -0,0 +1,6 @@ +-- Delete CIP-36 registration. +DELETE FROM cip36_registration +WHERE stake_address = :stake_address + AND nonce = :nonce + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql new file mode 100644 index 00000000000..571e289c803 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql @@ -0,0 +1,7 @@ +-- Delete CIP-36 registration by Stake Addres. +DELETE FROM cip36_registration_for_vote_key +WHERE vote_key = :vote_key + AND stake_address = :stake_address + AND slot_no = :slot_no + AND txn = :txn + AND valid = :valid diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql new file mode 100644 index 00000000000..ada3acdc2a7 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql @@ -0,0 +1,5 @@ +-- Delete invalid CIP-36 registration. +DELETE FROM cip36_registration_invalid +WHERE stake_address = :stake_address + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql new file mode 100644 index 00000000000..53eef2da725 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql @@ -0,0 +1,7 @@ +-- Delete RBAC 509 registration. +DELETE FROM rbac509_registration +WHERE chain_root = :chain_root + AND slot_no = :slot_no + AND txn = :txn + AND transaction_id = :transaction_id + AND purpose = :purpose diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql new file mode 100644 index 00000000000..03c85302892 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql @@ -0,0 +1,6 @@ +-- Delete Stake Registration. +DELETE FROM stake_registration +WHERE stake_hash = :stake_hash + AND script = :script + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql new file mode 100644 index 00000000000..87bf0a8b7a2 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql @@ -0,0 +1,5 @@ +-- Delete ADA or a native asset being spent. +-- This can represent a spend on either immutable data or volatile data. +DELETE FROM txi_by_txn_hash +WHERE txn_hash = :txn_hash + AND txo = :txo diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql new file mode 100644 index 00000000000..c46df7011bb --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql @@ -0,0 +1,8 @@ +-- Delete Transaction Output Assets by Stake Address. +DELETE FROM txo_assets_by_stake +WHERE stake_address = :stake_address + AND slot_no = :slot_no + AND txn = :txn + AND txo = :txo + AND policy_id = :policy_id + AND policy_name = :policy_name diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql new file mode 100644 index 00000000000..1b0c1fdbdf9 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql @@ -0,0 +1,6 @@ +-- Delete Transaction Output by Stake Address. +DELETE FROM txo_by_stake +WHERE stake_address = :stake_address + AND slot_no = :slot_no + AND txn = :txn + AND txo = :txo diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql new file mode 100644 index 00000000000..1c0270c3967 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql @@ -0,0 +1,6 @@ +-- Delete Transaction Outputs (Native Assets) by Transaction Hash. +DELETE FROM unstaked_txo_assets_by_txn_hash +WHERE txn_hash = :txn_hash + AND txo = :txo + AND policy_id = :policy_id + AND policy_name = :policy_name diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql new file mode 100644 index 00000000000..ae1cf76c88e --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql @@ -0,0 +1,4 @@ +-- Delete Unstaked Transaction Output Assets by Transaction Hash. +DELETE FROM unstaked_txo_by_txn_hash +WHERE txn_hash = :txn_hash + AND txo = :txo diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql new file mode 100644 index 00000000000..7987c2da104 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql @@ -0,0 +1,6 @@ +-- Get all primary keys from chain roots for a role0 key. +SELECT + role0_key, + slot_no, + txn +FROM chain_root_for_role0_key diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql new file mode 100644 index 00000000000..f3f9d0e2185 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql @@ -0,0 +1,3 @@ +-- Get all primary keys from Chain Root For TX ID. RBAC 509 Registrations. +SELECT transaction_id +FROM chain_root_for_txn_id diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql new file mode 100644 index 00000000000..84bb782303b --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql @@ -0,0 +1,6 @@ +-- Get all primary keys from Chain Roots for a Role0 Key. +SELECT + role0_key, + slot_no, + txn +FROM chain_root_for_role0_key diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql new file mode 100644 index 00000000000..2c0212301bb --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql @@ -0,0 +1,7 @@ +-- Get all primary keys from CIP-36 registration. +SELECT + stake_address, + nonce, + slot_no, + txn +FROM cip36_registration diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql new file mode 100644 index 00000000000..a57da1f281b --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql @@ -0,0 +1,8 @@ +-- Get all primary keys from CIP-36 registration by Stake Addres. +SELECT + vote_key, + stake_address, + slot_no, + txn, + valid +FROM cip36_registration_for_vote_key diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql new file mode 100644 index 00000000000..3b88a5d8023 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql @@ -0,0 +1,8 @@ +-- Get all primary keys from RBAC 509 registration. +SELECT + chain_root, + slot_no, + txn, + transaction_id, + purpose +FROM rbac509_registration diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql new file mode 100644 index 00000000000..a0f16522b5a --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql @@ -0,0 +1,7 @@ +-- Get all primary keys from Stake Registration. +SELECT + stake_hash, + script, + slot_no, + txn +FROM stake_registration diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql new file mode 100644 index 00000000000..062f3388847 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql @@ -0,0 +1,5 @@ +-- Get all primary keys from ADA or a native asset being spent. +SELECT + txn_hash, + txo +FROM txi_by_txn_hash diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql new file mode 100644 index 00000000000..07de857bd5a --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql @@ -0,0 +1,9 @@ +-- Get all primary keys from Transaction Output Assets by Stake Address. +SELECT + stake_address, + slot_no, + txn, + txo, + policy_id, + policy_name +FROM txo_assets_by_stake diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql new file mode 100644 index 00000000000..d83329ca7c6 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql @@ -0,0 +1,7 @@ +-- Get all primary keys from Transaction Output by Stake Address. +SELECT + stake_address, + slot_no, + txn, + txo +FROM txo_by_stake diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql new file mode 100644 index 00000000000..45ae2ef93b1 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql @@ -0,0 +1,7 @@ +-- Get all primary keys from Transaction Outputs (Native Assets) by Transaction Hash. +SELECT + txn_hash, + txo, + policy_id, + policy_name +FROM unstaked_txo_assets_by_txn_hash diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql new file mode 100644 index 00000000000..eceaac8e733 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql @@ -0,0 +1,5 @@ +-- Get all primary keys from Unstaked Transaction Output Assets by Transaction Hash. +SELECT + txn_hash, + txo +FROM unstaked_txo_by_txn_hash diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs new file mode 100644 index 00000000000..5182a795036 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs @@ -0,0 +1,131 @@ +//! Immutable Roll Forward logic. +#![allow(dead_code, clippy::unused_async, clippy::todo)] +use std::sync::Arc; + +use scylla::Session; + +use crate::{db::index::queries::SizedBatch, settings::cassandra_db::EnvVars}; + +type Wip = Vec<()>; + +/// Roll Forward Purge Query. +pub(crate) struct RollforwardPurgeQuery { + /// Get Primary Keys for TXO Purge Query. + pub(crate) get_txo: Wip, + /// Get Primary Keys for TXO Asset Purge Query. + pub(crate) get_txo_asset: Wip, + /// Get Primary Keys forUnstaked TXO Purge Query. + pub(crate) get_unstaked_txo: Wip, + /// Get Primary Keys for Unstaked TXO Asset Purge Query. + pub(crate) get_unstaked_txo_asset: Wip, + /// Get Primary Keys for TXI Purge Query. + pub(crate) get_txi: Wip, + /// Get Primary Keys for TXI Purge Query. + pub(crate) get_stake_registration: Wip, + /// Get Primary Keys for CIP36 Registrations Purge Query. + pub(crate) get_cip36_registration: Wip, + /// Get Primary Keys for CIP36 Registration errors Purge Query. + pub(crate) get_cip36_registration_error: Wip, + /// Get Primary Keys for CIP36 Registration for Stake Address Purge Query. + pub(crate) get_cip36_registration_for_stake_address: Wip, + /// Get Primary Keys for RBAC 509 Registrations Purge Query. + pub(crate) get_rbac509_registration: Wip, + /// Get Primary Keys for Chain Root for TX ID Purge Query.. + pub(crate) get_chain_root_for_txn_id: Wip, + /// Get Primary Keys for Chain Root for Role 0 Key Purge Query.. + pub(crate) get_chain_root_for_role0_key: Wip, + /// Get Primary Keys for Chain Root for Stake Address Purge Query. + pub(crate) get_chain_root_for_stake_address: Wip, + /// TXO Purge Query. + pub(crate) purge_txo: Wip, + /// TXO Asset Purge Query. + pub(crate) purge_txo_asset: Wip, + /// Unstaked TXO Purge Query. + pub(crate) purge_unstaked_txo: Wip, + /// Unstaked TXO Asset Purge Query. + pub(crate) purge_unstaked_txo_asset: Wip, + /// TXI Purge Query. + pub(crate) purge_txi: Wip, + /// TXI Purge Query. + pub(crate) purge_stake_registration: Wip, + /// CIP36 Registrations Purge Query. + pub(crate) purge_cip36_registration: Wip, + /// CIP36 Registration errors Purge Query. + pub(crate) purge_cip36_registration_error: Wip, + /// CIP36 Registration for Stake Address Purge Query. + pub(crate) purge_cip36_registration_for_stake_address: Wip, + /// RBAC 509 Registrations Purge Query. + pub(crate) purge_rbac509_registration: Wip, + /// Chain Root for TX ID Purge Query.. + pub(crate) purge_chain_root_for_txn_id: Wip, + /// Chain Root for Role 0 Key Purge Query.. + pub(crate) purge_chain_root_for_role0_key: Wip, + /// Chain Root for Stake Address Purge Query.. + pub(crate) purge_chain_root_for_stake_address: Wip, +} + +/// Purge batches. +#[allow(clippy::struct_field_names)] +pub(crate) struct PurgeBatches { + /// Get Primary Keys forTXO Purge Query. + pub(crate) get_txo_purge_queries: SizedBatch, + /// Get Primary Keys forTXO Asset Purge Query. + pub(crate) get_txo_asset_purge_queries: SizedBatch, + /// Get Primary Keys forUnstaked TXO Purge Query. + pub(crate) get_unstaked_txo_purge_queries: SizedBatch, + /// Get Primary Keys forUnstaked TXO Asset Purge Query. + pub(crate) get_unstaked_txo_asset_purge_queries: SizedBatch, + /// Get Primary Keys forTXI Purge Query. + pub(crate) get_txi_purge_queries: SizedBatch, + /// Get Primary Keys forTXI Purge Query. + pub(crate) get_stake_registration_purge_queries: SizedBatch, + /// Get Primary Keys forCIP36 Registrations Purge Query. + pub(crate) get_cip36_registration_purge_queries: SizedBatch, + /// Get Primary Keys forCIP36 Registration errors Purge Query. + pub(crate) get_cip36_registration_error_purge_queries: SizedBatch, + /// Get Primary Keys forCIP36 Registration for Stake Address Purge Query. + pub(crate) get_cip36_registration_for_stake_address_purge_queries: SizedBatch, + /// Get Primary Keys forRBAC 509 Registrations Purge Query. + pub(crate) get_rbac509_registration_purge_queries: SizedBatch, + /// Get Primary Keys forChain Root for TX ID Purge Query.. + pub(crate) get_chain_root_for_txn_id_purge_queries: SizedBatch, + /// Get Primary Keys forChain Root for Role 0 Key Purge Query.. + pub(crate) get_chain_root_for_role0_key_purge_queries: SizedBatch, + /// Get Primary Keys forChain Root for Stake Address Purge Query.. + pub(crate) get_chain_root_for_stake_address_purge_queries: SizedBatch, + /// TXO Purge Query. + pub(crate) txo_purge_queries: SizedBatch, + /// TXO Asset Purge Query. + pub(crate) txo_asset_purge_queries: SizedBatch, + /// Unstaked TXO Purge Query. + pub(crate) unstaked_txo_purge_queries: SizedBatch, + /// Unstaked TXO Asset Purge Query. + pub(crate) unstaked_txo_asset_purge_queries: SizedBatch, + /// TXI Purge Query. + pub(crate) txi_purge_queries: SizedBatch, + /// TXI Purge Query. + pub(crate) stake_registration_purge_queries: SizedBatch, + /// CIP36 Registrations Purge Query. + pub(crate) cip36_registration_purge_queries: SizedBatch, + /// CIP36 Registration errors Purge Query. + pub(crate) cip36_registration_error_purge_queries: SizedBatch, + /// CIP36 Registration for Stake Address Purge Query. + pub(crate) cip36_registration_for_stake_address_purge_queries: SizedBatch, + /// RBAC 509 Registrations Purge Query. + pub(crate) rbac509_registration_purge_queries: SizedBatch, + /// Chain Root for TX ID Purge Query.. + pub(crate) chain_root_for_txn_id_purge_queries: SizedBatch, + /// Chain Root for Role 0 Key Purge Query.. + pub(crate) chain_root_for_role0_key_purge_queries: SizedBatch, + /// Chain Root for Stake Address Purge Query.. + pub(crate) chain_root_for_stake_address_purge_queries: SizedBatch, +} + +impl RollforwardPurgeQuery { + /// Prepare the purge query. + pub(crate) async fn prepare_batch( + _session: &Arc, _cfg: &EnvVars, + ) -> anyhow::Result { + todo!(); + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/mod.rs b/catalyst-gateway/bin/src/db/index/queries/mod.rs index b257e855d7d..2c54df4faf0 100644 --- a/catalyst-gateway/bin/src/db/index/queries/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/mod.rs @@ -31,8 +31,12 @@ use staked_ada::{ use sync_status::update::SyncStatusInsertQuery; use super::block::{ - certs::CertInsertQuery, cip36::Cip36InsertQuery, rbac509::Rbac509InsertQuery, - txi::TxiInsertQuery, txo::TxoInsertQuery, + certs::CertInsertQuery, + cip36::Cip36InsertQuery, + rbac509::Rbac509InsertQuery, + roll_forward::{PurgeBatches, RollforwardPurgeQuery}, + txi::TxiInsertQuery, + txo::TxoInsertQuery, }; use crate::settings::cassandra_db; @@ -103,6 +107,38 @@ pub(crate) enum PreparedUpsertQuery { SyncStatusInsert, } +/// All prepared DELETE query statements (purge DB table rows). +#[derive(strum_macros::Display)] +#[allow(clippy::enum_variant_names)] +pub(crate) enum PreparedPurgeQuery { + /// TXO Purge query. + TxoAdaPurgeQuery, + /// TXO Asset Purge query. + TxoAssetPurgeQuery, + /// Unstaked TXO Purge query. + UnstakedTxoAdaPurgeQuery, + /// Unstaked TXO Asset Purge query. + UnstakedTxoAssetPurgeQuery, + /// TXI Purge query. + TxiPurgeQuery, + /// Stake Registration Purge query. + StakeRegistrationPurgeQuery, + /// CIP 36 Registration Purge Query. + Cip36RegistrationPurgeQuery, + /// CIP 36 Registration Error Purge query. + Cip36RegistrationPurgeErrorQuery, + /// CIP 36 Registration for stake address Purge query. + Cip36RegistrationForStakeAddrPurgeQuery, + /// RBAC 509 Registration Purge query. + Rbac509PurgeQuery, + /// Chain Root For Transaction ID Purge query. + ChainRootForTxnIdPurgeQuery, + /// Chain Root For Role0 Key Purge query. + ChainRootForRole0KeyPurgeQuery, + /// Chain Root For Stake Address Purge query. + ChainRootForStakeAddressPurgeQuery, +} + /// All prepared queries for a session. #[allow(clippy::struct_field_names)] pub(crate) struct PreparedQueries { @@ -156,6 +192,58 @@ pub(crate) struct PreparedQueries { registrations_by_chain_root_query: PreparedStatement, /// Get chain root by role0 key chain_root_by_role0_key_query: PreparedStatement, + /// TXO Purge Query. + get_txo_purge_queries: SizedBatch, + /// TXO Asset Purge Query. + get_txo_asset_purge_queries: SizedBatch, + /// Unstaked TXO Purge Query. + get_unstaked_txo_purge_queries: SizedBatch, + /// Unstaked TXO Asset Purge Query. + get_unstaked_txo_asset_purge_queries: SizedBatch, + /// TXI Purge Query. + get_txi_purge_queries: SizedBatch, + /// TXI Purge Query. + get_stake_registration_purge_queries: SizedBatch, + /// CIP36 Registrations Purge Query. + get_cip36_registration_purge_queries: SizedBatch, + /// CIP36 Registration errors Purge Query. + get_cip36_registration_error_purge_queries: SizedBatch, + /// CIP36 Registration for Stake Address Purge Query. + get_cip36_registration_for_stake_address_purge_queries: SizedBatch, + /// RBAC 509 Registrations Purge Query. + get_rbac509_registration_purge_queries: SizedBatch, + /// Chain Root for TX ID Purge Query.. + get_chain_root_for_txn_id_purge_queries: SizedBatch, + /// Chain Root for Role 0 Key Purge Query.. + get_chain_root_for_role0_key_purge_queries: SizedBatch, + /// Chain Root for Stake Address Purge Query.. + get_chain_root_for_stake_address_purge_queries: SizedBatch, + /// TXO Purge Query. + txo_purge_queries: SizedBatch, + /// TXO Asset Purge Query. + txo_asset_purge_queries: SizedBatch, + /// Unstaked TXO Purge Query. + unstaked_txo_purge_queries: SizedBatch, + /// Unstaked TXO Asset Purge Query. + unstaked_txo_asset_purge_queries: SizedBatch, + /// TXI Purge Query. + txi_purge_queries: SizedBatch, + /// TXI Purge Query. + stake_registration_purge_queries: SizedBatch, + /// CIP36 Registrations Purge Query. + cip36_registration_purge_queries: SizedBatch, + /// CIP36 Registration errors Purge Query. + cip36_registration_error_purge_queries: SizedBatch, + /// CIP36 Registration for Stake Address Purge Query. + cip36_registration_for_stake_address_purge_queries: SizedBatch, + /// RBAC 509 Registrations Purge Query. + rbac509_registration_purge_queries: SizedBatch, + /// Chain Root for TX ID Purge Query.. + chain_root_for_txn_id_purge_queries: SizedBatch, + /// Chain Root for Role 0 Key Purge Query.. + chain_root_for_role0_key_purge_queries: SizedBatch, + /// Chain Root for Stake Address Purge Query.. + chain_root_for_stake_address_purge_queries: SizedBatch, } /// An individual query response that can fail @@ -168,6 +256,7 @@ pub(crate) type FallibleQueryTasks = Vec, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { @@ -192,7 +281,8 @@ impl PreparedQueries { let chain_root_by_stake_address = GetChainRootQuery::prepare(session.clone()).await; let registrations_by_chain_root = GetRegistrationsByChainRootQuery::prepare(session.clone()).await; - let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session).await; + let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session.clone()).await; + let all_purge_queries = RollforwardPurgeQuery::prepare_batch(&session, cfg).await; let ( txo_insert_queries, @@ -214,6 +304,35 @@ impl PreparedQueries { chain_root_for_stake_address_insert_queries, ) = all_rbac_queries?; + let PurgeBatches { + get_txo_purge_queries, + get_txo_asset_purge_queries, + get_unstaked_txo_purge_queries, + get_unstaked_txo_asset_purge_queries, + get_txi_purge_queries, + get_stake_registration_purge_queries, + get_cip36_registration_purge_queries, + get_cip36_registration_error_purge_queries, + get_cip36_registration_for_stake_address_purge_queries, + get_rbac509_registration_purge_queries, + get_chain_root_for_txn_id_purge_queries, + get_chain_root_for_role0_key_purge_queries, + get_chain_root_for_stake_address_purge_queries, + txo_purge_queries, + txo_asset_purge_queries, + unstaked_txo_purge_queries, + unstaked_txo_asset_purge_queries, + txi_purge_queries, + stake_registration_purge_queries, + cip36_registration_purge_queries, + cip36_registration_error_purge_queries, + cip36_registration_for_stake_address_purge_queries, + rbac509_registration_purge_queries, + chain_root_for_txn_id_purge_queries, + chain_root_for_role0_key_purge_queries, + chain_root_for_stake_address_purge_queries, + }: PurgeBatches = all_purge_queries?; + Ok(Self { txo_insert_queries, txo_asset_insert_queries, @@ -240,6 +359,32 @@ impl PreparedQueries { chain_root_by_stake_address_query: chain_root_by_stake_address?, registrations_by_chain_root_query: registrations_by_chain_root?, chain_root_by_role0_key_query: chain_root_by_role0_key?, + get_txo_purge_queries, + get_txo_asset_purge_queries, + get_unstaked_txo_purge_queries, + get_unstaked_txo_asset_purge_queries, + get_txi_purge_queries, + get_stake_registration_purge_queries, + get_cip36_registration_purge_queries, + get_cip36_registration_error_purge_queries, + get_cip36_registration_for_stake_address_purge_queries, + get_rbac509_registration_purge_queries, + get_chain_root_for_txn_id_purge_queries, + get_chain_root_for_role0_key_purge_queries, + get_chain_root_for_stake_address_purge_queries, + txo_purge_queries, + txo_asset_purge_queries, + unstaked_txo_purge_queries, + unstaked_txo_asset_purge_queries, + txi_purge_queries, + stake_registration_purge_queries, + cip36_registration_purge_queries, + cip36_registration_error_purge_queries, + cip36_registration_for_stake_address_purge_queries, + rbac509_registration_purge_queries, + chain_root_for_txn_id_purge_queries, + chain_root_for_role0_key_purge_queries, + chain_root_for_stake_address_purge_queries, }) } @@ -397,4 +542,63 @@ impl PreparedQueries { Ok(results) } + + /// Execute a purge query with the given parameters. + pub(crate) async fn execute_purge( + &self, session: Arc, cfg: Arc, query: PreparedPurgeQuery, + values: Vec, + ) -> FallibleQueryResults { + let query_map = match query { + PreparedPurgeQuery::TxoAdaPurgeQuery => &self.txo_purge_queries, + PreparedPurgeQuery::TxoAssetPurgeQuery => &self.txo_asset_purge_queries, + PreparedPurgeQuery::UnstakedTxoAdaPurgeQuery => &self.unstaked_txo_purge_queries, + PreparedPurgeQuery::UnstakedTxoAssetPurgeQuery => { + &self.unstaked_txo_asset_purge_queries + }, + PreparedPurgeQuery::TxiPurgeQuery => &self.txi_purge_queries, + PreparedPurgeQuery::StakeRegistrationPurgeQuery => { + &self.stake_registration_purge_queries + }, + PreparedPurgeQuery::Cip36RegistrationPurgeQuery => { + &self.cip36_registration_purge_queries + }, + PreparedPurgeQuery::Cip36RegistrationPurgeErrorQuery => { + &self.cip36_registration_error_purge_queries + }, + PreparedPurgeQuery::Cip36RegistrationForStakeAddrPurgeQuery => { + &self.cip36_registration_for_stake_address_purge_queries + }, + PreparedPurgeQuery::Rbac509PurgeQuery => &self.rbac509_registration_purge_queries, + PreparedPurgeQuery::ChainRootForTxnIdPurgeQuery => { + &self.chain_root_for_txn_id_purge_queries + }, + PreparedPurgeQuery::ChainRootForRole0KeyPurgeQuery => { + &self.chain_root_for_role0_key_purge_queries + }, + PreparedPurgeQuery::ChainRootForStakeAddressPurgeQuery => { + &self.chain_root_for_stake_address_purge_queries + }, + }; + + let mut results: Vec = Vec::new(); + + let chunks = values.chunks(cfg.max_batch_size.try_into().unwrap_or(1)); + + for chunk in chunks { + let chunk_size: u16 = chunk.len().try_into()?; + let Some(batch_query) = query_map.get(&chunk_size) else { + // This should not actually occur. + bail!("No batch query found for size {}", chunk_size); + }; + let batch_query_statements = batch_query.value().clone(); + results.push( + session + .batch(&batch_query_statements, chunk) + .await + .context(format!("query={query}, chunk={chunk:?}"))?, + ); + } + + Ok(results) + } } From 14b48dbac6fe2c509445af3a388a4ac123304aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 18:34:30 -0600 Subject: [PATCH 02/37] wip(cat-gateway): refactor purge queries --- .../src/db/index/block/roll_forward/mod.rs | 55 +--- .../bin/src/db/index/queries/mod.rs | 149 +-------- .../bin/src/db/index/queries/purge.rs | 291 ++++++++++++++++++ 3 files changed, 295 insertions(+), 200 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge.rs diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs index 5182a795036..e6333474997 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs @@ -9,60 +9,7 @@ use crate::{db::index::queries::SizedBatch, settings::cassandra_db::EnvVars}; type Wip = Vec<()>; /// Roll Forward Purge Query. -pub(crate) struct RollforwardPurgeQuery { - /// Get Primary Keys for TXO Purge Query. - pub(crate) get_txo: Wip, - /// Get Primary Keys for TXO Asset Purge Query. - pub(crate) get_txo_asset: Wip, - /// Get Primary Keys forUnstaked TXO Purge Query. - pub(crate) get_unstaked_txo: Wip, - /// Get Primary Keys for Unstaked TXO Asset Purge Query. - pub(crate) get_unstaked_txo_asset: Wip, - /// Get Primary Keys for TXI Purge Query. - pub(crate) get_txi: Wip, - /// Get Primary Keys for TXI Purge Query. - pub(crate) get_stake_registration: Wip, - /// Get Primary Keys for CIP36 Registrations Purge Query. - pub(crate) get_cip36_registration: Wip, - /// Get Primary Keys for CIP36 Registration errors Purge Query. - pub(crate) get_cip36_registration_error: Wip, - /// Get Primary Keys for CIP36 Registration for Stake Address Purge Query. - pub(crate) get_cip36_registration_for_stake_address: Wip, - /// Get Primary Keys for RBAC 509 Registrations Purge Query. - pub(crate) get_rbac509_registration: Wip, - /// Get Primary Keys for Chain Root for TX ID Purge Query.. - pub(crate) get_chain_root_for_txn_id: Wip, - /// Get Primary Keys for Chain Root for Role 0 Key Purge Query.. - pub(crate) get_chain_root_for_role0_key: Wip, - /// Get Primary Keys for Chain Root for Stake Address Purge Query. - pub(crate) get_chain_root_for_stake_address: Wip, - /// TXO Purge Query. - pub(crate) purge_txo: Wip, - /// TXO Asset Purge Query. - pub(crate) purge_txo_asset: Wip, - /// Unstaked TXO Purge Query. - pub(crate) purge_unstaked_txo: Wip, - /// Unstaked TXO Asset Purge Query. - pub(crate) purge_unstaked_txo_asset: Wip, - /// TXI Purge Query. - pub(crate) purge_txi: Wip, - /// TXI Purge Query. - pub(crate) purge_stake_registration: Wip, - /// CIP36 Registrations Purge Query. - pub(crate) purge_cip36_registration: Wip, - /// CIP36 Registration errors Purge Query. - pub(crate) purge_cip36_registration_error: Wip, - /// CIP36 Registration for Stake Address Purge Query. - pub(crate) purge_cip36_registration_for_stake_address: Wip, - /// RBAC 509 Registrations Purge Query. - pub(crate) purge_rbac509_registration: Wip, - /// Chain Root for TX ID Purge Query.. - pub(crate) purge_chain_root_for_txn_id: Wip, - /// Chain Root for Role 0 Key Purge Query.. - pub(crate) purge_chain_root_for_role0_key: Wip, - /// Chain Root for Stake Address Purge Query.. - pub(crate) purge_chain_root_for_stake_address: Wip, -} +pub(crate) struct RollforwardPurgeQuery {} /// Purge batches. #[allow(clippy::struct_field_names)] diff --git a/catalyst-gateway/bin/src/db/index/queries/mod.rs b/catalyst-gateway/bin/src/db/index/queries/mod.rs index 2c54df4faf0..a06e126e4fd 100644 --- a/catalyst-gateway/bin/src/db/index/queries/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/mod.rs @@ -2,6 +2,7 @@ //! //! This improves query execution time. +pub(crate) mod purge; pub(crate) mod rbac; pub(crate) mod registrations; pub(crate) mod staked_ada; @@ -31,12 +32,8 @@ use staked_ada::{ use sync_status::update::SyncStatusInsertQuery; use super::block::{ - certs::CertInsertQuery, - cip36::Cip36InsertQuery, - rbac509::Rbac509InsertQuery, - roll_forward::{PurgeBatches, RollforwardPurgeQuery}, - txi::TxiInsertQuery, - txo::TxoInsertQuery, + certs::CertInsertQuery, cip36::Cip36InsertQuery, rbac509::Rbac509InsertQuery, + txi::TxiInsertQuery, txo::TxoInsertQuery, }; use crate::settings::cassandra_db; @@ -107,38 +104,6 @@ pub(crate) enum PreparedUpsertQuery { SyncStatusInsert, } -/// All prepared DELETE query statements (purge DB table rows). -#[derive(strum_macros::Display)] -#[allow(clippy::enum_variant_names)] -pub(crate) enum PreparedPurgeQuery { - /// TXO Purge query. - TxoAdaPurgeQuery, - /// TXO Asset Purge query. - TxoAssetPurgeQuery, - /// Unstaked TXO Purge query. - UnstakedTxoAdaPurgeQuery, - /// Unstaked TXO Asset Purge query. - UnstakedTxoAssetPurgeQuery, - /// TXI Purge query. - TxiPurgeQuery, - /// Stake Registration Purge query. - StakeRegistrationPurgeQuery, - /// CIP 36 Registration Purge Query. - Cip36RegistrationPurgeQuery, - /// CIP 36 Registration Error Purge query. - Cip36RegistrationPurgeErrorQuery, - /// CIP 36 Registration for stake address Purge query. - Cip36RegistrationForStakeAddrPurgeQuery, - /// RBAC 509 Registration Purge query. - Rbac509PurgeQuery, - /// Chain Root For Transaction ID Purge query. - ChainRootForTxnIdPurgeQuery, - /// Chain Root For Role0 Key Purge query. - ChainRootForRole0KeyPurgeQuery, - /// Chain Root For Stake Address Purge query. - ChainRootForStakeAddressPurgeQuery, -} - /// All prepared queries for a session. #[allow(clippy::struct_field_names)] pub(crate) struct PreparedQueries { @@ -192,58 +157,6 @@ pub(crate) struct PreparedQueries { registrations_by_chain_root_query: PreparedStatement, /// Get chain root by role0 key chain_root_by_role0_key_query: PreparedStatement, - /// TXO Purge Query. - get_txo_purge_queries: SizedBatch, - /// TXO Asset Purge Query. - get_txo_asset_purge_queries: SizedBatch, - /// Unstaked TXO Purge Query. - get_unstaked_txo_purge_queries: SizedBatch, - /// Unstaked TXO Asset Purge Query. - get_unstaked_txo_asset_purge_queries: SizedBatch, - /// TXI Purge Query. - get_txi_purge_queries: SizedBatch, - /// TXI Purge Query. - get_stake_registration_purge_queries: SizedBatch, - /// CIP36 Registrations Purge Query. - get_cip36_registration_purge_queries: SizedBatch, - /// CIP36 Registration errors Purge Query. - get_cip36_registration_error_purge_queries: SizedBatch, - /// CIP36 Registration for Stake Address Purge Query. - get_cip36_registration_for_stake_address_purge_queries: SizedBatch, - /// RBAC 509 Registrations Purge Query. - get_rbac509_registration_purge_queries: SizedBatch, - /// Chain Root for TX ID Purge Query.. - get_chain_root_for_txn_id_purge_queries: SizedBatch, - /// Chain Root for Role 0 Key Purge Query.. - get_chain_root_for_role0_key_purge_queries: SizedBatch, - /// Chain Root for Stake Address Purge Query.. - get_chain_root_for_stake_address_purge_queries: SizedBatch, - /// TXO Purge Query. - txo_purge_queries: SizedBatch, - /// TXO Asset Purge Query. - txo_asset_purge_queries: SizedBatch, - /// Unstaked TXO Purge Query. - unstaked_txo_purge_queries: SizedBatch, - /// Unstaked TXO Asset Purge Query. - unstaked_txo_asset_purge_queries: SizedBatch, - /// TXI Purge Query. - txi_purge_queries: SizedBatch, - /// TXI Purge Query. - stake_registration_purge_queries: SizedBatch, - /// CIP36 Registrations Purge Query. - cip36_registration_purge_queries: SizedBatch, - /// CIP36 Registration errors Purge Query. - cip36_registration_error_purge_queries: SizedBatch, - /// CIP36 Registration for Stake Address Purge Query. - cip36_registration_for_stake_address_purge_queries: SizedBatch, - /// RBAC 509 Registrations Purge Query. - rbac509_registration_purge_queries: SizedBatch, - /// Chain Root for TX ID Purge Query.. - chain_root_for_txn_id_purge_queries: SizedBatch, - /// Chain Root for Role 0 Key Purge Query.. - chain_root_for_role0_key_purge_queries: SizedBatch, - /// Chain Root for Stake Address Purge Query.. - chain_root_for_stake_address_purge_queries: SizedBatch, } /// An individual query response that can fail @@ -282,7 +195,6 @@ impl PreparedQueries { let registrations_by_chain_root = GetRegistrationsByChainRootQuery::prepare(session.clone()).await; let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session.clone()).await; - let all_purge_queries = RollforwardPurgeQuery::prepare_batch(&session, cfg).await; let ( txo_insert_queries, @@ -304,35 +216,6 @@ impl PreparedQueries { chain_root_for_stake_address_insert_queries, ) = all_rbac_queries?; - let PurgeBatches { - get_txo_purge_queries, - get_txo_asset_purge_queries, - get_unstaked_txo_purge_queries, - get_unstaked_txo_asset_purge_queries, - get_txi_purge_queries, - get_stake_registration_purge_queries, - get_cip36_registration_purge_queries, - get_cip36_registration_error_purge_queries, - get_cip36_registration_for_stake_address_purge_queries, - get_rbac509_registration_purge_queries, - get_chain_root_for_txn_id_purge_queries, - get_chain_root_for_role0_key_purge_queries, - get_chain_root_for_stake_address_purge_queries, - txo_purge_queries, - txo_asset_purge_queries, - unstaked_txo_purge_queries, - unstaked_txo_asset_purge_queries, - txi_purge_queries, - stake_registration_purge_queries, - cip36_registration_purge_queries, - cip36_registration_error_purge_queries, - cip36_registration_for_stake_address_purge_queries, - rbac509_registration_purge_queries, - chain_root_for_txn_id_purge_queries, - chain_root_for_role0_key_purge_queries, - chain_root_for_stake_address_purge_queries, - }: PurgeBatches = all_purge_queries?; - Ok(Self { txo_insert_queries, txo_asset_insert_queries, @@ -359,32 +242,6 @@ impl PreparedQueries { chain_root_by_stake_address_query: chain_root_by_stake_address?, registrations_by_chain_root_query: registrations_by_chain_root?, chain_root_by_role0_key_query: chain_root_by_role0_key?, - get_txo_purge_queries, - get_txo_asset_purge_queries, - get_unstaked_txo_purge_queries, - get_unstaked_txo_asset_purge_queries, - get_txi_purge_queries, - get_stake_registration_purge_queries, - get_cip36_registration_purge_queries, - get_cip36_registration_error_purge_queries, - get_cip36_registration_for_stake_address_purge_queries, - get_rbac509_registration_purge_queries, - get_chain_root_for_txn_id_purge_queries, - get_chain_root_for_role0_key_purge_queries, - get_chain_root_for_stake_address_purge_queries, - txo_purge_queries, - txo_asset_purge_queries, - unstaked_txo_purge_queries, - unstaked_txo_asset_purge_queries, - txi_purge_queries, - stake_registration_purge_queries, - cip36_registration_purge_queries, - cip36_registration_error_purge_queries, - cip36_registration_for_stake_address_purge_queries, - rbac509_registration_purge_queries, - chain_root_for_txn_id_purge_queries, - chain_root_for_role0_key_purge_queries, - chain_root_for_stake_address_purge_queries, }) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge.rs b/catalyst-gateway/bin/src/db/index/queries/purge.rs new file mode 100644 index 00000000000..33351f47edf --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge.rs @@ -0,0 +1,291 @@ +//! Queries for purging volatile data. + +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, serialize::row::SerializeRow, + transport::iterator::RowIterator, Session, +}; + +use super::{ + super::block::roll_forward::{PurgeBatches, RollforwardPurgeQuery}, + FallibleQueryResults, SizedBatch, +}; +use crate::{ + db::index::block::roll_forward::txo_by_stake_addr::get::TxoByStakeAddressPrimaryKeyQuery, + settings::cassandra_db, +}; + +/// All prepared DELETE query statements (purge DB table rows). +#[derive(strum_macros::Display)] +pub(crate) enum PreparedDeleteQuery { + /// TXO Delete query. + TxoAda, + /// TXO Asset Delete query. + TxoAsset, + /// Unstaked TXO Delete query. + UnstakedTxoAda, + /// Unstaked TXO Asset Delete query. + UnstakedTxoAsset, + /// TXI Delete query. + Txi, + /// Stake Registration Delete query. + StakeRegistration, + /// CIP 36 Registration Delete Query. + Cip36Registration, + /// CIP 36 Registration Error Delete query. + Cip36RegistrationError, + /// CIP 36 Registration for stake address Delete query. + Cip36RegistrationForStakeAddr, + /// RBAC 509 Registration Delete query. + Rbac509, + /// Chain Root For Transaction ID Delete query. + ChainRootForTxnId, + /// Chain Root For Role0 Key Delete query. + ChainRootForRole0Key, + /// Chain Root For Stake Address Delete query. + ChainRootForStakeAddress, +} + +/// All prepared SELECT query statements (primary keys from table). +#[derive(strum_macros::Display)] +pub(crate) enum PreparedSelectQuery { + /// TXO Select query. + TxoAda, + /// TXO Asset Select query. + TxoAsset, + /// Unstaked TXO Select query. + UnstakedTxoAda, + /// Unstaked TXO Asset Select query. + UnstakedTxoAsset, + /// TXI Select query. + Txi, + /// Stake Registration Select query. + StakeRegistration, + /// CIP 36 Registration Select Query. + Cip36Registration, + /// CIP 36 Registration Error Select query. + Cip36RegistrationError, + /// CIP 36 Registration for stake address Select query. + Cip36RegistrationForStakeAddr, + /// RBAC 509 Registration Select query. + Rbac509, + /// Chain Root For Transaction ID Select query. + ChainRootForTxnId, + /// Chain Root For Role0 Key Select query. + ChainRootForRole0Key, + /// Chain Root For Stake Address Select query. + ChainRootForStakeAddress, +} + +/// All prepared purge queries for a session. +#[allow(clippy::struct_field_names)] +pub(crate) struct PreparedQueries { + /// TXO Purge Query. + get_txo_purge_queries: PreparedStatement, + /// TXO Asset Purge Query. + get_txo_asset_purge_queries: PreparedStatement, + /// Unstaked TXO Purge Query. + get_unstaked_txo_purge_queries: PreparedStatement, + /// Unstaked TXO Asset Purge Query. + get_unstaked_txo_asset_purge_queries: PreparedStatement, + /// TXI Purge Query. + get_txi_purge_queries: PreparedStatement, + /// TXI Purge Query. + get_stake_registration_purge_queries: PreparedStatement, + /// CIP36 Registrations Purge Query. + get_cip36_registration_purge_queries: PreparedStatement, + /// CIP36 Registration errors Purge Query. + get_cip36_registration_error_purge_queries: PreparedStatement, + /// CIP36 Registration for Stake Address Purge Query. + get_cip36_registration_for_stake_address_purge_queries: PreparedStatement, + /// RBAC 509 Registrations Purge Query. + get_rbac509_registration_purge_queries: PreparedStatement, + /// Chain Root for TX ID Purge Query.. + get_chain_root_for_txn_id_purge_queries: PreparedStatement, + /// Chain Root for Role 0 Key Purge Query.. + get_chain_root_for_role0_key_purge_queries: PreparedStatement, + /// Chain Root for Stake Address Purge Query.. + get_chain_root_for_stake_address_purge_queries: PreparedStatement, + /// TXO Purge Query. + txo_purge_queries: SizedBatch, + /// TXO Asset Purge Query. + txo_asset_purge_queries: SizedBatch, + /// Unstaked TXO Purge Query. + unstaked_txo_purge_queries: SizedBatch, + /// Unstaked TXO Asset Purge Query. + unstaked_txo_asset_purge_queries: SizedBatch, + /// TXI Purge Query. + txi_purge_queries: SizedBatch, + /// TXI Purge Query. + stake_registration_purge_queries: SizedBatch, + /// CIP36 Registrations Purge Query. + cip36_registration_purge_queries: SizedBatch, + /// CIP36 Registration errors Purge Query. + cip36_registration_error_purge_queries: SizedBatch, + /// CIP36 Registration for Stake Address Purge Query. + cip36_registration_for_stake_address_purge_queries: SizedBatch, + /// RBAC 509 Registrations Purge Query. + rbac509_registration_purge_queries: SizedBatch, + /// Chain Root for TX ID Purge Query.. + chain_root_for_txn_id_purge_queries: SizedBatch, + /// Chain Root for Role 0 Key Purge Query.. + chain_root_for_role0_key_purge_queries: SizedBatch, + /// Chain Root for Stake Address Purge Query.. + chain_root_for_stake_address_purge_queries: SizedBatch, +} + +impl PreparedQueries { + /// Create new prepared queries for a given session. + #[allow(clippy::todo)] + pub(crate) async fn new( + session: Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + // We initialize like this, so that all errors preparing querys get shown before aborting. + let all_purge_queries = RollforwardPurgeQuery::prepare_batch(&session, cfg).await; + let get_txo_purge_queries = TxoByStakeAddressPrimaryKeyQuery::prepare(&session).await?; + let PurgeBatches { + get_txo_asset_purge_queries, + get_unstaked_txo_purge_queries, + get_unstaked_txo_asset_purge_queries, + get_txi_purge_queries, + get_stake_registration_purge_queries, + get_cip36_registration_purge_queries, + get_cip36_registration_error_purge_queries, + get_cip36_registration_for_stake_address_purge_queries, + get_rbac509_registration_purge_queries, + get_chain_root_for_txn_id_purge_queries, + get_chain_root_for_role0_key_purge_queries, + get_chain_root_for_stake_address_purge_queries, + txo_purge_queries, + txo_asset_purge_queries, + unstaked_txo_purge_queries, + unstaked_txo_asset_purge_queries, + txi_purge_queries, + stake_registration_purge_queries, + cip36_registration_purge_queries, + cip36_registration_error_purge_queries, + cip36_registration_for_stake_address_purge_queries, + rbac509_registration_purge_queries, + chain_root_for_txn_id_purge_queries, + chain_root_for_role0_key_purge_queries, + chain_root_for_stake_address_purge_queries, + .. + }: PurgeBatches = all_purge_queries?; + + Ok(Self { + get_txo_purge_queries, + get_txo_asset_purge_queries, + get_unstaked_txo_purge_queries, + get_unstaked_txo_asset_purge_queries, + get_txi_purge_queries, + get_stake_registration_purge_queries, + get_cip36_registration_purge_queries, + get_cip36_registration_error_purge_queries, + get_cip36_registration_for_stake_address_purge_queries, + get_rbac509_registration_purge_queries, + get_chain_root_for_txn_id_purge_queries, + get_chain_root_for_role0_key_purge_queries, + get_chain_root_for_stake_address_purge_queries, + txo_purge_queries, + txo_asset_purge_queries, + unstaked_txo_purge_queries, + unstaked_txo_asset_purge_queries, + txi_purge_queries, + stake_registration_purge_queries, + cip36_registration_purge_queries, + cip36_registration_error_purge_queries, + cip36_registration_for_stake_address_purge_queries, + rbac509_registration_purge_queries, + chain_root_for_txn_id_purge_queries, + chain_root_for_role0_key_purge_queries, + chain_root_for_stake_address_purge_queries, + }) + } + + /// Prepares a statement. + pub(crate) async fn prepare( + session: Arc, query: &str, consistency: scylla::statement::Consistency, + idempotent: bool, + ) -> anyhow::Result { + super::PreparedQueries::prepare(session, query, consistency, idempotent).await + } + + /// Prepares all permutations of the batch from 1 to max. + /// It is necessary to do this because batches are pre-sized, they can not be dynamic. + /// Preparing the batches in advance is a very larger performance increase. + pub(crate) async fn prepare_batch( + session: Arc, query: &str, cfg: &cassandra_db::EnvVars, + consistency: scylla::statement::Consistency, idempotent: bool, logged: bool, + ) -> anyhow::Result { + super::PreparedQueries::prepare_batch(session, query, cfg, consistency, idempotent, logged) + .await + } + + /// Executes a select query with the given parameters. + /// + /// Returns an iterator that iterates over all the result pages that the query + /// returns. + pub(crate) async fn execute_iter

( + &self, session: Arc, select_query: PreparedSelectQuery, params: P, + ) -> anyhow::Result + where P: SerializeRow { + let prepared_stmt = match select_query { + PreparedSelectQuery::TxoAda => &self.get_txo_purge_queries, + PreparedSelectQuery::TxoAsset => &self.get_txo_asset_purge_queries, + PreparedSelectQuery::UnstakedTxoAda => &self.get_unstaked_txo_purge_queries, + PreparedSelectQuery::UnstakedTxoAsset => &self.get_unstaked_txo_asset_purge_queries, + PreparedSelectQuery::Txi => &self.get_txi_purge_queries, + PreparedSelectQuery::StakeRegistration => &self.get_stake_registration_purge_queries, + PreparedSelectQuery::Cip36Registration => &self.get_cip36_registration_purge_queries, + PreparedSelectQuery::Cip36RegistrationError => { + &self.get_cip36_registration_error_purge_queries + }, + PreparedSelectQuery::Cip36RegistrationForStakeAddr => { + &self.get_cip36_registration_for_stake_address_purge_queries + }, + PreparedSelectQuery::Rbac509 => &self.get_rbac509_registration_purge_queries, + PreparedSelectQuery::ChainRootForTxnId => &self.get_chain_root_for_txn_id_purge_queries, + PreparedSelectQuery::ChainRootForRole0Key => { + &self.get_chain_root_for_role0_key_purge_queries + }, + PreparedSelectQuery::ChainRootForStakeAddress => { + &self.get_chain_root_for_stake_address_purge_queries + }, + }; + + super::session_execute_iter(session, prepared_stmt, params).await + } + + /// Execute a purge query with the given parameters. + pub(crate) async fn execute( + &self, session: Arc, cfg: Arc, query: PreparedDeleteQuery, + values: Vec, + ) -> FallibleQueryResults { + let query_map = match query { + PreparedDeleteQuery::TxoAda => &self.txo_purge_queries, + PreparedDeleteQuery::TxoAsset => &self.txo_asset_purge_queries, + PreparedDeleteQuery::UnstakedTxoAda => &self.unstaked_txo_purge_queries, + PreparedDeleteQuery::UnstakedTxoAsset => &self.unstaked_txo_asset_purge_queries, + PreparedDeleteQuery::Txi => &self.txi_purge_queries, + PreparedDeleteQuery::StakeRegistration => &self.stake_registration_purge_queries, + PreparedDeleteQuery::Cip36Registration => &self.cip36_registration_purge_queries, + PreparedDeleteQuery::Cip36RegistrationError => { + &self.cip36_registration_error_purge_queries + }, + PreparedDeleteQuery::Cip36RegistrationForStakeAddr => { + &self.cip36_registration_for_stake_address_purge_queries + }, + PreparedDeleteQuery::Rbac509 => &self.rbac509_registration_purge_queries, + PreparedDeleteQuery::ChainRootForTxnId => &self.chain_root_for_txn_id_purge_queries, + PreparedDeleteQuery::ChainRootForRole0Key => { + &self.chain_root_for_role0_key_purge_queries + }, + PreparedDeleteQuery::ChainRootForStakeAddress => { + &self.chain_root_for_stake_address_purge_queries + }, + }; + + super::session_execute_batch(session, query_map, cfg, query, values).await + } +} From cd5afba4de33c20144eca548666aa5c1d5080339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 18:38:30 -0600 Subject: [PATCH 03/37] feat(cat-gateway): refactor session execution code for reuse --- .../bin/src/db/index/queries/mod.rs | 126 +++++++----------- 1 file changed, 45 insertions(+), 81 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/mod.rs b/catalyst-gateway/bin/src/db/index/queries/mod.rs index a06e126e4fd..2a98d4dd6b7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/mod.rs @@ -333,11 +333,7 @@ impl PreparedQueries { }, PreparedSelectQuery::ChainRootByRole0Key => &self.chain_root_by_role0_key_query, }; - - session - .execute_iter(prepared_stmt.clone(), params) - .await - .map_err(|e| anyhow::anyhow!(e)) + session_execute_iter(session, prepared_stmt, params).await } /// Execute a Batch query with the given parameters. @@ -377,85 +373,53 @@ impl PreparedQueries { &self.chain_root_for_stake_address_insert_queries }, }; - - let mut results: Vec = Vec::new(); - - let chunks = values.chunks(cfg.max_batch_size.try_into().unwrap_or(1)); - - for chunk in chunks { - let chunk_size: u16 = chunk.len().try_into()?; - let Some(batch_query) = query_map.get(&chunk_size) else { - // This should not actually occur. - bail!("No batch query found for size {}", chunk_size); - }; - let batch_query_statements = batch_query.value().clone(); - results.push( - session - .batch(&batch_query_statements, chunk) - .await - .context(format!("query={query}, chunk={chunk:?}"))?, - ); - } - - Ok(results) + session_execute_batch(session, query_map, cfg, query, values).await } +} - /// Execute a purge query with the given parameters. - pub(crate) async fn execute_purge( - &self, session: Arc, cfg: Arc, query: PreparedPurgeQuery, - values: Vec, - ) -> FallibleQueryResults { - let query_map = match query { - PreparedPurgeQuery::TxoAdaPurgeQuery => &self.txo_purge_queries, - PreparedPurgeQuery::TxoAssetPurgeQuery => &self.txo_asset_purge_queries, - PreparedPurgeQuery::UnstakedTxoAdaPurgeQuery => &self.unstaked_txo_purge_queries, - PreparedPurgeQuery::UnstakedTxoAssetPurgeQuery => { - &self.unstaked_txo_asset_purge_queries - }, - PreparedPurgeQuery::TxiPurgeQuery => &self.txi_purge_queries, - PreparedPurgeQuery::StakeRegistrationPurgeQuery => { - &self.stake_registration_purge_queries - }, - PreparedPurgeQuery::Cip36RegistrationPurgeQuery => { - &self.cip36_registration_purge_queries - }, - PreparedPurgeQuery::Cip36RegistrationPurgeErrorQuery => { - &self.cip36_registration_error_purge_queries - }, - PreparedPurgeQuery::Cip36RegistrationForStakeAddrPurgeQuery => { - &self.cip36_registration_for_stake_address_purge_queries - }, - PreparedPurgeQuery::Rbac509PurgeQuery => &self.rbac509_registration_purge_queries, - PreparedPurgeQuery::ChainRootForTxnIdPurgeQuery => { - &self.chain_root_for_txn_id_purge_queries - }, - PreparedPurgeQuery::ChainRootForRole0KeyPurgeQuery => { - &self.chain_root_for_role0_key_purge_queries - }, - PreparedPurgeQuery::ChainRootForStakeAddressPurgeQuery => { - &self.chain_root_for_stake_address_purge_queries - }, +/// Execute a Batch query with the given parameters. +/// +/// Values should be a Vec of values which implement `SerializeRow` and they MUST be +/// the same, and must match the query being executed. +/// +/// This will divide the batch into optimal sized chunks and execute them until all +/// values have been executed or the first error is encountered. +async fn session_execute_batch( + session: Arc, query_map: &SizedBatch, cfg: Arc, query: Q, + values: Vec, +) -> FallibleQueryResults { + let mut results: Vec = Vec::new(); + + let chunks = values.chunks(cfg.max_batch_size.try_into().unwrap_or(1)); + + for chunk in chunks { + let chunk_size: u16 = chunk.len().try_into()?; + let Some(batch_query) = query_map.get(&chunk_size) else { + // This should not actually occur. + bail!("No batch query found for size {}", chunk_size); }; + let batch_query_statements = batch_query.value().clone(); + results.push( + session + .batch(&batch_query_statements, chunk) + .await + .context(format!("query={query}, chunk={chunk:?}"))?, + ); + } - let mut results: Vec = Vec::new(); - - let chunks = values.chunks(cfg.max_batch_size.try_into().unwrap_or(1)); - - for chunk in chunks { - let chunk_size: u16 = chunk.len().try_into()?; - let Some(batch_query) = query_map.get(&chunk_size) else { - // This should not actually occur. - bail!("No batch query found for size {}", chunk_size); - }; - let batch_query_statements = batch_query.value().clone(); - results.push( - session - .batch(&batch_query_statements, chunk) - .await - .context(format!("query={query}, chunk={chunk:?}"))?, - ); - } + Ok(results) +} - Ok(results) - } +/// Executes a select query with the given parameters. +/// +/// Returns an iterator that iterates over all the result pages that the query +/// returns. +pub(crate) async fn session_execute_iter

( + session: Arc, prepared_stmt: &PreparedStatement, params: P, +) -> anyhow::Result +where P: SerializeRow { + session + .execute_iter(prepared_stmt.clone(), params) + .await + .map_err(|e| anyhow::anyhow!(e)) } From 0510cb50d396ea2ac98f21ae20a4675428cdc1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 18:42:30 -0600 Subject: [PATCH 04/37] feat(cat-gateway): add purge_queries to CassandraSession --- catalyst-gateway/bin/src/db/index/session.rs | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index 6193a628dac..d6f6cf242d2 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -17,15 +17,12 @@ use tracing::{error, info}; use super::{ queries::{ - FallibleQueryResults, PreparedQueries, PreparedQuery, PreparedSelectQuery, + purge, FallibleQueryResults, PreparedQueries, PreparedQuery, PreparedSelectQuery, PreparedUpsertQuery, }, schema::create_schema, }; -use crate::{ - db::index::queries, - settings::{cassandra_db, Settings}, -}; +use crate::settings::{cassandra_db, Settings}; /// Configuration Choices for compression #[derive(Clone, strum::EnumString, strum::Display, strum::VariantNames)] @@ -63,6 +60,8 @@ pub(crate) struct CassandraSession { session: Arc, /// All prepared queries we can use on this session. queries: Arc, + /// All prepared purge queries we can use on this session. + purge_queries: Arc, } /// Persistent DB Session. @@ -269,7 +268,7 @@ async fn retry_init(cfg: cassandra_db::EnvVars, persistent: bool) { continue; } - let queries = match queries::PreparedQueries::new(session.clone(), &cfg).await { + let queries = match PreparedQueries::new(session.clone(), &cfg).await { Ok(queries) => Arc::new(queries), Err(error) => { error!( @@ -281,11 +280,24 @@ async fn retry_init(cfg: cassandra_db::EnvVars, persistent: bool) { }, }; + let purge_queries = match purge::PreparedQueries::new(session.clone(), &cfg).await { + Ok(queries) => Arc::new(queries), + Err(error) => { + error!( + db_type = db_type, + error = %error, + "Failed to Create Cassandra Prepared Purge Queries" + ); + continue; + }, + }; + let cassandra_session = CassandraSession { persistent, cfg: Arc::new(cfg), session, queries, + purge_queries, }; // Save the session so we can execute queries on the DB From 4abc97690001df04eac5b483ce7853e579260aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 21:34:58 -0600 Subject: [PATCH 05/37] fix(cat-gateway): refactor into queries::purge module --- .../src/db/index/block/{roll_forward/mod.rs => roll_forward.rs} | 0 .../purge}/cql/delete_chain_root_for_role0_key.cql | 0 .../purge}/cql/delete_chain_root_for_stake_addr.cql | 0 .../purge}/cql/delete_chain_root_for_txn_id.cql | 0 .../purge}/cql/delete_cip36_registration.cql | 0 .../purge}/cql/delete_cip36_registration_for_vote_key.cql | 0 .../purge}/cql/delete_cip36_registration_invalid.cql | 0 .../purge}/cql/delete_rbac509_registration.cql | 0 .../purge}/cql/delete_stake_registration.cql | 0 .../purge}/cql/delete_txi_by_txn_hashes.cql | 0 .../purge}/cql/delete_txo_assets_by_stake_addr.cql | 0 .../purge}/cql/delete_txo_by_stake_address.cql | 0 .../purge}/cql/delete_unstaked_txo_assets_by_txn_hash.cql | 0 .../purge}/cql/delete_unstaked_txo_by_txn_hash.cql | 0 .../purge}/cql/get_chain_root_for_role0_key.cql | 0 .../purge}/cql/get_chain_root_for_stake_addr.cql | 0 .../purge}/cql/get_chain_root_for_txn_id.cql | 0 .../purge}/cql/get_cip36_registration.cql | 0 .../purge}/cql/get_cip36_registration_for_vote_key.cql | 0 .../purge}/cql/get_rbac509_registration.cql | 0 .../purge}/cql/get_stake_registration.cql | 0 .../roll_forward => queries/purge}/cql/get_txi_by_txn_hashes.cql | 0 .../purge}/cql/get_txo_assets_by_stake_addr.cql | 0 .../purge}/cql/get_txo_by_stake_address.cql | 0 .../purge}/cql/get_unstaked_txo_assets_by_txn_hash.cql | 0 .../purge}/cql/get_unstaked_txo_by_txn_hash.cql | 0 .../bin/src/db/index/queries/{purge.rs => purge/mod.rs} | 0 catalyst-gateway/bin/src/db/index/session.rs | 1 - 28 files changed, 1 deletion(-) rename catalyst-gateway/bin/src/db/index/block/{roll_forward/mod.rs => roll_forward.rs} (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_chain_root_for_role0_key.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_chain_root_for_stake_addr.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_chain_root_for_txn_id.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_cip36_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_cip36_registration_for_vote_key.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_cip36_registration_invalid.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_rbac509_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_stake_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_txi_by_txn_hashes.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_txo_assets_by_stake_addr.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_txo_by_stake_address.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_unstaked_txo_assets_by_txn_hash.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/delete_unstaked_txo_by_txn_hash.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_chain_root_for_role0_key.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_chain_root_for_stake_addr.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_chain_root_for_txn_id.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_cip36_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_cip36_registration_for_vote_key.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_rbac509_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_stake_registration.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_txi_by_txn_hashes.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_txo_assets_by_stake_addr.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_txo_by_stake_address.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_unstaked_txo_assets_by_txn_hash.cql (100%) rename catalyst-gateway/bin/src/db/index/{block/roll_forward => queries/purge}/cql/get_unstaked_txo_by_txn_hash.cql (100%) rename catalyst-gateway/bin/src/db/index/queries/{purge.rs => purge/mod.rs} (100%) diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/mod.rs rename to catalyst-gateway/bin/src/db/index/block/roll_forward.rs diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_role0_key.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_stake_addr.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_chain_root_for_txn_id.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_for_vote_key.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_invalid.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_cip36_registration_invalid.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_invalid.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_rbac509_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_rbac509_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_rbac509_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_stake_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_stake_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_stake_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txi_by_txn_hashes.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txi_by_txn_hashes.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txi_by_txn_hashes.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txo_assets_by_stake_addr.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_assets_by_stake_addr.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txo_assets_by_stake_addr.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txo_by_stake_address.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_txo_by_stake_address.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_txo_by_stake_address.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_assets_by_txn_hash.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_by_txn_hash.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/delete_unstaked_txo_by_txn_hash.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_by_txn_hash.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_role0_key.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_stake_addr.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_chain_root_for_txn_id.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_cip36_registration_for_vote_key.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_rbac509_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_rbac509_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_rbac509_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_stake_registration.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_stake_registration.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_stake_registration.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txi_by_txn_hashes.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txi_by_txn_hashes.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txi_by_txn_hashes.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txo_assets_by_stake_addr.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_assets_by_stake_addr.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txo_assets_by_stake_addr.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txo_by_stake_address.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_txo_by_stake_address.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_txo_by_stake_address.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_assets_by_txn_hash.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_assets_by_txn_hash.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_assets_by_txn_hash.cql diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql similarity index 100% rename from catalyst-gateway/bin/src/db/index/block/roll_forward/cql/get_unstaked_txo_by_txn_hash.cql rename to catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql diff --git a/catalyst-gateway/bin/src/db/index/queries/purge.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs similarity index 100% rename from catalyst-gateway/bin/src/db/index/queries/purge.rs rename to catalyst-gateway/bin/src/db/index/queries/purge/mod.rs diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index d6f6cf242d2..289ad0b3a5a 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -52,7 +52,6 @@ pub(crate) enum TlsChoice { #[derive(Clone)] pub(crate) struct CassandraSession { /// Is the session to the persistent or volatile DB? - #[allow(dead_code)] persistent: bool, /// Configuration for this session. cfg: Arc, From e2bf64eb591734a6d799a30613b2a83951fe4668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 21:37:59 -0600 Subject: [PATCH 06/37] wip(cat-gateway): add purge methods to cassandra session --- .../bin/src/db/index/queries/purge/mod.rs | 11 +++-- catalyst-gateway/bin/src/db/index/session.rs | 43 ++++++++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 33351f47edf..3ddaf30d38b 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -226,10 +226,9 @@ impl PreparedQueries { /// /// Returns an iterator that iterates over all the result pages that the query /// returns. - pub(crate) async fn execute_iter

( - &self, session: Arc, select_query: PreparedSelectQuery, params: P, - ) -> anyhow::Result - where P: SerializeRow { + pub(crate) async fn execute_iter( + &self, session: Arc, select_query: PreparedSelectQuery, + ) -> anyhow::Result { let prepared_stmt = match select_query { PreparedSelectQuery::TxoAda => &self.get_txo_purge_queries, PreparedSelectQuery::TxoAsset => &self.get_txo_asset_purge_queries, @@ -254,11 +253,11 @@ impl PreparedQueries { }, }; - super::session_execute_iter(session, prepared_stmt, params).await + super::session_execute_iter(session, prepared_stmt, NO_PARAMS).await } /// Execute a purge query with the given parameters. - pub(crate) async fn execute( + pub(crate) async fn execute_batch( &self, session: Arc, cfg: Arc, query: PreparedDeleteQuery, values: Vec, ) -> FallibleQueryResults { diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index 289ad0b3a5a..99485086043 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -17,7 +17,8 @@ use tracing::{error, info}; use super::{ queries::{ - purge, FallibleQueryResults, PreparedQueries, PreparedQuery, PreparedSelectQuery, + purge::{self, PreparedDeleteQuery}, + FallibleQueryResults, PreparedQueries, PreparedSelectQuery, PreparedQuery, PreparedUpsertQuery, }, schema::create_schema, @@ -145,6 +146,46 @@ impl CassandraSession { queries.execute_upsert(session, query, value).await } + /// Execute a purge query with the given parameters. + /// + /// Values should be a Vec of values which implement `SerializeRow` and they MUST be + /// the same, and must match the query being executed. + /// + /// This will divide the batch into optimal sized chunks and execute them until all + /// values have been executed or the first error is encountered. + /// + /// NOTE: This is currently only used to purge volatile data. + pub(crate) async fn purge_execute_batch( + &self, query: PreparedDeleteQuery, values: Vec, + ) -> FallibleQueryResults { + // Only execute purge queries on the volatile session + let persistent = false; + let Some(volatile_db) = Self::get(persistent) else { + // This should never happen + anyhow::bail!("Volatile DB Session not found"); + }; + let cfg = self.cfg.clone(); + let queries = self.purge_queries.clone(); + let session = volatile_db.session.clone(); + + queries.execute_batch(session, cfg, query, values).await + } + + /// Execute a select query to gather primary keys for purging. + pub(crate) async fn purge_execute_iter( + &self, query: purge::PreparedSelectQuery + ) -> anyhow::Result { + // Only execute purge queries on the volatile session + let persistent = false; + let Some(volatile_db) = Self::get(persistent) else { + // This should never happen + anyhow::bail!("Volatile DB Session not found"); + }; + let queries = self.purge_queries.clone(); + + queries.execute_iter(volatile_db.session.clone(), query).await + } + /// Get underlying Raw Cassandra Session. pub(crate) fn get_raw_session(&self) -> Arc { self.session.clone() From 396972301ea64cb2d503f345fc5d459968a1f9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 21:41:13 -0600 Subject: [PATCH 07/37] wip(cat-gateway): rust code for txo by stake queries --- .../bin/src/db/index/block/roll_forward.rs | 77 ------------------ .../bin/src/db/index/queries/purge/mod.rs | 20 +++-- .../queries/purge/txo_by_stake_addr/delete.rs | 80 +++++++++++++++++++ .../queries/purge/txo_by_stake_addr/mod.rs | 4 + .../queries/purge/txo_by_stake_addr/select.rs | 64 +++++++++++++++ 5 files changed, 160 insertions(+), 85 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index e6333474997..f0d23fd92c1 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -1,78 +1 @@ //! Immutable Roll Forward logic. -#![allow(dead_code, clippy::unused_async, clippy::todo)] -use std::sync::Arc; - -use scylla::Session; - -use crate::{db::index::queries::SizedBatch, settings::cassandra_db::EnvVars}; - -type Wip = Vec<()>; - -/// Roll Forward Purge Query. -pub(crate) struct RollforwardPurgeQuery {} - -/// Purge batches. -#[allow(clippy::struct_field_names)] -pub(crate) struct PurgeBatches { - /// Get Primary Keys forTXO Purge Query. - pub(crate) get_txo_purge_queries: SizedBatch, - /// Get Primary Keys forTXO Asset Purge Query. - pub(crate) get_txo_asset_purge_queries: SizedBatch, - /// Get Primary Keys forUnstaked TXO Purge Query. - pub(crate) get_unstaked_txo_purge_queries: SizedBatch, - /// Get Primary Keys forUnstaked TXO Asset Purge Query. - pub(crate) get_unstaked_txo_asset_purge_queries: SizedBatch, - /// Get Primary Keys forTXI Purge Query. - pub(crate) get_txi_purge_queries: SizedBatch, - /// Get Primary Keys forTXI Purge Query. - pub(crate) get_stake_registration_purge_queries: SizedBatch, - /// Get Primary Keys forCIP36 Registrations Purge Query. - pub(crate) get_cip36_registration_purge_queries: SizedBatch, - /// Get Primary Keys forCIP36 Registration errors Purge Query. - pub(crate) get_cip36_registration_error_purge_queries: SizedBatch, - /// Get Primary Keys forCIP36 Registration for Stake Address Purge Query. - pub(crate) get_cip36_registration_for_stake_address_purge_queries: SizedBatch, - /// Get Primary Keys forRBAC 509 Registrations Purge Query. - pub(crate) get_rbac509_registration_purge_queries: SizedBatch, - /// Get Primary Keys forChain Root for TX ID Purge Query.. - pub(crate) get_chain_root_for_txn_id_purge_queries: SizedBatch, - /// Get Primary Keys forChain Root for Role 0 Key Purge Query.. - pub(crate) get_chain_root_for_role0_key_purge_queries: SizedBatch, - /// Get Primary Keys forChain Root for Stake Address Purge Query.. - pub(crate) get_chain_root_for_stake_address_purge_queries: SizedBatch, - /// TXO Purge Query. - pub(crate) txo_purge_queries: SizedBatch, - /// TXO Asset Purge Query. - pub(crate) txo_asset_purge_queries: SizedBatch, - /// Unstaked TXO Purge Query. - pub(crate) unstaked_txo_purge_queries: SizedBatch, - /// Unstaked TXO Asset Purge Query. - pub(crate) unstaked_txo_asset_purge_queries: SizedBatch, - /// TXI Purge Query. - pub(crate) txi_purge_queries: SizedBatch, - /// TXI Purge Query. - pub(crate) stake_registration_purge_queries: SizedBatch, - /// CIP36 Registrations Purge Query. - pub(crate) cip36_registration_purge_queries: SizedBatch, - /// CIP36 Registration errors Purge Query. - pub(crate) cip36_registration_error_purge_queries: SizedBatch, - /// CIP36 Registration for Stake Address Purge Query. - pub(crate) cip36_registration_for_stake_address_purge_queries: SizedBatch, - /// RBAC 509 Registrations Purge Query. - pub(crate) rbac509_registration_purge_queries: SizedBatch, - /// Chain Root for TX ID Purge Query.. - pub(crate) chain_root_for_txn_id_purge_queries: SizedBatch, - /// Chain Root for Role 0 Key Purge Query.. - pub(crate) chain_root_for_role0_key_purge_queries: SizedBatch, - /// Chain Root for Stake Address Purge Query.. - pub(crate) chain_root_for_stake_address_purge_queries: SizedBatch, -} - -impl RollforwardPurgeQuery { - /// Prepare the purge query. - pub(crate) async fn prepare_batch( - _session: &Arc, _cfg: &EnvVars, - ) -> anyhow::Result { - todo!(); - } -} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 3ddaf30d38b..c311c0ef0a3 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,5 +1,7 @@ //! Queries for purging volatile data. +pub(crate) mod txo_by_stake_addr; + use std::{fmt::Debug, sync::Arc}; use scylla::{ @@ -8,13 +10,12 @@ use scylla::{ }; use super::{ - super::block::roll_forward::{PurgeBatches, RollforwardPurgeQuery}, FallibleQueryResults, SizedBatch, }; -use crate::{ - db::index::block::roll_forward::txo_by_stake_addr::get::TxoByStakeAddressPrimaryKeyQuery, - settings::cassandra_db, -}; +use crate::settings::cassandra_db; + +/// No parameters +const NO_PARAMS = (); /// All prepared DELETE query statements (purge DB table rows). #[derive(strum_macros::Display)] @@ -137,13 +138,14 @@ pub(crate) struct PreparedQueries { impl PreparedQueries { /// Create new prepared queries for a given session. - #[allow(clippy::todo)] + #[allow(clippy::todo, unused_variables)] pub(crate) async fn new( session: Arc, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { // We initialize like this, so that all errors preparing querys get shown before aborting. - let all_purge_queries = RollforwardPurgeQuery::prepare_batch(&session, cfg).await; - let get_txo_purge_queries = TxoByStakeAddressPrimaryKeyQuery::prepare(&session).await?; + let get_txo_purge_queries = + txo_by_stake_addr::select::PrimaryKeyQuery::prepare(&session).await?; + let _unused = " let PurgeBatches { get_txo_asset_purge_queries, get_unstaked_txo_purge_queries, @@ -201,6 +203,8 @@ impl PreparedQueries { chain_root_for_role0_key_purge_queries, chain_root_for_stake_address_purge_queries, }) + "; + todo!("WIP"); } /// Prepares a statement. diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs new file mode 100644 index 00000000000..3f608538bf2 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs @@ -0,0 +1,80 @@ +//! Delete queries for TXO by Stake Address. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{SerializeRow, Session}; + +use crate::{ + db::index::{queries::{purge::{PreparedDeleteQuery, PreparedQueries}, FallibleQueryResults, SizedBatch}, session::CassandraSession}, + settings::cassandra_db, +}; + +/// Delete TXO by Stake Address +const DELETE_QUERY: &str = include_str!("../cql/delete_txo_by_stake_address.cql"); + +/// Get TXO by Stake Address Query Parameters +#[derive(SerializeRow, Clone)] +pub(super) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + stake_address: Vec, + /// Block Slot Number + slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + txn: i16, + /// Transaction Output Offset inside the transaction. + txo: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("txo", &self.txo) + .finish() + } +} + +impl Params { + /// Create a new record for this transaction. + pub(super) fn new(stake_address: &[u8], slot_no: u64, txn: i16, txo: i16) -> Self { + Self { + stake_address: stake_address.to_vec(), + slot_no: slot_no.into(), + txn, + txo, + } + } +} + +/// Delet;e TXO by Stake Address Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(super) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs new file mode 100644 index 00000000000..edcdcdb5749 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs @@ -0,0 +1,4 @@ +//! TXO by Stake Address Queries used in purging data. + +pub(crate) mod delete; +pub(crate) mod select; diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs new file mode 100644 index 00000000000..c795637f0c2 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs @@ -0,0 +1,64 @@ +//! Select queries for TXO by Stake Address. + +use std::sync::Arc; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, FromRow, + Session, +}; +use tracing::error; + +use crate::db::index::{ + queries::purge::{PreparedQueries, PreparedSelectQuery}, + session::CassandraSession, +}; + +/// Select primary keys for TXO by Stake Address. +const SELECT_QUERY: &str = include_str!("../cql/get_txo_by_stake_address.cql"); + +/// Primary Key Row +#[derive(FromRow)] +pub(crate) struct PrimaryKey { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + stake_address: Vec, + /// Block Slot Number + slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + txn: i16, + /// Transaction Output Offset inside the transaction. + txo: i16, +} + +/// Get primary key for TXO by Stake Address query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all TXO by stake address primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get TXO by stake address primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all TXO by stake address primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} From 1ba35b125261ea32dded6f8db83815e5678444aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 22:45:24 -0600 Subject: [PATCH 08/37] wip(cat-gateway): refactor queries for purging volatile data --- .../bin/src/db/index/queries/purge/mod.rs | 15 +-- .../queries/purge/txo_by_stake_addr/delete.rs | 79 ------------ .../queries/purge/txo_by_stake_addr/mod.rs | 118 +++++++++++++++++- .../queries/purge/txo_by_stake_addr/select.rs | 63 ---------- catalyst-gateway/bin/src/db/index/session.rs | 9 +- 5 files changed, 130 insertions(+), 154 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index c311c0ef0a3..6652a817b82 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -9,16 +9,15 @@ use scylla::{ transport::iterator::RowIterator, Session, }; -use super::{ - FallibleQueryResults, SizedBatch, -}; +use super::{FallibleQueryResults, SizedBatch}; use crate::settings::cassandra_db; /// No parameters -const NO_PARAMS = (); +const NO_PARAMS: () = (); /// All prepared DELETE query statements (purge DB table rows). #[derive(strum_macros::Display)] +#[allow(dead_code)] pub(crate) enum PreparedDeleteQuery { /// TXO Delete query. TxoAda, @@ -50,6 +49,7 @@ pub(crate) enum PreparedDeleteQuery { /// All prepared SELECT query statements (primary keys from table). #[derive(strum_macros::Display)] +#[allow(dead_code)] pub(crate) enum PreparedSelectQuery { /// TXO Select query. TxoAda, @@ -138,13 +138,14 @@ pub(crate) struct PreparedQueries { impl PreparedQueries { /// Create new prepared queries for a given session. - #[allow(clippy::todo, unused_variables)] + #[allow(clippy::todo, clippy::no_effect_underscore_binding, unused_variables)] pub(crate) async fn new( session: Arc, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { // We initialize like this, so that all errors preparing querys get shown before aborting. - let get_txo_purge_queries = - txo_by_stake_addr::select::PrimaryKeyQuery::prepare(&session).await?; + let get_txo_purge_queries = txo_by_stake_addr::PrimaryKeyQuery::prepare(&session).await?; + let txo_purge_queries = + txo_by_stake_addr::DeleteQuery::prepare_batch(&session, cfg).await?; let _unused = " let PurgeBatches { get_txo_asset_purge_queries, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs index 3f608538bf2..8b137891791 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs @@ -1,80 +1 @@ -//! Delete queries for TXO by Stake Address. -use std::{fmt::Debug, sync::Arc}; -use scylla::{SerializeRow, Session}; - -use crate::{ - db::index::{queries::{purge::{PreparedDeleteQuery, PreparedQueries}, FallibleQueryResults, SizedBatch}, session::CassandraSession}, - settings::cassandra_db, -}; - -/// Delete TXO by Stake Address -const DELETE_QUERY: &str = include_str!("../cql/delete_txo_by_stake_address.cql"); - -/// Get TXO by Stake Address Query Parameters -#[derive(SerializeRow, Clone)] -pub(super) struct Params { - /// Stake Address - Binary 28 bytes. 0 bytes = not staked. - stake_address: Vec, - /// Block Slot Number - slot_no: num_bigint::BigInt, - /// Transaction Offset inside the block. - txn: i16, - /// Transaction Output Offset inside the transaction. - txo: i16, -} - -impl Debug for Params { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Params") - .field("stake_address", &self.stake_address) - .field("slot_no", &self.slot_no) - .field("txn", &self.txn) - .field("txo", &self.txo) - .finish() - } -} - -impl Params { - /// Create a new record for this transaction. - pub(super) fn new(stake_address: &[u8], slot_no: u64, txn: i16, txo: i16) -> Self { - Self { - stake_address: stake_address.to_vec(), - slot_no: slot_no.into(), - txn, - txo, - } - } -} - -/// Delet;e TXO by Stake Address Query -pub(crate) struct DeleteQuery; - -impl DeleteQuery { - /// Prepare Batch of Delete Queries - pub(super) async fn prepare_batch( - session: &Arc, cfg: &cassandra_db::EnvVars, - ) -> anyhow::Result { - let delete_queries = PreparedQueries::prepare_batch( - session.clone(), - DELETE_QUERY, - cfg, - scylla::statement::Consistency::Any, - true, - false, - ) - .await?; - Ok(delete_queries) - } - - /// Executes a DELETE Query - pub(crate) async fn execute( - session: &CassandraSession, params: Vec, - ) -> FallibleQueryResults { - let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) - .await?; - - Ok(results) - } -} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs index edcdcdb5749..a2ddb1264b8 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs @@ -1,4 +1,118 @@ //! TXO by Stake Address Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; -pub(crate) mod delete; -pub(crate) mod select; +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, FromRow, + SerializeRow, Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +/// Select primary keys for TXO by Stake Address. +const SELECT_QUERY: &str = include_str!("../cql/get_txo_by_stake_address.cql"); + +/// Primary Key Row +#[derive(FromRow, SerializeRow, Clone)] +#[allow(dead_code)] +pub(crate) struct PrimaryKey { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, + /// Transaction Output Offset inside the transaction. + pub(crate) txo: i16, +} + +impl Debug for PrimaryKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("txo", &self.txo) + .finish() + } +} + +/// Get primary key for TXO by Stake Address query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all TXO by stake address primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get TXO by stake address primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all TXO by stake address primary keys. + #[allow(dead_code)] + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete TXO by Stake Address +const DELETE_QUERY: &str = include_str!("../cql/delete_txo_by_stake_address.cql"); + +/// Delet;e TXO by Stake Address Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + #[allow(dead_code)] + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs index c795637f0c2..40f5cf3e8b6 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs @@ -1,64 +1 @@ //! Select queries for TXO by Stake Address. - -use std::sync::Arc; - -use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, FromRow, - Session, -}; -use tracing::error; - -use crate::db::index::{ - queries::purge::{PreparedQueries, PreparedSelectQuery}, - session::CassandraSession, -}; - -/// Select primary keys for TXO by Stake Address. -const SELECT_QUERY: &str = include_str!("../cql/get_txo_by_stake_address.cql"); - -/// Primary Key Row -#[derive(FromRow)] -pub(crate) struct PrimaryKey { - /// Stake Address - Binary 28 bytes. 0 bytes = not staked. - stake_address: Vec, - /// Block Slot Number - slot_no: num_bigint::BigInt, - /// Transaction Offset inside the block. - txn: i16, - /// Transaction Output Offset inside the transaction. - txo: i16, -} - -/// Get primary key for TXO by Stake Address query. -pub(crate) struct PrimaryKeyQuery; - -impl PrimaryKeyQuery { - /// Prepares a query to get all TXO by stake address primary keys. - pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { - let select_primary_key = PreparedQueries::prepare( - session.clone(), - SELECT_QUERY, - scylla::statement::Consistency::All, - true, - ) - .await; - - if let Err(ref error) = select_primary_key { - error!(error=%error, "Failed to prepare get TXO by stake address primary key query"); - }; - - select_primary_key - } - - /// Executes a query to get all TXO by stake address primary keys. - pub(crate) async fn execute( - session: &CassandraSession, - ) -> anyhow::Result> { - let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) - .await? - .into_typed::(); - - Ok(iter) - } -} diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index 99485086043..bd4ef4c0df0 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -18,7 +18,7 @@ use tracing::{error, info}; use super::{ queries::{ purge::{self, PreparedDeleteQuery}, - FallibleQueryResults, PreparedQueries, PreparedSelectQuery, PreparedQuery, + FallibleQueryResults, PreparedQueries, PreparedQuery, PreparedSelectQuery, PreparedUpsertQuery, }, schema::create_schema, @@ -53,6 +53,7 @@ pub(crate) enum TlsChoice { #[derive(Clone)] pub(crate) struct CassandraSession { /// Is the session to the persistent or volatile DB? + #[allow(dead_code)] persistent: bool, /// Configuration for this session. cfg: Arc, @@ -173,7 +174,7 @@ impl CassandraSession { /// Execute a select query to gather primary keys for purging. pub(crate) async fn purge_execute_iter( - &self, query: purge::PreparedSelectQuery + &self, query: purge::PreparedSelectQuery, ) -> anyhow::Result { // Only execute purge queries on the volatile session let persistent = false; @@ -183,7 +184,9 @@ impl CassandraSession { }; let queries = self.purge_queries.clone(); - queries.execute_iter(volatile_db.session.clone(), query).await + queries + .execute_iter(volatile_db.session.clone(), query) + .await } /// Get underlying Raw Cassandra Session. From a73a1f674055903ca87fc3917691015cc3f21b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 11 Nov 2024 22:46:13 -0600 Subject: [PATCH 09/37] wip(cat-gateway): refactor queries for purging volatile data --- .../purge/{txo_by_stake_addr/mod.rs => txo_by_stake_addr.rs} | 0 .../bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs | 1 - .../bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs | 1 - 3 files changed, 2 deletions(-) rename catalyst-gateway/bin/src/db/index/queries/purge/{txo_by_stake_addr/mod.rs => txo_by_stake_addr.rs} (100%) delete mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs delete mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs similarity index 100% rename from catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/mod.rs rename to catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs deleted file mode 100644 index 8b137891791..00000000000 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/delete.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs deleted file mode 100644 index 40f5cf3e8b6..00000000000 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr/select.rs +++ /dev/null @@ -1 +0,0 @@ -//! Select queries for TXO by Stake Address. From 535aabfc80d6832ca96297071cd12ff9ee8b0870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 09:48:14 -0600 Subject: [PATCH 10/37] fix(cat-gateway): modify types for query result --- .../index/queries/purge/txo_by_stake_addr.rs | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs index a2ddb1264b8..53c6ca47894 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs @@ -2,8 +2,8 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, FromRow, - SerializeRow, Session, + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, }; use tracing::error; @@ -18,13 +18,19 @@ use crate::{ settings::cassandra_db, }; +pub(crate) mod result { + //! Return values for TXO by Stake Address purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16, i16); +} + /// Select primary keys for TXO by Stake Address. -const SELECT_QUERY: &str = include_str!("../cql/get_txo_by_stake_address.cql"); +const SELECT_QUERY: &str = include_str!("./cql/get_txo_by_stake_address.cql"); -/// Primary Key Row -#[derive(FromRow, SerializeRow, Clone)] -#[allow(dead_code)] -pub(crate) struct PrimaryKey { +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { /// Stake Address - Binary 28 bytes. 0 bytes = not staked. pub(crate) stake_address: Vec, /// Block Slot Number @@ -35,7 +41,7 @@ pub(crate) struct PrimaryKey { pub(crate) txo: i16, } -impl Debug for PrimaryKey { +impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") .field("stake_address", &self.stake_address) @@ -46,6 +52,16 @@ impl Debug for PrimaryKey { } } +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_address: value.0, + slot_no: value.1, + txn: value.2, + txo: value.3, + } + } +} /// Get primary key for TXO by Stake Address query. pub(crate) struct PrimaryKeyQuery; @@ -71,18 +87,18 @@ impl PrimaryKeyQuery { #[allow(dead_code)] pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .into_typed::(); Ok(iter) } } /// Delete TXO by Stake Address -const DELETE_QUERY: &str = include_str!("../cql/delete_txo_by_stake_address.cql"); +const DELETE_QUERY: &str = include_str!("./cql/delete_txo_by_stake_address.cql"); /// Delet;e TXO by Stake Address Query pub(crate) struct DeleteQuery; @@ -107,7 +123,7 @@ impl DeleteQuery { /// Executes a DELETE Query #[allow(dead_code)] pub(crate) async fn execute( - session: &CassandraSession, params: Vec, + session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) From 7aa90f15cb48ed2efcf38918f0c5ba12133e04e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 10:03:58 -0600 Subject: [PATCH 11/37] fix(cat-gateway): cleanup naming types --- .../bin/src/db/index/queries/purge/mod.rs | 129 +++++------------- .../{txo_by_stake_addr.rs => txo_ada.rs} | 0 2 files changed, 36 insertions(+), 93 deletions(-) rename catalyst-gateway/bin/src/db/index/queries/purge/{txo_by_stake_addr.rs => txo_ada.rs} (100%) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 6652a817b82..470467b5306 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,6 +1,6 @@ //! Queries for purging volatile data. -pub(crate) mod txo_by_stake_addr; +pub(crate) mod txo_ada; use std::{fmt::Debug, sync::Arc}; @@ -21,8 +21,8 @@ const NO_PARAMS: () = (); pub(crate) enum PreparedDeleteQuery { /// TXO Delete query. TxoAda, - /// TXO Asset Delete query. - TxoAsset, + /// TXO Assets Delete query. + TxoAssets, /// Unstaked TXO Delete query. UnstakedTxoAda, /// Unstaked TXO Asset Delete query. @@ -54,7 +54,7 @@ pub(crate) enum PreparedSelectQuery { /// TXO Select query. TxoAda, /// TXO Asset Select query. - TxoAsset, + TxoAssets, /// Unstaked TXO Select query. UnstakedTxoAda, /// Unstaked TXO Asset Select query. @@ -83,33 +83,33 @@ pub(crate) enum PreparedSelectQuery { #[allow(clippy::struct_field_names)] pub(crate) struct PreparedQueries { /// TXO Purge Query. - get_txo_purge_queries: PreparedStatement, + select_txo_ada: PreparedStatement, /// TXO Asset Purge Query. - get_txo_asset_purge_queries: PreparedStatement, + select_txo_assets: PreparedStatement, /// Unstaked TXO Purge Query. - get_unstaked_txo_purge_queries: PreparedStatement, + select_unstaked_txo_purge_queries: PreparedStatement, /// Unstaked TXO Asset Purge Query. - get_unstaked_txo_asset_purge_queries: PreparedStatement, + select_unstaked_txo_asset_purge_queries: PreparedStatement, /// TXI Purge Query. - get_txi_purge_queries: PreparedStatement, + select_txi_purge_queries: PreparedStatement, /// TXI Purge Query. - get_stake_registration_purge_queries: PreparedStatement, + select_stake_registration_purge_queries: PreparedStatement, /// CIP36 Registrations Purge Query. - get_cip36_registration_purge_queries: PreparedStatement, + select_cip36_registration_purge_queries: PreparedStatement, /// CIP36 Registration errors Purge Query. - get_cip36_registration_error_purge_queries: PreparedStatement, + select_cip36_registration_error_purge_queries: PreparedStatement, /// CIP36 Registration for Stake Address Purge Query. - get_cip36_registration_for_stake_address_purge_queries: PreparedStatement, + select_cip36_registration_for_stake_address_purge_queries: PreparedStatement, /// RBAC 509 Registrations Purge Query. - get_rbac509_registration_purge_queries: PreparedStatement, + select_rbac509_registration_purge_queries: PreparedStatement, /// Chain Root for TX ID Purge Query.. - get_chain_root_for_txn_id_purge_queries: PreparedStatement, + select_chain_root_for_txn_id_purge_queries: PreparedStatement, /// Chain Root for Role 0 Key Purge Query.. - get_chain_root_for_role0_key_purge_queries: PreparedStatement, + select_chain_root_for_role0_key_purge_queries: PreparedStatement, /// Chain Root for Stake Address Purge Query.. - get_chain_root_for_stake_address_purge_queries: PreparedStatement, + select_chain_root_for_stake_address_purge_queries: PreparedStatement, /// TXO Purge Query. - txo_purge_queries: SizedBatch, + delete_txo_ada: SizedBatch, /// TXO Asset Purge Query. txo_asset_purge_queries: SizedBatch, /// Unstaked TXO Purge Query. @@ -143,68 +143,9 @@ impl PreparedQueries { session: Arc, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { // We initialize like this, so that all errors preparing querys get shown before aborting. - let get_txo_purge_queries = txo_by_stake_addr::PrimaryKeyQuery::prepare(&session).await?; - let txo_purge_queries = - txo_by_stake_addr::DeleteQuery::prepare_batch(&session, cfg).await?; - let _unused = " - let PurgeBatches { - get_txo_asset_purge_queries, - get_unstaked_txo_purge_queries, - get_unstaked_txo_asset_purge_queries, - get_txi_purge_queries, - get_stake_registration_purge_queries, - get_cip36_registration_purge_queries, - get_cip36_registration_error_purge_queries, - get_cip36_registration_for_stake_address_purge_queries, - get_rbac509_registration_purge_queries, - get_chain_root_for_txn_id_purge_queries, - get_chain_root_for_role0_key_purge_queries, - get_chain_root_for_stake_address_purge_queries, - txo_purge_queries, - txo_asset_purge_queries, - unstaked_txo_purge_queries, - unstaked_txo_asset_purge_queries, - txi_purge_queries, - stake_registration_purge_queries, - cip36_registration_purge_queries, - cip36_registration_error_purge_queries, - cip36_registration_for_stake_address_purge_queries, - rbac509_registration_purge_queries, - chain_root_for_txn_id_purge_queries, - chain_root_for_role0_key_purge_queries, - chain_root_for_stake_address_purge_queries, - .. - }: PurgeBatches = all_purge_queries?; + let select_txo_ada = txo_ada::PrimaryKeyQuery::prepare(&session).await?; + let delete_txo_ada = txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; - Ok(Self { - get_txo_purge_queries, - get_txo_asset_purge_queries, - get_unstaked_txo_purge_queries, - get_unstaked_txo_asset_purge_queries, - get_txi_purge_queries, - get_stake_registration_purge_queries, - get_cip36_registration_purge_queries, - get_cip36_registration_error_purge_queries, - get_cip36_registration_for_stake_address_purge_queries, - get_rbac509_registration_purge_queries, - get_chain_root_for_txn_id_purge_queries, - get_chain_root_for_role0_key_purge_queries, - get_chain_root_for_stake_address_purge_queries, - txo_purge_queries, - txo_asset_purge_queries, - unstaked_txo_purge_queries, - unstaked_txo_asset_purge_queries, - txi_purge_queries, - stake_registration_purge_queries, - cip36_registration_purge_queries, - cip36_registration_error_purge_queries, - cip36_registration_for_stake_address_purge_queries, - rbac509_registration_purge_queries, - chain_root_for_txn_id_purge_queries, - chain_root_for_role0_key_purge_queries, - chain_root_for_stake_address_purge_queries, - }) - "; todo!("WIP"); } @@ -235,26 +176,28 @@ impl PreparedQueries { &self, session: Arc, select_query: PreparedSelectQuery, ) -> anyhow::Result { let prepared_stmt = match select_query { - PreparedSelectQuery::TxoAda => &self.get_txo_purge_queries, - PreparedSelectQuery::TxoAsset => &self.get_txo_asset_purge_queries, + PreparedSelectQuery::TxoAda => &self.select_txo_ada, + PreparedSelectQuery::TxoAssets => &self.select_txo_assets, PreparedSelectQuery::UnstakedTxoAda => &self.get_unstaked_txo_purge_queries, - PreparedSelectQuery::UnstakedTxoAsset => &self.get_unstaked_txo_asset_purge_queries, - PreparedSelectQuery::Txi => &self.get_txi_purge_queries, - PreparedSelectQuery::StakeRegistration => &self.get_stake_registration_purge_queries, - PreparedSelectQuery::Cip36Registration => &self.get_cip36_registration_purge_queries, + PreparedSelectQuery::UnstakedTxoAsset => &self.select_unstaked_txo_asset_purge_queries, + PreparedSelectQuery::Txi => &self.select_txi_purge_queries, + PreparedSelectQuery::StakeRegistration => &self.select_stake_registration_purge_queries, + PreparedSelectQuery::Cip36Registration => &self.select_cip36_registration_purge_queries, PreparedSelectQuery::Cip36RegistrationError => { - &self.get_cip36_registration_error_purge_queries + &self.select_cip36_registration_error_purge_queries }, PreparedSelectQuery::Cip36RegistrationForStakeAddr => { - &self.get_cip36_registration_for_stake_address_purge_queries + &self.select_cip36_registration_for_stake_address_purge_queries + }, + PreparedSelectQuery::Rbac509 => &self.select_rbac509_registration_purge_queries, + PreparedSelectQuery::ChainRootForTxnId => { + &self.select_chain_root_for_txn_id_purge_queries }, - PreparedSelectQuery::Rbac509 => &self.get_rbac509_registration_purge_queries, - PreparedSelectQuery::ChainRootForTxnId => &self.get_chain_root_for_txn_id_purge_queries, PreparedSelectQuery::ChainRootForRole0Key => { - &self.get_chain_root_for_role0_key_purge_queries + &self.select_chain_root_for_role0_key_purge_queries }, PreparedSelectQuery::ChainRootForStakeAddress => { - &self.get_chain_root_for_stake_address_purge_queries + &self.select_chain_root_for_stake_address_purge_queries }, }; @@ -267,8 +210,8 @@ impl PreparedQueries { values: Vec, ) -> FallibleQueryResults { let query_map = match query { - PreparedDeleteQuery::TxoAda => &self.txo_purge_queries, - PreparedDeleteQuery::TxoAsset => &self.txo_asset_purge_queries, + PreparedDeleteQuery::TxoAda => &self.delete_txo_ada, + PreparedDeleteQuery::TxoAssets => &self.txo_asset_purge_queries, PreparedDeleteQuery::UnstakedTxoAda => &self.unstaked_txo_purge_queries, PreparedDeleteQuery::UnstakedTxoAsset => &self.unstaked_txo_asset_purge_queries, PreparedDeleteQuery::Txi => &self.txi_purge_queries, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs similarity index 100% rename from catalyst-gateway/bin/src/db/index/queries/purge/txo_by_stake_addr.rs rename to catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs From bb2a4b086a200b544ee847e44236b330be6efa1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 10:07:40 -0600 Subject: [PATCH 12/37] chore(cat-gateway): fix spelling --- .../purge/cql/delete_cip36_registration_for_vote_key.cql | 2 +- .../queries/purge/cql/get_cip36_registration_for_vote_key.cql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql index 571e289c803..6cfa670f048 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_cip36_registration_for_vote_key.cql @@ -1,4 +1,4 @@ --- Delete CIP-36 registration by Stake Addres. +-- Delete CIP-36 registration by Stake Address. DELETE FROM cip36_registration_for_vote_key WHERE vote_key = :vote_key AND stake_address = :stake_address diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql index a57da1f281b..8769a22cde1 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql @@ -1,4 +1,4 @@ --- Get all primary keys from CIP-36 registration by Stake Addres. +-- Get all primary keys from CIP-36 registration by Stake Address. SELECT vote_key, stake_address, From 617a9c716353cef8906203c1f2a9b16ece38e742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 10:51:08 -0600 Subject: [PATCH 13/37] chore(cat-gateway): fix spelling --- catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs index 53c6ca47894..dcb842ac4a5 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs @@ -100,7 +100,7 @@ impl PrimaryKeyQuery { /// Delete TXO by Stake Address const DELETE_QUERY: &str = include_str!("./cql/delete_txo_by_stake_address.cql"); -/// Delet;e TXO by Stake Address Query +/// Delete TXO by Stake Address Query pub(crate) struct DeleteQuery; impl DeleteQuery { From a36778b9a682944524100689d0584803ee2de872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 11:14:12 -0600 Subject: [PATCH 14/37] wip(cat-gateway): rust code for txo assets by stake queries --- .../bin/src/db/index/queries/purge/mod.rs | 5 +- .../src/db/index/queries/purge/txo_assets.rs | 141 ++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 470467b5306..4a160c404d0 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,6 +1,7 @@ //! Queries for purging volatile data. pub(crate) mod txo_ada; +pub(crate) mod txo_assets; use std::{fmt::Debug, sync::Arc}; @@ -138,13 +139,15 @@ pub(crate) struct PreparedQueries { impl PreparedQueries { /// Create new prepared queries for a given session. - #[allow(clippy::todo, clippy::no_effect_underscore_binding, unused_variables)] + #[allow(clippy::todo, unused_variables)] pub(crate) async fn new( session: Arc, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { // We initialize like this, so that all errors preparing querys get shown before aborting. let select_txo_ada = txo_ada::PrimaryKeyQuery::prepare(&session).await?; let delete_txo_ada = txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_txo_assets = txo_assets::PrimaryKeyQuery::prepare(&session).await?; + let delete_txo_assets = txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs new file mode 100644 index 00000000000..f724e147ea4 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs @@ -0,0 +1,141 @@ +//! TXO Assets by Stake Address Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for TXO Assets by Stake Address purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16, i16, Vec, String); +} + +/// Select primary keys for TXO Assets by Stake Address. +const SELECT_QUERY: &str = include_str!("./cql/get_txo_by_stake_address.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, + /// Transaction Output Offset inside the transaction. + pub(crate) txo: i16, + /// Asset Policy Hash - Binayr 28 bytes. + policy_id: Vec, + /// Name of the Policy (UTF8) + // TODO: https://github.com/input-output-hk/catalyst-voices/issues/1121 + policy_name: String, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("txo", &self.txo) + .field("policy_id", &self.policy_id) + .field("policy_name", &self.policy_name) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_address: value.0, + slot_no: value.1, + txn: value.2, + txo: value.3, + policy_id: value.4, + policy_name: value.5, + } + } +} +/// Get primary key for TXO Assets by Stake Address query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all TXO Assets by stake address primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get TXO Assets by stake address primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all TXO Assets by stake address primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete TXO Assets by Stake Address +const DELETE_QUERY: &str = include_str!("./cql/delete_txo_by_stake_address.cql"); + +/// Delete TXO Assets by Stake Address Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} From 79228ae9446dc29ce946fd19ccb28c61a95d1987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 11:50:50 -0600 Subject: [PATCH 15/37] fix(cat-gateway): cleanup field names --- .../bin/src/db/index/queries/purge/mod.rs | 162 ++++++++---------- 1 file changed, 76 insertions(+), 86 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 4a160c404d0..a161807a737 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -28,7 +28,7 @@ pub(crate) enum PreparedDeleteQuery { UnstakedTxoAda, /// Unstaked TXO Asset Delete query. UnstakedTxoAsset, - /// TXI Delete query. + /// TXI by TXN Hash Delete query. Txi, /// Stake Registration Delete query. StakeRegistration, @@ -60,7 +60,7 @@ pub(crate) enum PreparedSelectQuery { UnstakedTxoAda, /// Unstaked TXO Asset Select query. UnstakedTxoAsset, - /// TXI Select query. + /// TXI by TXN Hash Select query. Txi, /// Stake Registration Select query. StakeRegistration, @@ -81,60 +81,60 @@ pub(crate) enum PreparedSelectQuery { } /// All prepared purge queries for a session. -#[allow(clippy::struct_field_names)] +#[allow(dead_code)] pub(crate) struct PreparedQueries { - /// TXO Purge Query. + /// TXO ADA Primary Key Query. select_txo_ada: PreparedStatement, - /// TXO Asset Purge Query. - select_txo_assets: PreparedStatement, - /// Unstaked TXO Purge Query. - select_unstaked_txo_purge_queries: PreparedStatement, - /// Unstaked TXO Asset Purge Query. - select_unstaked_txo_asset_purge_queries: PreparedStatement, - /// TXI Purge Query. - select_txi_purge_queries: PreparedStatement, - /// TXI Purge Query. - select_stake_registration_purge_queries: PreparedStatement, - /// CIP36 Registrations Purge Query. - select_cip36_registration_purge_queries: PreparedStatement, - /// CIP36 Registration errors Purge Query. - select_cip36_registration_error_purge_queries: PreparedStatement, - /// CIP36 Registration for Stake Address Purge Query. - select_cip36_registration_for_stake_address_purge_queries: PreparedStatement, - /// RBAC 509 Registrations Purge Query. - select_rbac509_registration_purge_queries: PreparedStatement, - /// Chain Root for TX ID Purge Query.. - select_chain_root_for_txn_id_purge_queries: PreparedStatement, - /// Chain Root for Role 0 Key Purge Query.. - select_chain_root_for_role0_key_purge_queries: PreparedStatement, - /// Chain Root for Stake Address Purge Query.. - select_chain_root_for_stake_address_purge_queries: PreparedStatement, - /// TXO Purge Query. + /// TXO Delete Query. delete_txo_ada: SizedBatch, - /// TXO Asset Purge Query. - txo_asset_purge_queries: SizedBatch, - /// Unstaked TXO Purge Query. - unstaked_txo_purge_queries: SizedBatch, - /// Unstaked TXO Asset Purge Query. - unstaked_txo_asset_purge_queries: SizedBatch, - /// TXI Purge Query. - txi_purge_queries: SizedBatch, - /// TXI Purge Query. - stake_registration_purge_queries: SizedBatch, - /// CIP36 Registrations Purge Query. - cip36_registration_purge_queries: SizedBatch, - /// CIP36 Registration errors Purge Query. - cip36_registration_error_purge_queries: SizedBatch, - /// CIP36 Registration for Stake Address Purge Query. - cip36_registration_for_stake_address_purge_queries: SizedBatch, - /// RBAC 509 Registrations Purge Query. - rbac509_registration_purge_queries: SizedBatch, - /// Chain Root for TX ID Purge Query.. - chain_root_for_txn_id_purge_queries: SizedBatch, - /// Chain Root for Role 0 Key Purge Query.. - chain_root_for_role0_key_purge_queries: SizedBatch, - /// Chain Root for Stake Address Purge Query.. - chain_root_for_stake_address_purge_queries: SizedBatch, + /// TXO Asset Primary Key Query. + select_txo_assets: PreparedStatement, + /// TXO Assets Delete Query. + delete_txo_assets: SizedBatch, + /// Unstaked TXO ADA Primary Key Query. + select_unstaked_txo_ada: PreparedStatement, + /// Unstaked TXO Assets Primary Key Query. + select_unstaked_txo_assets: PreparedStatement, + /// TXI by TXN Hash by TXN Hash Primary Key Query. + select_txi_by_hash: PreparedStatement, + /// Stake Registration Primary Key Query. + select_stake_registration: PreparedStatement, + /// CIP36 Registrations Primary Key Query. + select_cip36_registration: PreparedStatement, + /// CIP36 Registration errors Primary Key Query. + select_cip36_registration_error: PreparedStatement, + /// CIP36 Registration for Stake Address Primary Key Query. + select_cip36_registration_for_stake_address: PreparedStatement, + /// RBAC 509 Registrations Primary Key Query. + select_rbac509_registration: PreparedStatement, + /// Chain Root for TX ID Primary Key Query.. + select_chain_root_for_txn_id: PreparedStatement, + /// Chain Root for Role 0 Key Primary Key Query.. + select_chain_root_for_role0_key: PreparedStatement, + /// Chain Root for Stake Address Primary Key Query.. + select_chain_root_for_stake_address: PreparedStatement, + /// Unstaked TXO ADA Delete Query. + delete_unstaked_txo_ada: SizedBatch, + /// Unstaked TXO Asset Delete Query. + delete_unstaked_txo_assets: SizedBatch, + /// TXI by TXN Hash Delete Query. + delete_txi: SizedBatch, + /// Stake Registration Delete Query. + delete_stake_registration: SizedBatch, + /// CIP36 Registrations Delete Query. + delete_cip36_registration: SizedBatch, + /// CIP36 Registration errors Delete Query. + delete_cip36_registration_error: SizedBatch, + /// CIP36 Registration for Stake Address Delete Query. + delete_cip36_registration_for_stake_addr: SizedBatch, + /// RBAC 509 Registrations Delete Query. + delete_rbac509_registration: SizedBatch, + /// Chain Root for TX ID Delete Query.. + delete_chain_root_for_txn_id: SizedBatch, + /// Chain Root for Role 0 Key Delete Query.. + delete_chain_root_for_role0_key: SizedBatch, + /// Chain Root for Stake Address Delete Query.. + delete_chain_root_for_stake_address: SizedBatch, } impl PreparedQueries { @@ -181,26 +181,20 @@ impl PreparedQueries { let prepared_stmt = match select_query { PreparedSelectQuery::TxoAda => &self.select_txo_ada, PreparedSelectQuery::TxoAssets => &self.select_txo_assets, - PreparedSelectQuery::UnstakedTxoAda => &self.get_unstaked_txo_purge_queries, - PreparedSelectQuery::UnstakedTxoAsset => &self.select_unstaked_txo_asset_purge_queries, - PreparedSelectQuery::Txi => &self.select_txi_purge_queries, - PreparedSelectQuery::StakeRegistration => &self.select_stake_registration_purge_queries, - PreparedSelectQuery::Cip36Registration => &self.select_cip36_registration_purge_queries, - PreparedSelectQuery::Cip36RegistrationError => { - &self.select_cip36_registration_error_purge_queries - }, + PreparedSelectQuery::UnstakedTxoAda => &self.select_unstaked_txo_ada, + PreparedSelectQuery::UnstakedTxoAsset => &self.select_unstaked_txo_assets, + PreparedSelectQuery::Txi => &self.select_txi_by_hash, + PreparedSelectQuery::StakeRegistration => &self.select_stake_registration, + PreparedSelectQuery::Cip36Registration => &self.select_cip36_registration, + PreparedSelectQuery::Cip36RegistrationError => &self.select_cip36_registration_error, PreparedSelectQuery::Cip36RegistrationForStakeAddr => { - &self.select_cip36_registration_for_stake_address_purge_queries - }, - PreparedSelectQuery::Rbac509 => &self.select_rbac509_registration_purge_queries, - PreparedSelectQuery::ChainRootForTxnId => { - &self.select_chain_root_for_txn_id_purge_queries - }, - PreparedSelectQuery::ChainRootForRole0Key => { - &self.select_chain_root_for_role0_key_purge_queries + &self.select_cip36_registration_for_stake_address }, + PreparedSelectQuery::Rbac509 => &self.select_rbac509_registration, + PreparedSelectQuery::ChainRootForTxnId => &self.select_chain_root_for_txn_id, + PreparedSelectQuery::ChainRootForRole0Key => &self.select_chain_root_for_role0_key, PreparedSelectQuery::ChainRootForStakeAddress => { - &self.select_chain_root_for_stake_address_purge_queries + &self.select_chain_root_for_stake_address }, }; @@ -214,25 +208,21 @@ impl PreparedQueries { ) -> FallibleQueryResults { let query_map = match query { PreparedDeleteQuery::TxoAda => &self.delete_txo_ada, - PreparedDeleteQuery::TxoAssets => &self.txo_asset_purge_queries, - PreparedDeleteQuery::UnstakedTxoAda => &self.unstaked_txo_purge_queries, - PreparedDeleteQuery::UnstakedTxoAsset => &self.unstaked_txo_asset_purge_queries, - PreparedDeleteQuery::Txi => &self.txi_purge_queries, - PreparedDeleteQuery::StakeRegistration => &self.stake_registration_purge_queries, - PreparedDeleteQuery::Cip36Registration => &self.cip36_registration_purge_queries, - PreparedDeleteQuery::Cip36RegistrationError => { - &self.cip36_registration_error_purge_queries - }, + PreparedDeleteQuery::TxoAssets => &self.delete_txo_assets, + PreparedDeleteQuery::UnstakedTxoAda => &self.delete_unstaked_txo_ada, + PreparedDeleteQuery::UnstakedTxoAsset => &self.delete_unstaked_txo_assets, + PreparedDeleteQuery::Txi => &self.delete_txi, + PreparedDeleteQuery::StakeRegistration => &self.delete_stake_registration, + PreparedDeleteQuery::Cip36Registration => &self.delete_cip36_registration, + PreparedDeleteQuery::Cip36RegistrationError => &self.delete_cip36_registration_error, PreparedDeleteQuery::Cip36RegistrationForStakeAddr => { - &self.cip36_registration_for_stake_address_purge_queries - }, - PreparedDeleteQuery::Rbac509 => &self.rbac509_registration_purge_queries, - PreparedDeleteQuery::ChainRootForTxnId => &self.chain_root_for_txn_id_purge_queries, - PreparedDeleteQuery::ChainRootForRole0Key => { - &self.chain_root_for_role0_key_purge_queries + &self.delete_cip36_registration_for_stake_addr }, + PreparedDeleteQuery::Rbac509 => &self.delete_rbac509_registration, + PreparedDeleteQuery::ChainRootForTxnId => &self.delete_chain_root_for_txn_id, + PreparedDeleteQuery::ChainRootForRole0Key => &self.delete_chain_root_for_role0_key, PreparedDeleteQuery::ChainRootForStakeAddress => { - &self.chain_root_for_stake_address_purge_queries + &self.delete_chain_root_for_stake_address }, }; From bb9fb0a2d6125a9c2ca3c3119d9bf9bd9d3f6bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 13:26:49 -0600 Subject: [PATCH 16/37] fix(cat-gateway): incorporate unstaked txo queries, cleanup cql comments --- ...delete_unstaked_txo_assets_by_txn_hash.cql | 2 +- .../cql/get_unstaked_txo_by_txn_hash.cql | 2 +- .../bin/src/db/index/queries/purge/mod.rs | 9 ++ .../index/queries/purge/unstaked_txo_ada.rs | 125 ++++++++++++++++ .../queries/purge/unstaked_txo_assets.rs | 141 ++++++++++++++++++ .../cql/unstaked_txo_assets_by_txn_hash.cql | 2 +- 6 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql index 1c0270c3967..8b975bf963d 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_unstaked_txo_assets_by_txn_hash.cql @@ -1,4 +1,4 @@ --- Delete Transaction Outputs (Native Assets) by Transaction Hash. +-- Delete Primary Keys from Unstaked Transaction Outputs (Native Assets) by their transaction hash. DELETE FROM unstaked_txo_assets_by_txn_hash WHERE txn_hash = :txn_hash AND txo = :txo diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql index eceaac8e733..dffba316e50 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_unstaked_txo_by_txn_hash.cql @@ -1,4 +1,4 @@ --- Get all primary keys from Unstaked Transaction Output Assets by Transaction Hash. +-- Get ALL Primary Keys from Unstaked Transaction Outputs (Native Assets) per stake address. SELECT txn_hash, txo diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index a161807a737..f86e8e1522f 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -2,6 +2,8 @@ pub(crate) mod txo_ada; pub(crate) mod txo_assets; +pub(crate) mod unstaked_txo_ada; +pub(crate) mod unstaked_txo_assets; use std::{fmt::Debug, sync::Arc}; @@ -148,6 +150,13 @@ impl PreparedQueries { let delete_txo_ada = txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; let select_txo_assets = txo_assets::PrimaryKeyQuery::prepare(&session).await?; let delete_txo_assets = txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_unstaked_txo_ada = unstaked_txo_ada::PrimaryKeyQuery::prepare(&session).await?; + let delete_unstaked_txo_ada = + unstaked_txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_unstaked_txo_assets = + unstaked_txo_assets::PrimaryKeyQuery::prepare(&session).await?; + let delete_unstaked_txo_assets = + unstaked_txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs new file mode 100644 index 00000000000..09ef36572e5 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs @@ -0,0 +1,125 @@ +//! Unstaked Transaction Outputs (ADA), by their transaction hash, queries used in purging +//! data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for Unstaked TXO ADA purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, i16); +} + +/// Select primary keys for Unstaked TXO ADA. +const SELECT_QUERY: &str = include_str!("./cql/get_unstaked_txo_by_txn_hash.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// 32 byte hash of this transaction. + pub(crate) txn_hash: Vec, + /// Transaction Output Offset inside the transaction. + pub(crate) txo: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("txn_hash", &self.txn_hash) + .field("txo", &self.txo) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + txn_hash: value.0, + txo: value.1, + } + } +} +/// Get primary key for Unstaked TXO ADA query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all Unstaked TXO ADA primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get Unstaked TXO ADA primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all Unstaked TXO ADA primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::UnstakedTxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete Unstaked TXO by Stake Address +const DELETE_QUERY: &str = include_str!("./cql/delete_unstaked_txo_by_txn_hash.cql"); + +/// Delete TXO by Stake Address Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs new file mode 100644 index 00000000000..63c9af73706 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs @@ -0,0 +1,141 @@ +//! TXO Assets by TXN Hash Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for TXO Assets by TXN Hash purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16, i16, Vec, String); +} + +/// Select primary keys for TXO Assets by TXN Hash. +const SELECT_QUERY: &str = include_str!("./cql/get_unstaked_txo_assets_by_txn_hash.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, + /// Transaction Output Offset inside the transaction. + pub(crate) txo: i16, + /// Asset Policy Hash - Binayr 28 bytes. + policy_id: Vec, + /// Name of the Policy (UTF8) + // TODO: https://github.com/input-output-hk/catalyst-voices/issues/1121 + policy_name: String, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("txo", &self.txo) + .field("policy_id", &self.policy_id) + .field("policy_name", &self.policy_name) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_address: value.0, + slot_no: value.1, + txn: value.2, + txo: value.3, + policy_id: value.4, + policy_name: value.5, + } + } +} +/// Get primary key for TXO Assets by TXN Hash query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all TXO Assets by TXN Hash primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get TXO Assets by TXN Hash primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all TXO Assets by TXN Hash primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete TXO Assets by TXN Hash +const DELETE_QUERY: &str = include_str!("./cql/delete_unstaked_txo_assets_by_txn_hash.cql"); + +/// Delete TXO Assets by TXN Hash Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/unstaked_txo_assets_by_txn_hash.cql b/catalyst-gateway/bin/src/db/index/schema/cql/unstaked_txo_assets_by_txn_hash.cql index 4628e236ec4..c3dffcfbd75 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/unstaked_txo_assets_by_txn_hash.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/unstaked_txo_assets_by_txn_hash.cql @@ -1,4 +1,4 @@ --- Transaction Outputs (Native Assets) per stake address. +-- Transaction Outputs (Native Assets) by their transaction hash. CREATE TABLE IF NOT EXISTS unstaked_txo_assets_by_txn_hash ( -- Primary Key Fields txn_hash blob, -- 32 byte hash of this transaction. From 7d2ee85af2985fabf3ef4f30fc72ab440e0399a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 14:01:48 -0600 Subject: [PATCH 17/37] fix(cat-gateway): incorporate unstaked txi queries, cleanup cql comments --- .../bin/src/db/index/queries/purge/mod.rs | 17 ++- .../src/db/index/queries/purge/txi_by_hash.rs | 124 ++++++++++++++++++ 2 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index f86e8e1522f..69778349be9 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,5 +1,6 @@ //! Queries for purging volatile data. +pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; pub(crate) mod txo_assets; pub(crate) mod unstaked_txo_ada; @@ -95,10 +96,16 @@ pub(crate) struct PreparedQueries { delete_txo_assets: SizedBatch, /// Unstaked TXO ADA Primary Key Query. select_unstaked_txo_ada: PreparedStatement, + /// Unstaked TXO ADA Delete Query. + delete_unstaked_txo_ada: SizedBatch, /// Unstaked TXO Assets Primary Key Query. select_unstaked_txo_assets: PreparedStatement, + /// Unstaked TXO Asset Delete Query. + delete_unstaked_txo_assets: SizedBatch, /// TXI by TXN Hash by TXN Hash Primary Key Query. select_txi_by_hash: PreparedStatement, + /// TXI by TXN Hash Delete Query. + delete_txi_by_hash: SizedBatch, /// Stake Registration Primary Key Query. select_stake_registration: PreparedStatement, /// CIP36 Registrations Primary Key Query. @@ -115,12 +122,6 @@ pub(crate) struct PreparedQueries { select_chain_root_for_role0_key: PreparedStatement, /// Chain Root for Stake Address Primary Key Query.. select_chain_root_for_stake_address: PreparedStatement, - /// Unstaked TXO ADA Delete Query. - delete_unstaked_txo_ada: SizedBatch, - /// Unstaked TXO Asset Delete Query. - delete_unstaked_txo_assets: SizedBatch, - /// TXI by TXN Hash Delete Query. - delete_txi: SizedBatch, /// Stake Registration Delete Query. delete_stake_registration: SizedBatch, /// CIP36 Registrations Delete Query. @@ -157,6 +158,8 @@ impl PreparedQueries { unstaked_txo_assets::PrimaryKeyQuery::prepare(&session).await?; let delete_unstaked_txo_assets = unstaked_txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_txi_by_hash = txi_by_hash::PrimaryKeyQuery::prepare(&session).await?; + let delete_txi_by_hash = txi_by_hash::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } @@ -220,7 +223,7 @@ impl PreparedQueries { PreparedDeleteQuery::TxoAssets => &self.delete_txo_assets, PreparedDeleteQuery::UnstakedTxoAda => &self.delete_unstaked_txo_ada, PreparedDeleteQuery::UnstakedTxoAsset => &self.delete_unstaked_txo_assets, - PreparedDeleteQuery::Txi => &self.delete_txi, + PreparedDeleteQuery::Txi => &self.delete_txi_by_hash, PreparedDeleteQuery::StakeRegistration => &self.delete_stake_registration, PreparedDeleteQuery::Cip36Registration => &self.delete_cip36_registration, PreparedDeleteQuery::Cip36RegistrationError => &self.delete_cip36_registration_error, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs new file mode 100644 index 00000000000..7477bcc61d0 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs @@ -0,0 +1,124 @@ +//! Transaction Inputs (ADA or a native asset) queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for TXI by hash purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, i16); +} + +/// Select primary keys for TXI by hash. +const SELECT_QUERY: &str = include_str!("./cql/get_txi_by_txn_hashes.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// 32 byte hash of this transaction. + pub(crate) txn_hash: Vec, + /// Transaction Output Offset inside the transaction. + pub(crate) txo: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("txn_hash", &self.txn_hash) + .field("txo", &self.txo) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + txn_hash: value.0, + txo: value.1, + } + } +} +/// Get primary key for TXI by hash query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all TXI by hash primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get TXI by hash primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all TXI by hash primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::Txi) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete TXI by hash Query +const DELETE_QUERY: &str = include_str!("./cql/delete_txi_by_txn_hashes.cql"); + +/// Delete TXI by hash Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::Txi, params) + .await?; + + Ok(results) + } +} From 80810faccb8e56a1aaebaa9b4ae5744d78d0a815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 14:33:11 -0600 Subject: [PATCH 18/37] fix(cat-gateway): incorporate stake registration queries, cleanup cql comments --- .../bin/src/db/index/queries/purge/mod.rs | 9 +- .../index/queries/purge/stake_registration.rs | 128 ++++++++++++++++++ 2 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 69778349be9..d1697f4dfa1 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,5 +1,6 @@ //! Queries for purging volatile data. +pub(crate) mod stake_registration; pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; pub(crate) mod txo_assets; @@ -108,6 +109,8 @@ pub(crate) struct PreparedQueries { delete_txi_by_hash: SizedBatch, /// Stake Registration Primary Key Query. select_stake_registration: PreparedStatement, + /// Stake Registration Delete Query. + delete_stake_registration: SizedBatch, /// CIP36 Registrations Primary Key Query. select_cip36_registration: PreparedStatement, /// CIP36 Registration errors Primary Key Query. @@ -122,8 +125,6 @@ pub(crate) struct PreparedQueries { select_chain_root_for_role0_key: PreparedStatement, /// Chain Root for Stake Address Primary Key Query.. select_chain_root_for_stake_address: PreparedStatement, - /// Stake Registration Delete Query. - delete_stake_registration: SizedBatch, /// CIP36 Registrations Delete Query. delete_cip36_registration: SizedBatch, /// CIP36 Registration errors Delete Query. @@ -160,6 +161,10 @@ impl PreparedQueries { unstaked_txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; let select_txi_by_hash = txi_by_hash::PrimaryKeyQuery::prepare(&session).await?; let delete_txi_by_hash = txi_by_hash::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_stake_registration = + stake_registration::PrimaryKeyQuery::prepare(&session).await?; + let delete_stake_registration = + stake_registration::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs new file mode 100644 index 00000000000..4bb8c81e20b --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs @@ -0,0 +1,128 @@ +//! Stake Registration Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for Stake Registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); +} + +/// Select primary keys for Stake Registration. +const SELECT_QUERY: &str = include_str!("./cql/get_stake_registration.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_hash: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_hash", &self.stake_hash) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_hash: value.0, + slot_no: value.1, + txn: value.2, + } + } +} +/// Get primary key for Stake Registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all Stake Registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get Stake Registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all Stake Registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete Stake Registration +const DELETE_QUERY: &str = include_str!("./cql/delete_stake_registration.cql"); + +/// Delete Stake Registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} From 3c03af2d1b7fa3724b4a7c3611444b122dbfc8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 12 Nov 2024 17:20:03 -0600 Subject: [PATCH 19/37] fix(cat-gateway): incorporate cip36 registration queries --- .../index/queries/purge/cip36_registration.rs | 132 ++++++++++++++++++ .../bin/src/db/index/queries/purge/mod.rs | 5 + 2 files changed, 137 insertions(+) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs new file mode 100644 index 00000000000..9acccb3e0a8 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs @@ -0,0 +1,132 @@ +//! CIP-36 registration Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for CIP-36 registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, num_bigint::BigInt, i16); +} + +/// Select primary keys for CIP-36 registration. +const SELECT_QUERY: &str = include_str!("./cql/get_cip36_registration.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Nonce that has been slot corrected. + pub(crate) nonce: num_bigint::BigInt, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("nonce", &self.nonce) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_address: value.0, + nonce: value.1, + slot_no: value.2, + txn: value.3, + } + } +} +/// Get primary key for CIP-36 registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all CIP-36 registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get CIP-36 registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all CIP-36 registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete CIP-36 registration +const DELETE_QUERY: &str = include_str!("./cql/delete_cip36_registration.cql"); + +/// Delete CIP-36 registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index d1697f4dfa1..fe38bf6e78b 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,5 +1,6 @@ //! Queries for purging volatile data. +pub(crate) mod cip36_registration; pub(crate) mod stake_registration; pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; @@ -165,6 +166,10 @@ impl PreparedQueries { stake_registration::PrimaryKeyQuery::prepare(&session).await?; let delete_stake_registration = stake_registration::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_cip36_registration = + cip36_registration::PrimaryKeyQuery::prepare(&session).await?; + let delete_cip36_registration = + cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } From bab86cfba21c14908fe88aaa3b2f6cd7fa1fda47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 07:27:31 -0600 Subject: [PATCH 20/37] fix(cat-gateway): incorporate cip36 registration for votekey queries --- .../purge/cip36_registration_for_vote_key.rs | 136 ++++++++++++++++++ .../get_cip36_registration_for_vote_key.cql | 2 +- .../bin/src/db/index/queries/purge/mod.rs | 5 + .../cql/cip36_registration_for_vote_key.cql | 4 +- 4 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs new file mode 100644 index 00000000000..2eac8d18b46 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs @@ -0,0 +1,136 @@ +//! CIP-36 registration by Vote Key Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for CIP-36 registration by Vote key purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, Vec, num_bigint::BigInt, i16, bool); +} + +/// Select primary keys for CIP-36 registration. +const SELECT_QUERY: &str = include_str!("./cql/get_cip36_registration_for_vote_key.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Vote key - Binary 28 bytes. + pub(crate) vote_key: Vec, + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, + /// True if registration is valid. + pub(crate) valid: bool, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("vote_key", &self.vote_key) + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("valid", &self.valid) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + vote_key: value.0, + stake_address: value.1, + slot_no: value.2, + txn: value.3, + valid: value.4, + } + } +} +/// Get primary key for CIP-36 registration by Vote key query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all CIP-36 registration by Vote key primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get CIP-36 registration by Vote key primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all CIP-36 registration by Vote key primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete CIP-36 registration +const DELETE_QUERY: &str = include_str!("./cql/delete_cip36_registration_for_vote_key.cql"); + +/// Delete CIP-36 registration by Vote key Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql index 8769a22cde1..795f00c415b 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_for_vote_key.cql @@ -1,4 +1,4 @@ --- Get all primary keys from CIP-36 registration by Stake Address. +-- Get all primary keys from CIP-36 registration by Vote Key. SELECT vote_key, stake_address, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index fe38bf6e78b..9b3a1f2b8a0 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,6 +1,7 @@ //! Queries for purging volatile data. pub(crate) mod cip36_registration; +pub(crate) mod cip36_registration_for_vote_key; pub(crate) mod stake_registration; pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; @@ -170,6 +171,10 @@ impl PreparedQueries { cip36_registration::PrimaryKeyQuery::prepare(&session).await?; let delete_cip36_registration = cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_cip36_registration = + cip36_registration::PrimaryKeyQuery::prepare(&session).await?; + let delete_cip36_registration = + cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql b/catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql index c3ba5f6dfce..066245e2672 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql @@ -1,5 +1,5 @@ --- Index of CIP-36 registrations searchable by Stake Address. --- Full registration data needs to be queried from the man cip36 registration tables. +-- Index of CIP-36 registrations searchable by Vote Key. +-- Full registration data needs to be queried from the main CIP-36 registration tables. -- Includes both Valid and Invalid registrations. CREATE TABLE IF NOT EXISTS cip36_registration_for_vote_key ( -- Primary Key Data From df85e7b0a263cefdc702014e8278db2e71e63f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 07:52:48 -0600 Subject: [PATCH 21/37] chore(cat-gateway): fix spelling --- catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs | 2 +- .../bin/src/db/index/queries/purge/unstaked_txo_assets.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs index f724e147ea4..8a2bd6143c4 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs @@ -39,7 +39,7 @@ pub(crate) struct Params { pub(crate) txn: i16, /// Transaction Output Offset inside the transaction. pub(crate) txo: i16, - /// Asset Policy Hash - Binayr 28 bytes. + /// Asset Policy Hash - Binary 28 bytes. policy_id: Vec, /// Name of the Policy (UTF8) // TODO: https://github.com/input-output-hk/catalyst-voices/issues/1121 diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs index 63c9af73706..cd960de6635 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs @@ -39,7 +39,7 @@ pub(crate) struct Params { pub(crate) txn: i16, /// Transaction Output Offset inside the transaction. pub(crate) txo: i16, - /// Asset Policy Hash - Binayr 28 bytes. + /// Asset Policy Hash - Binary 28 bytes. policy_id: Vec, /// Name of the Policy (UTF8) // TODO: https://github.com/input-output-hk/catalyst-voices/issues/1121 From 030d29800b1618f6dcd260afaf0739dd8deca3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 08:52:55 -0600 Subject: [PATCH 22/37] fix(cat-gateway): incorporate cip36 registration for votekey queries --- .../purge/cip36_registration_for_vote_key.rs | 6 ++-- .../bin/src/db/index/queries/purge/mod.rs | 36 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs index 2eac8d18b46..311b19cd963 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs @@ -25,7 +25,7 @@ pub(crate) mod result { pub(crate) type PrimaryKey = (Vec, Vec, num_bigint::BigInt, i16, bool); } -/// Select primary keys for CIP-36 registration. +/// Select primary keys for CIP-36 registration by Vote key. const SELECT_QUERY: &str = include_str!("./cql/get_cip36_registration_for_vote_key.cql"); /// Primary Key Value. @@ -92,7 +92,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::Cip36RegistrationForVoteKey) .await? .into_typed::(); @@ -128,7 +128,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::Cip36RegistrationForVoteKey, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 9b3a1f2b8a0..051e17ab8ec 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -40,10 +40,10 @@ pub(crate) enum PreparedDeleteQuery { StakeRegistration, /// CIP 36 Registration Delete Query. Cip36Registration, - /// CIP 36 Registration Error Delete query. - Cip36RegistrationError, - /// CIP 36 Registration for stake address Delete query. - Cip36RegistrationForStakeAddr, + /// CIP 36 Registration Invalid Delete query. + Cip36RegistrationInvalid, + /// CIP 36 Registration for vote key Delete query. + Cip36RegistrationForVoteKey, /// RBAC 509 Registration Delete query. Rbac509, /// Chain Root For Transaction ID Delete query. @@ -72,10 +72,10 @@ pub(crate) enum PreparedSelectQuery { StakeRegistration, /// CIP 36 Registration Select Query. Cip36Registration, - /// CIP 36 Registration Error Select query. - Cip36RegistrationError, - /// CIP 36 Registration for stake address Select query. - Cip36RegistrationForStakeAddr, + /// CIP 36 Registration Invalid Select query. + Cip36RegistrationInvalid, + /// CIP 36 Registration for vote key Select query. + Cip36RegistrationForVoteKey, /// RBAC 509 Registration Select query. Rbac509, /// Chain Root For Transaction ID Select query. @@ -117,8 +117,8 @@ pub(crate) struct PreparedQueries { select_cip36_registration: PreparedStatement, /// CIP36 Registration errors Primary Key Query. select_cip36_registration_error: PreparedStatement, - /// CIP36 Registration for Stake Address Primary Key Query. - select_cip36_registration_for_stake_address: PreparedStatement, + /// CIP36 Registration for Vote Key Primary Key Query. + select_cip36_registration_for_vote_key: PreparedStatement, /// RBAC 509 Registrations Primary Key Query. select_rbac509_registration: PreparedStatement, /// Chain Root for TX ID Primary Key Query.. @@ -131,8 +131,8 @@ pub(crate) struct PreparedQueries { delete_cip36_registration: SizedBatch, /// CIP36 Registration errors Delete Query. delete_cip36_registration_error: SizedBatch, - /// CIP36 Registration for Stake Address Delete Query. - delete_cip36_registration_for_stake_addr: SizedBatch, + /// CIP36 Registration for Vote Key Delete Query. + delete_cip36_registration_for_vote_key: SizedBatch, /// RBAC 509 Registrations Delete Query. delete_rbac509_registration: SizedBatch, /// Chain Root for TX ID Delete Query.. @@ -213,9 +213,9 @@ impl PreparedQueries { PreparedSelectQuery::Txi => &self.select_txi_by_hash, PreparedSelectQuery::StakeRegistration => &self.select_stake_registration, PreparedSelectQuery::Cip36Registration => &self.select_cip36_registration, - PreparedSelectQuery::Cip36RegistrationError => &self.select_cip36_registration_error, - PreparedSelectQuery::Cip36RegistrationForStakeAddr => { - &self.select_cip36_registration_for_stake_address + PreparedSelectQuery::Cip36RegistrationInvalid => &self.select_cip36_registration_error, + PreparedSelectQuery::Cip36RegistrationForVoteKey => { + &self.select_cip36_registration_for_vote_key }, PreparedSelectQuery::Rbac509 => &self.select_rbac509_registration, PreparedSelectQuery::ChainRootForTxnId => &self.select_chain_root_for_txn_id, @@ -241,9 +241,9 @@ impl PreparedQueries { PreparedDeleteQuery::Txi => &self.delete_txi_by_hash, PreparedDeleteQuery::StakeRegistration => &self.delete_stake_registration, PreparedDeleteQuery::Cip36Registration => &self.delete_cip36_registration, - PreparedDeleteQuery::Cip36RegistrationError => &self.delete_cip36_registration_error, - PreparedDeleteQuery::Cip36RegistrationForStakeAddr => { - &self.delete_cip36_registration_for_stake_addr + PreparedDeleteQuery::Cip36RegistrationInvalid => &self.delete_cip36_registration_error, + PreparedDeleteQuery::Cip36RegistrationForVoteKey => { + &self.delete_cip36_registration_for_vote_key }, PreparedDeleteQuery::Rbac509 => &self.delete_rbac509_registration, PreparedDeleteQuery::ChainRootForTxnId => &self.delete_chain_root_for_txn_id, From 23371157d1483835bc9ca72223a9b2330f5b8b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 12:36:40 -0600 Subject: [PATCH 23/37] fix(cat-gateway): incorporate cip36 invalid registration queries --- .../purge/cip36_registration_invalid.rs | 128 ++++++++++++++++++ .../cql/get_cip36_registration_invalid.cql | 6 + .../bin/src/db/index/queries/purge/mod.rs | 37 +++-- 3 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs new file mode 100644 index 00000000000..7b4175fbb6d --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs @@ -0,0 +1,128 @@ +//! CIP-36 Registration (Invalid) Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for CIP-36 invalid registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); +} + +/// Select primary keys for CIP-36 invalid registration. +const SELECT_QUERY: &str = include_str!("./cql/get_cip36_registration_invalid.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Stake Address - Binary 28 bytes. 0 bytes = not staked. + pub(crate) stake_address: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("stake_address", &self.stake_address) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + stake_address: value.0, + slot_no: value.1, + txn: value.2, + } + } +} +/// Get primary key for CIP-36 invalid registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all CIP-36 invalid registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get CIP-36 invalid registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all CIP-36 invalid registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete CIP-36 invalid registration +const DELETE_QUERY: &str = include_str!("./cql/delete_cip36_registration_invalid.cql"); + +/// Delete CIP-36 invalid registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql new file mode 100644 index 00000000000..3ebfc5d6eff --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql @@ -0,0 +1,6 @@ +-- Get all primary keys from CIP-36 invalid registration by Stake Address. +SELECT + stake_address, + slot_no, + txn, +FROM cip36_registration_invalid diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 051e17ab8ec..f7a08883b07 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod cip36_registration; pub(crate) mod cip36_registration_for_vote_key; +pub(crate) mod cip36_registration_invalid; pub(crate) mod stake_registration; pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; @@ -115,10 +116,16 @@ pub(crate) struct PreparedQueries { delete_stake_registration: SizedBatch, /// CIP36 Registrations Primary Key Query. select_cip36_registration: PreparedStatement, - /// CIP36 Registration errors Primary Key Query. - select_cip36_registration_error: PreparedStatement, + /// CIP36 Registrations Delete Query. + delete_cip36_registration: SizedBatch, + /// CIP36 Registration Invalid Primary Key Query. + select_cip36_registration_invalid: PreparedStatement, + /// CIP36 Registration Invalid Delete Query. + delete_cip36_registration_invalid: SizedBatch, /// CIP36 Registration for Vote Key Primary Key Query. select_cip36_registration_for_vote_key: PreparedStatement, + /// CIP36 Registration for Vote Key Delete Query. + delete_cip36_registration_for_vote_key: SizedBatch, /// RBAC 509 Registrations Primary Key Query. select_rbac509_registration: PreparedStatement, /// Chain Root for TX ID Primary Key Query.. @@ -127,12 +134,6 @@ pub(crate) struct PreparedQueries { select_chain_root_for_role0_key: PreparedStatement, /// Chain Root for Stake Address Primary Key Query.. select_chain_root_for_stake_address: PreparedStatement, - /// CIP36 Registrations Delete Query. - delete_cip36_registration: SizedBatch, - /// CIP36 Registration errors Delete Query. - delete_cip36_registration_error: SizedBatch, - /// CIP36 Registration for Vote Key Delete Query. - delete_cip36_registration_for_vote_key: SizedBatch, /// RBAC 509 Registrations Delete Query. delete_rbac509_registration: SizedBatch, /// Chain Root for TX ID Delete Query.. @@ -171,10 +172,14 @@ impl PreparedQueries { cip36_registration::PrimaryKeyQuery::prepare(&session).await?; let delete_cip36_registration = cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_cip36_registration = - cip36_registration::PrimaryKeyQuery::prepare(&session).await?; - let delete_cip36_registration = - cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_cip36_registration_invalid = + cip36_registration_invalid::PrimaryKeyQuery::prepare(&session).await?; + let delete_cip36_registration_invalid = + cip36_registration_invalid::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_cip36_registration_for_vote_key = + cip36_registration_for_vote_key::PrimaryKeyQuery::prepare(&session).await?; + let delete_cip36_registration_for_vote_key = + cip36_registration_for_vote_key::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } @@ -213,7 +218,9 @@ impl PreparedQueries { PreparedSelectQuery::Txi => &self.select_txi_by_hash, PreparedSelectQuery::StakeRegistration => &self.select_stake_registration, PreparedSelectQuery::Cip36Registration => &self.select_cip36_registration, - PreparedSelectQuery::Cip36RegistrationInvalid => &self.select_cip36_registration_error, + PreparedSelectQuery::Cip36RegistrationInvalid => { + &self.select_cip36_registration_invalid + }, PreparedSelectQuery::Cip36RegistrationForVoteKey => { &self.select_cip36_registration_for_vote_key }, @@ -241,7 +248,9 @@ impl PreparedQueries { PreparedDeleteQuery::Txi => &self.delete_txi_by_hash, PreparedDeleteQuery::StakeRegistration => &self.delete_stake_registration, PreparedDeleteQuery::Cip36Registration => &self.delete_cip36_registration, - PreparedDeleteQuery::Cip36RegistrationInvalid => &self.delete_cip36_registration_error, + PreparedDeleteQuery::Cip36RegistrationInvalid => { + &self.delete_cip36_registration_invalid + }, PreparedDeleteQuery::Cip36RegistrationForVoteKey => { &self.delete_cip36_registration_for_vote_key }, From 843d9c634477e641964b9fed7d0c97b84a2bc47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 13:28:20 -0600 Subject: [PATCH 24/37] fix(cat-gateway): incorporate rbac509 registration queries --- .../bin/src/db/index/queries/purge/mod.rs | 5 + .../queries/purge/rbac509_registration.rs | 136 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index f7a08883b07..6341330941d 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod cip36_registration; pub(crate) mod cip36_registration_for_vote_key; pub(crate) mod cip36_registration_invalid; +pub(crate) mod rbac509_registration; pub(crate) mod stake_registration; pub(crate) mod txi_by_hash; pub(crate) mod txo_ada; @@ -180,6 +181,10 @@ impl PreparedQueries { cip36_registration_for_vote_key::PrimaryKeyQuery::prepare(&session).await?; let delete_cip36_registration_for_vote_key = cip36_registration_for_vote_key::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_rbac509_registration = + rbac509_registration::PrimaryKeyQuery::prepare(&session).await?; + let delete_rbac509_registration = + rbac509_registration::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs new file mode 100644 index 00000000000..18eba9cf33e --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs @@ -0,0 +1,136 @@ +//! RBAC 509 Registration Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for RBAC 509 registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16, Vec, Vec); +} + +/// Select primary keys for RBAC 509 registration. +const SELECT_QUERY: &str = include_str!("./cql/get_rbac509_registration.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Chain Root - Binary 32 bytes. + pub(crate) chain_root: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, + /// Transaction Hash ID - Binary 32 bytes. + transaction_id: Vec, + /// `UUIDv4` Purpose - Binary 16 bytes. + purpose: Vec, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("chain_root", &self.chain_root) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .field("transaction_id", &self.transaction_id) + .field("purpose", &self.purpose) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + chain_root: value.0, + slot_no: value.1, + txn: value.2, + transaction_id: value.3, + purpose: value.4, + } + } +} +/// Get primary key for RBAC 509 registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all RBAC 509 registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get RBAC 509 registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all RBAC 509 registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::TxoAda) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete RBAC 509 registration +const DELETE_QUERY: &str = include_str!("./cql/delete_rbac509_registration.cql"); + +/// Delete RBAC 509 registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .await?; + + Ok(results) + } +} From 43f2e8d65eb982df04fcd06586561e8352d99c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 19:52:32 -0600 Subject: [PATCH 25/37] fix(cat-gateway): incorporate chain root for role0 registration queries --- .../queries/purge/chain_root_for_role0_key.rs | 128 ++++++++++++++++++ .../cql/delete_chain_root_for_role0_key.cql | 2 +- .../cql/get_chain_root_for_role0_key.cql | 2 +- .../bin/src/db/index/queries/purge/mod.rs | 17 ++- .../schema/cql/chain_root_for_role0_key.cql | 4 +- 5 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs new file mode 100644 index 00000000000..2d3e40d5ba7 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs @@ -0,0 +1,128 @@ +//! Chain Root For Role0 Key (RBAC 509 registrations) Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for Chain Root For Role0 Key registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); +} + +/// Select primary keys for Chain Root For Role0 Key registration. +const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_role0_key.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Role0 Key - Binary 16 bytes. + pub(crate) role0_key: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("role0_key", &self.role0_key) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + role0_key: value.0, + slot_no: value.1, + txn: value.2, + } + } +} +/// Get primary key for Chain Root For Role0 Key registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get Chain Root For Role0 Key registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete Chain Root For Role0 Key registration +const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_role0_key.cql"); + +/// Delete Chain Root For Role0 Key registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::ChainRootForRole0Key, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql index 63ef2aedba8..5285b565f7c 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql @@ -1,4 +1,4 @@ --- Delete all the chain roots for a role0 key +-- Delete Chain Root For Role0 Key. RBAC 509 registrations. DELETE FROM chain_root_for_role0_key WHERE role0_key = :role0_key AND slot_no = :slot_no diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql index 7987c2da104..b817f18bc79 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql @@ -1,4 +1,4 @@ --- Get all primary keys from chain roots for a role0 key. +-- Get all primary keys from Chain Root For Role0 Key. RBAC 509 registrations. SELECT role0_key, slot_no, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 6341330941d..ef594a2ce0a 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,5 +1,6 @@ //! Queries for purging volatile data. +pub(crate) mod chain_root_for_role0_key; pub(crate) mod cip36_registration; pub(crate) mod cip36_registration_for_vote_key; pub(crate) mod cip36_registration_invalid; @@ -129,18 +130,18 @@ pub(crate) struct PreparedQueries { delete_cip36_registration_for_vote_key: SizedBatch, /// RBAC 509 Registrations Primary Key Query. select_rbac509_registration: PreparedStatement, - /// Chain Root for TX ID Primary Key Query.. - select_chain_root_for_txn_id: PreparedStatement, - /// Chain Root for Role 0 Key Primary Key Query.. - select_chain_root_for_role0_key: PreparedStatement, - /// Chain Root for Stake Address Primary Key Query.. - select_chain_root_for_stake_address: PreparedStatement, /// RBAC 509 Registrations Delete Query. delete_rbac509_registration: SizedBatch, + /// Chain Root for TX ID Primary Key Query.. + select_chain_root_for_txn_id: PreparedStatement, /// Chain Root for TX ID Delete Query.. delete_chain_root_for_txn_id: SizedBatch, + /// Chain Root for Role 0 Key Primary Key Query.. + select_chain_root_for_role0_key: PreparedStatement, /// Chain Root for Role 0 Key Delete Query.. delete_chain_root_for_role0_key: SizedBatch, + /// Chain Root for Stake Address Primary Key Query.. + select_chain_root_for_stake_address: PreparedStatement, /// Chain Root for Stake Address Delete Query.. delete_chain_root_for_stake_address: SizedBatch, } @@ -185,6 +186,10 @@ impl PreparedQueries { rbac509_registration::PrimaryKeyQuery::prepare(&session).await?; let delete_rbac509_registration = rbac509_registration::DeleteQuery::prepare_batch(&session, cfg).await?; + let select_chain_root_for_role0_key = + chain_root_for_role0_key::PrimaryKeyQuery::prepare(&session).await?; + let delete_chain_root_for_role0_key = + chain_root_for_role0_key::DeleteQuery::prepare_batch(&session, cfg).await?; todo!("WIP"); } diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_role0_key.cql index 6d5ff854621..fb34c2a4131 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_role0_key.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_role0_key.cql @@ -1,9 +1,11 @@ --- Index of Chain Root For Role0 Key. RBAC 509 registrations. +-- Index of Chain Root For Role0 Key. RBAC 509 registrations. CREATE TABLE IF NOT EXISTS chain_root_for_role0_key ( -- Primary Key Data role0_key blob, -- 16 Bytes of Role0 Key. slot_no varint, -- slot number when the key_was_registered. txn smallint, -- Index of the TX which holds the registration data. + + -- Non-primary Key Data chain_root blob, -- 32 Bytes of Chain Root. PRIMARY KEY (role0_key, slot_no, txn) From 6bba6f657773103e04a574de9cab6027e196eb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 19:55:06 -0600 Subject: [PATCH 26/37] chore(cat-gateway): fix CQL comments --- .../index/queries/purge/cql/delete_chain_root_for_role0_key.cql | 2 +- .../db/index/queries/purge/cql/get_chain_root_for_role0_key.cql | 2 +- .../index/queries/purge/cql/get_cip36_registration_invalid.cql | 2 +- .../bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql index 5285b565f7c..dfffe6d83d7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_role0_key.cql @@ -1,4 +1,4 @@ --- Delete Chain Root For Role0 Key. RBAC 509 registrations. +-- Delete Chain Root For Role0 Key (RBAC 509 registrations). DELETE FROM chain_root_for_role0_key WHERE role0_key = :role0_key AND slot_no = :slot_no diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql index b817f18bc79..6f64ffc05e7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_role0_key.cql @@ -1,4 +1,4 @@ --- Get all primary keys from Chain Root For Role0 Key. RBAC 509 registrations. +-- Get all primary keys from Chain Root For Role0 Key (RBAC 509 registrations). SELECT role0_key, slot_no, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql index 3ebfc5d6eff..7883352bca7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_cip36_registration_invalid.cql @@ -1,4 +1,4 @@ --- Get all primary keys from CIP-36 invalid registration by Stake Address. +-- Get all primary keys from CIP-36 Invalid Registration by Stake Address. SELECT stake_address, slot_no, diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql index 0bf88a7321e..0ccaf81cf0d 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql @@ -1,4 +1,4 @@ --- Index of Chain Root For Stake Address. RBAC 509 registrations. +-- Index of Chain Root For Stake Address (RBAC 509 registrations). CREATE TABLE IF NOT EXISTS chain_root_for_stake_addr ( -- Primary Key Data stake_addr blob, -- 32 Bytes of Stake Address. From 82ed9b49de743e24eb1338f60604fab818a951ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 20:03:53 -0600 Subject: [PATCH 27/37] fix(cat-gateway): incorporate chain root for stake addr and txn id registration queries --- .../purge/chain_root_for_stake_address.rs | 128 ++++++++++++++++++ .../queries/purge/chain_root_for_txn_id.rs | 128 ++++++++++++++++++ .../cql/delete_chain_root_for_stake_addr.cql | 8 +- .../cql/delete_chain_root_for_txn_id.cql | 8 +- .../cql/get_chain_root_for_stake_addr.cql | 9 +- .../purge/cql/get_chain_root_for_txn_id.cql | 9 +- .../bin/src/db/index/queries/purge/mod.rs | 105 ++++++++------ .../bin/src/db/index/queries/purge/txo_ada.rs | 2 - .../schema/cql/chain_root_for_stake_addr.cql | 2 + .../schema/cql/chain_root_for_txn_id.cql | 2 +- 10 files changed, 339 insertions(+), 62 deletions(-) create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs create mode 100644 catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs new file mode 100644 index 00000000000..2d3e40d5ba7 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs @@ -0,0 +1,128 @@ +//! Chain Root For Role0 Key (RBAC 509 registrations) Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for Chain Root For Role0 Key registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); +} + +/// Select primary keys for Chain Root For Role0 Key registration. +const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_role0_key.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Role0 Key - Binary 16 bytes. + pub(crate) role0_key: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("role0_key", &self.role0_key) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + role0_key: value.0, + slot_no: value.1, + txn: value.2, + } + } +} +/// Get primary key for Chain Root For Role0 Key registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get Chain Root For Role0 Key registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete Chain Root For Role0 Key registration +const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_role0_key.cql"); + +/// Delete Chain Root For Role0 Key registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::ChainRootForRole0Key, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs new file mode 100644 index 00000000000..2d3e40d5ba7 --- /dev/null +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs @@ -0,0 +1,128 @@ +//! Chain Root For Role0 Key (RBAC 509 registrations) Queries used in purging data. +use std::{fmt::Debug, sync::Arc}; + +use scylla::{ + prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + Session, +}; +use tracing::error; + +use crate::{ + db::index::{ + queries::{ + purge::{PreparedDeleteQuery, PreparedQueries, PreparedSelectQuery}, + FallibleQueryResults, SizedBatch, + }, + session::CassandraSession, + }, + settings::cassandra_db, +}; + +pub(crate) mod result { + //! Return values for Chain Root For Role0 Key registration purge queries. + + /// Primary Key Row + pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); +} + +/// Select primary keys for Chain Root For Role0 Key registration. +const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_role0_key.cql"); + +/// Primary Key Value. +#[derive(SerializeRow)] +pub(crate) struct Params { + /// Role0 Key - Binary 16 bytes. + pub(crate) role0_key: Vec, + /// Block Slot Number + pub(crate) slot_no: num_bigint::BigInt, + /// Transaction Offset inside the block. + pub(crate) txn: i16, +} + +impl Debug for Params { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Params") + .field("role0_key", &self.role0_key) + .field("slot_no", &self.slot_no) + .field("txn", &self.txn) + .finish() + } +} + +impl From for Params { + fn from(value: result::PrimaryKey) -> Self { + Self { + role0_key: value.0, + slot_no: value.1, + txn: value.2, + } + } +} +/// Get primary key for Chain Root For Role0 Key registration query. +pub(crate) struct PrimaryKeyQuery; + +impl PrimaryKeyQuery { + /// Prepares a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { + let select_primary_key = PreparedQueries::prepare( + session.clone(), + SELECT_QUERY, + scylla::statement::Consistency::All, + true, + ) + .await; + + if let Err(ref error) = select_primary_key { + error!(error=%error, "Failed to prepare get Chain Root For Role0 Key registration primary key query"); + }; + + select_primary_key + } + + /// Executes a query to get all Chain Root For Role0 Key registration primary keys. + pub(crate) async fn execute( + session: &CassandraSession, + ) -> anyhow::Result> { + let iter = session + .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) + .await? + .into_typed::(); + + Ok(iter) + } +} + +/// Delete Chain Root For Role0 Key registration +const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_role0_key.cql"); + +/// Delete Chain Root For Role0 Key registration Query +pub(crate) struct DeleteQuery; + +impl DeleteQuery { + /// Prepare Batch of Delete Queries + pub(crate) async fn prepare_batch( + session: &Arc, cfg: &cassandra_db::EnvVars, + ) -> anyhow::Result { + let delete_queries = PreparedQueries::prepare_batch( + session.clone(), + DELETE_QUERY, + cfg, + scylla::statement::Consistency::Any, + true, + false, + ) + .await?; + Ok(delete_queries) + } + + /// Executes a DELETE Query + pub(crate) async fn execute( + session: &CassandraSession, params: Vec, + ) -> FallibleQueryResults { + let results = session + .purge_execute_batch(PreparedDeleteQuery::ChainRootForRole0Key, params) + .await?; + + Ok(results) + } +} diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql index 41a776e4ef4..d964bb11812 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_stake_addr.cql @@ -1,3 +1,5 @@ --- Delete Chain Root For TX ID. RBAC 509 registrations. -DELETE FROM chain_root_for_txn_id -WHERE transaction_id = :transaction_id +-- Delete Chain Root For Stake Address (RBAC 509 registrations). +DELETE FROM chain_root_for_stake_addr +WHERE stake_addr = :stake_addr + AND slot_no = :slot_no + AND txn = :txn diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql index 2a992be601a..5a4ca4d773c 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/delete_chain_root_for_txn_id.cql @@ -1,5 +1,3 @@ --- Delete all the chain roots for a stake address -DELETE FROM chain_root_for_stake_addr -WHERE stake_addr = :stake_addr - AND slot_no = :slot_no - AND txn = :txn +-- Delete Chain Root For TX ID (RBAC 509 registrations). +DELETE FROM chain_root_for_txn_id +WHERE transaction_id = :transaction_id diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql index f3f9d0e2185..90dad7264b6 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql @@ -1,3 +1,6 @@ --- Get all primary keys from Chain Root For TX ID. RBAC 509 Registrations. -SELECT transaction_id -FROM chain_root_for_txn_id +-- Get all primary keys from Chain Root For Stake Address (RBAC 509 registrations). +SELECT + role0_key, + slot_no, + txn +FROM chain_root_for_role0_key diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql index 84bb782303b..176a29975f5 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_txn_id.cql @@ -1,6 +1,3 @@ --- Get all primary keys from Chain Roots for a Role0 Key. -SELECT - role0_key, - slot_no, - txn -FROM chain_root_for_role0_key +-- Get all primary keys from Chain Root For TX ID (RBAC 509 Registrations). +SELECT transaction_id +FROM chain_root_for_txn_id diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index ef594a2ce0a..978a81d7962 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -1,6 +1,8 @@ //! Queries for purging volatile data. pub(crate) mod chain_root_for_role0_key; +pub(crate) mod chain_root_for_stake_address; +pub(crate) mod chain_root_for_txn_id; pub(crate) mod cip36_registration; pub(crate) mod cip36_registration_for_vote_key; pub(crate) mod cip36_registration_invalid; @@ -27,7 +29,6 @@ const NO_PARAMS: () = (); /// All prepared DELETE query statements (purge DB table rows). #[derive(strum_macros::Display)] -#[allow(dead_code)] pub(crate) enum PreparedDeleteQuery { /// TXO Delete query. TxoAda, @@ -59,7 +60,6 @@ pub(crate) enum PreparedDeleteQuery { /// All prepared SELECT query statements (primary keys from table). #[derive(strum_macros::Display)] -#[allow(dead_code)] pub(crate) enum PreparedSelectQuery { /// TXO Select query. TxoAda, @@ -148,50 +148,71 @@ pub(crate) struct PreparedQueries { impl PreparedQueries { /// Create new prepared queries for a given session. - #[allow(clippy::todo, unused_variables)] pub(crate) async fn new( session: Arc, cfg: &cassandra_db::EnvVars, ) -> anyhow::Result { // We initialize like this, so that all errors preparing querys get shown before aborting. - let select_txo_ada = txo_ada::PrimaryKeyQuery::prepare(&session).await?; - let delete_txo_ada = txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_txo_assets = txo_assets::PrimaryKeyQuery::prepare(&session).await?; - let delete_txo_assets = txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_unstaked_txo_ada = unstaked_txo_ada::PrimaryKeyQuery::prepare(&session).await?; - let delete_unstaked_txo_ada = - unstaked_txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_unstaked_txo_assets = - unstaked_txo_assets::PrimaryKeyQuery::prepare(&session).await?; - let delete_unstaked_txo_assets = - unstaked_txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_txi_by_hash = txi_by_hash::PrimaryKeyQuery::prepare(&session).await?; - let delete_txi_by_hash = txi_by_hash::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_stake_registration = - stake_registration::PrimaryKeyQuery::prepare(&session).await?; - let delete_stake_registration = - stake_registration::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_cip36_registration = - cip36_registration::PrimaryKeyQuery::prepare(&session).await?; - let delete_cip36_registration = - cip36_registration::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_cip36_registration_invalid = - cip36_registration_invalid::PrimaryKeyQuery::prepare(&session).await?; - let delete_cip36_registration_invalid = - cip36_registration_invalid::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_cip36_registration_for_vote_key = - cip36_registration_for_vote_key::PrimaryKeyQuery::prepare(&session).await?; - let delete_cip36_registration_for_vote_key = - cip36_registration_for_vote_key::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_rbac509_registration = - rbac509_registration::PrimaryKeyQuery::prepare(&session).await?; - let delete_rbac509_registration = - rbac509_registration::DeleteQuery::prepare_batch(&session, cfg).await?; - let select_chain_root_for_role0_key = - chain_root_for_role0_key::PrimaryKeyQuery::prepare(&session).await?; - let delete_chain_root_for_role0_key = - chain_root_for_role0_key::DeleteQuery::prepare_batch(&session, cfg).await?; - - todo!("WIP"); + Ok(Self { + select_txo_ada: txo_ada::PrimaryKeyQuery::prepare(&session).await?, + delete_txo_ada: txo_ada::DeleteQuery::prepare_batch(&session, cfg).await?, + select_txo_assets: txo_assets::PrimaryKeyQuery::prepare(&session).await?, + delete_txo_assets: txo_assets::DeleteQuery::prepare_batch(&session, cfg).await?, + select_unstaked_txo_ada: unstaked_txo_ada::PrimaryKeyQuery::prepare(&session).await?, + delete_unstaked_txo_ada: unstaked_txo_ada::DeleteQuery::prepare_batch(&session, cfg) + .await?, + select_unstaked_txo_assets: unstaked_txo_assets::PrimaryKeyQuery::prepare(&session) + .await?, + delete_unstaked_txo_assets: unstaked_txo_assets::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_txi_by_hash: txi_by_hash::PrimaryKeyQuery::prepare(&session).await?, + delete_txi_by_hash: txi_by_hash::DeleteQuery::prepare_batch(&session, cfg).await?, + select_stake_registration: stake_registration::PrimaryKeyQuery::prepare(&session) + .await?, + delete_stake_registration: stake_registration::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_cip36_registration: cip36_registration::PrimaryKeyQuery::prepare(&session) + .await?, + delete_cip36_registration: cip36_registration::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_cip36_registration_invalid: + cip36_registration_invalid::PrimaryKeyQuery::prepare(&session).await?, + delete_cip36_registration_invalid: + cip36_registration_invalid::DeleteQuery::prepare_batch(&session, cfg).await?, + select_cip36_registration_for_vote_key: + cip36_registration_for_vote_key::PrimaryKeyQuery::prepare(&session).await?, + delete_cip36_registration_for_vote_key: + cip36_registration_for_vote_key::DeleteQuery::prepare_batch(&session, cfg).await?, + select_rbac509_registration: rbac509_registration::PrimaryKeyQuery::prepare(&session) + .await?, + delete_rbac509_registration: rbac509_registration::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_chain_root_for_role0_key: chain_root_for_role0_key::PrimaryKeyQuery::prepare( + &session, + ) + .await?, + delete_chain_root_for_role0_key: chain_root_for_role0_key::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_chain_root_for_txn_id: chain_root_for_txn_id::PrimaryKeyQuery::prepare(&session) + .await?, + delete_chain_root_for_txn_id: chain_root_for_txn_id::DeleteQuery::prepare_batch( + &session, cfg, + ) + .await?, + select_chain_root_for_stake_address: + chain_root_for_stake_address::PrimaryKeyQuery::prepare(&session).await?, + delete_chain_root_for_stake_address: + chain_root_for_stake_address::DeleteQuery::prepare_batch(&session, cfg).await?, + }) } /// Prepares a statement. diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs index dcb842ac4a5..4ad61f27ffb 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs @@ -84,7 +84,6 @@ impl PrimaryKeyQuery { } /// Executes a query to get all TXO by stake address primary keys. - #[allow(dead_code)] pub(crate) async fn execute( session: &CassandraSession, ) -> anyhow::Result> { @@ -121,7 +120,6 @@ impl DeleteQuery { } /// Executes a DELETE Query - #[allow(dead_code)] pub(crate) async fn execute( session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql index 0ccaf81cf0d..e1f23ac0b8f 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_stake_addr.cql @@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS chain_root_for_stake_addr ( stake_addr blob, -- 32 Bytes of Stake Address. slot_no varint, -- slot number when the key_was_registered. txn smallint, -- Index of the TX which holds the registration data. + + -- Non-primary Key Data chain_root blob, -- 32 Bytes of Chain Root. PRIMARY KEY (stake_addr, slot_no, txn) diff --git a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_txn_id.cql b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_txn_id.cql index 4c8324b0883..a051b76d76e 100644 --- a/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_txn_id.cql +++ b/catalyst-gateway/bin/src/db/index/schema/cql/chain_root_for_txn_id.cql @@ -1,4 +1,4 @@ --- Index of Chain Root For TX ID. RBAC 509 registrations. +-- Index of Chain Root For TX ID (RBAC 509 registrations). CREATE TABLE IF NOT EXISTS chain_root_for_txn_id ( -- Primary Key Data transaction_id blob, -- 32 Bytes of Transaction Hash. From c08a4f1afd03b134ec74be81bc596107d774c388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 13 Nov 2024 21:26:10 -0600 Subject: [PATCH 28/37] fix(cat-gateway): box large futures lint --- catalyst-gateway/bin/src/db/index/session.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index bd4ef4c0df0..15003a8c174 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -76,8 +76,10 @@ impl CassandraSession { pub(crate) fn init() { let (persistent, volatile) = Settings::cassandra_db_cfg(); - let _join_handle = tokio::task::spawn(async move { retry_init(persistent, true).await }); - let _join_handle = tokio::task::spawn(async move { retry_init(volatile, false).await }); + let _join_handle = + tokio::task::spawn(async move { Box::pin(retry_init(persistent, true)).await }); + let _join_handle = + tokio::task::spawn(async move { Box::pin(retry_init(volatile, false)).await }); } /// Check to see if the Cassandra Indexing DB is ready for use @@ -323,7 +325,8 @@ async fn retry_init(cfg: cassandra_db::EnvVars, persistent: bool) { }, }; - let purge_queries = match purge::PreparedQueries::new(session.clone(), &cfg).await { + let purge_queries = match Box::pin(purge::PreparedQueries::new(session.clone(), &cfg)).await + { Ok(queries) => Arc::new(queries), Err(error) => { error!( From 40ee8e1d9bdf699e78a080b3888fe15337f351f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 3 Dec 2024 13:05:21 -0600 Subject: [PATCH 29/37] feat(cat-gateway): add PURGE_SLOT_BUFFER parameter --- catalyst-gateway/bin/src/settings/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/catalyst-gateway/bin/src/settings/mod.rs b/catalyst-gateway/bin/src/settings/mod.rs index 28726514680..2b5f2d6c1d9 100644 --- a/catalyst-gateway/bin/src/settings/mod.rs +++ b/catalyst-gateway/bin/src/settings/mod.rs @@ -56,6 +56,9 @@ const CHECK_CONFIG_TICK_DEFAULT: &str = "5s"; const EVENT_DB_URL_DEFAULT: &str = "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable"; +/// Default number of slots used as overlap when purging Live Index data. +const PURGE_SLOT_BUFFER_DEFAULT: u64 = 100; + /// Hash the Public IPv4 and IPv6 address of the machine, and convert to a 128 bit V4 /// UUID. fn calculate_service_uuid() -> String { @@ -148,6 +151,9 @@ struct EnvVars { /// Tick every N seconds until config exists in db #[allow(unused)] check_config_tick: Duration, + + /// Slot buffer used as overlap when purging Live Index data. + purge_slot_buffer: u64, } // Lazy initialization of all env vars which are not command line parameters. @@ -173,6 +179,8 @@ static ENV_VARS: LazyLock = LazyLock::new(|| { Duration::from_secs(5) }, }; + let purge_slot_buffer = + StringEnvVar::new_as("PURGE_SLOT_BUFFER", PURGE_SLOT_BUFFER_DEFAULT, 0, u64::MAX); EnvVars { github_repo_owner: StringEnvVar::new("GITHUB_REPO_OWNER", GITHUB_REPO_OWNER_DEFAULT.into()), @@ -199,6 +207,7 @@ static ENV_VARS: LazyLock = LazyLock::new(|| { chain_follower: chain_follower::EnvVars::new(), internal_api_key: StringEnvVar::new_optional("INTERNAL_API_KEY", true), check_config_tick, + purge_slot_buffer, } }); @@ -374,6 +383,11 @@ impl Settings { false } } + + /// Slot buffer used as overlap when purging Live Index data. + pub(crate) fn purge_slot_buffer() -> u64 { + ENV_VARS.purge_slot_buffer + } } /// Transform a string list of host names into a vec of host names. From 53876d264d947ef667f09751033363777d6a34e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 4 Dec 2024 15:52:13 -0600 Subject: [PATCH 30/37] fix(cat-gateway): update scylla type usage --- catalyst-gateway/bin/src/db/index/queries/mod.rs | 2 +- .../src/db/index/queries/purge/chain_root_for_role0_key.rs | 6 +++--- .../db/index/queries/purge/chain_root_for_stake_address.rs | 6 +++--- .../bin/src/db/index/queries/purge/chain_root_for_txn_id.rs | 6 +++--- .../bin/src/db/index/queries/purge/cip36_registration.rs | 6 +++--- .../index/queries/purge/cip36_registration_for_vote_key.rs | 6 +++--- .../db/index/queries/purge/cip36_registration_invalid.rs | 6 +++--- catalyst-gateway/bin/src/db/index/queries/purge/mod.rs | 4 ++-- .../bin/src/db/index/queries/purge/rbac509_registration.rs | 6 +++--- .../bin/src/db/index/queries/purge/stake_registration.rs | 6 +++--- .../bin/src/db/index/queries/purge/txi_by_hash.rs | 6 +++--- catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs | 6 +++--- .../bin/src/db/index/queries/purge/txo_assets.rs | 6 +++--- .../bin/src/db/index/queries/purge/unstaked_txo_ada.rs | 6 +++--- .../bin/src/db/index/queries/purge/unstaked_txo_assets.rs | 6 +++--- catalyst-gateway/bin/src/db/index/session.rs | 2 +- 16 files changed, 43 insertions(+), 43 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/mod.rs b/catalyst-gateway/bin/src/db/index/queries/mod.rs index 2a98d4dd6b7..0d0393ea4ed 100644 --- a/catalyst-gateway/bin/src/db/index/queries/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/mod.rs @@ -416,7 +416,7 @@ async fn session_execute_batch( /// returns. pub(crate) async fn session_execute_iter

( session: Arc, prepared_stmt: &PreparedStatement, params: P, -) -> anyhow::Result +) -> anyhow::Result where P: SerializeRow { session .execute_iter(prepared_stmt.clone(), params) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs index 2d3e40d5ba7..b13a4103e69 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_role0_key.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -82,11 +82,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all Chain Root For Role0 Key registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs index 2d3e40d5ba7..b13a4103e69 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -82,11 +82,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all Chain Root For Role0 Key registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs index 2d3e40d5ba7..b13a4103e69 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -82,11 +82,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all Chain Root For Role0 Key registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs index 9acccb3e0a8..45d21156f26 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -86,11 +86,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all CIP-36 registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs index 311b19cd963..7f4dbd02dd7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_for_vote_key.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -90,11 +90,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all CIP-36 registration by Vote key primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::Cip36RegistrationForVoteKey) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs index 7b4175fbb6d..3b513dc0ee7 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -82,11 +82,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all CIP-36 invalid registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs index 978a81d7962..6958c96b748 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/mod.rs @@ -18,7 +18,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ prepared_statement::PreparedStatement, serialize::row::SerializeRow, - transport::iterator::RowIterator, Session, + transport::iterator::QueryPager, Session, }; use super::{FallibleQueryResults, SizedBatch}; @@ -240,7 +240,7 @@ impl PreparedQueries { /// returns. pub(crate) async fn execute_iter( &self, session: Arc, select_query: PreparedSelectQuery, - ) -> anyhow::Result { + ) -> anyhow::Result { let prepared_stmt = match select_query { PreparedSelectQuery::TxoAda => &self.select_txo_ada, PreparedSelectQuery::TxoAssets => &self.select_txo_assets, diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs index 18eba9cf33e..a371739161a 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -90,11 +90,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all RBAC 509 registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs index 4bb8c81e20b..0e4540a6e48 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -82,11 +82,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all Stake Registration primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs index 7477bcc61d0..87b70e17ffc 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -78,11 +78,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all TXI by hash primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::Txi) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs index 4ad61f27ffb..e2ae31d8637 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_ada.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -86,11 +86,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all TXO by stake address primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs index 8a2bd6143c4..bbd9e8884f2 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -95,11 +95,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all TXO Assets by stake address primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs index 09ef36572e5..253919dd1c9 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs @@ -3,7 +3,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -79,11 +79,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all Unstaked TXO ADA primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::UnstakedTxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs index cd960de6635..b9c03459d12 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; use scylla::{ - prepared_statement::PreparedStatement, transport::iterator::TypedRowIterator, SerializeRow, + prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, SerializeRow, Session, }; use tracing::error; @@ -95,11 +95,11 @@ impl PrimaryKeyQuery { /// Executes a query to get all TXO Assets by TXN Hash primary keys. pub(crate) async fn execute( session: &CassandraSession, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let iter = session .purge_execute_iter(PreparedSelectQuery::TxoAda) .await? - .into_typed::(); + .rows_stream::()?; Ok(iter) } diff --git a/catalyst-gateway/bin/src/db/index/session.rs b/catalyst-gateway/bin/src/db/index/session.rs index 15003a8c174..a5531664984 100644 --- a/catalyst-gateway/bin/src/db/index/session.rs +++ b/catalyst-gateway/bin/src/db/index/session.rs @@ -177,7 +177,7 @@ impl CassandraSession { /// Execute a select query to gather primary keys for purging. pub(crate) async fn purge_execute_iter( &self, query: purge::PreparedSelectQuery, - ) -> anyhow::Result { + ) -> anyhow::Result { // Only execute purge queries on the volatile session let persistent = false; let Some(volatile_db) = Self::get(persistent) else { From e0e3791366206e199e85ccc6bd7238fa58c48f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 4 Dec 2024 18:27:51 -0600 Subject: [PATCH 31/37] fix(cat-gateway): fix prepared query variants --- .../db/index/queries/purge/chain_root_for_stake_address.rs | 4 ++-- .../bin/src/db/index/queries/purge/chain_root_for_txn_id.rs | 4 ++-- .../bin/src/db/index/queries/purge/cip36_registration.rs | 4 ++-- .../src/db/index/queries/purge/cip36_registration_invalid.rs | 4 ++-- .../bin/src/db/index/queries/purge/rbac509_registration.rs | 4 ++-- .../bin/src/db/index/queries/purge/stake_registration.rs | 4 ++-- catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs | 4 ++-- .../bin/src/db/index/queries/purge/unstaked_txo_ada.rs | 2 +- .../bin/src/db/index/queries/purge/unstaked_txo_assets.rs | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs index b13a4103e69..cfb057060a1 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs @@ -84,7 +84,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) + .purge_execute_iter(PreparedSelectQuery::ChainRootForStakeAddress) .await? .rows_stream::()?; @@ -120,7 +120,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::ChainRootForRole0Key, params) + .purge_execute_batch(PreparedDeleteQuery::ChainRootForStakeAddress, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs index b13a4103e69..9dcea888db3 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs @@ -84,7 +84,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::ChainRootForRole0Key) + .purge_execute_iter(PreparedSelectQuery::ChainRootForTxnId) .await? .rows_stream::()?; @@ -120,7 +120,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::ChainRootForRole0Key, params) + .purge_execute_batch(PreparedDeleteQuery::ChainRootForTxnId, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs index 45d21156f26..3ba4b2b4d83 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration.rs @@ -88,7 +88,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::Cip36Registration) .await? .rows_stream::()?; @@ -124,7 +124,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::Cip36Registration, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs index 3b513dc0ee7..13e7edd70ad 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cip36_registration_invalid.rs @@ -84,7 +84,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::Cip36RegistrationInvalid) .await? .rows_stream::()?; @@ -120,7 +120,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::Cip36RegistrationInvalid, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs index a371739161a..4ca10bb3a55 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/rbac509_registration.rs @@ -92,7 +92,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::Rbac509) .await? .rows_stream::()?; @@ -128,7 +128,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::Rbac509, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs index 0e4540a6e48..ad5e2270c6f 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/stake_registration.rs @@ -84,7 +84,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::StakeRegistration) .await? .rows_stream::()?; @@ -120,7 +120,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::StakeRegistration, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs index bbd9e8884f2..50df27451aa 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txo_assets.rs @@ -97,7 +97,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::TxoAssets) .await? .rows_stream::()?; @@ -133,7 +133,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::TxoAssets, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs index 253919dd1c9..6fbb870337e 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs @@ -117,7 +117,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::UnstakedTxoAda, params) .await?; Ok(results) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs index b9c03459d12..89f277056ea 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_assets.rs @@ -97,7 +97,7 @@ impl PrimaryKeyQuery { session: &CassandraSession, ) -> anyhow::Result> { let iter = session - .purge_execute_iter(PreparedSelectQuery::TxoAda) + .purge_execute_iter(PreparedSelectQuery::UnstakedTxoAsset) .await? .rows_stream::()?; @@ -133,7 +133,7 @@ impl DeleteQuery { session: &CassandraSession, params: Vec, ) -> FallibleQueryResults { let results = session - .purge_execute_batch(PreparedDeleteQuery::TxoAda, params) + .purge_execute_batch(PreparedDeleteQuery::UnstakedTxoAsset, params) .await?; Ok(results) From 955607462a0b0ccd75c7c1f06818f9040d81d8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 4 Dec 2024 18:30:23 -0600 Subject: [PATCH 32/37] wip(cat-gateway): add purge command when only one chain follower is left --- catalyst-gateway/bin/src/cardano/mod.rs | 23 ++++++---- .../bin/src/db/index/block/roll_forward.rs | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/catalyst-gateway/bin/src/cardano/mod.rs b/catalyst-gateway/bin/src/cardano/mod.rs index 41e775865f4..0f8e5e6384a 100644 --- a/catalyst-gateway/bin/src/cardano/mod.rs +++ b/catalyst-gateway/bin/src/cardano/mod.rs @@ -12,7 +12,7 @@ use tracing::{debug, error, info, warn}; use crate::{ db::index::{ - block::index_block, + block::{index_block, roll_forward}, queries::sync_status::{ get::{get_sync_status, SyncStatus}, update::update_sync_status, @@ -451,12 +451,21 @@ impl SyncTask { // TODO: IF there is only 1 chain follower left in sync_tasks, then all // immutable followers have finished. - // When this happens we need to purge the live index of any records that exist - // before the current immutable tip. - // Note: to prevent a data race when multiple nodes are syncing, we probably - // want to put a gap in this, so that there are X slots of overlap - // between the live chain and immutable chain. This gap should be - // a parameter. + // When this happens we need to purge the live index of any records that + // exist before the current immutable tip. + // Note: to prevent a data race when multiple nodes are syncing, we + // probably want to put a gap in this, so that there are X + // slots of overlap between the live chain and immutable + // chain. This gap should be a parameter. + + // WIP: How do we check that only one follower is left, and that the slot buffer + // (overlap) criteria is met? + let only_one_chain_follower_left = false; + if only_one_chain_follower_left { + if let Err(error) = roll_forward::purge_live_index(self.immutable_tip_slot).await { + error!(chain=%self.cfg.chain, error=%error, "BUG: Purging volatile data task failed."); + } + } } error!(chain=%self.cfg.chain,"BUG: Sync tasks have all stopped. This is an unexpected error!"); diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index f0d23fd92c1..7e0c19d4733 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -1 +1,43 @@ //! Immutable Roll Forward logic. + +use crate::{ + db::index::{block::CassandraSession, queries::purge}, + settings::Settings, +}; + +/// Purge Data from Live Index +#[allow(unused_variables)] +pub(crate) async fn purge_live_index(purge_slot: u64) -> anyhow::Result<()> { + let persistent = false; // get volatile session + let Some(session) = CassandraSession::get(persistent) else { + anyhow::bail!("Failed to acquire db session"); + }; + + let chain_root_for_role0_keys = + purge::chain_root_for_role0_key::PrimaryKeyQuery::execute(&session).await?; + let chain_root_for_stake_address_keys = + purge::chain_root_for_stake_address::PrimaryKeyQuery::execute(&session).await?; + let chain_root_for_txn_id_keys = + purge::chain_root_for_txn_id::PrimaryKeyQuery::execute(&session).await?; + let cip36_registration_keys = + purge::cip36_registration::PrimaryKeyQuery::execute(&session).await?; + let cip36_registration_for_vote_keys = + purge::cip36_registration_for_vote_key::PrimaryKeyQuery::execute(&session).await?; + let cip36_registration_invalid_keys = + purge::cip36_registration_invalid::PrimaryKeyQuery::execute(&session).await?; + let rbac509_registration_keys = + purge::rbac509_registration::PrimaryKeyQuery::execute(&session).await?; + let stake_registration_keys = + purge::stake_registration::PrimaryKeyQuery::execute(&session).await?; + let txi_by_hash_keys = purge::txi_by_hash::PrimaryKeyQuery::execute(&session).await?; + let txo_ada_keys = purge::txo_ada::PrimaryKeyQuery::execute(&session).await?; + let txo_assets_keys = purge::txo_assets::PrimaryKeyQuery::execute(&session).await?; + let unstaked_txo_ada_keys = purge::unstaked_txo_ada::PrimaryKeyQuery::execute(&session).await?; + let unstaked_txo_assets_keys = + purge::unstaked_txo_assets::PrimaryKeyQuery::execute(&session).await?; + + // WIP: delete filtered keys + // let purge_to_slot: u64 = purge_slot.saturating_sub(Settings::purge_slot_buffer()); + + Ok(()) +} From 7b170e0bb1e5bb2dd379f2f2d5c6b4011e8a4af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 4 Dec 2024 22:02:43 -0600 Subject: [PATCH 33/37] feat(cat-gateway): purge roll forward volatile data --- .../bin/src/db/index/block/roll_forward.rs | 297 ++++++++++++++++-- 1 file changed, 272 insertions(+), 25 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index 7e0c19d4733..513efa36218 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -1,5 +1,9 @@ //! Immutable Roll Forward logic. +use std::sync::Arc; + +use futures::StreamExt; + use crate::{ db::index::{block::CassandraSession, queries::purge}, settings::Settings, @@ -13,31 +17,274 @@ pub(crate) async fn purge_live_index(purge_slot: u64) -> anyhow::Result<()> { anyhow::bail!("Failed to acquire db session"); }; - let chain_root_for_role0_keys = - purge::chain_root_for_role0_key::PrimaryKeyQuery::execute(&session).await?; - let chain_root_for_stake_address_keys = - purge::chain_root_for_stake_address::PrimaryKeyQuery::execute(&session).await?; - let chain_root_for_txn_id_keys = - purge::chain_root_for_txn_id::PrimaryKeyQuery::execute(&session).await?; - let cip36_registration_keys = - purge::cip36_registration::PrimaryKeyQuery::execute(&session).await?; - let cip36_registration_for_vote_keys = - purge::cip36_registration_for_vote_key::PrimaryKeyQuery::execute(&session).await?; - let cip36_registration_invalid_keys = - purge::cip36_registration_invalid::PrimaryKeyQuery::execute(&session).await?; - let rbac509_registration_keys = - purge::rbac509_registration::PrimaryKeyQuery::execute(&session).await?; - let stake_registration_keys = - purge::stake_registration::PrimaryKeyQuery::execute(&session).await?; - let txi_by_hash_keys = purge::txi_by_hash::PrimaryKeyQuery::execute(&session).await?; - let txo_ada_keys = purge::txo_ada::PrimaryKeyQuery::execute(&session).await?; - let txo_assets_keys = purge::txo_assets::PrimaryKeyQuery::execute(&session).await?; - let unstaked_txo_ada_keys = purge::unstaked_txo_ada::PrimaryKeyQuery::execute(&session).await?; - let unstaked_txo_assets_keys = - purge::unstaked_txo_assets::PrimaryKeyQuery::execute(&session).await?; - - // WIP: delete filtered keys - // let purge_to_slot: u64 = purge_slot.saturating_sub(Settings::purge_slot_buffer()); + // Purge data up to this slot + let purge_to_slot: num_bigint::BigInt = purge_slot + .saturating_sub(Settings::purge_slot_buffer()) + .into(); + + purge_chain_root_for_role0_key(&session, &purge_to_slot).await?; + purge_chain_root_for_stake_address(&session, &purge_to_slot).await?; + purge_chain_root_for_txn_id(&session, &purge_to_slot).await?; + purge_cip36_registration(&session, &purge_to_slot).await?; + purge_cip36_registration_for_vote_key(&session, &purge_to_slot).await?; + purge_cip36_registration_invalid(&session, &purge_to_slot).await?; + purge_rbac509_registration(&session, &purge_to_slot).await?; + purge_stake_registration(&session, &purge_to_slot).await?; + purge_txi_by_hash(&session, &purge_to_slot).await?; // WIP + purge_txo_ada(&session, &purge_to_slot).await?; + purge_txo_assets(&session, &purge_to_slot).await?; + purge_unstaked_txo_ada(&session, &purge_to_slot).await?; // WIP + purge_unstaked_txo_assets(&session, &purge_to_slot).await?; + + Ok(()) +} + +/// Purge data from `chain_root_for_role0_key`. +async fn purge_chain_root_for_role0_key( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::chain_root_for_role0_key::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `chain_root_for_stake_address`. +async fn purge_chain_root_for_stake_address( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::chain_root_for_stake_address::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `chain_root_for_txn_id`. +async fn purge_chain_root_for_txn_id( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::chain_root_for_txn_id::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `cip36_registration`. +async fn purge_cip36_registration( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::cip36_registration::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `cip36_registration_for_vote_key`. +async fn purge_cip36_registration_for_vote_key( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::cip36_registration_for_vote_key::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `cip36_registration_invalid`. +async fn purge_cip36_registration_invalid( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::cip36_registration_invalid::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `rbac509_registration`. +async fn purge_rbac509_registration( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::rbac509_registration::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `stake_registration`. +async fn purge_stake_registration( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::stake_registration::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `txi_by_hash`. +#[allow(clippy::unused_async)] +async fn purge_txi_by_hash( + _session: &Arc, _purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + // WIP: Get TXN hashes from other queries + Ok(()) +} + +/// Purge data from `txo_ada`. +async fn purge_txo_ada( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::txo_ada::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `txo_assets`. +async fn purge_txo_assets( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::txo_assets::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(()) +} + +/// Purge data from `unstaked_txo_ada`. +#[allow(clippy::unused_async)] +async fn purge_unstaked_txo_ada( + _session: &Arc, _purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + // use purge::unstaked_txo_ada::{DeleteQuery, Params, PrimaryKeyQuery}; + // WIP: Get TXN hashes from other queries + Ok(()) +} + +/// Purge data from `unstaked_txo_assets`. +async fn purge_unstaked_txo_assets( + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result<()> { + use purge::unstaked_txo_assets::{DeleteQuery, Params, PrimaryKeyQuery}; + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + let params: Params = primary_key.into(); + if ¶ms.slot_no <= purge_to_slot { + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; Ok(()) } From 34af5ef7d2ada965da458deaa6deaf5e02d259ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Sun, 8 Dec 2024 21:17:02 -0600 Subject: [PATCH 34/37] fix(cat-gateway): correct names for chain root queries --- .../purge/chain_root_for_stake_address.rs | 32 ++++++++-------- .../queries/purge/chain_root_for_txn_id.rs | 38 ++++++++----------- .../cql/get_chain_root_for_stake_addr.cql | 4 +- 3 files changed, 34 insertions(+), 40 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs index cfb057060a1..a9a2c007a84 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs @@ -1,4 +1,4 @@ -//! Chain Root For Role0 Key (RBAC 509 registrations) Queries used in purging data. +//! Chain Root For Stake Address (RBAC 509 registrations) Queries used in purging data. use std::{fmt::Debug, sync::Arc}; use scylla::{ @@ -19,20 +19,20 @@ use crate::{ }; pub(crate) mod result { - //! Return values for Chain Root For Role0 Key registration purge queries. + //! Return values for Chain Root For Stake Address registration purge queries. /// Primary Key Row pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); } -/// Select primary keys for Chain Root For Role0 Key registration. -const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_role0_key.cql"); +/// Select primary keys for Chain Root For Stake Address registration. +const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_stake_addr.cql"); /// Primary Key Value. #[derive(SerializeRow)] pub(crate) struct Params { - /// Role0 Key - Binary 16 bytes. - pub(crate) role0_key: Vec, + /// Stake Address - Binary 32 bytes. + pub(crate) stake_addr: Vec, /// Block Slot Number pub(crate) slot_no: num_bigint::BigInt, /// Transaction Offset inside the block. @@ -42,7 +42,7 @@ pub(crate) struct Params { impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") - .field("role0_key", &self.role0_key) + .field("stake_addr", &self.stake_addr) .field("slot_no", &self.slot_no) .field("txn", &self.txn) .finish() @@ -52,17 +52,18 @@ impl Debug for Params { impl From for Params { fn from(value: result::PrimaryKey) -> Self { Self { - role0_key: value.0, + stake_addr: value.0, slot_no: value.1, txn: value.2, } } } -/// Get primary key for Chain Root For Role0 Key registration query. +/// Get primary key for Chain Root For Stake Address registration query. pub(crate) struct PrimaryKeyQuery; impl PrimaryKeyQuery { - /// Prepares a query to get all Chain Root For Role0 Key registration primary keys. + /// Prepares a query to get all Chain Root For Stake Address registration primary + /// keys. pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { let select_primary_key = PreparedQueries::prepare( session.clone(), @@ -73,13 +74,14 @@ impl PrimaryKeyQuery { .await; if let Err(ref error) = select_primary_key { - error!(error=%error, "Failed to prepare get Chain Root For Role0 Key registration primary key query"); + error!(error=%error, "Failed to prepare get Chain Root For Stake Address registration primary key query"); }; select_primary_key } - /// Executes a query to get all Chain Root For Role0 Key registration primary keys. + /// Executes a query to get all Chain Root For Stake Address registration primary + /// keys. pub(crate) async fn execute( session: &CassandraSession, ) -> anyhow::Result> { @@ -92,10 +94,10 @@ impl PrimaryKeyQuery { } } -/// Delete Chain Root For Role0 Key registration -const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_role0_key.cql"); +/// Delete Chain Root For Stake Address registration +const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_stake_addr.cql"); -/// Delete Chain Root For Role0 Key registration Query +/// Delete Chain Root For Stake Address registration Query pub(crate) struct DeleteQuery; impl DeleteQuery { diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs index 9dcea888db3..fde1efe0b73 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs @@ -1,4 +1,4 @@ -//! Chain Root For Role0 Key (RBAC 509 registrations) Queries used in purging data. +//! Chain Root For TX ID (RBAC 509 registrations) Queries used in purging data. use std::{fmt::Debug, sync::Arc}; use scylla::{ @@ -19,32 +19,26 @@ use crate::{ }; pub(crate) mod result { - //! Return values for Chain Root For Role0 Key registration purge queries. + //! Return values for Chain Root For TX ID registration purge queries. /// Primary Key Row pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); } -/// Select primary keys for Chain Root For Role0 Key registration. -const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_role0_key.cql"); +/// Select primary keys for Chain Root For TX ID registration. +const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_txn_id.cql"); /// Primary Key Value. #[derive(SerializeRow)] pub(crate) struct Params { - /// Role0 Key - Binary 16 bytes. - pub(crate) role0_key: Vec, - /// Block Slot Number - pub(crate) slot_no: num_bigint::BigInt, - /// Transaction Offset inside the block. - pub(crate) txn: i16, + /// TX ID - Binary 32 bytes. + pub(crate) transaction_id: Vec, } impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") - .field("role0_key", &self.role0_key) - .field("slot_no", &self.slot_no) - .field("txn", &self.txn) + .field("transaction_id", &self.transaction_id) .finish() } } @@ -52,17 +46,15 @@ impl Debug for Params { impl From for Params { fn from(value: result::PrimaryKey) -> Self { Self { - role0_key: value.0, - slot_no: value.1, - txn: value.2, + transaction_id: value.0, } } } -/// Get primary key for Chain Root For Role0 Key registration query. +/// Get primary key for Chain Root For TX ID registration query. pub(crate) struct PrimaryKeyQuery; impl PrimaryKeyQuery { - /// Prepares a query to get all Chain Root For Role0 Key registration primary keys. + /// Prepares a query to get all Chain Root For TX ID registration primary keys. pub(crate) async fn prepare(session: &Arc) -> anyhow::Result { let select_primary_key = PreparedQueries::prepare( session.clone(), @@ -73,13 +65,13 @@ impl PrimaryKeyQuery { .await; if let Err(ref error) = select_primary_key { - error!(error=%error, "Failed to prepare get Chain Root For Role0 Key registration primary key query"); + error!(error=%error, "Failed to prepare get Chain Root For TX ID registration primary key query"); }; select_primary_key } - /// Executes a query to get all Chain Root For Role0 Key registration primary keys. + /// Executes a query to get all Chain Root For TX ID registration primary keys. pub(crate) async fn execute( session: &CassandraSession, ) -> anyhow::Result> { @@ -92,10 +84,10 @@ impl PrimaryKeyQuery { } } -/// Delete Chain Root For Role0 Key registration -const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_role0_key.cql"); +/// Delete Chain Root For TX ID registration +const DELETE_QUERY: &str = include_str!("./cql/delete_chain_root_for_txn_id.cql"); -/// Delete Chain Root For Role0 Key registration Query +/// Delete Chain Root For TX ID registration Query pub(crate) struct DeleteQuery; impl DeleteQuery { diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql index 90dad7264b6..fdc872b488d 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql +++ b/catalyst-gateway/bin/src/db/index/queries/purge/cql/get_chain_root_for_stake_addr.cql @@ -1,6 +1,6 @@ -- Get all primary keys from Chain Root For Stake Address (RBAC 509 registrations). SELECT - role0_key, + stake_addr, slot_no, txn -FROM chain_root_for_role0_key +FROM chain_root_for_stake_addr From 572aa9bfd459fd58e415a816a4d1011db0737f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 10 Dec 2024 17:58:10 -0600 Subject: [PATCH 35/37] feat(cat-gateway): wrap up queries to purge volatile data --- .../bin/src/db/index/block/roll_forward.rs | 53 ++++++++++++++----- .../purge/chain_root_for_stake_address.rs | 2 +- .../queries/purge/chain_root_for_txn_id.rs | 13 +++-- .../src/db/index/queries/purge/txi_by_hash.rs | 4 +- .../index/queries/purge/unstaked_txo_ada.rs | 2 +- 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index 513efa36218..9a649a73f66 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -1,6 +1,6 @@ //! Immutable Roll Forward logic. -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use futures::StreamExt; @@ -22,18 +22,18 @@ pub(crate) async fn purge_live_index(purge_slot: u64) -> anyhow::Result<()> { .saturating_sub(Settings::purge_slot_buffer()) .into(); + let txn_hashes = purge_txi_by_hash(&session, &purge_to_slot).await?; purge_chain_root_for_role0_key(&session, &purge_to_slot).await?; purge_chain_root_for_stake_address(&session, &purge_to_slot).await?; - purge_chain_root_for_txn_id(&session, &purge_to_slot).await?; + purge_chain_root_for_txn_id(&session, &txn_hashes).await?; purge_cip36_registration(&session, &purge_to_slot).await?; purge_cip36_registration_for_vote_key(&session, &purge_to_slot).await?; purge_cip36_registration_invalid(&session, &purge_to_slot).await?; purge_rbac509_registration(&session, &purge_to_slot).await?; purge_stake_registration(&session, &purge_to_slot).await?; - purge_txi_by_hash(&session, &purge_to_slot).await?; // WIP purge_txo_ada(&session, &purge_to_slot).await?; purge_txo_assets(&session, &purge_to_slot).await?; - purge_unstaked_txo_ada(&session, &purge_to_slot).await?; // WIP + purge_unstaked_txo_ada(&session, &purge_to_slot).await?; purge_unstaked_txo_assets(&session, &purge_to_slot).await?; Ok(()) @@ -83,7 +83,7 @@ async fn purge_chain_root_for_stake_address( /// Purge data from `chain_root_for_txn_id`. async fn purge_chain_root_for_txn_id( - session: &Arc, purge_to_slot: &num_bigint::BigInt, + session: &Arc, txn_hashes: &HashSet>, ) -> anyhow::Result<()> { use purge::chain_root_for_txn_id::{DeleteQuery, Params, PrimaryKeyQuery}; @@ -93,7 +93,7 @@ async fn purge_chain_root_for_txn_id( let mut delete_params: Vec = Vec::new(); while let Some(Ok(primary_key)) = primary_keys_stream.next().await { let params: Params = primary_key.into(); - if ¶ms.slot_no <= purge_to_slot { + if txn_hashes.contains(¶ms.transaction_id) { delete_params.push(params); } } @@ -210,10 +210,25 @@ async fn purge_stake_registration( /// Purge data from `txi_by_hash`. #[allow(clippy::unused_async)] async fn purge_txi_by_hash( - _session: &Arc, _purge_to_slot: &num_bigint::BigInt, -) -> anyhow::Result<()> { - // WIP: Get TXN hashes from other queries - Ok(()) + session: &Arc, purge_to_slot: &num_bigint::BigInt, +) -> anyhow::Result>> { + use purge::txi_by_hash::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + let mut txn_hashes: HashSet> = HashSet::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + if &primary_key.2 <= purge_to_slot { + let params: Params = primary_key.into(); + txn_hashes.insert(params.txn_hash.clone()); + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; + Ok(txn_hashes) } /// Purge data from `txo_ada`. @@ -261,10 +276,22 @@ async fn purge_txo_assets( /// Purge data from `unstaked_txo_ada`. #[allow(clippy::unused_async)] async fn purge_unstaked_txo_ada( - _session: &Arc, _purge_to_slot: &num_bigint::BigInt, + session: &Arc, purge_to_slot: &num_bigint::BigInt, ) -> anyhow::Result<()> { - // use purge::unstaked_txo_ada::{DeleteQuery, Params, PrimaryKeyQuery}; - // WIP: Get TXN hashes from other queries + use purge::unstaked_txo_ada::{DeleteQuery, Params, PrimaryKeyQuery}; + + // Get all keys + let mut primary_keys_stream = PrimaryKeyQuery::execute(session).await?; + // Filter + let mut delete_params: Vec = Vec::new(); + while let Some(Ok(primary_key)) = primary_keys_stream.next().await { + if &primary_key.2 <= purge_to_slot { + let params: Params = primary_key.into(); + delete_params.push(params); + } + } + // Delete filtered keys + DeleteQuery::execute(session, delete_params).await?; Ok(()) } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs index a9a2c007a84..ac925a2c350 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_stake_address.rs @@ -42,7 +42,7 @@ pub(crate) struct Params { impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") - .field("stake_addr", &self.stake_addr) + .field("stake_addr", &hex::encode(&self.stake_addr)) .field("slot_no", &self.slot_no) .field("txn", &self.txn) .finish() diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs index fde1efe0b73..aeb258daf56 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/chain_root_for_txn_id.rs @@ -20,9 +20,14 @@ use crate::{ pub(crate) mod result { //! Return values for Chain Root For TX ID registration purge queries. + use scylla::DeserializeRow; /// Primary Key Row - pub(crate) type PrimaryKey = (Vec, num_bigint::BigInt, i16); + #[derive(DeserializeRow)] + pub(crate) struct PrimaryKey { + /// TXN ID HASH - Binary 32 bytes. + pub(crate) transaction_id: Vec, + } } /// Select primary keys for Chain Root For TX ID registration. @@ -31,14 +36,14 @@ const SELECT_QUERY: &str = include_str!("./cql/get_chain_root_for_txn_id.cql"); /// Primary Key Value. #[derive(SerializeRow)] pub(crate) struct Params { - /// TX ID - Binary 32 bytes. + /// TXN ID HASH - Binary 32 bytes. pub(crate) transaction_id: Vec, } impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") - .field("transaction_id", &self.transaction_id) + .field("transaction_id", &hex::encode(&self.transaction_id)) .finish() } } @@ -46,7 +51,7 @@ impl Debug for Params { impl From for Params { fn from(value: result::PrimaryKey) -> Self { Self { - transaction_id: value.0, + transaction_id: value.transaction_id, } } } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs index 87b70e17ffc..002f8be8a5c 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/txi_by_hash.rs @@ -22,7 +22,7 @@ pub(crate) mod result { //! Return values for TXI by hash purge queries. /// Primary Key Row - pub(crate) type PrimaryKey = (Vec, i16); + pub(crate) type PrimaryKey = (Vec, i16, num_bigint::BigInt); } /// Select primary keys for TXI by hash. @@ -40,7 +40,7 @@ pub(crate) struct Params { impl Debug for Params { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Params") - .field("txn_hash", &self.txn_hash) + .field("txn_hash", &hex::encode(&self.txn_hash)) .field("txo", &self.txo) .finish() } diff --git a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs index 6fbb870337e..99c4d5da22f 100644 --- a/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs +++ b/catalyst-gateway/bin/src/db/index/queries/purge/unstaked_txo_ada.rs @@ -23,7 +23,7 @@ pub(crate) mod result { //! Return values for Unstaked TXO ADA purge queries. /// Primary Key Row - pub(crate) type PrimaryKey = (Vec, i16); + pub(crate) type PrimaryKey = (Vec, i16, num_bigint::BigInt); } /// Select primary keys for Unstaked TXO ADA. From 8d98546240a97304277e21be8087bcf14eff1729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 18 Dec 2024 22:35:02 -0600 Subject: [PATCH 36/37] fix(cat-gateway): cleanup --- catalyst-gateway/bin/src/cardano/mod.rs | 14 +------------- .../bin/src/db/index/block/roll_forward.rs | 1 - 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/catalyst-gateway/bin/src/cardano/mod.rs b/catalyst-gateway/bin/src/cardano/mod.rs index 0f8e5e6384a..41aadf69695 100644 --- a/catalyst-gateway/bin/src/cardano/mod.rs +++ b/catalyst-gateway/bin/src/cardano/mod.rs @@ -449,19 +449,7 @@ impl SyncTask { }, } - // TODO: IF there is only 1 chain follower left in sync_tasks, then all - // immutable followers have finished. - // When this happens we need to purge the live index of any records that - // exist before the current immutable tip. - // Note: to prevent a data race when multiple nodes are syncing, we - // probably want to put a gap in this, so that there are X - // slots of overlap between the live chain and immutable - // chain. This gap should be a parameter. - - // WIP: How do we check that only one follower is left, and that the slot buffer - // (overlap) criteria is met? - let only_one_chain_follower_left = false; - if only_one_chain_follower_left { + if self.sync_tasks.len() == 1 { if let Err(error) = roll_forward::purge_live_index(self.immutable_tip_slot).await { error!(chain=%self.cfg.chain, error=%error, "BUG: Purging volatile data task failed."); } diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index 9a649a73f66..4cd98b16aec 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -10,7 +10,6 @@ use crate::{ }; /// Purge Data from Live Index -#[allow(unused_variables)] pub(crate) async fn purge_live_index(purge_slot: u64) -> anyhow::Result<()> { let persistent = false; // get volatile session let Some(session) = CassandraSession::get(persistent) else { From 3e0043b4279f2ac99cf7bec815fbdff4b85b005e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 18 Dec 2024 22:35:54 -0600 Subject: [PATCH 37/37] fix(cat-gateway): update cardano-chain-follower to v0.0.6 --- catalyst-gateway/bin/Cargo.toml | 2 +- catalyst-gateway/bin/src/db/index/block/roll_forward.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 4f664ec101d..75c578f6d92 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -15,7 +15,7 @@ repository.workspace = true workspace = true [dependencies] -cardano-chain-follower = { version = "0.0.5", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.9" } +cardano-chain-follower = { version = "0.0.6", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.10" } c509-certificate = { version = "0.0.3", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.3" } rbac-registration = { version = "0.0.2", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.8" } diff --git a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs index 4cd98b16aec..46994b5d4c7 100644 --- a/catalyst-gateway/bin/src/db/index/block/roll_forward.rs +++ b/catalyst-gateway/bin/src/db/index/block/roll_forward.rs @@ -207,7 +207,6 @@ async fn purge_stake_registration( } /// Purge data from `txi_by_hash`. -#[allow(clippy::unused_async)] async fn purge_txi_by_hash( session: &Arc, purge_to_slot: &num_bigint::BigInt, ) -> anyhow::Result>> { @@ -273,7 +272,6 @@ async fn purge_txo_assets( } /// Purge data from `unstaked_txo_ada`. -#[allow(clippy::unused_async)] async fn purge_unstaked_txo_ada( session: &Arc, purge_to_slot: &num_bigint::BigInt, ) -> anyhow::Result<()> {