From 163f2abcaa1342284eb808a2e2f20473b7265320 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 7 Jul 2020 06:17:07 +0000 Subject: [PATCH 01/10] Roll back (tip - 1) prune locks when doing a reorg --- src/validation.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index a6cab6b0954d6..1d4a5f1ec3313 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2795,13 +2795,13 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra Ticks(SteadyClock::now() - time_start)); { - // Prune locks that began at or after the tip should be moved backward so they get a chance to reorg + // Prune locks that began around the tip should be moved backward so they get a chance to reorg const int max_height_first{pindexDelete->nHeight - 1}; for (auto& prune_lock : m_blockman.m_prune_locks) { - if (prune_lock.second.height_first <= max_height_first) continue; + if (prune_lock.second.height_first < max_height_first) continue; - prune_lock.second.height_first = max_height_first; - LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, max_height_first); + --prune_lock.second.height_first; + LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, prune_lock.second.height_first); } } From 11f3bca5f016ae7ad5b943606b896d68dd447807 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 18 Dec 2023 16:59:31 +0000 Subject: [PATCH 02/10] Move prune lock checking into BlockManager --- src/node/blockstorage.cpp | 26 ++++++++++++++++++++++++++ src/node/blockstorage.h | 2 ++ src/validation.cpp | 21 --------------------- test/functional/feature_index_prune.py | 4 ++-- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 53f616de23b68..f635cd2be285c 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -165,6 +165,13 @@ bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CB return pa->nHeight < pb->nHeight; } +/** The number of blocks to keep below the deepest prune lock. + * There is nothing special about this number. It is higher than what we + * expect to see in regular mainnet reorgs, but not so high that it would + * noticeably interfere with the pruning mechanism. + * */ +static constexpr int PRUNE_LOCK_BUFFER{10}; + std::vector BlockManager::GetAllBlockIndices() { AssertLockHeld(cs_main); @@ -258,6 +265,21 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) m_dirty_fileinfo.insert(fileNumber); } +bool BlockManager::DoPruneLocksForbidPruning(const CBlockFileInfo& block_file_info) +{ + AssertLockHeld(cs_main); + for (const auto& prune_lock : m_prune_locks) { + if (prune_lock.second.height_first == std::numeric_limits::max()) continue; + // Remove the buffer and one additional block here to get actual height that is outside of the buffer + const unsigned int lock_height{(unsigned)std::max(1, prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1)}; + if (block_file_info.nHeightLast > lock_height) { + LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", prune_lock.first, lock_height); + return true; + } + } + return false; +} + void BlockManager::FindFilesToPruneManual( std::set& setFilesToPrune, int nManualPruneHeight, @@ -280,6 +302,8 @@ void BlockManager::FindFilesToPruneManual( continue; } + if (DoPruneLocksForbidPruning(m_blockfile_info[fileNumber])) continue; + PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; @@ -344,6 +368,8 @@ void BlockManager::FindFilesToPrune( continue; } + if (DoPruneLocksForbidPruning(m_blockfile_info[fileNumber])) continue; + PruneOneBlockFile(fileNumber); // Queue up the files for removal setFilesToPrune.insert(fileNumber); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index ba44d31581cc8..17c9e939e4e4f 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -167,6 +167,8 @@ class BlockManager bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; + bool DoPruneLocksForbidPruning(const CBlockFileInfo& block_file_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual( std::set& setFilesToPrune, diff --git a/src/validation.cpp b/src/validation.cpp index 1d4a5f1ec3313..831ef1147031a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -98,12 +98,6 @@ const std::vector CHECKLEVEL_DOC { "level 4 tries to reconnect the blocks", "each level includes the checks of the previous levels", }; -/** The number of blocks to keep below the deepest prune lock. - * There is nothing special about this number. It is higher than what we - * expect to see in regular mainnet reorgs, but not so high that it would - * noticeably interfere with the pruning mechanism. - * */ -static constexpr int PRUNE_LOCK_BUFFER{10}; GlobalMutex g_best_block_mutex; std::condition_variable g_best_block_cv; @@ -2540,21 +2534,6 @@ bool Chainstate::FlushStateToDisk( // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune - std::optional limiting_lock; // prune lock that actually was the limiting factor, only used for logging - - for (const auto& prune_lock : m_blockman.m_prune_locks) { - if (prune_lock.second.height_first == std::numeric_limits::max()) continue; - // Remove the buffer and one additional block here to get actual height that is outside of the buffer - const int lock_height{prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1}; - last_prune = std::max(1, std::min(last_prune, lock_height)); - if (last_prune == lock_height) { - limiting_lock = prune_lock.first; - } - } - - if (limiting_lock) { - LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", limiting_lock.value(), last_prune); - } if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py index d6e802b399e9d..d9642f543441f 100755 --- a/test/functional/feature_index_prune.py +++ b/test/functional/feature_index_prune.py @@ -70,7 +70,7 @@ def run_test(self): self.log.info("prune some blocks") for node in self.nodes[:2]: - with node.assert_debug_log(['limited pruning to height 689']): + with node.assert_debug_log(['Prune: UnlinkPrunedFiles deleted blk/rev (00000)']): pruneheight_new = node.pruneblockchain(400) # the prune heights used here and below are magic numbers that are determined by the # thresholds at which block files wrap, so they depend on disk serialization and default block file size. @@ -143,7 +143,7 @@ def run_test(self): self.sync_index(height=2500) for node in self.nodes[:2]: - with node.assert_debug_log(['limited pruning to height 2489']): + with node.assert_debug_log(['Prune: UnlinkPrunedFiles deleted blk/rev (00007)']): pruneheight_new = node.pruneblockchain(2500) assert_equal(pruneheight_new, 2005) From 0cc7b5c02ed6f303d80523649a68f13c3bda8135 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 21 Mar 2022 15:36:22 +0000 Subject: [PATCH 03/10] blockstorage: Add height_last to PruneLockInfo --- src/node/blockstorage.cpp | 12 ++++++++---- src/node/blockstorage.h | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f635cd2be285c..073a0712789fe 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -272,10 +273,13 @@ bool BlockManager::DoPruneLocksForbidPruning(const CBlockFileInfo& block_file_in if (prune_lock.second.height_first == std::numeric_limits::max()) continue; // Remove the buffer and one additional block here to get actual height that is outside of the buffer const unsigned int lock_height{(unsigned)std::max(1, prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1)}; - if (block_file_info.nHeightLast > lock_height) { - LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", prune_lock.first, lock_height); - return true; - } + const unsigned int lock_height_last{(unsigned)std::max(1, SaturatingAdd(prune_lock.second.height_last, PRUNE_LOCK_BUFFER))}; + if (block_file_info.nHeightFirst > lock_height_last) continue; + if (block_file_info.nHeightLast <= lock_height) continue; + // TODO: Check each block within the file against the prune_lock range + + LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", prune_lock.first, lock_height); + return true; } return false; } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 17c9e939e4e4f..0c20455e02ed5 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -95,6 +95,7 @@ struct CBlockIndexHeightOnlyComparator { struct PruneLockInfo { int height_first{std::numeric_limits::max()}; //! Height of earliest block that should be kept and not pruned + int height_last{std::numeric_limits::max()}; //! Height of latest block that should be kept and not pruned }; enum BlockfileType { @@ -241,8 +242,9 @@ class BlockManager /** * Map from external index name to oldest block that must not be pruned. * - * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and - * below will be pruned, but callers should avoid assuming any particular buffer size. + * @note Internally, only blocks before height (height_first - PRUNE_LOCK_BUFFER - 1) and + * after height (height_last + PRUNE_LOCK_BUFFER) will be pruned, but callers should + * avoid assuming any particular buffer size. */ std::unordered_map m_prune_locks GUARDED_BY(::cs_main); From c03b9e359df95db49248cd37cef19be7eb136de4 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 7 Jul 2021 00:02:16 +0000 Subject: [PATCH 04/10] Add internal interfaces for prune locks Including desc member of PruneLockInfo for a human-readable description --- src/interfaces/chain.h | 5 +++++ src/node/blockstorage.cpp | 10 ++++++++++ src/node/blockstorage.h | 3 +++ src/node/interfaces.cpp | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index dea868f844da4..450d8b741c862 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -33,6 +33,7 @@ struct CBlockLocator; struct FeeCalculation; namespace node { struct NodeContext; +struct PruneLockInfo; } // namespace node namespace interfaces { @@ -136,6 +137,10 @@ class Chain //! pruned), and contains transactions. virtual bool haveBlockOnDisk(int height) = 0; + virtual bool pruneLockExists(const std::string& name) const = 0; + virtual void updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) = 0; + virtual void deletePruneLock(const std::string& name) = 0; + //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 073a0712789fe..f3621cd6412c3 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -388,11 +388,21 @@ void BlockManager::FindFilesToPrune( min_block_to_prune, last_block_can_prune, count); } +bool BlockManager::PruneLockExists(const std::string& name) const { + return m_prune_locks.count(name); +} + void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) { AssertLockHeld(::cs_main); m_prune_locks[name] = lock_info; } +void BlockManager::DeletePruneLock(const std::string& name) +{ + AssertLockHeld(::cs_main); + m_prune_locks.erase(name); +} + CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 0c20455e02ed5..e4ff706f84f6f 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -94,6 +94,7 @@ struct CBlockIndexHeightOnlyComparator { }; struct PruneLockInfo { + std::string desc; //! Arbitrary human-readable description of the lock purpose int height_first{std::numeric_limits::max()}; //! Height of earliest block that should be kept and not pruned int height_last{std::numeric_limits::max()}; //! Height of latest block that should be kept and not pruned }; @@ -350,8 +351,10 @@ class BlockManager //! Check whether the block associated with this index entry is pruned or not. bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool PruneLockExists(const std::string& name) const SHARED_LOCKS_REQUIRED(::cs_main); //! Create or update a prune lock identified by its name void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void DeletePruneLock(const std::string& name) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Open a block file (blk?????.dat) */ CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f6dbe4f008707..420df7506fc01 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -530,6 +530,24 @@ class ChainImpl : public Chain const CBlockIndex* block{chainman().ActiveChain()[height]}; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } + bool pruneLockExists(const std::string& name) const override + { + LOCK(cs_main); + auto& blockman = m_node.chainman->m_blockman; + return blockman.PruneLockExists(name); + } + void updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) override + { + LOCK(cs_main); + auto& blockman = m_node.chainman->m_blockman; + blockman.UpdatePruneLock(name, lock_info); + } + void deletePruneLock(const std::string& name) override + { + LOCK(cs_main); + auto& blockman = m_node.chainman->m_blockman; + blockman.DeletePruneLock(name); + } CBlockLocator getTipLocator() override { LOCK(::cs_main); From 4190ba6986fb48b64fa32081e411b2435e50dbef Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 7 Jul 2021 00:08:41 +0000 Subject: [PATCH 05/10] Support for persisting prune locks in blocks/index db --- src/interfaces/chain.h | 4 +-- src/node/blockstorage.cpp | 69 ++++++++++++++++++++++++++++++++++----- src/node/blockstorage.h | 24 +++++++++++--- src/node/interfaces.cpp | 8 ++--- src/validation.cpp | 3 +- 5 files changed, 88 insertions(+), 20 deletions(-) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 450d8b741c862..fa06bb22ef71e 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -138,8 +138,8 @@ class Chain virtual bool haveBlockOnDisk(int height) = 0; virtual bool pruneLockExists(const std::string& name) const = 0; - virtual void updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) = 0; - virtual void deletePruneLock(const std::string& name) = 0; + virtual bool updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info, bool sync=false) = 0; + virtual bool deletePruneLock(const std::string& name) = 0; //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f3621cd6412c3..866da157b10cd 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -37,6 +37,7 @@ static constexpr uint8_t DB_BLOCK_INDEX{'b'}; static constexpr uint8_t DB_FLAG{'F'}; static constexpr uint8_t DB_REINDEX_FLAG{'R'}; static constexpr uint8_t DB_LAST_BLOCK{'l'}; +static constexpr uint8_t DB_PRUNE_LOCK{'L'}; // Keys used in previous version that might still be found in the DB: // BlockTreeDB::DB_TXINDEX_BLOCK{'T'}; // BlockTreeDB::DB_TXINDEX{'t'} @@ -66,7 +67,7 @@ bool BlockTreeDB::ReadLastBlockFile(int& nFile) return Read(DB_LAST_BLOCK, nFile); } -bool BlockTreeDB::WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo) +bool BlockTreeDB::WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo, const std::unordered_map& prune_locks) { CDBBatch batch(*this); for (const auto& [file, info] : fileInfo) { @@ -76,9 +77,40 @@ bool BlockTreeDB::WriteBatchSync(const std::vectorGetBlockHash()), CDiskBlockIndex{bi}); } + for (const auto& prune_lock : prune_locks) { + if (prune_lock.second.temporary) continue; + batch.Write(std::make_pair(DB_PRUNE_LOCK, prune_lock.first), prune_lock.second); + } return WriteBatch(batch, true); } +bool BlockTreeDB::WritePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) { + if (lock_info.temporary) return true; + return Write(std::make_pair(DB_PRUNE_LOCK, name), lock_info); +} + +bool BlockTreeDB::DeletePruneLock(const std::string& name) { + return Erase(std::make_pair(DB_PRUNE_LOCK, name)); +} + +bool BlockTreeDB::LoadPruneLocks(std::unordered_map& prune_locks, const util::SignalInterrupt& interrupt) { + std::unique_ptr pcursor(NewIterator()); + for (pcursor->Seek(DB_PRUNE_LOCK); pcursor->Valid(); pcursor->Next()) { + if (interrupt) return false; + + std::pair key; + if ((!pcursor->GetKey(key)) || key.first != DB_PRUNE_LOCK) break; + + node::PruneLockInfo& lock_info = prune_locks[key.second]; + if (!pcursor->GetValue(lock_info)) { + return error("%s: failed to %s prune lock '%s'", __func__, "read", key.second); + } + lock_info.temporary = false; + } + + return true; +} + bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'}); @@ -270,10 +302,10 @@ bool BlockManager::DoPruneLocksForbidPruning(const CBlockFileInfo& block_file_in { AssertLockHeld(cs_main); for (const auto& prune_lock : m_prune_locks) { - if (prune_lock.second.height_first == std::numeric_limits::max()) continue; + if (prune_lock.second.height_first == std::numeric_limits::max()) continue; // Remove the buffer and one additional block here to get actual height that is outside of the buffer - const unsigned int lock_height{(unsigned)std::max(1, prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1)}; - const unsigned int lock_height_last{(unsigned)std::max(1, SaturatingAdd(prune_lock.second.height_last, PRUNE_LOCK_BUFFER))}; + const uint64_t lock_height{(prune_lock.second.height_first <= PRUNE_LOCK_BUFFER + 1) ? 1 : (prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1)}; + const uint64_t lock_height_last{SaturatingAdd(prune_lock.second.height_last, (uint64_t)PRUNE_LOCK_BUFFER)}; if (block_file_info.nHeightFirst > lock_height_last) continue; if (block_file_info.nHeightLast <= lock_height) continue; // TODO: Check each block within the file against the prune_lock range @@ -392,15 +424,34 @@ bool BlockManager::PruneLockExists(const std::string& name) const { return m_prune_locks.count(name); } -void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) { +bool BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info, const bool sync) { AssertLockHeld(::cs_main); - m_prune_locks[name] = lock_info; + if (sync) { + if (!m_block_tree_db->WritePruneLock(name, lock_info)) { + return error("%s: failed to %s prune lock '%s'", __func__, "write", name); + } + } + PruneLockInfo& stored_lock_info = m_prune_locks[name]; + if (lock_info.temporary && !stored_lock_info.temporary) { + // Erase non-temporary lock from disk + if (!m_block_tree_db->DeletePruneLock(name)) { + return error("%s: failed to %s prune lock '%s'", __func__, "erase", name); + } + } + stored_lock_info = lock_info; + return true; } -void BlockManager::DeletePruneLock(const std::string& name) +bool BlockManager::DeletePruneLock(const std::string& name) { AssertLockHeld(::cs_main); m_prune_locks.erase(name); + + // Since there is no reasonable expectation for any follow-up to this prune lock, actually ensure it gets committed to disk immediately + if (!m_block_tree_db->DeletePruneLock(name)) { + return error("%s: failed to %s prune lock '%s'", __func__, "erase", name); + } + return true; } CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) @@ -426,6 +477,8 @@ bool BlockManager::LoadBlockIndex(const std::optional& snapshot_blockha return false; } + if (!m_block_tree_db->LoadPruneLocks(m_prune_locks, m_interrupt)) return false; + if (snapshot_blockhash) { const AssumeutxoData au_data = *Assert(GetParams().AssumeutxoForBlockhash(*snapshot_blockhash)); m_snapshot_height = au_data.height; @@ -508,7 +561,7 @@ bool BlockManager::WriteBlockIndexDB() m_dirty_blockindex.erase(it++); } int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum()); - if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks)) { + if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks, m_prune_locks)) { return false; } return true; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index e4ff706f84f6f..c76f961d3bc6e 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -41,6 +41,9 @@ struct FlatFilePos; namespace Consensus { struct Params; } +namespace node { +struct PruneLockInfo; +}; namespace util { class SignalInterrupt; } // namespace util @@ -51,15 +54,18 @@ class BlockTreeDB : public CDBWrapper { public: using CDBWrapper::CDBWrapper; - bool WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo); + bool WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo, const std::unordered_map& prune_locks); bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info); bool ReadLastBlockFile(int& nFile); bool WriteReindexing(bool fReindexing); void ReadReindexing(bool& fReindexing); + bool WritePruneLock(const std::string& name, const node::PruneLockInfo&); + bool DeletePruneLock(const std::string& name); bool WriteFlag(const std::string& name, bool fValue); bool ReadFlag(const std::string& name, bool& fValue); bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex, const util::SignalInterrupt& interrupt) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadPruneLocks(std::unordered_map& prune_locks, const util::SignalInterrupt& interrupt); }; } // namespace kernel @@ -95,8 +101,16 @@ struct CBlockIndexHeightOnlyComparator { struct PruneLockInfo { std::string desc; //! Arbitrary human-readable description of the lock purpose - int height_first{std::numeric_limits::max()}; //! Height of earliest block that should be kept and not pruned - int height_last{std::numeric_limits::max()}; //! Height of latest block that should be kept and not pruned + uint64_t height_first{std::numeric_limits::max()}; //! Height of earliest block that should be kept and not pruned + uint64_t height_last{std::numeric_limits::max()}; //! Height of latest block that should be kept and not pruned + bool temporary{true}; + + SERIALIZE_METHODS(PruneLockInfo, obj) + { + READWRITE(obj.desc); + READWRITE(VARINT(obj.height_first)); + READWRITE(VARINT(obj.height_last)); + } }; enum BlockfileType { @@ -353,8 +367,8 @@ class BlockManager bool PruneLockExists(const std::string& name) const SHARED_LOCKS_REQUIRED(::cs_main); //! Create or update a prune lock identified by its name - void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - void DeletePruneLock(const std::string& name) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info, bool sync=false) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool DeletePruneLock(const std::string& name) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Open a block file (blk?????.dat) */ CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 420df7506fc01..00832b2013667 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -536,17 +536,17 @@ class ChainImpl : public Chain auto& blockman = m_node.chainman->m_blockman; return blockman.PruneLockExists(name); } - void updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) override + bool updatePruneLock(const std::string& name, const node::PruneLockInfo& lock_info, bool sync) override { LOCK(cs_main); auto& blockman = m_node.chainman->m_blockman; - blockman.UpdatePruneLock(name, lock_info); + return blockman.UpdatePruneLock(name, lock_info, sync); } - void deletePruneLock(const std::string& name) override + bool deletePruneLock(const std::string& name) override { LOCK(cs_main); auto& blockman = m_node.chainman->m_blockman; - blockman.DeletePruneLock(name); + return blockman.DeletePruneLock(name); } CBlockLocator getTipLocator() override { diff --git a/src/validation.cpp b/src/validation.cpp index 831ef1147031a..ebcc0a5110bcb 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2775,12 +2775,13 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra { // Prune locks that began around the tip should be moved backward so they get a chance to reorg - const int max_height_first{pindexDelete->nHeight - 1}; + const uint64_t max_height_first{static_cast(pindexDelete->nHeight - 1)}; for (auto& prune_lock : m_blockman.m_prune_locks) { if (prune_lock.second.height_first < max_height_first) continue; --prune_lock.second.height_first; LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, prune_lock.second.height_first); + // NOTE: Don't need to write to db here, since it will get synced with the rest of the chainstate } } From 977027d11c394dfebc32fbb3d8055d2c5c8e0999 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 7 Jul 2021 00:10:44 +0000 Subject: [PATCH 06/10] RPC: blockchain: Add listprunelocks and setprunelock methods --- src/node/blockstorage.h | 2 + src/rpc/blockchain.cpp | 161 ++++++++++++++++++++++++++++++++++++++++ src/rpc/client.cpp | 1 + 3 files changed, 164 insertions(+) diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index c76f961d3bc6e..fb1c90d89ed92 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -254,6 +254,7 @@ class BlockManager /** Dirty block file entries. */ std::set m_dirty_fileinfo; +public: /** * Map from external index name to oldest block that must not be pruned. * @@ -263,6 +264,7 @@ class BlockManager */ std::unordered_map m_prune_locks GUARDED_BY(::cs_main); +private: BlockfileType BlockfileTypeForHeight(int height); const kernel::BlockManagerOpts m_opts; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b84747a3fd8b..6ab5ead43302c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -760,6 +760,165 @@ static RPCHelpMan getblock() }; } +static RPCHelpMan listprunelocks() +{ + return RPCHelpMan{"listprunelocks", + "\nReturns a list of pruning locks.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "prune_locks", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "id", "A unique identifier for the lock"}, + {RPCResult::Type::STR, "desc", "A description of the lock's purpose"}, + {RPCResult::Type::ARR_FIXED, "height", "Range of blocks prevented from being pruned", + { + {RPCResult::Type::NUM, "height_first", "Height of first block that may not be pruned"}, + {RPCResult::Type::NUM, "height_last", "Height of last block that may not be pruned (omitted if unbounded)"}, + }}, + {RPCResult::Type::BOOL, "temporary", "Indicates the lock will not remain after a restart of the node"}, + }}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("listprunelocks", "") + + HelpExampleRpc("listprunelocks", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + Chainstate& active_chainstate = chainman.ActiveChainstate(); + + UniValue locks_uv(UniValue::VARR); + { + LOCK(::cs_main); + BlockManager * const blockman = &active_chainstate.m_blockman; + for (const auto& prune_lock : blockman->m_prune_locks) { + UniValue prune_lock_uv(UniValue::VOBJ); + const auto& lock_info = prune_lock.second; + prune_lock_uv.pushKV("id", prune_lock.first); + prune_lock_uv.pushKV("desc", lock_info.desc); + UniValue heights_uv(UniValue::VARR); + heights_uv.push_back(lock_info.height_first); + if (lock_info.height_last < std::numeric_limits::max()) { + heights_uv.push_back(lock_info.height_last); + } + prune_lock_uv.pushKV("height", heights_uv); + locks_uv.push_back(prune_lock_uv); + } + } + + UniValue result(UniValue::VOBJ); + result.pushKV("prune_locks", locks_uv); + return result; +}, + }; +} + +static RPCHelpMan setprunelock() +{ + return RPCHelpMan{"setprunelock", + "\nManipulate pruning locks.\n", + { + {"id", RPCArg::Type::STR, RPCArg::Optional::NO, "The unique id of the manipulated prune lock (or \"*\" if deleting all)"}, + {"lock_info", RPCArg::Type::OBJ, RPCArg::Optional::NO, "An object describing the desired lock", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "Description of the lock"}, + {"height", RPCArg::Type::RANGE, RPCArg::DefaultHint("deletes the lock"), "The range of block heights to prevent pruning"}, + {"sync", RPCArg::Type::BOOL, RPCArg::Default(false), "If true, success indicates the lock change was stored to disk (if non-temporary). If false, it is possible for a subsequent node crash to lose the lock."}, + {"temporary", RPCArg::Type::BOOL, RPCArg::Default(false), "If true, the lock will not persist across node restart."}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "success", "Whether the change was successful"}, + }}, + RPCExamples{ + HelpExampleCli("setprunelock", "\"test\" \"{\\\"desc\\\": \\\"Just a test\\\", \\\"height\\\": [0,100]}\"") + + HelpExampleCli("setprunelock", "\"test-2\" \"{\\\"desc\\\": \\\"Second RPC-created prunelock test\\\", \\\"height\\\": [100]}\"") + + HelpExampleRpc("setprunelock", "\"test\", {\"desc\": \"Just a test\", \"height\": [0,100]}") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + Chainstate& active_chainstate = chainman.ActiveChainstate(); + + const auto& lock_info_json = request.params[1]; + RPCTypeCheckObj(lock_info_json, + { + {"desc", UniValueType(UniValue::VSTR)}, + {"height", UniValueType()}, // will be checked below + {"sync", UniValueType(UniValue::VBOOL)}, + {"temporary", UniValueType(UniValue::VBOOL)}, + }, + /*fAllowNull=*/ true, /*fStrict=*/ true); + + const auto& lockid = request.params[0].get_str(); + + node::PruneLockInfo lock_info; + + auto height_param = lock_info_json["height"]; + if (!height_param.isArray()) { + UniValue new_height_param(UniValue::VARR); + new_height_param.push_back(std::move(height_param)); + height_param = std::move(new_height_param); + } + bool success; + if (height_param[0].isNull() && height_param[1].isNull()) { + // Delete + LOCK(::cs_main); + BlockManager * const blockman = &active_chainstate.m_blockman; + if (lockid == "*") { + // Delete all + success = true; + std::vector all_ids; + for (const auto& prune_lock : blockman->m_prune_locks) { + all_ids.push_back(prune_lock.first); + } + for (auto& lockid : all_ids) { + success |= blockman->DeletePruneLock(lockid); + } + } else { + success = blockman->PruneLockExists(lockid) && blockman->DeletePruneLock(lockid); + } + } else { + if (lockid == "*") throw JSONRPCError(RPC_INVALID_PARAMETER, "id \"*\" only makes sense when deleting"); + if (!height_param[0].isNum()) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid start height"); + lock_info.height_first = height_param[0].getInt(); + if (!height_param[1].isNull()) { + if (!height_param[1].isNum()) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid end height"); + lock_info.height_last = height_param[1].getInt(); + } + lock_info.desc = lock_info_json["desc"].get_str(); + if (lock_info_json["temporary"].isNull()) { + lock_info.temporary = false; + } else { + lock_info.temporary = lock_info_json["temporary"].get_bool(); + } + bool sync = false; + if (!lock_info_json["sync"].isNull()) { + sync = lock_info_json["sync"].get_bool(); + } + LOCK(::cs_main); + BlockManager * const blockman = &active_chainstate.m_blockman; + success = blockman->UpdatePruneLock(lockid, lock_info, sync); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("success", success); + return result; +}, + }; +} + static RPCHelpMan pruneblockchain() { return RPCHelpMan{"pruneblockchain", "", @@ -2887,6 +3046,8 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &getdeploymentinfo}, {"blockchain", &gettxout}, {"blockchain", &gettxoutsetinfo}, + {"blockchain", &listprunelocks}, + {"blockchain", &setprunelock}, {"blockchain", &pruneblockchain}, {"blockchain", &verifychain}, {"blockchain", &preciousblock}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 49820f25a35a5..ba539f1b2ff8e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -242,6 +242,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "verifychain", 1, "nblocks" }, { "getblockstats", 0, "hash_or_height" }, { "getblockstats", 1, "stats" }, + { "setprunelock", 1, "lock_info" }, { "pruneblockchain", 0, "height" }, { "keypoolrefill", 0, "newsize" }, { "getrawmempool", 0, "verbose" }, From ecbbb74626514287e4eb2c513710c68a8304d39c Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 29 Jul 2020 04:19:37 +0000 Subject: [PATCH 07/10] QA: Test prune locks via RPC --- test/functional/feature_pruning.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 4b548ef0f33a6..33da9de6efbf6 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -324,6 +324,17 @@ def has_block(index): node.pruneblockchain(height(0)) assert has_block(0), "blk00000.dat is missing when should still be there" + # height=500 shouldn't prune first file if there's a prune lock + node.setprunelock("test", { + "desc": "Testing", + "height": [2, 2], + }) + assert_equal(node.listprunelocks(), {'prune_locks': [{'id': 'test', 'desc': 'Testing', 'height': [2, 2]}]}) + prune(500) + assert has_block(0), "blk00000.dat is missing when should still be there" + node.setprunelock("test", {}) # delete prune lock + assert_equal(node.listprunelocks(), {'prune_locks': []}) + # height=500 should prune first file prune(500) assert not has_block(0), "blk00000.dat is still there, should be pruned by now" From 2319e15a6a73a3aee1f3cddd66d1ab620278d8ab Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Fri, 6 Oct 2023 00:47:39 +0000 Subject: [PATCH 08/10] RPC/Blockchain: Optimise setprunelock "delete all" slightly --- src/rpc/blockchain.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 6ab5ead43302c..ed7f4cacf9cfc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -880,6 +880,7 @@ static RPCHelpMan setprunelock() // Delete all success = true; std::vector all_ids; + all_ids.reserve(blockman->m_prune_locks.size()); for (const auto& prune_lock : blockman->m_prune_locks) { all_ids.push_back(prune_lock.first); } From 427353a89ddaba122428e918782b209414293665 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 10 Oct 2023 00:31:37 +0000 Subject: [PATCH 09/10] Bugfix: QA/fuzz: Add listprunelocks and setprunelock to RPC_COMMANDS_SAFE_FOR_FUZZING --- src/test/fuzz/rpc.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e29e6..7a09860c49736 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -153,6 +153,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "invalidateblock", "joinpsbts", "listbanned", + "listprunelocks", "logging", "mockscheduler", "ping", @@ -166,6 +167,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "sendrawtransaction", "setmocktime", "setnetworkactive", + "setprunelock", "signmessagewithprivkey", "signrawtransactionwithkey", "submitblock", From 31874a0aeea64c2ce24fe2396fc78d5c8ac1e90b Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 20 Dec 2023 21:58:08 +0000 Subject: [PATCH 10/10] Bugfix: RPC: blockchain: Actually include "temporary" flag in listprunelocks result --- src/rpc/blockchain.cpp | 1 + test/functional/feature_pruning.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ed7f4cacf9cfc..a41b6cf7af501 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -809,6 +809,7 @@ static RPCHelpMan listprunelocks() heights_uv.push_back(lock_info.height_last); } prune_lock_uv.pushKV("height", heights_uv); + prune_lock_uv.pushKV("temporary", lock_info.temporary); locks_uv.push_back(prune_lock_uv); } } diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 33da9de6efbf6..0e5e3100eb65e 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -329,7 +329,7 @@ def has_block(index): "desc": "Testing", "height": [2, 2], }) - assert_equal(node.listprunelocks(), {'prune_locks': [{'id': 'test', 'desc': 'Testing', 'height': [2, 2]}]}) + assert_equal(node.listprunelocks(), {'prune_locks': [{'id': 'test', 'desc': 'Testing', 'height': [2, 2], 'temporary': False}]}) prune(500) assert has_block(0), "blk00000.dat is missing when should still be there" node.setprunelock("test", {}) # delete prune lock