From 9e35ea77199080f9128cced0facb500d5bbbff44 Mon Sep 17 00:00:00 2001 From: sunerok Date: Mon, 6 Nov 2017 14:50:17 -0500 Subject: [PATCH] Revert "BIP 37 - Bloom Filters" This reverts commit b1a3b27b6369028023dcf7974a16d3a6bc70930c. --- src/Makefile.am | 4 - src/addrman.h | 1 - src/bloom.cpp | 183 -------------------------- src/bloom.h | 91 ------------- src/hash.cpp | 58 --------- src/hash.h | 14 +- src/init.cpp | 6 - src/main.cpp | 261 ++------------------------------------ src/main.h | 234 +++++----------------------------- src/net.cpp | 39 ------ src/net.h | 69 +++++++--- src/protocol.cpp | 1 - src/protocol.h | 1 - src/rpcrawtransaction.cpp | 2 +- src/script.h | 1 - src/uint256.h | 12 -- src/util.cpp | 1 - src/util.h | 1 - src/wallet.cpp | 4 +- 19 files changed, 99 insertions(+), 884 deletions(-) delete mode 100644 src/bloom.cpp delete mode 100644 src/bloom.h delete mode 100644 src/hash.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 96adbc83a..9854c49a1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -59,7 +59,6 @@ BITCOIN_CORE_H = \ base58.h \ bignum.h \ bitcoinrpc.h \ - bloom.h \ checkpoints.h \ clientversion.h \ compat.h \ @@ -94,7 +93,6 @@ BITCOIN_CORE_H = \ blake2.h \ blake2-impl.h \ db.h \ - hash.h \ txdb-leveldb.h \ init.h \ kernel.h \ @@ -141,7 +139,6 @@ libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) libbitcoin_server_a_SOURCES = \ addrman.cpp \ alert.cpp \ - bloom.cpp \ checkpoints.cpp \ init.cpp \ db.cpp \ @@ -178,7 +175,6 @@ libbitcoin_common_a_SOURCES = \ netbase.cpp \ protocol.cpp \ script.cpp \ - hash.cpp \ kernel.cpp \ scrypt.cpp \ groestl.c \ diff --git a/src/addrman.h b/src/addrman.h index 650a0773f..7af6afd78 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -8,7 +8,6 @@ #include "protocol.h" #include "util.h" #include "sync.h" -#include "hash.h" #include diff --git a/src/bloom.cpp b/src/bloom.cpp deleted file mode 100644 index ab61e6de2..000000000 --- a/src/bloom.cpp +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) 2012 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include - -#include "bloom.h" -#include "main.h" -#include "script.h" -#include "hash.h" - -#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 -#define LN2 0.6931471805599453094172321214581765680755001343602552 - -using namespace std; - -static const unsigned char bit_mask[8] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80}; - -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : -// The ideal size for a bloom filter with a given number of elements and false positive rate is: -// - nElements * log(fp rate) / ln(2)^2 -// We ignore filter parameters which will create a bloom filter larger than the protocol limits -vData(min((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8), -// The ideal number of hash functions is filter size * ln(2) / number of elements -// Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits -// See http://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas -isFull(false), -isEmpty(false), -nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), -nTweak(nTweakIn), -nFlags(nFlagsIn) -{ -} - -inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector& vDataToHash) const -{ - // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values. - return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8); -} - -void CBloomFilter::insert(const vector& vKey) -{ - if (isFull) - return; - for (unsigned int i = 0; i < nHashFuncs; i++) - { - unsigned int nIndex = Hash(i, vKey); - // Sets bit nIndex of vData - vData[nIndex >> 3] |= bit_mask[7 & nIndex]; - } - isEmpty = false; -} - -void CBloomFilter::insert(const COutPoint& outpoint) -{ - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << outpoint; - vector data(stream.begin(), stream.end()); - insert(data); -} - -void CBloomFilter::insert(const uint256& hash) -{ - vector data(hash.begin(), hash.end()); - insert(data); -} - -bool CBloomFilter::contains(const vector& vKey) const -{ - if (isFull) - return true; - if (isEmpty) - return false; - for (unsigned int i = 0; i < nHashFuncs; i++) - { - unsigned int nIndex = Hash(i, vKey); - // Checks bit nIndex of vData - if (!(vData[nIndex >> 3] & bit_mask[7 & nIndex])) - return false; - } - return true; -} - -bool CBloomFilter::contains(const COutPoint& outpoint) const -{ - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << outpoint; - vector data(stream.begin(), stream.end()); - return contains(data); -} - -bool CBloomFilter::contains(const uint256& hash) const -{ - vector data(hash.begin(), hash.end()); - return contains(data); -} - -bool CBloomFilter::IsWithinSizeConstraints() const -{ - return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS; -} - -bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx, const uint256& hash) -{ - bool fFound = false; - // Match if the filter contains the hash of tx - // for finding tx when they appear in a block - if (isFull) - return true; - if (isEmpty) - return false; - if (contains(hash)) - fFound = true; - - for (unsigned int i = 0; i < tx.vout.size(); i++) - { - const CTxOut& txout = tx.vout[i]; - // Match if the filter contains any arbitrary script data element in any scriptPubKey in tx - // If this matches, also add the specific output that was matched. - // This means clients don't have to update the filter themselves when a new relevant tx - // is discovered in order to find spending transactions, which avoids round-tripping and race conditions. - CScript::const_iterator pc = txout.scriptPubKey.begin(); - vector data; - while (pc < txout.scriptPubKey.end()) - { - opcodetype opcode; - if (!txout.scriptPubKey.GetOp(pc, opcode, data)) - break; - if (data.size() != 0 && contains(data)) - { - fFound = true; - if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_ALL) - insert(COutPoint(hash, i)); - else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY) - { - txnouttype type; - vector > vSolutions; - if (Solver(txout.scriptPubKey, type, vSolutions) && - (type == TX_PUBKEY || type == TX_MULTISIG)) - insert(COutPoint(hash, i)); - } - break; - } - } - } - - if (fFound) - return true; - - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - // Match if the filter contains an outpoint tx spends - if (contains(txin.prevout)) - return true; - - // Match if the filter contains any arbitrary script data element in any scriptSig in tx - CScript::const_iterator pc = txin.scriptSig.begin(); - vector data; - while (pc < txin.scriptSig.end()) - { - opcodetype opcode; - if (!txin.scriptSig.GetOp(pc, opcode, data)) - break; - if (data.size() != 0 && contains(data)) - return true; - } - } - - return false; -} - -void CBloomFilter::UpdateEmptyFull() -{ - bool full = true; - bool empty = true; - for (unsigned int i = 0; i < vData.size(); i++) - { - full &= vData[i] == 0xff; - empty &= vData[i] == 0; - } - isFull = full; - isEmpty = empty; -} \ No newline at end of file diff --git a/src/bloom.h b/src/bloom.h deleted file mode 100644 index 77e802716..000000000 --- a/src/bloom.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2012 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_BLOOM_H -#define BITCOIN_BLOOM_H - -#include - -#include "uint256.h" -#include "serialize.h" - -class COutPoint; -class CTransaction; - -// 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001% -static const unsigned int MAX_BLOOM_FILTER_SIZE = 36000; // bytes -static const unsigned int MAX_HASH_FUNCS = 50; - -// First two bits of nFlags control how much IsRelevantAndUpdate actually updates -// The remaining bits are reserved -enum bloomflags -{ - BLOOM_UPDATE_NONE = 0, - BLOOM_UPDATE_ALL = 1, - // Only adds outpoints to the filter if the output is a pay-to-pubkey/pay-to-multisig script - BLOOM_UPDATE_P2PUBKEY_ONLY = 2, - BLOOM_UPDATE_MASK = 3, -}; - -/** - * BloomFilter is a probabilistic filter which SPV clients provide - * so that we can filter the transactions we sends them. - * - * This allows for significantly more efficient transaction and block downloads. - * - * Because bloom filters are probabilistic, an SPV node can increase the false- - * positive rate, making us send them transactions which aren't actually theirs, - * allowing clients to trade more bandwidth for more privacy by obfuscating which - * keys are owned by them. - */ -class CBloomFilter -{ -private: - std::vector vData; - bool isFull; - bool isEmpty; - unsigned int nHashFuncs; - unsigned int nTweak; - unsigned char nFlags; - - unsigned int Hash(unsigned int nHashNum, const std::vector& vDataToHash) const; - -public: - // Creates a new bloom filter which will provide the given fp rate when filled with the given number of elements - // Note that if the given parameters will result in a filter outside the bounds of the protocol limits, - // the filter created will be as close to the given parameters as possible within the protocol limits. - // This will apply if nFPRate is very low or nElements is unreasonably high. - // nTweak is a constant which is added to the seed value passed to the hash function - // It should generally always be a random value (and is largely only exposed for unit testing) - // nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); - CBloomFilter() : isFull(true) {} - - IMPLEMENT_SERIALIZE - ( - READWRITE(vData); - READWRITE(nHashFuncs); - READWRITE(nTweak); - READWRITE(nFlags); - ) - - void insert(const std::vector& vKey); - void insert(const COutPoint& outpoint); - void insert(const uint256& hash); - - bool contains(const std::vector& vKey) const; - bool contains(const COutPoint& outpoint) const; - bool contains(const uint256& hash) const; - - // True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS - // (catch a filter which was just deserialized which was too big) - bool IsWithinSizeConstraints() const; - - // Also adds any outputs which match the filter to the filter (to match their spending txes) - bool IsRelevantAndUpdate(const CTransaction& tx, const uint256& hash); - - // Checks for empty and full filters to avoid wasting cpu - void UpdateEmptyFull(); -}; - -#endif /* BITCOIN_BLOOM_H */ \ No newline at end of file diff --git a/src/hash.cpp b/src/hash.cpp deleted file mode 100644 index b99b514d9..000000000 --- a/src/hash.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "hash.h" - -inline uint32_t ROTL32 ( uint32_t x, int8_t r ) -{ - return (x << r) | (x >> (32 - r)); -} - -unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector& vDataToHash) -{ - // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp - uint32_t h1 = nHashSeed; - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; - - const int nblocks = vDataToHash.size() / 4; - - //---------- - // body - const uint32_t * blocks = (const uint32_t *)(&vDataToHash[0] + nblocks*4); - - for(int i = -nblocks; i; i++) - { - uint32_t k1 = blocks[i]; - - k1 *= c1; - k1 = ROTL32(k1,15); - k1 *= c2; - - h1 ^= k1; - h1 = ROTL32(h1,13); - h1 = h1*5+0xe6546b64; - } - - //---------- - // tail - const uint8_t * tail = (const uint8_t*)(&vDataToHash[0] + nblocks*4); - - uint32_t k1 = 0; - - switch(vDataToHash.size() & 3) - { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; - case 1: k1 ^= tail[0]; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - h1 ^= vDataToHash.size(); - h1 ^= h1 >> 16; - h1 *= 0x85ebca6b; - h1 ^= h1 >> 13; - h1 *= 0xc2b2ae35; - h1 ^= h1 >> 16; - - return h1; -} \ No newline at end of file diff --git a/src/hash.h b/src/hash.h index 89d3bf45b..97ce9b2b3 100644 --- a/src/hash.h +++ b/src/hash.h @@ -10,7 +10,6 @@ #include #include -#include template inline uint256 Hash(const T1 pbegin, const T1 pend) @@ -105,22 +104,13 @@ uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL return ss.GetHash(); } -template -inline uint160 Hash160(const T1 pbegin, const T1 pend) +inline uint160 Hash160(const std::vector& vch) { - static unsigned char pblank[1]; uint256 hash1; - SHA256((pbegin == pend ? pblank : (unsigned char*)&pbegin[0]), (pend - pbegin) * sizeof(pbegin[0]), (unsigned char*)&hash1); + SHA256(&vch[0], vch.size(), (unsigned char*)&hash1); uint160 hash2; RIPEMD160((unsigned char*)&hash1, sizeof(hash1), (unsigned char*)&hash2); return hash2; } -inline uint160 Hash160(const std::vector& vch) -{ - return Hash160(vch.begin(), vch.end()); -} - -unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector& vDataToHash); - #endif \ No newline at end of file diff --git a/src/init.cpp b/src/init.cpp index 6722272ba..07b86cf12 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,7 +243,6 @@ std::string HelpMessage() " -bantime= " + _("Number of seconds to keep misbehaving peers from reconnecting (default: 86400)") + "\n" + " -maxreceivebuffer= " + _("Maximum per-connection receive buffer, *1000 bytes (default: 5000)") + "\n" + " -maxsendbuffer= " + _("Maximum per-connection send buffer, *1000 bytes (default: 1000)") + "\n" + - " -bloomfilters " + _("Allow peers to set bloom filters (default: 1)") + "\n" + #ifdef USE_UPNP #if USE_UPNP " -upnp " + _("Use UPnP to map the listening port (default: 1 when listening)") + "\n" + @@ -351,11 +350,6 @@ bool AppInit2() // ********************************************************* Step 2: parameter interactions fTestNet = GetBoolArg("-testnet"); - - fBloomFilters = GetBoolArg("-bloomfilters", true); - if (fBloomFilters) { - nLocalServices |= NODE_BLOOM; - } if (mapArgs.count("-bind")) { // when specifying an explicit binding address, you want to listen on it diff --git a/src/main.cpp b/src/main.cpp index fbbcc71c7..e6ccd000d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2534,146 +2534,6 @@ bool CBlock::CheckBlockSignature() const return false; } -CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter& filter) -{ - header = block.GetBlockHeader(); - - vector vMatch; - vector vHashes; - - vMatch.reserve(block.vtx.size()); - vHashes.reserve(block.vtx.size()); - - for (unsigned int i = 0; i < block.vtx.size(); i++) - { - uint256 hash = block.vtx[i].GetHash(); - if (filter.IsRelevantAndUpdate(block.vtx[i], hash)) - { - vMatch.push_back(true); - vMatchedTxn.push_back(make_pair(i, hash)); - } - else - vMatch.push_back(false); - vHashes.push_back(hash); - } - - txn = CPartialMerkleTree(vHashes, vMatch); -} - - -uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector &vTxid) { - if (height == 0) { - // hash at height 0 is the txids themself - return vTxid[pos]; - } else { - // calculate left hash - uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyong the end of the array - copy left hash otherwise1 - if (pos*2+1 < CalcTreeWidth(height-1)) - right = CalcHash(height-1, pos*2+1, vTxid); - else - right = left; - // combine subhashes - return Hash(BEGIN(left), END(left), BEGIN(right), END(right)); - } -} - -void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const std::vector &vTxid, const std::vector &vMatch) { - // determine whether this node is the parent of at least one matched txid - bool fParentOfMatch = false; - for (unsigned int p = pos << height; p < (pos+1) << height && p < nTransactions; p++) - fParentOfMatch |= vMatch[p]; - // store as flag bit - vBits.push_back(fParentOfMatch); - if (height==0 || !fParentOfMatch) { - // if at height 0, or nothing interesting below, store hash and stop - vHash.push_back(CalcHash(height, pos, vTxid)); - } else { - // otherwise, don't store any hash, but descend into the subtrees - TraverseAndBuild(height-1, pos*2, vTxid, vMatch); - if (pos*2+1 < CalcTreeWidth(height-1)) - TraverseAndBuild(height-1, pos*2+1, vTxid, vMatch); - } -} - -uint256 CPartialMerkleTree::TraverseAndExtract(int height, unsigned int pos, unsigned int &nBitsUsed, unsigned int &nHashUsed, std::vector &vMatch) { - if (nBitsUsed >= vBits.size()) { - // overflowed the bits array - failure - fBad = true; - return 0; - } - bool fParentOfMatch = vBits[nBitsUsed++]; - if (height==0 || !fParentOfMatch) { - // if at height 0, or nothing interesting below, use stored hash and do not descend - if (nHashUsed >= vHash.size()) { - // overflowed the hash array - failure - fBad = true; - return 0; - } - const uint256 &hash = vHash[nHashUsed++]; - if (height==0 && fParentOfMatch) // in case of height 0, we have a matched txid - vMatch.push_back(hash); - return hash; - } else { - // otherwise, descend into the subtrees to extract matched txids and hashes - uint256 left = TraverseAndExtract(height-1, pos*2, nBitsUsed, nHashUsed, vMatch), right; - if (pos*2+1 < CalcTreeWidth(height-1)) - right = TraverseAndExtract(height-1, pos*2+1, nBitsUsed, nHashUsed, vMatch); - else - right = left; - // and combine them before returning - return Hash(BEGIN(left), END(left), BEGIN(right), END(right)); - } -} - -CPartialMerkleTree::CPartialMerkleTree(const std::vector &vTxid, const std::vector &vMatch) : nTransactions(vTxid.size()), fBad(false) { - // reset state - vBits.clear(); - vHash.clear(); - - // calculate height of tree - int nHeight = 0; - while (CalcTreeWidth(nHeight) > 1) - nHeight++; - - // traverse the partial tree - TraverseAndBuild(nHeight, 0, vTxid, vMatch); -} - -CPartialMerkleTree::CPartialMerkleTree() : nTransactions(0), fBad(true) {} - -uint256 CPartialMerkleTree::ExtractMatches(std::vector &vMatch) { - vMatch.clear(); - // An empty set will not work - if (nTransactions == 0) - return 0; - // check for excessively high numbers of transactions - if (nTransactions > MAX_BLOCK_SIZE / 60) // 60 is the lower bound for the size of a serialized CTransaction+ return 0; - // there can never be more hashes provided than one for every txid - if (vHash.size() > nTransactions) - return 0; - // there must be at least one bit per node in the partial tree, and at least one node per hash - if (vBits.size() < vHash.size()) - return 0; - // calculate height of tree - int nHeight = 0; - while (CalcTreeWidth(nHeight) > 1) - nHeight++; - // traverse the partial tree - unsigned int nBitsUsed = 0, nHashUsed = 0; - uint256 hashMerkleRoot = TraverseAndExtract(nHeight, 0, nBitsUsed, nHashUsed, vMatch); - // verify that no problems occured during the tree traversal - if (fBad) - return 0; - // verify that all bits were consumed (except for the padding caused by serializing it as a byte sequence) - if ((nBitsUsed+7)/8 != (vBits.size()+7)/8) - return 0; - // verify that all hashes were consumed - if (nHashUsed != vHash.size()) - return 0; - return hashMerkleRoot; -} - bool CheckDiskSpace(uint64 nAdditionalBytes) { uint64 nFreeBytesAvailable = filesystem::space(GetDataDir()).available; @@ -3170,6 +3030,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) return true; } + + + + if (strCommand == "version") { // Each connection can only send one version message @@ -3191,10 +3055,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) vRecv >> pfrom->strSubVer; if (!vRecv.empty()) vRecv >> pfrom->nStartingHeight; - if (!vRecv.empty()) - vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message - else - pfrom->fRelayTxes = true; if (pfrom->fInbound && addrMe.IsRoutable()) { @@ -3439,7 +3299,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (fDebugNet || (vInv.size() == 1)) printf("received getdata for: %s\n", inv.ToString().c_str()); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK) { // Send block from disk map::iterator mi = mapBlockIndex.find(inv.hash); @@ -3447,30 +3307,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) { CBlock block; block.ReadFromDisk((*mi).second); - if (inv.type == MSG_BLOCK) - pfrom->PushMessage("block", block); - - else // MSG_FILTERED_BLOCK) - { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) - { - CMerkleBlock merkleBlock(block, *pfrom->pfilter); - pfrom->PushMessage("merkleblock", merkleBlock); - // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see - // This avoids hurting performance by pointlessly requiring a round-trip - // Note that there is currently no way for a node to request any single transactions we didnt send here - - // they must either disconnect and retry or request the full block. - // Thus, the protocol spec specified allows for us to provide duplicate txn here, - // however we MUST always provide at least what the remote peer needs - typedef std::pair PairType; - BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - if (!pfrom->setInventoryKnown.count(CInv(MSG_TX, pair.second))) - pfrom->PushMessage("tx", block.vtx[pair.first]); - } - // else - // no response - } + pfrom->PushMessage("block", block); // Trigger them to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) @@ -3507,23 +3344,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->PushMessage("tx", ss); } } - if (!pushed) - vNotFound.push_back(inv); } // Track requests for our stuff Inventory(inv.hash); - } - - if (!vNotFound.empty()) { - // Let the peer know that we didn't find what it asked for, so it doesn't - // have to wait around forever. Currently only SPV clients actually care - // about this message: it's needed when they are recursively walking the - // dependencies of relevant unconfirmed transactions. SPV clients want to - // do that because they want to know about (and store and rebroadcast and - // risk analyze) the dependencies of transactions relevant to them, without - // having to download the entire memory pool. - pfrom->PushMessage("notfound", vNotFound); } } @@ -3642,7 +3466,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (tx.AcceptToMemoryPool(txdb, true, &fMissingInputs)) { SyncWithWallets(tx, NULL, true); - RelayTransaction(tx, inv.hash, vMsg); + RelayMessage(inv, vMsg); mapAlreadyAskedFor.erase(inv); vWorkQueue.push_back(inv.hash); vEraseQueue.push_back(inv.hash); @@ -3665,7 +3489,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) { printf(" accepted orphan tx %s\n", inv.hash.ToString().substr(0,10).c_str()); SyncWithWallets(tx, NULL, true); - RelayTransaction(tx, inv.hash, vMsg); + RelayMessage(inv, vMsg); mapAlreadyAskedFor.erase(inv); vWorkQueue.push_back(inv.hash); vEraseQueue.push_back(inv.hash); @@ -3724,16 +3548,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else if (strCommand == "mempool") { std::vector vtxid; - LOCK2(mempool.cs, pfrom->cs_filter); mempool.queryHashes(vtxid); vector vInv; - BOOST_FOREACH(uint256& hash, vtxid) { - CInv inv(MSG_TX, hash); - if ((pfrom->pfilter && pfrom->pfilter->IsRelevantAndUpdate(mempool.lookup(hash), hash)) || - (!pfrom->pfilter)) - vInv.push_back(inv); - if (vInv.size() == MAX_INV_SZ) - break; + for (unsigned int i = 0; i < vtxid.size(); i++) { + CInv inv(MSG_TX, vtxid[i]); + vInv.push_back(inv); + if (i == (MAX_INV_SZ - 1)) + break; } if (vInv.size() > 0) pfrom->PushMessage("inv", vInv); @@ -3835,63 +3656,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) } } } - - else if (!fBloomFilters && - (strCommand == "filterload" || - strCommand == "filteradd" || - strCommand == "filterclear")) - { - pfrom->CloseSocketDisconnect(); - return error("peer %s attempted to set a bloom filter even though we do not advertise that service", - pfrom->addr.ToString().c_str()); - } - - else if (strCommand == "filterload") - { - CBloomFilter filter; - vRecv >> filter; - - if (!filter.IsWithinSizeConstraints()) - // There is no excuse for sending a too-large filter - pfrom->Misbehaving(100); - else - { - LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(filter); - pfrom->pfilter->UpdateEmptyFull(); - } - pfrom->fRelayTxes = true; - } - - else if (strCommand == "filteradd") - { - vector vData; - vRecv >> vData; - - // Nodes must NEVER send a data item > 520 bytes (the max size for a script data object, - // and thus, the maximum size any matched object can have) in a filteradd message - if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) - { - pfrom->Misbehaving(100); - } else { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) - pfrom->pfilter->insert(vData); - else - pfrom->Misbehaving(100); - } - } - - - else if (strCommand == "filterclear") - { - LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); - pfrom->fRelayTxes = true; - } else { diff --git a/src/main.h b/src/main.h index ebb70927f..a50e96375 100644 --- a/src/main.h +++ b/src/main.h @@ -899,7 +899,7 @@ class CTxIndex * Blocks are appended to blk0001.dat files on disk. Their location on disk * is indexed by CBlockIndex objects in memory. */ -class CBlockHeader +class CBlock { public: // header @@ -911,11 +911,26 @@ class CBlockHeader unsigned int nBits; unsigned int nNonce; - CBlockHeader() + // network and disk + std::vector vtx; + + // ppcoin: block signature - signed by one of the coin base txout[N]'s owner + std::vector vchBlockSig; + + // memory only + mutable std::vector vMerkleTree; + + // Denial-of-service detection: + mutable int nDoS; + bool DoS(int nDoSIn, bool fIn) const { nDoS += nDoSIn; return fIn; } + + CBlock() { SetNull(); } + int GetAlgo() const { return ::GetAlgo(nVersion); } + IMPLEMENT_SERIALIZE ( READWRITE(this->nVersion); @@ -926,16 +941,31 @@ class CBlockHeader READWRITE(nBits); READWRITE(nNonce); + // ConnectBlock depends on vtx following header to generate CDiskTxPos + if (!(nType & (SER_GETHASH|SER_BLOCKHEADERONLY))) + { + READWRITE(vtx); + READWRITE(vchBlockSig); + } + else if (fRead) + { + const_cast(this)->vtx.clear(); + const_cast(this)->vchBlockSig.clear(); + } ) void SetNull() { - nVersion = CBlockHeader::CURRENT_VERSION; + nVersion = CBlock::CURRENT_VERSION; hashPrevBlock = 0; hashMerkleRoot = 0; nTime = 0; nBits = 0; nNonce = 0; + vtx.clear(); + vchBlockSig.clear(); + vMerkleTree.clear(); + nDoS = 0; } bool IsNull() const @@ -982,68 +1012,6 @@ class CBlockHeader return (int64)nTime; } - void UpdateTime(const CBlockIndex* pindexPrev); - }; - -class CBlock : public CBlockHeader -{ -public: - // network and disk - std::vector vtx; - - // ppcoin: block signature - signed by one of the coin base txout[N]'s owner - std::vector vchBlockSig; - - // memory only - mutable std::vector vMerkleTree; - - // Denial-of-service detection: - mutable int nDoS; - bool DoS(int nDoSIn, bool fIn) const { nDoS += nDoSIn; return fIn; } - - CBlock() - { - SetNull(); - } - - int GetAlgo() const { return ::GetAlgo(nVersion); } - - IMPLEMENT_SERIALIZE - ( - READWRITE(*(CBlockHeader*)this); - - // ConnectBlock depends on vtx following header to generate CDiskTxPos - if (!(nType & (SER_GETHASH|SER_BLOCKHEADERONLY))) - { - READWRITE(vtx); - READWRITE(vchBlockSig); - } - else if (fRead) - { - const_cast(this)->vtx.clear(); - const_cast(this)->vchBlockSig.clear(); - } - ) - - void SetNull() - { - CBlockHeader::SetNull(); - vtx.clear(); - vchBlockSig.clear(); - vMerkleTree.clear(); - nDoS = 0; - } - - bool IsNull() const - { - return (nBits == 0); - } - - int64 GetBlockTime() const - { - return (int64)nTime; - } - void UpdateTime(const CBlockIndex* pindexPrev); // ppcoin: entropy bit for stake modifier if chosen by modifier @@ -1092,18 +1060,6 @@ class CBlock : public CBlockHeader maxTransactionTime = std::max(maxTransactionTime, (int64)tx.nTime); return maxTransactionTime; } - - CBlockHeader GetBlockHeader() const - { - CBlockHeader block; - block.nVersion = nVersion; - block.hashPrevBlock = hashPrevBlock; - block.hashMerkleRoot = hashMerkleRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - return block; - } uint256 BuildMerkleTree() const { @@ -1751,128 +1707,4 @@ class CTxMemPool extern CTxMemPool mempool; -/** Data structure that represents a partial merkle tree. - * - * It respresents a subset of the txid's of a known block, in a way that - * allows recovery of the list of txid's and the merkle root, in an - * authenticated way. - * - * The encoding works as follows: we traverse the tree in depth-first order, - * storing a bit for each traversed node, signifying whether the node is the - * parent of at least one matched leaf txid (or a matched txid itself). In - * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is - * stored, but we recurse into both (or the only) child branch. During - * decoding, the same depth-first traversal is performed, consuming bits and - * hashes as they written during encoding. - * - * The serialization is fixed and provides a hard guarantee about the - * encoded size: - * - * SIZE <= 10 + ceil(32.25*N) - * - * Where N represents the number of leaf nodes of the partial tree. N itself - * is bounded by: - * - * N <= total_transactions - * N <= 1 + matched_transactions*tree_height - * - * The serialization format: - * - uint32 total_transactions (4 bytes) - * - varint number of hashes (1-3 bytes) - * - uint256[] hashes in depth-first order (<= 32*N bytes) - * - varint number of bytes of flag bits (1-3 bytes) - * - byte[] flag bits, packed per 8 in a byte, least significant bit first (<= 2*N-1 bits) - * The size constraints follow from this. - */ -class CPartialMerkleTree -{ -protected: - // the total number of transactions in the block - unsigned int nTransactions; - - // node-is-parent-of-matched-txid bits - std::vector vBits; - - // txids and internal hashes - std::vector vHash; - - // flag set when encountering invalid data - bool fBad; - - // helper function to efficiently calculate the number of nodes at given height in the merkle tree - unsigned int CalcTreeWidth(int height) { - return (nTransactions+(1 << height)-1) >> height; - } - - // calculate the hash of a node in the merkle tree (at leaf level: the txid's themself) - uint256 CalcHash(int height, unsigned int pos, const std::vector &vTxid); - - // recursive function that traverses tree nodes, storing the data as bits and hashes - void TraverseAndBuild(int height, unsigned int pos, const std::vector &vTxid, const std::vector &vMatch); - - // recursive function that traverses tree nodes, consuming the bits and hashes produced by TraverseAndBuild. - // it returns the hash of the respective node. - uint256 TraverseAndExtract(int height, unsigned int pos, unsigned int &nBitsUsed, unsigned int &nHashUsed, std::vector &vMatch); - -public: - - // serialization implementation - IMPLEMENT_SERIALIZE( - READWRITE(nTransactions); - READWRITE(vHash); - std::vector vBytes; - if (fRead) { - READWRITE(vBytes); - CPartialMerkleTree &us = *(const_cast(this)); - us.vBits.resize(vBytes.size() * 8); - for (unsigned int p = 0; p < us.vBits.size(); p++) - us.vBits[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - us.fBad = false; - } else { - vBytes.resize((vBits.size()+7)/8); - for (unsigned int p = 0; p < vBits.size(); p++) - vBytes[p / 8] |= vBits[p] << (p % 8); - READWRITE(vBytes); - } - ) - - // Construct a partial merkle tree from a list of transaction id's, and a mask that selects a subset of them - CPartialMerkleTree(const std::vector &vTxid, const std::vector &vMatch); - - CPartialMerkleTree(); - - // extract the matching txid's represented by this partial merkle tree. - // returns the merkle root, or 0 in case of failure - uint256 ExtractMatches(std::vector &vMatch); -}; - -/** Used to relay blocks as header + vector - * to filtered nodes. - */ -class CMerkleBlock -{ -public: - // Public only for unit testing - CBlockHeader - header; - CPartialMerkleTree txn; - -public: - // Public only for unit testing and relay testing - // (not relayed) - std::vector > vMatchedTxn; - - // Create from a CBlock, filtering transactions according to filter - // Note that this will call IsRelevantAndUpdate on the filter for each transaction, - // thus the filter will likely be modified. - CMerkleBlock(const CBlock& block, CBloomFilter& filter); - - IMPLEMENT_SERIALIZE - ( - READWRITE(header); - READWRITE(txn); - ) -}; - #endif diff --git a/src/net.cpp b/src/net.cpp index 7758da207..8e4a51201 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1954,42 +1954,3 @@ class CNetCleanup } } instance_of_cnetcleanup; - -void RelayTransaction(const CTransaction& tx, const uint256& hash) -{ - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss.reserve(10000); - ss << tx; - RelayTransaction(tx, hash, ss); -} - -void RelayTransaction(const CTransaction& tx, const uint256& hash, const CDataStream& ss) -{ - CInv inv(MSG_TX, hash); - { - LOCK(cs_mapRelay); - // Expire old relay messages - while (!vRelayExpiration.empty() && vRelayExpiration.front().first < GetTime()) - { - mapRelay.erase(vRelayExpiration.front().second); - vRelayExpiration.pop_front(); - } - - // Save original serialized message so newer versions are preserved - mapRelay.insert(std::make_pair(inv, ss)); - vRelayExpiration.push_back(std::make_pair(GetTime() + 15 * 60, inv)); - } - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - { - if(!pnode->fRelayTxes) - continue; - LOCK(pnode->cs_filter); - if (pnode->pfilter) - { - if (pnode->pfilter->IsRelevantAndUpdate(tx, hash)) - pnode->PushInventory(inv); - } else - pnode->PushInventory(inv); - } -} diff --git a/src/net.h b/src/net.h index eb7f8307b..7e4c62240 100644 --- a/src/net.h +++ b/src/net.h @@ -18,7 +18,6 @@ #include "netbase.h" #include "protocol.h" #include "addrman.h" -#include "bloom.h" #include "hash.h" class CRequestTracker; @@ -73,9 +72,6 @@ enum { MSG_TX = 1, MSG_BLOCK, - // Nodes may always request a MSG_FILTERED_BLOCK in a getdata, however, - // MSG_FILTERED_BLOCK should not appear in any invs except as a part of getdata. - MSG_FILTERED_BLOCK, }; class CRequestTracker @@ -241,14 +237,7 @@ class CNode bool fNetworkNode; bool fSuccessfullyConnected; bool fDisconnect; - // We use fRelayTxes for two purposes - - // a) it allows us to not relay tx invs before receiving the peer's version message - // b) the peer may tell us in their version message that we should not relay tx invs - // until they have initialized their bloom filter. - bool fRelayTxes; CSemaphoreGrant grantOutbound; - CCriticalSection cs_filter; - CBloomFilter* pfilter; protected: int nRefCount; @@ -309,13 +298,11 @@ class CNode nStartingHeight = -1; fGetAddr = false; nMisbehavior = 0; - fRelayTxes = false; hashCheckpointKnown = 0; setInventoryKnown.max_size(SendBufferSize() / 1000); - pfilter = new CBloomFilter(); // Be shy and don't send version until we hear - if (hSocket != INVALID_SOCKET && !fInbound) + if (!fInbound) PushVersion(); } @@ -326,8 +313,6 @@ class CNode closesocket(hSocket); hSocket = INVALID_SOCKET; } - if (pfilter) - delete pfilter; } private: @@ -722,11 +707,53 @@ class CNode void copyStats(CNodeStats &stats); }; -class CTransaction; -class CTxIn; -class CTxOut; -void RelayTransaction(const CTransaction& tx, const uint256& hash); -void RelayTransaction(const CTransaction& tx, const uint256& hash, const CDataStream& ss); + + + + + + + + +inline void RelayInventory(const CInv& inv) +{ + // Put on lists to offer to the other nodes + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + pnode->PushInventory(inv); + } +} + +template +void RelayMessage(const CInv& inv, const T& a) +{ + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss.reserve(10000); + ss << a; + RelayMessage(inv, ss); +} + +template<> +inline void RelayMessage<>(const CInv& inv, const CDataStream& ss) +{ + { + LOCK(cs_mapRelay); + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < GetTime()) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + // Save original serialized message so newer versions are preserved + mapRelay.insert(std::make_pair(inv, ss)); + vRelayExpiration.push_back(std::make_pair(GetTime() + 15 * 60, inv)); + } + + RelayInventory(inv); +} + #endif diff --git a/src/protocol.cpp b/src/protocol.cpp index 48a826d16..d6e340e36 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -16,7 +16,6 @@ static const char* ppszTypeName[] = "ERROR", "tx", "block", - "filtered block", }; CMessageHeader::CMessageHeader() diff --git a/src/protocol.h b/src/protocol.h index ba346d9a3..b5e856748 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -69,7 +69,6 @@ class CMessageHeader enum { NODE_NETWORK = (1 << 0), - NODE_BLOOM = (1 << 1), }; /** A CService with information about it as peer */ diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 77118b0ba..cac14d92e 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -526,7 +526,7 @@ Value sendrawtransaction(const Array& params, bool fHelp) SyncWithWallets(tx, NULL, true); } - RelayTransaction(tx, hashTx); + RelayMessage(CInv(MSG_TX, hashTx), tx); return hashTx.GetHex(); } diff --git a/src/script.h b/src/script.h index 029c2ae34..cf3d5ee70 100644 --- a/src/script.h +++ b/src/script.h @@ -17,7 +17,6 @@ typedef std::vector valtype; class CTransaction; -static const unsigned int MAX_SCRIPT_ELEMENT_SIZE = 520; // bytes static const unsigned int LOCKTIME_THRESHOLD = 500000000; /** Signature hash types/flags */ diff --git a/src/uint256.h b/src/uint256.h index 5e715400e..6f6593e37 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -343,18 +343,6 @@ class base_uint { return (unsigned char*)&pn[WIDTH]; } - - const unsigned char* begin() const - { - return (unsigned char*)&pn[0]; - } - - const unsigned char* end() const - { - return (unsigned char*)&pn[WIDTH]; - } - - unsigned int size() unsigned int size() { diff --git a/src/util.cpp b/src/util.cpp index 246d94ae6..25a9a9e07 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -91,7 +91,6 @@ bool fHeadless = false; bool fCommandLine = false; string strMiscWarning; bool fTestNet = false; -bool fBloomFilters = true; bool fNoListen = false; bool fLogTimestamps = false; CMedianFilter vTimeOffsets(200,0); diff --git a/src/util.h b/src/util.h index 04c8b5f01..3f4e459e1 100644 --- a/src/util.h +++ b/src/util.h @@ -156,7 +156,6 @@ extern bool fHeadless; extern bool fCommandLine; extern std::string strMiscWarning; extern bool fTestNet; -extern bool fBloomFilters; extern bool fNoListen; extern bool fLogTimestamps; extern bool fReopenDebugLog; diff --git a/src/wallet.cpp b/src/wallet.cpp index 802b7117d..d7c2c8599 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -886,7 +886,7 @@ void CWalletTx::RelayWalletTransaction(CTxDB& txdb) { uint256 hash = tx.GetHash(); if (!txdb.ContainsTx(hash)) - RelayTransaction((CTransaction)tx, tx.GetHash()); + RelayMessage(CInv(MSG_TX, hash), (CTransaction)tx); } } if (!(IsCoinBase() || IsCoinStake())) @@ -895,7 +895,7 @@ void CWalletTx::RelayWalletTransaction(CTxDB& txdb) if (!txdb.ContainsTx(hash)) { printf("Relaying wtx %s\n", hash.ToString().substr(0,10).c_str()); - RelayTransaction((CTransaction)*this, hash); + RelayMessage(CInv(MSG_TX, hash), (CTransaction)*this); } } }