From 05b1fd1610a837f1eb351f487d47a8ad19f5db10 Mon Sep 17 00:00:00 2001 From: Sami Ahmed Date: Fri, 13 Mar 2026 02:40:13 -0700 Subject: [PATCH] Dramatically speed up startup and sync: headers-first, fast block index, indexed rescan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Startup optimization (fixes multi-minute hang on 2M+ block chain): - Skip computing SHA-256 stake modifier checksum for blocks below last checkpoint (was doing 2M+ unnecessary hashes on every startup) - Reduce default -checkblocks from 2500 to 50 (disk read savings) - Add progress percentage reporting during block index load Headers-first sync (faster initial sync): - Switch initial sync from getblocks to getheaders protocol - Add PushGetHeaders and headers message handler in net/main - Downloads 80-byte headers first, then requests blocks — allows the node to learn chain structure before downloading full blocks Address-index accelerated wallet rescan: - New ScanForWalletTransactionsFromIndex() uses address index to find only transactions touching wallet keys (skips scanning every block) - Follows spend chains to find related transactions - Falls back to full scan if index unavailable - Track index sync state with addressIndexBestChain/StartHeight in LevelDB Co-Authored-By: Claude Opus 4.6 --- src/init.cpp | 26 +++++- src/main.cpp | 82 +++++++++++++++++- src/net.cpp | 10 +++ src/net.h | 5 ++ src/txdb-leveldb.cpp | 58 +++++++++++-- src/txdb-leveldb.h | 4 + src/wallet.cpp | 200 +++++++++++++++++++++++++++++++++++++++++++ src/wallet.h | 1 + 8 files changed, 376 insertions(+), 10 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 75b09b0..70060c1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -949,7 +949,31 @@ bool AppInit2() uiInterface.InitMessage(_("Rescanning...")); printf("Rescanning last %i blocks (from block %i)...\n", pindexBest->nHeight - pindexRescan->nHeight, pindexRescan->nHeight); nStart = GetTimeMillis(); - pwalletMain->ScanForWalletTransactions(pindexRescan, true); + bool fScannedWithIndex = false; + if (GetBoolArg("-addressindex", false) && !GetBoolArg("-rescan")) + { + CTxDB txdb("r"); + int nAddressIndexStartHeight = 0; + uint256 hashAddressIndexBestChain = 0; + if (txdb.ReadAddressIndexStartHeight(nAddressIndexStartHeight) && + txdb.ReadAddressIndexBestChain(hashAddressIndexBestChain) && + hashAddressIndexBestChain == hashBestChain && + pindexRescan->nHeight >= nAddressIndexStartHeight) + { + int nFound = 0; + fScannedWithIndex = pwalletMain->ScanForWalletTransactionsFromIndex(pindexRescan, true, &nFound); + if (!fScannedWithIndex) + printf("Indexed wallet rescan failed, falling back to full rescan.\n"); + } + else + { + printf("Address index wallet rescan unavailable from block %i.\n", pindexRescan->nHeight); + } + } + + if (!fScannedWithIndex) + pwalletMain->ScanForWalletTransactions(pindexRescan, true); + printf(" rescan %15"PRId64"ms\n", GetTimeMillis() - nStart); } diff --git a/src/main.cpp b/src/main.cpp index 74df385..accdf4f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -168,6 +168,21 @@ void static SetBestChain(const CBlockLocator& loc) pwallet->SetBestChain(loc); } +static bool UpdateAddressIndexSyncState(CTxDB& txdb, const CBlockIndex* pindexNew) +{ + if (!fAddressIndex || pindexNew == NULL) + return true; + + int nStartHeight = 0; + if (!txdb.ReadAddressIndexStartHeight(nStartHeight)) + { + if (!txdb.WriteAddressIndexStartHeight(pindexNew->nHeight)) + return false; + } + + return txdb.WriteAddressIndexBestChain(pindexNew->GetBlockHash()); +} + // notify wallets about an updated transaction void static UpdatedTransaction(const uint256& hashTx) { @@ -1898,6 +1913,8 @@ bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew) } if (!txdb.WriteHashBestChain(pindexNew->GetBlockHash())) return error("Reorganize() : WriteHashBestChain failed"); + if (!UpdateAddressIndexSyncState(txdb, pindexNew)) + return error("Reorganize() : WriteAddressIndexBestChain failed"); // Make sure it's successfully written to disk before changing memory structure if (!txdb.TxnCommit()) @@ -1935,7 +1952,7 @@ bool CBlock::SetBestChainInner(CTxDB& txdb, CBlockIndex *pindexNew) uint256 hash = GetHash(); // Adding to current best branch - if (!ConnectBlock(txdb, pindexNew) || !txdb.WriteHashBestChain(hash)) + if (!ConnectBlock(txdb, pindexNew) || !txdb.WriteHashBestChain(hash) || !UpdateAddressIndexSyncState(txdb, pindexNew)) { txdb.TxnAbort(); InvalidChainFound(pindexNew); @@ -1964,6 +1981,8 @@ bool CBlock::SetBestChain(CTxDB& txdb, CBlockIndex* pindexNew) if (pindexGenesisBlock == NULL && hash == (!fTestNet ? hashGenesisBlockOfficial : hashGenesisBlockTestNet)) { txdb.WriteHashBestChain(hash); + if (!UpdateAddressIndexSyncState(txdb, pindexNew)) + return error("SetBestChain() : WriteAddressIndexBestChain failed"); if (!txdb.TxnCommit()) return error("SetBestChain() : TxnCommit failed"); pindexGenesisBlock = pindexNew; @@ -3207,7 +3226,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) (nAskedForBlocks < 1 || vNodes.size() <= 1)) { nAskedForBlocks++; - pfrom->PushGetBlocks(pindexBest, uint256(0)); + pfrom->PushGetHeaders(pindexBest, uint256(0)); } // Relay alerts @@ -3525,6 +3544,63 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->PushMessage("headers", vHeaders); } + else if (strCommand == "headers") + { + vector vHeaders; + vRecv >> vHeaders; + if (vHeaders.size() > 2000) + { + pfrom->Misbehaving(20); + return error("message headers size() = %"PRIszu"", vHeaders.size()); + } + + CTxDB txdb("r"); + uint256 hashChainTip = 0; + int nRequested = 0; + BOOST_FOREACH(const CBlock& header, vHeaders) + { + if (!header.vtx.empty()) + { + pfrom->Misbehaving(20); + return error("headers message includes transactions"); + } + + const uint256 hashHeader = header.GetHash(); + if (mapBlockIndex.count(hashHeader)) + { + hashChainTip = hashHeader; + continue; + } + + if (hashChainTip != 0) + { + if (header.hashPrevBlock != hashChainTip) + { + pfrom->Misbehaving(20); + return error("non-continuous headers sequence"); + } + } + else + { + map::iterator miPrev = mapBlockIndex.find(header.hashPrevBlock); + if (miPrev == mapBlockIndex.end()) + break; + hashChainTip = header.hashPrevBlock; + } + + CInv inv(MSG_BLOCK, hashHeader); + if (!AlreadyHave(txdb, inv)) + { + pfrom->AskFor(inv); + nRequested++; + } + hashChainTip = hashHeader; + } + + if (nRequested > 0 && fDebug) + printf("requested %d blocks from headers announcement\n", nRequested); + } + else if (strCommand == "tx") { @@ -4078,5 +4154,3 @@ bool SendMessages(CNode* pto, bool fSendTrickle) } - - diff --git a/src/net.cpp b/src/net.cpp index 47e6a44..40b8bac 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -106,6 +106,16 @@ void CNode::PushGetBlocks(CBlockIndex* pindexBegin, uint256 hashEnd) PushMessage("getblocks", CBlockLocator(pindexBegin), hashEnd); } +void CNode::PushGetHeaders(CBlockIndex* pindexBegin, uint256 hashEnd) +{ + if (pindexBegin == pindexLastGetHeadersBegin && hashEnd == hashLastGetHeadersEnd) + return; + pindexLastGetHeadersBegin = pindexBegin; + hashLastGetHeadersEnd = hashEnd; + + PushMessage("getheaders", CBlockLocator(pindexBegin), hashEnd); +} + // find 'best' local address for a particular peer bool GetLocal(CService& addr, const CNetAddr *paddrPeer) { diff --git a/src/net.h b/src/net.h index 88bd403..f2598ab 100644 --- a/src/net.h +++ b/src/net.h @@ -268,6 +268,8 @@ public: uint256 hashContinue; CBlockIndex* pindexLastGetBlocksBegin; uint256 hashLastGetBlocksEnd; + CBlockIndex* pindexLastGetHeadersBegin; + uint256 hashLastGetHeadersEnd; int nStartingHeight; // flood relay @@ -312,6 +314,8 @@ public: hashContinue = 0; pindexLastGetBlocksBegin = 0; hashLastGetBlocksEnd = 0; + pindexLastGetHeadersBegin = 0; + hashLastGetHeadersEnd = 0; nStartingHeight = -1; fGetAddr = false; nMisbehavior = 0; @@ -698,6 +702,7 @@ public: void PushGetBlocks(CBlockIndex* pindexBegin, uint256 hashEnd); + void PushGetHeaders(CBlockIndex* pindexBegin, uint256 hashEnd); bool IsSubscribed(unsigned int nChannel); void Subscribe(unsigned int nChannel, unsigned int nHops=0); void CancelSubscribe(unsigned int nChannel); diff --git a/src/txdb-leveldb.cpp b/src/txdb-leveldb.cpp index 099b681..7e95019 100644 --- a/src/txdb-leveldb.cpp +++ b/src/txdb-leveldb.cpp @@ -19,6 +19,7 @@ #include "checkpoints.h" #include "txdb.h" #include "util.h" +#include "ui_interface.h" #include "addressindex.h" #include "main.h" @@ -277,6 +278,26 @@ bool CTxDB::WriteHashBestChain(uint256 hashBestChain) return Write(string("hashBestChain"), hashBestChain); } +bool CTxDB::ReadAddressIndexBestChain(uint256& hashBestChain) +{ + return Read(string("addressIndexBestChain"), hashBestChain); +} + +bool CTxDB::WriteAddressIndexBestChain(uint256 hashBestChain) +{ + return Write(string("addressIndexBestChain"), hashBestChain); +} + +bool CTxDB::ReadAddressIndexStartHeight(int& nHeight) +{ + return Read(string("addressIndexStartHeight"), nHeight); +} + +bool CTxDB::WriteAddressIndexStartHeight(int nHeight) +{ + return Write(string("addressIndexStartHeight"), nHeight); +} + bool CTxDB::ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust) { return Read(string("bnBestInvalidTrust"), bnBestInvalidTrust); @@ -343,8 +364,16 @@ bool CTxDB::LoadBlockIndex() ssStartKey << make_pair(string("blockindex"), uint256(0)); iterator->Seek(ssStartKey.str()); // Now read each entry. + int nBlocksLoaded = 0; while (iterator->Valid()) { + // Report progress every 100k blocks + if (++nBlocksLoaded % 100000 == 0) + { + std::string strMsg = strprintf(_("Loading block index... (%d blocks)"), nBlocksLoaded); + uiInterface.InitMessage(strMsg); + } + // Unpack keys and values. CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.write(iterator->key().data(), iterator->key().size()); @@ -409,14 +438,33 @@ bool CTxDB::LoadBlockIndex() vSortedByHeight.push_back(make_pair(pindex->nHeight, pindex)); } sort(vSortedByHeight.begin(), vSortedByHeight.end()); + + int nLastCheckpointHeight = Checkpoints::GetTotalBlocksEstimate(); + int nProgressInterval = std::max((int)vSortedByHeight.size() / 20, 1); + int nCount = 0; + BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) { CBlockIndex* pindex = item.second; pindex->nChainTrust = (pindex->pprev ? pindex->pprev->nChainTrust : 0) + pindex->GetBlockTrust(); - // triangles: calculate stake modifier checksum - pindex->nStakeModifierChecksum = GetStakeModifierChecksum(pindex); - if (!CheckStakeModifierCheckpoints(pindex->nHeight, pindex->nStakeModifierChecksum)) - return error("CTxDB::LoadBlockIndex() : Failed stake modifier checkpoint height=%d, modifier=0x%016"PRIx64, pindex->nHeight, pindex->nStakeModifier); + + // Only compute the expensive SHA-256 stake modifier checksum for blocks + // at or beyond the last hardcoded checkpoint. Blocks well below the + // checkpoint have already been validated — recomputing 2M+ hashes on + // every startup was the main cause of multi-minute load times. + if (pindex->nHeight >= nLastCheckpointHeight) + { + pindex->nStakeModifierChecksum = GetStakeModifierChecksum(pindex); + if (!CheckStakeModifierCheckpoints(pindex->nHeight, pindex->nStakeModifierChecksum)) + return error("CTxDB::LoadBlockIndex() : Failed stake modifier checkpoint height=%d, modifier=0x%016"PRIx64, pindex->nHeight, pindex->nStakeModifier); + } + + // Report progress for UI responsiveness + if (++nCount % nProgressInterval == 0) + { + std::string strMsg = strprintf(_("Loading block index... (%d%%)"), nCount * 100 / vSortedByHeight.size()); + uiInterface.InitMessage(strMsg); + } } // Load hashBestChain pointer to end of best chain @@ -448,7 +496,7 @@ bool CTxDB::LoadBlockIndex() // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); - int nCheckDepth = GetArg( "-checkblocks", 2500); + int nCheckDepth = GetArg( "-checkblocks", 50); if (nCheckDepth == 0) nCheckDepth = 1000000000; // suffices until the year 19000 if (nCheckDepth > nBestHeight) diff --git a/src/txdb-leveldb.h b/src/txdb-leveldb.h index 71513e3..919f65c 100644 --- a/src/txdb-leveldb.h +++ b/src/txdb-leveldb.h @@ -195,6 +195,10 @@ public: bool WriteBlockIndex(const CDiskBlockIndex& blockindex); bool ReadHashBestChain(uint256& hashBestChain); bool WriteHashBestChain(uint256 hashBestChain); + bool ReadAddressIndexBestChain(uint256& hashBestChain); + bool WriteAddressIndexBestChain(uint256 hashBestChain); + bool ReadAddressIndexStartHeight(int& nHeight); + bool WriteAddressIndexStartHeight(int nHeight); bool ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust); bool WriteBestInvalidTrust(CBigNum bnBestInvalidTrust); bool ReadSyncCheckpoint(uint256& hashCheckpoint); diff --git a/src/wallet.cpp b/src/wallet.cpp index 0b01be2..509695e 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -11,7 +11,10 @@ #include "base58.h" #include "kernel.h" #include "coincontrol.h" +#include "addressindex.h" #include +#include +#include using namespace std; extern unsigned int nStakeMaxAge; @@ -63,6 +66,57 @@ static CBlockIndex* GetWalletRescanStart(const CWallet& wallet) return pindexStart ? pindexStart : pindexGenesisBlock; } +struct IndexedWalletTx +{ + int nHeight; + uint256 hashTx; + + IndexedWalletTx(int nHeightIn, const uint256& hashTxIn) + : nHeight(nHeightIn), hashTx(hashTxIn) + { + } +}; + +static bool IndexedWalletTxLess(const IndexedWalletTx& a, const IndexedWalletTx& b) +{ + if (a.nHeight < b.nHeight) + return true; + if (a.nHeight > b.nHeight) + return false; + return a.hashTx < b.hashTx; +} + +static bool GetIndexedWalletTxHeight(const CTxIndex& txindex, int& nHeight) +{ + CBlock block; + if (!block.ReadFromDisk(txindex.pos.nFile, txindex.pos.nBlockPos, false)) + return false; + + map::iterator mi = mapBlockIndex.find(block.GetHash()); + if (mi == mapBlockIndex.end()) + return false; + + nHeight = (*mi).second->nHeight; + return true; +} + +static bool ReadIndexedWalletTransaction(CTxDB& txdb, const uint256& hashTx, CTransaction& tx, CTxIndex& txindex, int& nHeight) +{ + if (!txdb.ReadDiskTx(hashTx, tx, txindex)) + return false; + return GetIndexedWalletTxHeight(txindex, nHeight); +} + +static bool ReadIndexedWalletTransaction(CTxDB& txdb, const CDiskTxPos& txPos, CTransaction& tx, CTxIndex& txindex, int& nHeight) +{ + tx.SetNull(); + if (!tx.ReadFromDisk(txPos)) + return false; + if (!txdb.ReadTxIndex(tx.GetHash(), txindex)) + return false; + return GetIndexedWalletTxHeight(txindex, nHeight); +} + CPubKey CWallet::GenerateNewKey() { bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets @@ -928,6 +982,152 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) return ret; } +bool CWallet::ScanForWalletTransactionsFromIndex(CBlockIndex* pindexStart, bool fUpdate, int* pnFound) +{ + int nFound = 0; + if (pnFound) + *pnFound = 0; + if (!pindexBest) + return false; + + const int nStartHeight = pindexStart ? pindexStart->nHeight : 0; + + set setKeys; + GetKeys(setKeys); + + set setScripts; + { + LOCK(cs_KeyStore); + for (ScriptMap::const_iterator it = mapScripts.begin(); it != mapScripts.end(); ++it) + setScripts.insert((*it).first); + } + + CTxDB txdb("r"); + set setSeedTxIds; + + BOOST_FOREACH(const CKeyID& keyId, setKeys) + { + vector vTxIds; + if (!txdb.GetAddressTxIds(ADDR_TYPE_P2PKH, keyId, nStartHeight, nBestHeight, vTxIds)) + return false; + setSeedTxIds.insert(vTxIds.begin(), vTxIds.end()); + } + + BOOST_FOREACH(const CScriptID& scriptId, setScripts) + { + vector vTxIds; + if (!txdb.GetAddressTxIds(ADDR_TYPE_P2SH, scriptId, nStartHeight, nBestHeight, vTxIds)) + return false; + setSeedTxIds.insert(vTxIds.begin(), vTxIds.end()); + } + + vector vOrderedSeedTxs; + BOOST_FOREACH(const uint256& hashTx, setSeedTxIds) + { + CTransaction tx; + CTxIndex txindex; + int nHeight = 0; + if (!ReadIndexedWalletTransaction(txdb, hashTx, tx, txindex, nHeight)) + return false; + if (nHeight < nStartHeight) + continue; + vOrderedSeedTxs.push_back(IndexedWalletTx(nHeight, hashTx)); + } + sort(vOrderedSeedTxs.begin(), vOrderedSeedTxs.end(), IndexedWalletTxLess); + + deque vWorkQueue; + set setQueuedTxs; + BOOST_FOREACH(const IndexedWalletTx& indexedTx, vOrderedSeedTxs) + { + bool fExists = false; + { + LOCK(cs_wallet); + fExists = mapWallet.count(indexedTx.hashTx) > 0; + } + if (fExists && !fUpdate) + continue; + + CTransaction tx; + CTxIndex txindex; + int nHeight = 0; + if (!ReadIndexedWalletTransaction(txdb, indexedTx.hashTx, tx, txindex, nHeight)) + return false; + + CWalletTx wtx(this, tx); + wtx.SetMerkleBranch(); + if (!AddToWallet(wtx)) + return false; + nFound++; + + if (setQueuedTxs.insert(indexedTx.hashTx).second) + vWorkQueue.push_back(indexedTx.hashTx); + } + + set setProcessedTxs; + while (!vWorkQueue.empty()) + { + uint256 hashTx = vWorkQueue.front(); + vWorkQueue.pop_front(); + + if (!setProcessedTxs.insert(hashTx).second) + continue; + + CWalletTx wtx; + { + LOCK(cs_wallet); + map::const_iterator mi = mapWallet.find(hashTx); + if (mi == mapWallet.end()) + continue; + wtx = (*mi).second; + } + + CTxIndex txindex; + if (!txdb.ReadTxIndex(hashTx, txindex)) + continue; + if (txindex.vSpent.size() != wtx.vout.size()) + { + printf("ERROR: ScanForWalletTransactionsFromIndex() : txindex.vSpent.size() %"PRIszu" != wtx.vout.size() %"PRIszu" for %s\n", + txindex.vSpent.size(), wtx.vout.size(), hashTx.ToString().c_str()); + continue; + } + + for (unsigned int i = 0; i < txindex.vSpent.size(); i++) + { + if (txindex.vSpent[i].IsNull() || !IsMine(wtx.vout[i])) + continue; + + CTransaction txSpend; + CTxIndex txindexSpend; + int nSpendHeight = 0; + if (!ReadIndexedWalletTransaction(txdb, txindex.vSpent[i], txSpend, txindexSpend, nSpendHeight)) + return false; + + bool fSpendExists = false; + { + LOCK(cs_wallet); + fSpendExists = mapWallet.count(txSpend.GetHash()) > 0; + } + if (fSpendExists && !fUpdate) + continue; + + CWalletTx wtxSpend(this, txSpend); + wtxSpend.SetMerkleBranch(); + if (!AddToWallet(wtxSpend)) + return false; + nFound++; + + if (setQueuedTxs.insert(txSpend.GetHash()).second) + vWorkQueue.push_back(txSpend.GetHash()); + } + } + + SetBestChain(CBlockLocator(pindexBest)); + + if (pnFound) + *pnFound = nFound; + return true; +} + int CWallet::ScanForWalletTransaction(const uint256& hashTx) { CTransaction tx; diff --git a/src/wallet.h b/src/wallet.h index d5e382a..b872212 100644 --- a/src/wallet.h +++ b/src/wallet.h @@ -181,6 +181,7 @@ public: bool EraseFromWallet(uint256 hash); void WalletUpdateSpent(const CTransaction& prevout, bool fBlock = false); int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); + bool ScanForWalletTransactionsFromIndex(CBlockIndex* pindexStart, bool fUpdate, int* pnFound = NULL); int ScanForWalletTransaction(const uint256& hashTx); void ReacceptWalletTransactions(); void ResendWalletTransactions(bool fForce = false);