From b506a481927c3c9a77055d56d769ddd1886b76ea Mon Sep 17 00:00:00 2001 From: Sami Ahmed Date: Thu, 23 Apr 2026 01:17:51 -0700 Subject: [PATCH] UTXO snapshot support, script verify cache, and Tor sync tuning (v5.8.7) - Add UTXO snapshot dump/load system (utxosnapshot.cpp/h) for fast initial sync - Add dumputxoset RPC command to create snapshots from current chain state - Add script verification cache (sigcache.h) to skip re-verifying scripts already validated during mempool acceptance - Bootstrap: try UTXO snapshot first (fast path), fall back to full bootstrap - Support manual utxo-snapshot.bin loading on startup - Tune sync parameters for Tor: increase timeouts, reduce buffer sizes - Header sync cache: TTL-based eviction instead of full cache clear - Reduce orphan block limits and script check batch size for lower memory usage Co-Authored-By: Claude Opus 4.6 --- src/CMakeLists.txt | 1 + src/bootstrap.cpp | 35 +++ src/bootstrap.h | 8 + src/init.cpp | 66 +++++- src/main.cpp | 69 +++++- src/main.h | 13 +- src/net.h | 2 +- src/rpcblockchain.cpp | 47 ++++ src/sigcache.h | 72 +++++++ src/trianglesrpc.cpp | 1 + src/trianglesrpc.h | 1 + src/utxosnapshot.cpp | 487 ++++++++++++++++++++++++++++++++++++++++++ src/utxosnapshot.h | 39 ++++ 13 files changed, 815 insertions(+), 26 deletions(-) create mode 100644 src/sigcache.h create mode 100644 src/utxosnapshot.cpp create mode 100644 src/utxosnapshot.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1c27174..8425535 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,6 +72,7 @@ set(CORE_SOURCES rpcsmessage.cpp zmqpublishnotifier.cpp txdb-leveldb.cpp + utxosnapshot.cpp lz4/lz4.c tor/onion_v3.cpp tor/tor_process.cpp diff --git a/src/bootstrap.cpp b/src/bootstrap.cpp index 4af7584..7a16db7 100644 --- a/src/bootstrap.cpp +++ b/src/bootstrap.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT/X11 software license #include "bootstrap.h" +#include "utxosnapshot.h" #include #include @@ -766,4 +767,38 @@ bool DownloadBootstrap(const std::string& host, return true; } +bool DownloadUtxoSnapshot(const std::string& host, + const fs::path& dataDir, + ProgressCallback progressFn, + std::string& strError) +{ + const bool noProxy = true; + const char* snapshotFilename = "utxo-snapshot.bin"; + + // Download utxo-snapshot.bin to a temp file + fs::path tmpPath = dataDir / "utxo-snapshot.bin.tmp"; + std::string urlPath = std::string(BASE_PATH) + snapshotFilename; + + printf("Bootstrap: downloading UTXO snapshot from %s%s...\n", host.c_str(), urlPath.c_str()); + + if (!DownloadFile(host, urlPath, tmpPath, progressFn, strError, noProxy)) { + fs::remove(tmpPath); + return false; + } + + printf("Bootstrap: UTXO snapshot downloaded, loading into database...\n"); + + // Load the snapshot into a fresh txleveldb + if (!UtxoSnapshot::LoadSnapshot(tmpPath, dataDir, strError)) { + fs::remove(tmpPath); + return false; + } + + // Clean up the temp file + fs::remove(tmpPath); + + printf("Bootstrap: UTXO snapshot loaded successfully.\n"); + return true; +} + } // namespace Bootstrap diff --git a/src/bootstrap.h b/src/bootstrap.h index 6e3b116..0b69bba 100644 --- a/src/bootstrap.h +++ b/src/bootstrap.h @@ -62,6 +62,14 @@ namespace Bootstrap { bool VerifyManifest(const SnapshotManifest& manifest, std::string& strError); + // Download a UTXO snapshot and load it into a fresh txleveldb. + // This is much faster than downloading the full bootstrap archive. + // Returns true if snapshot was downloaded and loaded successfully. + bool DownloadUtxoSnapshot(const std::string& host, + const boost::filesystem::path& dataDir, + ProgressCallback progressFn, + std::string& strError); + } // namespace Bootstrap #endif // TRIANGLES_BOOTSTRAP_H diff --git a/src/init.cpp b/src/init.cpp index 5cdd32a..379dd2a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -14,6 +14,7 @@ #include "smessage.h" #include "openssl_compat.h" #include "bootstrap.h" +#include "utxosnapshot.h" #include "tor/tor_embedded.h" #include "tor/onion_v3.h" #include "tor/tor_process.h" @@ -669,7 +670,7 @@ bool AppInit2() nScriptCheckThreads = 16; if (nScriptCheckThreads > 1) { - pScriptCheckQueue = new CCheckQueue(128); + pScriptCheckQueue = new CCheckQueue(32); pScriptCheckThreads = new boost::thread_group(); for (int i = 0; i < nScriptCheckThreads - 1; ++i) pScriptCheckThreads->create_thread(&ThreadScriptCheck); @@ -907,9 +908,6 @@ bool AppInit2() std::string host = Bootstrap::DEFAULT_HOST; std::string strError; - uiInterface.InitMessage(_("Downloading blockchain snapshot...")); - printf("Bootstrap: contacting %s...\n", host.c_str()); - auto progressFn = [](int64_t bytesDownloaded, int64_t totalBytes) { if (totalBytes > 0) { printf("\rBootstrap: %lld / %lld MB (%lld%%)", @@ -920,20 +918,66 @@ bool AppInit2() } }; - bool success = Bootstrap::DownloadBootstrap(host, dataPath, progressFn, strError); + // Try UTXO snapshot first (fast: ~2-10 MB download) + bool success = false; + bool triedUtxoSnapshot = false; + if (needsBootstrap && !fs::exists(dataPath / "txleveldb")) { + uiInterface.InitMessage(_("Downloading UTXO snapshot...")); + printf("Bootstrap: trying UTXO snapshot from %s (fast path)...\n", host.c_str()); - if (!success) { - printf("\nBootstrap: failed: %s\n", strError.c_str()); - printf("Bootstrap: skipping, will sync from network.\n"); - } else { - printf("\nBootstrap: done.\n"); + std::string utxoError; + if (Bootstrap::DownloadUtxoSnapshot(host, dataPath, progressFn, utxoError)) { + printf("\nBootstrap: UTXO snapshot loaded — will sync remaining blocks from network.\n"); + success = true; + } else { + printf("\nBootstrap: UTXO snapshot unavailable: %s\n", utxoError.c_str()); + printf("Bootstrap: falling back to full bootstrap download...\n"); + } + triedUtxoSnapshot = true; } + + // Fall back to full bootstrap.tar.gz if UTXO snapshot failed + if (!success) { + uiInterface.InitMessage(_("Downloading blockchain snapshot...")); + printf("Bootstrap: contacting %s...\n", host.c_str()); + + success = Bootstrap::DownloadBootstrap(host, dataPath, progressFn, strError); + + if (!success) { + printf("\nBootstrap: failed: %s\n", strError.c_str()); + printf("Bootstrap: skipping, will sync from network.\n"); + } else { + printf("\nBootstrap: done.\n"); + } + } + StartupPerfLog("bootstrap_download", GetTimeMillis() - nBootstrapStart, - strprintf("host=%s success=%d", host.c_str(), success)); + strprintf("host=%s success=%d utxo_snapshot=%d", host.c_str(), success, triedUtxoSnapshot)); } } // end bootstrap scope #endif + // ********************************************************* Step 6c: manual UTXO snapshot loading + // If utxo-snapshot.bin exists in data dir and no txleveldb, load it. + { + fs::path dataPath = GetDataDir(); + fs::path snapshotFile = dataPath / "utxo-snapshot.bin"; + fs::path txleveldbDir = dataPath / "txleveldb"; + + if (fs::exists(snapshotFile) && !fs::exists(txleveldbDir)) { + printf("Found utxo-snapshot.bin — loading UTXO snapshot...\n"); + uiInterface.InitMessage(_("Loading UTXO snapshot...")); + + std::string strError; + if (UtxoSnapshot::LoadSnapshot(snapshotFile, dataPath, strError)) { + printf("UTXO snapshot loaded successfully.\n"); + } else { + printf("UTXO snapshot load failed: %s\n", strError.c_str()); + printf("Will proceed with normal sync.\n"); + } + } + } + // ********************************************************* Step 7: load blockchain if (!bitdb.Open(GetDataDir())) diff --git a/src/main.cpp b/src/main.cpp index eae816b..176d229 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -77,6 +77,8 @@ int64_t nTimeBestReceived = 0; CMedianFilter cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have +CScriptVerifyCache scriptVerifyCache; + map mapOrphanBlocks; multimap mapOrphanBlocksByPrev; set > setStakeSeenOrphan; @@ -110,6 +112,7 @@ struct CHeaderSyncNode uint256 nChainTrust; bool fRequested; int64_t nLastRequestTime; + int64_t nInsertTime; }; static std::map mapHeaderSync; @@ -117,11 +120,12 @@ static uint256 hashBestHeaderSync = 0; static CCriticalSection cs_PostIbdWork; static bool fPostIbdWorkStarted = false; -static const unsigned int MAX_HEADER_SYNC_CACHE = 50000; +static const unsigned int MAX_HEADER_SYNC_CACHE = 15000; static const unsigned int HEADER_DOWNLOAD_WINDOW = 512; // Increased from 128 for parallel downloads -static const unsigned int HEADER_DOWNLOAD_PER_PEER = 64; // Max blocks to request from each peer -static const int64_t HEADER_REQUEST_TIMEOUT_MICROS = 30 * 1000000; -static const int64_t HEADER_REDUNDANT_REQUEST_MICROS = 10 * 1000000; // Request from another peer after 10s +static const unsigned int HEADER_DOWNLOAD_PER_PEER = 32; // Reduced from 64 for Tor circuit stability +static const int64_t HEADER_REQUEST_TIMEOUT_MICROS = 60 * 1000000; // 60s for Tor latency (was 30s) +static const int64_t HEADER_REDUNDANT_REQUEST_MICROS = 20 * 1000000; // 20s redundant request (was 10s) +static const int64_t HEADER_SYNC_TTL_MICROS = 5 * 60 * 1000000; // 5-minute TTL for cache entries static void ThreadPostIbdWork(void* parg) { @@ -232,12 +236,47 @@ static void RecomputeBestHeaderSync() static void PruneHeaderSync() { - if (mapHeaderSync.size() <= MAX_HEADER_SYNC_CACHE) - return; + const int64_t nNow = GetTime() * 1000000; - printf("IBD-DIAG: header sync cache exceeded %u entries, clearing planner state\n", MAX_HEADER_SYNC_CACHE); - mapHeaderSync.clear(); - hashBestHeaderSync = 0; + // TTL eviction: remove entries older than 5 minutes + if (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE / 2) + { + unsigned int nEvicted = 0; + for (auto it = mapHeaderSync.begin(); it != mapHeaderSync.end(); ) + { + if (nNow - it->second.nInsertTime >= HEADER_SYNC_TTL_MICROS) + { + it = mapHeaderSync.erase(it); + ++nEvicted; + } + else + ++it; + } + if (nEvicted > 0) + { + printf("IBD-DIAG: TTL-evicted %u stale header sync entries, %u remain\n", + nEvicted, (unsigned int)mapHeaderSync.size()); + RecomputeBestHeaderSync(); + } + } + + // Hard limit: if still over max, evict oldest entries + if (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE) + { + printf("IBD-DIAG: header sync cache exceeded %u entries, evicting oldest\n", MAX_HEADER_SYNC_CACHE); + while (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE * 3 / 4) + { + // Find oldest entry by insert time + auto oldest = mapHeaderSync.begin(); + for (auto it = mapHeaderSync.begin(); it != mapHeaderSync.end(); ++it) + { + if (it->second.nInsertTime < oldest->second.nInsertTime) + oldest = it; + } + mapHeaderSync.erase(oldest); + } + RecomputeBestHeaderSync(); + } } static bool AddHeaderSyncNode(const CBlock& header, const uint256& hashHeader) @@ -283,6 +322,7 @@ static bool AddHeaderSyncNode(const CBlock& header, const uint256& hashHeader) node.nChainTrust = nPrevChainTrust + GetHeaderSyncTrust(header.nBits); node.fRequested = false; node.nLastRequestTime = 0; + node.nInsertTime = GetTime() * 1000000; mapHeaderSync.insert(std::make_pair(hashHeader, node)); @@ -1900,6 +1940,7 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs, // The first loop above does all the inexpensive checks. // Only if ALL inputs pass do we perform expensive ECDSA signature checks. // Helps prevent CPU exhaustion attacks. + const uint256 hashTx = GetHash(); for (unsigned int i = 0; i < vin.size(); i++) { COutPoint prevout = vin[i].prevout; @@ -1910,6 +1951,11 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs, // still computed and checked, and any change will be caught at the next checkpoint. if (!(fBlock && (nBestHeight < Checkpoints::GetTotalBlocksEstimate()))) { + // Check signature cache: skip re-verification for scripts already + // validated during mempool acceptance or prior block connections. + if (scriptVerifyCache.Get(hashTx, i)) + continue; + if (pvChecks) { pvChecks->push_back(CScriptCheck(entry.scriptPubKey, vin[i].scriptSig, *this, i, 0)); @@ -1919,6 +1965,7 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs, // Verify signature using scriptPubKey from UTXO entry if (!VerifyScript(vin[i].scriptSig, entry.scriptPubKey, *this, i, 0)) return DoS(100, error("ConnectInputs() : %s VerifySignature failed", GetHash().ToString().substr(0,10).c_str())); + scriptVerifyCache.Set(hashTx, i); } } } @@ -2286,7 +2333,7 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) if (!tx.ConnectInputs(txdb, mapInputs, pindex, true, false, pScriptCheckQueue ? &vChecks : NULL)) return false; - if (pScriptCheckQueue && vChecks.size() >= 128) + if (pScriptCheckQueue && vChecks.size() >= 32) { scriptcheckcontrol.Add(vChecks); vChecks.clear(); @@ -5510,7 +5557,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle) static int64_t nLastBlockReceived = 0; static int nLastHeight = 0; static int64_t nLastStallLog = 0; - int nStallTimeout = IsInitialBlockDownload() ? 5 : 15; + int nStallTimeout = IsInitialBlockDownload() ? 10 : 30; // Higher for Tor latency if (nBestHeight > nLastHeight) { nLastHeight = nBestHeight; nLastBlockReceived = GetTime(); diff --git a/src/main.h b/src/main.h index 8c4804e..7e736ec 100644 --- a/src/main.h +++ b/src/main.h @@ -12,6 +12,7 @@ #include "scrypt.h" #include "hashblock.h" #include "checkqueue.h" +#include "sigcache.h" #include @@ -37,8 +38,8 @@ static const unsigned int MAX_BLOCK_SIZE = 1000000; static const unsigned int MAX_BLOCK_SIZE_GEN = MAX_BLOCK_SIZE/2; static const unsigned int MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50; static const unsigned int MAX_ORPHAN_TRANSACTIONS = MAX_BLOCK_SIZE/100; -static const unsigned int MAX_ORPHAN_BLOCKS = 2000; -static const unsigned int MAX_ORPHAN_BLOCKS_IBD = 4000; +static const unsigned int MAX_ORPHAN_BLOCKS = 750; +static const unsigned int MAX_ORPHAN_BLOCKS_IBD = 1500; static const unsigned int MAX_REORG_DEPTH = 100; // reject reorgs deeper than this (finality) static const unsigned int MAX_INV_SZ = 50000; static const int64_t MIN_TX_FEE = (1 * CENT) / 100; @@ -1687,6 +1688,7 @@ public: }; extern CTxMemPool mempool; +extern CScriptVerifyCache scriptVerifyCache; /** * Closure representing one script check for parallel verification. @@ -1711,7 +1713,12 @@ public: bool operator()() { - return ptxTo && VerifyScript(scriptSig, scriptPubKey, *ptxTo, nIn, nHashType); + if (!ptxTo) + return false; + if (!VerifyScript(scriptSig, scriptPubKey, *ptxTo, nIn, nHashType)) + return false; + scriptVerifyCache.Set(ptxTo->GetHash(), nIn); + return true; } void swap(CScriptCheck& other) diff --git a/src/net.h b/src/net.h index ac80e1d..ca89665 100644 --- a/src/net.h +++ b/src/net.h @@ -26,7 +26,7 @@ extern int nBestHeight; -inline unsigned int ReceiveFloodSize() { return 100 * 1024 * 1024; } // 100 MB +inline unsigned int ReceiveFloodSize() { return 50 * 1024 * 1024; } // 50 MB (reduced for Tor-only network) inline unsigned int SendBufferSize() { return 32 * 1024 * 1024; } // 32 MB void AddOneShot(std::string strDest); diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index c6a4c48..8bba87d 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -9,6 +9,7 @@ #include "addressindex.h" #include "txdb.h" #include "base58.h" +#include "utxosnapshot.h" using namespace json_spirit; using namespace std; @@ -849,3 +850,49 @@ Value reconsiderblock(const Array& params, bool fHelp) return Value::null; } + +Value dumputxoset(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 1 || params.size() > 2) + throw runtime_error( + "dumputxoset [nheaders]\n" + "Dumps the current UTXO set and recent block headers to a binary snapshot file.\n" + "The snapshot can be used by new nodes to skip initial block download.\n" + "\nArguments:\n" + "1. filename (string, required) Destination file path\n" + "2. nheaders (int, optional, default=2000) Number of block headers to include\n" + "\nResult:\n" + "{\n" + " \"filename\": \"...\",\n" + " \"height\": n,\n" + " \"blockhash\": \"...\",\n" + " \"file_size\": n\n" + "}"); + + string filename = params[0].get_str(); + unsigned int nHeaders = UTXO_SNAPSHOT_DEFAULT_HEADERS; + if (params.size() > 1) + nHeaders = params[1].get_int(); + + if (nHeaders < 100) + throw JSONRPCError(RPC_INVALID_PARAMETER, "nheaders must be at least 100"); + + boost::filesystem::path destPath(filename); + std::string strError; + + if (!UtxoSnapshot::DumpSnapshot(destPath, nHeaders, strError)) + throw runtime_error("dumputxoset failed: " + strError); + + // Get file size + int64_t nFileSize = 0; + if (boost::filesystem::exists(destPath)) + nFileSize = (int64_t)boost::filesystem::file_size(destPath); + + Object result; + result.push_back(Pair("filename", filename)); + result.push_back(Pair("height", nBestHeight)); + result.push_back(Pair("blockhash", hashBestChain.GetHex())); + result.push_back(Pair("file_size", nFileSize)); + + return result; +} diff --git a/src/sigcache.h b/src/sigcache.h new file mode 100644 index 0000000..39df3c3 --- /dev/null +++ b/src/sigcache.h @@ -0,0 +1,72 @@ +// Copyright (c) 2024 The Triangles developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef TRIANGLES_SCRIPT_VERIFY_CACHE_H +#define TRIANGLES_SCRIPT_VERIFY_CACHE_H + +#include "uint256.h" +#include "sync.h" + +#include +#include +#include + +/** + * High-level script verification cache keyed by (txid, input_index). + * Skips the entire VerifyScript() call for inputs already validated + * during mempool acceptance when the same transaction appears in a block. + * + * Complements the lower-level CSignatureCache in script.cpp which caches + * individual ECDSA signature checks. + * + * ~256KB memory footprint at 32K entries. + */ +class CScriptVerifyCache +{ +private: + static const unsigned int MAX_CACHE_SIZE = 32768; + + struct Uint256Hasher { + size_t operator()(const uint256& v) const { + return *reinterpret_cast(v.begin()); + } + }; + + mutable CCriticalSection cs; + std::unordered_set setValid; + + uint256 ComputeKey(const uint256& txid, unsigned int nIn) const + { + unsigned char data[36]; // 32 bytes txid + 4 bytes input index + memcpy(data, txid.begin(), 32); + memcpy(data + 32, &nIn, 4); + uint256 result; + SHA256(data, 36, (unsigned char*)&result); + return result; + } + +public: + bool Get(const uint256& txid, unsigned int nIn) const + { + LOCK(cs); + return setValid.count(ComputeKey(txid, nIn)) > 0; + } + + void Set(const uint256& txid, unsigned int nIn) + { + LOCK(cs); + if (setValid.size() >= MAX_CACHE_SIZE) + { + // Evict half the cache when full + auto it = setValid.begin(); + unsigned int nEvict = MAX_CACHE_SIZE / 2; + while (nEvict > 0 && it != setValid.end()) { + it = setValid.erase(it); + --nEvict; + } + } + setValid.insert(ComputeKey(txid, nIn)); + } +}; + +#endif // TRIANGLES_SCRIPT_VERIFY_CACHE_H diff --git a/src/trianglesrpc.cpp b/src/trianglesrpc.cpp index ceac2b1..b81800d 100644 --- a/src/trianglesrpc.cpp +++ b/src/trianglesrpc.cpp @@ -319,6 +319,7 @@ static const CRPCCommand vRPCCommands[] = { "invalidateblock", &invalidateblock, false, false }, { "reconsiderblock", &reconsiderblock, false, false }, { "recalculatesupply", &recalculatesupply, false, false }, + { "dumputxoset", &dumputxoset, false, false }, { "reservebalance", &reservebalance, false, true}, { "checkwallet", &checkwallet, false, true}, { "repairwallet", &repairwallet, false, true}, diff --git a/src/trianglesrpc.h b/src/trianglesrpc.h index 8c0b9ad..1e72af2 100644 --- a/src/trianglesrpc.h +++ b/src/trianglesrpc.h @@ -227,6 +227,7 @@ extern json_spirit::Value getchaintips(const json_spirit::Array& params, bool fH extern json_spirit::Value invalidateblock(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value recalculatesupply(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value dumputxoset(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddressbalance(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddressutxos(const json_spirit::Array& params, bool fHelp); diff --git a/src/utxosnapshot.cpp b/src/utxosnapshot.cpp new file mode 100644 index 0000000..7f37fb7 --- /dev/null +++ b/src/utxosnapshot.cpp @@ -0,0 +1,487 @@ +// Copyright (c) 2024-2025 Triangles developers +// Distributed under the MIT/X11 software license + +#include "utxosnapshot.h" + +#include "main.h" +#include "txdb.h" +#include "checkpoints.h" +#include "util.h" +#include "ui_interface.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace fs = boost::filesystem; + +namespace UtxoSnapshot { + +// --------------------------------------------------------------------------- +// DumpSnapshot - create a UTXO snapshot from the current chain state +// --------------------------------------------------------------------------- + +bool DumpSnapshot(const fs::path& destPath, + unsigned int nHeaders, + std::string& strError) +{ + LOCK(cs_main); + + if (!pindexBest) { + strError = "No best block - chain not loaded"; + return false; + } + + // Collect block index entries (last nHeaders blocks, height ascending) + std::vector> vHeaders; + vHeaders.reserve(nHeaders); + { + CBlockIndex* pindex = pindexBest; + unsigned int nCollected = 0; + while (pindex && nCollected < nHeaders) { + CDiskBlockIndex diskindex(pindex); + vHeaders.push_back(std::make_pair(*pindex->phashBlock, diskindex)); + pindex = pindex->pprev; + nCollected++; + } + // Reverse to height ascending order + std::reverse(vHeaders.begin(), vHeaders.end()); + } + + // Count UTXOs first + int nUtxoCount = 0; + { + CTxDB txdbRead("r"); + txdbRead.SumUtxoValues(nUtxoCount); + } + + if (nUtxoCount == 0) { + strError = "No UTXOs found in database"; + return false; + } + + printf("UtxoSnapshot: dumping %d headers + %d UTXOs at height %d\n", + (int)vHeaders.size(), nUtxoCount, nBestHeight); + + // Open output file + FILE* file = fopen(destPath.string().c_str(), "wb"); + if (!file) { + strError = "Cannot create file: " + destPath.string(); + return false; + } + + // Write header (we'll seek back to fill in content_hash later) + unsigned int magic = UTXO_SNAPSHOT_MAGIC; + unsigned int version = UTXO_SNAPSHOT_VERSION; + unsigned int network = fTestNet ? 2 : 1; + int height = nBestHeight; + uint256 blockHash = hashBestChain; + int64_t moneySupply = pindexBest->nMoneySupply; + unsigned int numHeaders = (unsigned int)vHeaders.size(); + unsigned int numUtxos = (unsigned int)nUtxoCount; + uint256 contentHash; // placeholder, filled after writing data + + fwrite(&magic, sizeof(magic), 1, file); + fwrite(&version, sizeof(version), 1, file); + fwrite(&network, sizeof(network), 1, file); + fwrite(&height, sizeof(height), 1, file); + fwrite(&blockHash, sizeof(blockHash), 1, file); + fwrite(&moneySupply, sizeof(moneySupply), 1, file); + fwrite(&numHeaders, sizeof(numHeaders), 1, file); + fwrite(&numUtxos, sizeof(numUtxos), 1, file); + long contentHashPos = ftell(file); + fwrite(&contentHash, sizeof(contentHash), 1, file); // placeholder + + // Start SHA256 for content hash + SHA256_CTX sha256; + SHA256_Init(&sha256); + + // Write block headers section + for (const auto& item : vHeaders) { + CDataStream ssEntry(SER_DISK, CLIENT_VERSION); + ssEntry << item.first; // block hash + ssEntry << item.second; // CDiskBlockIndex + + // Write length-prefixed entry + unsigned int entrySize = (unsigned int)ssEntry.size(); + std::string strEntry = ssEntry.str(); + fwrite(&entrySize, sizeof(entrySize), 1, file); + fwrite(strEntry.data(), 1, entrySize, file); + + SHA256_Update(&sha256, &entrySize, sizeof(entrySize)); + SHA256_Update(&sha256, strEntry.data(), entrySize); + } + + // Write UTXO section using LevelDB iterator (same pattern as SumUtxoValues) + { + // Access the global LevelDB directly via a CTxDB instance + // We need a raw iterator, so we re-implement the iteration pattern + extern leveldb::DB *txdb; + + CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); + ssKeyPrefix << std::make_pair(std::string("u"), std::make_pair(uint256(0), (unsigned int)0)); + std::string strPrefixBegin = ssKeyPrefix.str(); + + leveldb::Iterator* it = txdb->NewIterator(leveldb::ReadOptions()); + unsigned int nWritten = 0; + for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) { + // Check key prefix is still "u" + CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION); + std::string strKeyType; + ssKey >> strKeyType; + if (strKeyType != "u") + break; + + // Extract outpoint from key + uint256 txhash; + unsigned int nIndex; + ssKey >> txhash; + ssKey >> nIndex; + + // Extract UTXO entry from value + CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION); + CUtxoEntry entry; + ssValue >> entry; + + // Serialize the UTXO record + CDataStream ssRecord(SER_DISK, CLIENT_VERSION); + ssRecord << txhash; + ssRecord << nIndex; + ssRecord << entry; + + unsigned int recordSize = (unsigned int)ssRecord.size(); + std::string strRecord = ssRecord.str(); + fwrite(&recordSize, sizeof(recordSize), 1, file); + fwrite(strRecord.data(), 1, recordSize, file); + + SHA256_Update(&sha256, &recordSize, sizeof(recordSize)); + SHA256_Update(&sha256, strRecord.data(), recordSize); + + nWritten++; + if (nWritten % 10000 == 0) + printf("UtxoSnapshot: wrote %d / %d UTXOs\n", nWritten, nUtxoCount); + } + delete it; + + // Update actual count (in case it changed during iteration) + if (nWritten != numUtxos) { + numUtxos = nWritten; + // Seek back and update numUtxos in header + long currentPos = ftell(file); + fseek(file, contentHashPos - sizeof(numUtxos), SEEK_SET); + fwrite(&numUtxos, sizeof(numUtxos), 1, file); + fseek(file, currentPos, SEEK_SET); + } + } + + // Finalize content hash and write it to the header + SHA256_Final((unsigned char*)&contentHash, &sha256); + fseek(file, contentHashPos, SEEK_SET); + fwrite(&contentHash, sizeof(contentHash), 1, file); + + fclose(file); + + printf("UtxoSnapshot: wrote %s (%d headers, %d UTXOs, hash=%s)\n", + destPath.string().c_str(), numHeaders, numUtxos, + contentHash.ToString().c_str()); + + return true; +} + +// --------------------------------------------------------------------------- +// LoadSnapshot - load a UTXO snapshot into a fresh LevelDB +// --------------------------------------------------------------------------- + +bool LoadSnapshot(const fs::path& snapshotPath, + const fs::path& dataDir, + std::string& strError) +{ + FILE* file = fopen(snapshotPath.string().c_str(), "rb"); + if (!file) { + strError = "Cannot open snapshot file: " + snapshotPath.string(); + return false; + } + + // Read header + unsigned int magic, version, network; + int height; + uint256 blockHash; + int64_t moneySupply; + unsigned int numHeaders, numUtxos; + uint256 expectedContentHash; + + if (fread(&magic, sizeof(magic), 1, file) != 1 || + fread(&version, sizeof(version), 1, file) != 1 || + fread(&network, sizeof(network), 1, file) != 1 || + fread(&height, sizeof(height), 1, file) != 1 || + fread(&blockHash, sizeof(blockHash), 1, file) != 1 || + fread(&moneySupply, sizeof(moneySupply), 1, file) != 1 || + fread(&numHeaders, sizeof(numHeaders), 1, file) != 1 || + fread(&numUtxos, sizeof(numUtxos), 1, file) != 1 || + fread(&expectedContentHash, sizeof(expectedContentHash), 1, file) != 1) { + fclose(file); + strError = "Truncated snapshot header"; + return false; + } + + // Validate header + if (magic != UTXO_SNAPSHOT_MAGIC) { + fclose(file); + strError = "Invalid snapshot magic (not a UTXO snapshot file)"; + return false; + } + + if (version != UTXO_SNAPSHOT_VERSION) { + fclose(file); + strError = "Unsupported snapshot version: " + std::to_string(version); + return false; + } + + unsigned int expectedNetwork = fTestNet ? 2 : 1; + if (network != expectedNetwork) { + fclose(file); + strError = "Network mismatch: snapshot is " + std::string(network == 1 ? "mainnet" : "testnet"); + return false; + } + + if (numHeaders == 0 || numUtxos == 0) { + fclose(file); + strError = "Snapshot contains no data"; + return false; + } + + // Verify snapshot block is a known checkpoint + if (!Checkpoints::IsKnownCheckpoint(height, blockHash)) { + fclose(file); + strError = "Snapshot block " + blockHash.ToString() + " at height " + + std::to_string(height) + " is not a known checkpoint"; + return false; + } + + printf("UtxoSnapshot: loading snapshot at height %d (%d headers, %d UTXOs)\n", + height, numHeaders, numUtxos); + + // Create fresh LevelDB directory + fs::path txleveldbPath = dataDir / "txleveldb"; + if (fs::exists(txleveldbPath)) + fs::remove_all(txleveldbPath); + fs::create_directories(txleveldbPath); + + // Open LevelDB directly (not via CTxDB - it's not initialized yet) + leveldb::Options options; + int nCacheSizeMB = GetArg("-dbcache", 2048); + options.block_cache = leveldb::NewLRUCache(nCacheSizeMB * 1048576); + options.filter_policy = leveldb::NewBloomFilterPolicy(10); + options.write_buffer_size = 64 * 1048576; + options.max_open_files = 1000; + options.create_if_missing = true; + + leveldb::DB* pdb = NULL; + leveldb::Status status = leveldb::DB::Open(options, txleveldbPath.string(), &pdb); + if (!status.ok()) { + fclose(file); + delete options.filter_policy; + delete options.block_cache; + strError = "Cannot create LevelDB: " + status.ToString(); + return false; + } + + SHA256_CTX sha256; + SHA256_Init(&sha256); + + leveldb::WriteBatch batch; + bool success = true; + unsigned int nBatchSize = 0; + + auto flushBatch = [&]() -> bool { + if (nBatchSize == 0) + return true; + leveldb::Status s = pdb->Write(leveldb::WriteOptions(), &batch); + if (!s.ok()) { + strError = "LevelDB write failed: " + s.ToString(); + return false; + } + batch.Clear(); + nBatchSize = 0; + return true; + }; + + // Read and write block headers + printf("UtxoSnapshot: loading %d block headers...\n", numHeaders); + uiInterface.InitMessage(_("Loading UTXO snapshot (headers)...")); + + for (unsigned int i = 0; i < numHeaders; i++) { + unsigned int entrySize; + if (fread(&entrySize, sizeof(entrySize), 1, file) != 1 || entrySize > 10000) { + success = false; + strError = "Invalid header entry size at index " + std::to_string(i); + break; + } + + std::vector buf(entrySize); + if (fread(buf.data(), 1, entrySize, file) != entrySize) { + success = false; + strError = "Truncated header entry at index " + std::to_string(i); + break; + } + + SHA256_Update(&sha256, &entrySize, sizeof(entrySize)); + SHA256_Update(&sha256, buf.data(), entrySize); + + // Parse: block_hash + CDiskBlockIndex + CDataStream ssEntry(buf.data(), buf.data() + buf.size(), SER_DISK, CLIENT_VERSION); + uint256 entryHash; + CDiskBlockIndex diskindex; + ssEntry >> entryHash; + ssEntry >> diskindex; + + // Write to LevelDB as "blockindex" key + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey << std::make_pair(std::string("blockindex"), entryHash); + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + ssValue << diskindex; + + batch.Put(ssKey.str(), ssValue.str()); + nBatchSize++; + + if (nBatchSize >= 1000) { + if (!flushBatch()) { success = false; break; } + } + } + + if (success && !flushBatch()) + success = false; + + // Read and write UTXOs + if (success) { + printf("UtxoSnapshot: loading %d UTXOs...\n", numUtxos); + + for (unsigned int i = 0; i < numUtxos; i++) { + unsigned int recordSize; + if (fread(&recordSize, sizeof(recordSize), 1, file) != 1 || recordSize > 100000) { + success = false; + strError = "Invalid UTXO record size at index " + std::to_string(i); + break; + } + + std::vector buf(recordSize); + if (fread(buf.data(), 1, recordSize, file) != recordSize) { + success = false; + strError = "Truncated UTXO record at index " + std::to_string(i); + break; + } + + SHA256_Update(&sha256, &recordSize, sizeof(recordSize)); + SHA256_Update(&sha256, buf.data(), recordSize); + + // Parse: txid + output_index + CUtxoEntry + CDataStream ssRecord(buf.data(), buf.data() + buf.size(), SER_DISK, CLIENT_VERSION); + uint256 txhash; + unsigned int nIndex; + CUtxoEntry entry; + ssRecord >> txhash; + ssRecord >> nIndex; + ssRecord >> entry; + + // Write to LevelDB with "u" prefix key + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey << std::make_pair(std::string("u"), std::make_pair(txhash, nIndex)); + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + ssValue << entry; + + batch.Put(ssKey.str(), ssValue.str()); + nBatchSize++; + + if (nBatchSize >= 50000) { + if (!flushBatch()) { success = false; break; } + + if (i % 50000 == 0) { + std::string strMsg = strprintf(_("Loading UTXO snapshot (%d%%)..."), + i * 100 / numUtxos); + uiInterface.InitMessage(strMsg); + printf("UtxoSnapshot: loaded %d / %d UTXOs\n", i, numUtxos); + } + } + } + + if (success && !flushBatch()) + success = false; + } + + // Verify content hash + if (success) { + uint256 actualHash; + SHA256_Final((unsigned char*)&actualHash, &sha256); + + if (actualHash != expectedContentHash) { + success = false; + strError = "Content hash mismatch - snapshot may be corrupted"; + } + } + + // Write metadata + if (success) { + leveldb::WriteBatch metaBatch; + + // hashBestChain + CDataStream ssKey1(SER_DISK, CLIENT_VERSION); + ssKey1 << std::string("hashBestChain"); + CDataStream ssVal1(SER_DISK, CLIENT_VERSION); + ssVal1 << blockHash; + metaBatch.Put(ssKey1.str(), ssVal1.str()); + + // dbformat = 3 + CDataStream ssKey2(SER_DISK, CLIENT_VERSION); + ssKey2 << std::string("dbformat"); + CDataStream ssVal2(SER_DISK, CLIENT_VERSION); + ssVal2 << (int)3; + metaBatch.Put(ssKey2.str(), ssVal2.str()); + + // version + CDataStream ssKey3(SER_DISK, CLIENT_VERSION); + ssKey3 << std::string("version"); + CDataStream ssVal3(SER_DISK, CLIENT_VERSION); + ssVal3 << DATABASE_VERSION; + metaBatch.Put(ssKey3.str(), ssVal3.str()); + + leveldb::Status s = pdb->Write(leveldb::WriteOptions(), &metaBatch); + if (!s.ok()) { + success = false; + strError = "Failed to write metadata: " + s.ToString(); + } + } + + // Clean up LevelDB + delete pdb; + delete options.filter_policy; + delete options.block_cache; + + fclose(file); + + if (!success) { + // Remove corrupted/incomplete database + printf("UtxoSnapshot: load failed: %s\n", strError.c_str()); + if (fs::exists(txleveldbPath)) + fs::remove_all(txleveldbPath); + return false; + } + + printf("UtxoSnapshot: successfully loaded %d headers + %d UTXOs at height %d\n", + numHeaders, numUtxos, height); + + return true; +} + +} // namespace UtxoSnapshot diff --git a/src/utxosnapshot.h b/src/utxosnapshot.h new file mode 100644 index 0000000..e4ef4f4 --- /dev/null +++ b/src/utxosnapshot.h @@ -0,0 +1,39 @@ +// Copyright (c) 2024-2025 Triangles developers +// Distributed under the MIT/X11 software license + +#ifndef TRIANGLES_UTXOSNAPSHOT_H +#define TRIANGLES_UTXOSNAPSHOT_H + +#include +#include + +// UTXO snapshot file magic bytes +static const unsigned int UTXO_SNAPSHOT_MAGIC = 0x53585455; // "UTXS" little-endian + +// UTXO snapshot format version +static const unsigned int UTXO_SNAPSHOT_VERSION = 1; + +// Number of block index entries to include in snapshot (covers difficulty, +// median time, stake modifier, and reorg depth requirements) +static const unsigned int UTXO_SNAPSHOT_DEFAULT_HEADERS = 2000; + +namespace UtxoSnapshot { + + // Create a UTXO snapshot from the current chain state. + // Writes last nHeaders block index entries + all UTXOs to destPath. + // Returns true on success, sets strError on failure. + bool DumpSnapshot(const boost::filesystem::path& destPath, + unsigned int nHeaders, + std::string& strError); + + // Load a UTXO snapshot from a file into a fresh LevelDB. + // Writes block index entries, UTXOs, hashBestChain, and dbformat. + // The LevelDB must NOT be open yet (call before LoadBlockIndex). + // Returns true on success, sets strError on failure. + bool LoadSnapshot(const boost::filesystem::path& snapshotPath, + const boost::filesystem::path& dataDir, + std::string& strError); + +} // namespace UtxoSnapshot + +#endif // TRIANGLES_UTXOSNAPSHOT_H