UTXO snapshot support, script verify cache, and Tor sync tuning (v5.8.7)

- Add UTXO snapshot dump/load system (utxosnapshot.cpp/h) for fast initial sync
- Add dumputxoset RPC command to create snapshots from current chain state
- Add script verification cache (sigcache.h) to skip re-verifying scripts
  already validated during mempool acceptance
- Bootstrap: try UTXO snapshot first (fast path), fall back to full bootstrap
- Support manual utxo-snapshot.bin loading on startup
- Tune sync parameters for Tor: increase timeouts, reduce buffer sizes
- Header sync cache: TTL-based eviction instead of full cache clear
- Reduce orphan block limits and script check batch size for lower memory usage

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-23 01:17:51 -07:00
parent dee0d9ef62
commit b506a48192
13 changed files with 815 additions and 26 deletions
+1
View File
@@ -72,6 +72,7 @@ set(CORE_SOURCES
rpcsmessage.cpp
zmqpublishnotifier.cpp
txdb-leveldb.cpp
utxosnapshot.cpp
lz4/lz4.c
tor/onion_v3.cpp
tor/tor_process.cpp
+35
View File
@@ -2,6 +2,7 @@
// Distributed under the MIT/X11 software license
#include "bootstrap.h"
#include "utxosnapshot.h"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
@@ -766,4 +767,38 @@ bool DownloadBootstrap(const std::string& host,
return true;
}
bool DownloadUtxoSnapshot(const std::string& host,
const fs::path& dataDir,
ProgressCallback progressFn,
std::string& strError)
{
const bool noProxy = true;
const char* snapshotFilename = "utxo-snapshot.bin";
// Download utxo-snapshot.bin to a temp file
fs::path tmpPath = dataDir / "utxo-snapshot.bin.tmp";
std::string urlPath = std::string(BASE_PATH) + snapshotFilename;
printf("Bootstrap: downloading UTXO snapshot from %s%s...\n", host.c_str(), urlPath.c_str());
if (!DownloadFile(host, urlPath, tmpPath, progressFn, strError, noProxy)) {
fs::remove(tmpPath);
return false;
}
printf("Bootstrap: UTXO snapshot downloaded, loading into database...\n");
// Load the snapshot into a fresh txleveldb
if (!UtxoSnapshot::LoadSnapshot(tmpPath, dataDir, strError)) {
fs::remove(tmpPath);
return false;
}
// Clean up the temp file
fs::remove(tmpPath);
printf("Bootstrap: UTXO snapshot loaded successfully.\n");
return true;
}
} // namespace Bootstrap
+8
View File
@@ -62,6 +62,14 @@ namespace Bootstrap {
bool VerifyManifest(const SnapshotManifest& manifest,
std::string& strError);
// Download a UTXO snapshot and load it into a fresh txleveldb.
// This is much faster than downloading the full bootstrap archive.
// Returns true if snapshot was downloaded and loaded successfully.
bool DownloadUtxoSnapshot(const std::string& host,
const boost::filesystem::path& dataDir,
ProgressCallback progressFn,
std::string& strError);
} // namespace Bootstrap
#endif // TRIANGLES_BOOTSTRAP_H
+50 -6
View File
@@ -14,6 +14,7 @@
#include "smessage.h"
#include "openssl_compat.h"
#include "bootstrap.h"
#include "utxosnapshot.h"
#include "tor/tor_embedded.h"
#include "tor/onion_v3.h"
#include "tor/tor_process.h"
@@ -669,7 +670,7 @@ bool AppInit2()
nScriptCheckThreads = 16;
if (nScriptCheckThreads > 1)
{
pScriptCheckQueue = new CCheckQueue<CScriptCheck>(128);
pScriptCheckQueue = new CCheckQueue<CScriptCheck>(32);
pScriptCheckThreads = new boost::thread_group();
for (int i = 0; i < nScriptCheckThreads - 1; ++i)
pScriptCheckThreads->create_thread(&ThreadScriptCheck);
@@ -907,9 +908,6 @@ bool AppInit2()
std::string host = Bootstrap::DEFAULT_HOST;
std::string strError;
uiInterface.InitMessage(_("Downloading blockchain snapshot..."));
printf("Bootstrap: contacting %s...\n", host.c_str());
auto progressFn = [](int64_t bytesDownloaded, int64_t totalBytes) {
if (totalBytes > 0) {
printf("\rBootstrap: %lld / %lld MB (%lld%%)",
@@ -920,7 +918,30 @@ bool AppInit2()
}
};
bool success = Bootstrap::DownloadBootstrap(host, dataPath, progressFn, strError);
// Try UTXO snapshot first (fast: ~2-10 MB download)
bool success = false;
bool triedUtxoSnapshot = false;
if (needsBootstrap && !fs::exists(dataPath / "txleveldb")) {
uiInterface.InitMessage(_("Downloading UTXO snapshot..."));
printf("Bootstrap: trying UTXO snapshot from %s (fast path)...\n", host.c_str());
std::string utxoError;
if (Bootstrap::DownloadUtxoSnapshot(host, dataPath, progressFn, utxoError)) {
printf("\nBootstrap: UTXO snapshot loaded — will sync remaining blocks from network.\n");
success = true;
} else {
printf("\nBootstrap: UTXO snapshot unavailable: %s\n", utxoError.c_str());
printf("Bootstrap: falling back to full bootstrap download...\n");
}
triedUtxoSnapshot = true;
}
// Fall back to full bootstrap.tar.gz if UTXO snapshot failed
if (!success) {
uiInterface.InitMessage(_("Downloading blockchain snapshot..."));
printf("Bootstrap: contacting %s...\n", host.c_str());
success = Bootstrap::DownloadBootstrap(host, dataPath, progressFn, strError);
if (!success) {
printf("\nBootstrap: failed: %s\n", strError.c_str());
@@ -928,12 +949,35 @@ bool AppInit2()
} else {
printf("\nBootstrap: done.\n");
}
}
StartupPerfLog("bootstrap_download", GetTimeMillis() - nBootstrapStart,
strprintf("host=%s success=%d", host.c_str(), success));
strprintf("host=%s success=%d utxo_snapshot=%d", host.c_str(), success, triedUtxoSnapshot));
}
} // end bootstrap scope
#endif
// ********************************************************* Step 6c: manual UTXO snapshot loading
// If utxo-snapshot.bin exists in data dir and no txleveldb, load it.
{
fs::path dataPath = GetDataDir();
fs::path snapshotFile = dataPath / "utxo-snapshot.bin";
fs::path txleveldbDir = dataPath / "txleveldb";
if (fs::exists(snapshotFile) && !fs::exists(txleveldbDir)) {
printf("Found utxo-snapshot.bin — loading UTXO snapshot...\n");
uiInterface.InitMessage(_("Loading UTXO snapshot..."));
std::string strError;
if (UtxoSnapshot::LoadSnapshot(snapshotFile, dataPath, strError)) {
printf("UTXO snapshot loaded successfully.\n");
} else {
printf("UTXO snapshot load failed: %s\n", strError.c_str());
printf("Will proceed with normal sync.\n");
}
}
}
// ********************************************************* Step 7: load blockchain
if (!bitdb.Open(GetDataDir()))
+58 -11
View File
@@ -77,6 +77,8 @@ int64_t nTimeBestReceived = 0;
CMedianFilter<int> cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have
CScriptVerifyCache scriptVerifyCache;
map<uint256, CBlock*> mapOrphanBlocks;
multimap<uint256, CBlock*> mapOrphanBlocksByPrev;
set<pair<COutPoint, unsigned int> > setStakeSeenOrphan;
@@ -110,6 +112,7 @@ struct CHeaderSyncNode
uint256 nChainTrust;
bool fRequested;
int64_t nLastRequestTime;
int64_t nInsertTime;
};
static std::map<uint256, CHeaderSyncNode> mapHeaderSync;
@@ -117,11 +120,12 @@ static uint256 hashBestHeaderSync = 0;
static CCriticalSection cs_PostIbdWork;
static bool fPostIbdWorkStarted = false;
static const unsigned int MAX_HEADER_SYNC_CACHE = 50000;
static const unsigned int MAX_HEADER_SYNC_CACHE = 15000;
static const unsigned int HEADER_DOWNLOAD_WINDOW = 512; // Increased from 128 for parallel downloads
static const unsigned int HEADER_DOWNLOAD_PER_PEER = 64; // Max blocks to request from each peer
static const int64_t HEADER_REQUEST_TIMEOUT_MICROS = 30 * 1000000;
static const int64_t HEADER_REDUNDANT_REQUEST_MICROS = 10 * 1000000; // Request from another peer after 10s
static const unsigned int HEADER_DOWNLOAD_PER_PEER = 32; // Reduced from 64 for Tor circuit stability
static const int64_t HEADER_REQUEST_TIMEOUT_MICROS = 60 * 1000000; // 60s for Tor latency (was 30s)
static const int64_t HEADER_REDUNDANT_REQUEST_MICROS = 20 * 1000000; // 20s redundant request (was 10s)
static const int64_t HEADER_SYNC_TTL_MICROS = 5 * 60 * 1000000; // 5-minute TTL for cache entries
static void ThreadPostIbdWork(void* parg)
{
@@ -232,12 +236,47 @@ static void RecomputeBestHeaderSync()
static void PruneHeaderSync()
{
if (mapHeaderSync.size() <= MAX_HEADER_SYNC_CACHE)
return;
const int64_t nNow = GetTime() * 1000000;
printf("IBD-DIAG: header sync cache exceeded %u entries, clearing planner state\n", MAX_HEADER_SYNC_CACHE);
mapHeaderSync.clear();
hashBestHeaderSync = 0;
// TTL eviction: remove entries older than 5 minutes
if (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE / 2)
{
unsigned int nEvicted = 0;
for (auto it = mapHeaderSync.begin(); it != mapHeaderSync.end(); )
{
if (nNow - it->second.nInsertTime >= HEADER_SYNC_TTL_MICROS)
{
it = mapHeaderSync.erase(it);
++nEvicted;
}
else
++it;
}
if (nEvicted > 0)
{
printf("IBD-DIAG: TTL-evicted %u stale header sync entries, %u remain\n",
nEvicted, (unsigned int)mapHeaderSync.size());
RecomputeBestHeaderSync();
}
}
// Hard limit: if still over max, evict oldest entries
if (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE)
{
printf("IBD-DIAG: header sync cache exceeded %u entries, evicting oldest\n", MAX_HEADER_SYNC_CACHE);
while (mapHeaderSync.size() > MAX_HEADER_SYNC_CACHE * 3 / 4)
{
// Find oldest entry by insert time
auto oldest = mapHeaderSync.begin();
for (auto it = mapHeaderSync.begin(); it != mapHeaderSync.end(); ++it)
{
if (it->second.nInsertTime < oldest->second.nInsertTime)
oldest = it;
}
mapHeaderSync.erase(oldest);
}
RecomputeBestHeaderSync();
}
}
static bool AddHeaderSyncNode(const CBlock& header, const uint256& hashHeader)
@@ -283,6 +322,7 @@ static bool AddHeaderSyncNode(const CBlock& header, const uint256& hashHeader)
node.nChainTrust = nPrevChainTrust + GetHeaderSyncTrust(header.nBits);
node.fRequested = false;
node.nLastRequestTime = 0;
node.nInsertTime = GetTime() * 1000000;
mapHeaderSync.insert(std::make_pair(hashHeader, node));
@@ -1900,6 +1940,7 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs,
// The first loop above does all the inexpensive checks.
// Only if ALL inputs pass do we perform expensive ECDSA signature checks.
// Helps prevent CPU exhaustion attacks.
const uint256 hashTx = GetHash();
for (unsigned int i = 0; i < vin.size(); i++)
{
COutPoint prevout = vin[i].prevout;
@@ -1910,6 +1951,11 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs,
// still computed and checked, and any change will be caught at the next checkpoint.
if (!(fBlock && (nBestHeight < Checkpoints::GetTotalBlocksEstimate())))
{
// Check signature cache: skip re-verification for scripts already
// validated during mempool acceptance or prior block connections.
if (scriptVerifyCache.Get(hashTx, i))
continue;
if (pvChecks)
{
pvChecks->push_back(CScriptCheck(entry.scriptPubKey, vin[i].scriptSig, *this, i, 0));
@@ -1919,6 +1965,7 @@ bool CTransaction::ConnectInputs(CTxDB& txdb, const MapPrevTx& inputs,
// Verify signature using scriptPubKey from UTXO entry
if (!VerifyScript(vin[i].scriptSig, entry.scriptPubKey, *this, i, 0))
return DoS(100, error("ConnectInputs() : %s VerifySignature failed", GetHash().ToString().substr(0,10).c_str()));
scriptVerifyCache.Set(hashTx, i);
}
}
}
@@ -2286,7 +2333,7 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck)
if (!tx.ConnectInputs(txdb, mapInputs, pindex, true, false,
pScriptCheckQueue ? &vChecks : NULL))
return false;
if (pScriptCheckQueue && vChecks.size() >= 128)
if (pScriptCheckQueue && vChecks.size() >= 32)
{
scriptcheckcontrol.Add(vChecks);
vChecks.clear();
@@ -5510,7 +5557,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle)
static int64_t nLastBlockReceived = 0;
static int nLastHeight = 0;
static int64_t nLastStallLog = 0;
int nStallTimeout = IsInitialBlockDownload() ? 5 : 15;
int nStallTimeout = IsInitialBlockDownload() ? 10 : 30; // Higher for Tor latency
if (nBestHeight > nLastHeight) {
nLastHeight = nBestHeight;
nLastBlockReceived = GetTime();
+10 -3
View File
@@ -12,6 +12,7 @@
#include "scrypt.h"
#include "hashblock.h"
#include "checkqueue.h"
#include "sigcache.h"
#include <list>
@@ -37,8 +38,8 @@ static const unsigned int MAX_BLOCK_SIZE = 1000000;
static const unsigned int MAX_BLOCK_SIZE_GEN = MAX_BLOCK_SIZE/2;
static const unsigned int MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50;
static const unsigned int MAX_ORPHAN_TRANSACTIONS = MAX_BLOCK_SIZE/100;
static const unsigned int MAX_ORPHAN_BLOCKS = 2000;
static const unsigned int MAX_ORPHAN_BLOCKS_IBD = 4000;
static const unsigned int MAX_ORPHAN_BLOCKS = 750;
static const unsigned int MAX_ORPHAN_BLOCKS_IBD = 1500;
static const unsigned int MAX_REORG_DEPTH = 100; // reject reorgs deeper than this (finality)
static const unsigned int MAX_INV_SZ = 50000;
static const int64_t MIN_TX_FEE = (1 * CENT) / 100;
@@ -1687,6 +1688,7 @@ public:
};
extern CTxMemPool mempool;
extern CScriptVerifyCache scriptVerifyCache;
/**
* Closure representing one script check for parallel verification.
@@ -1711,7 +1713,12 @@ public:
bool operator()()
{
return ptxTo && VerifyScript(scriptSig, scriptPubKey, *ptxTo, nIn, nHashType);
if (!ptxTo)
return false;
if (!VerifyScript(scriptSig, scriptPubKey, *ptxTo, nIn, nHashType))
return false;
scriptVerifyCache.Set(ptxTo->GetHash(), nIn);
return true;
}
void swap(CScriptCheck& other)
+1 -1
View File
@@ -26,7 +26,7 @@ extern int nBestHeight;
inline unsigned int ReceiveFloodSize() { return 100 * 1024 * 1024; } // 100 MB
inline unsigned int ReceiveFloodSize() { return 50 * 1024 * 1024; } // 50 MB (reduced for Tor-only network)
inline unsigned int SendBufferSize() { return 32 * 1024 * 1024; } // 32 MB
void AddOneShot(std::string strDest);
+47
View File
@@ -9,6 +9,7 @@
#include "addressindex.h"
#include "txdb.h"
#include "base58.h"
#include "utxosnapshot.h"
using namespace json_spirit;
using namespace std;
@@ -849,3 +850,49 @@ Value reconsiderblock(const Array& params, bool fHelp)
return Value::null;
}
Value dumputxoset(const Array& params, bool fHelp)
{
if (fHelp || params.size() < 1 || params.size() > 2)
throw runtime_error(
"dumputxoset <filename> [nheaders]\n"
"Dumps the current UTXO set and recent block headers to a binary snapshot file.\n"
"The snapshot can be used by new nodes to skip initial block download.\n"
"\nArguments:\n"
"1. filename (string, required) Destination file path\n"
"2. nheaders (int, optional, default=2000) Number of block headers to include\n"
"\nResult:\n"
"{\n"
" \"filename\": \"...\",\n"
" \"height\": n,\n"
" \"blockhash\": \"...\",\n"
" \"file_size\": n\n"
"}");
string filename = params[0].get_str();
unsigned int nHeaders = UTXO_SNAPSHOT_DEFAULT_HEADERS;
if (params.size() > 1)
nHeaders = params[1].get_int();
if (nHeaders < 100)
throw JSONRPCError(RPC_INVALID_PARAMETER, "nheaders must be at least 100");
boost::filesystem::path destPath(filename);
std::string strError;
if (!UtxoSnapshot::DumpSnapshot(destPath, nHeaders, strError))
throw runtime_error("dumputxoset failed: " + strError);
// Get file size
int64_t nFileSize = 0;
if (boost::filesystem::exists(destPath))
nFileSize = (int64_t)boost::filesystem::file_size(destPath);
Object result;
result.push_back(Pair("filename", filename));
result.push_back(Pair("height", nBestHeight));
result.push_back(Pair("blockhash", hashBestChain.GetHex()));
result.push_back(Pair("file_size", nFileSize));
return result;
}
+72
View File
@@ -0,0 +1,72 @@
// Copyright (c) 2024 The Triangles developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef TRIANGLES_SCRIPT_VERIFY_CACHE_H
#define TRIANGLES_SCRIPT_VERIFY_CACHE_H
#include "uint256.h"
#include "sync.h"
#include <openssl/sha.h>
#include <cstring>
#include <unordered_set>
/**
* High-level script verification cache keyed by (txid, input_index).
* Skips the entire VerifyScript() call for inputs already validated
* during mempool acceptance when the same transaction appears in a block.
*
* Complements the lower-level CSignatureCache in script.cpp which caches
* individual ECDSA signature checks.
*
* ~256KB memory footprint at 32K entries.
*/
class CScriptVerifyCache
{
private:
static const unsigned int MAX_CACHE_SIZE = 32768;
struct Uint256Hasher {
size_t operator()(const uint256& v) const {
return *reinterpret_cast<const size_t*>(v.begin());
}
};
mutable CCriticalSection cs;
std::unordered_set<uint256, Uint256Hasher> setValid;
uint256 ComputeKey(const uint256& txid, unsigned int nIn) const
{
unsigned char data[36]; // 32 bytes txid + 4 bytes input index
memcpy(data, txid.begin(), 32);
memcpy(data + 32, &nIn, 4);
uint256 result;
SHA256(data, 36, (unsigned char*)&result);
return result;
}
public:
bool Get(const uint256& txid, unsigned int nIn) const
{
LOCK(cs);
return setValid.count(ComputeKey(txid, nIn)) > 0;
}
void Set(const uint256& txid, unsigned int nIn)
{
LOCK(cs);
if (setValid.size() >= MAX_CACHE_SIZE)
{
// Evict half the cache when full
auto it = setValid.begin();
unsigned int nEvict = MAX_CACHE_SIZE / 2;
while (nEvict > 0 && it != setValid.end()) {
it = setValid.erase(it);
--nEvict;
}
}
setValid.insert(ComputeKey(txid, nIn));
}
};
#endif // TRIANGLES_SCRIPT_VERIFY_CACHE_H
+1
View File
@@ -319,6 +319,7 @@ static const CRPCCommand vRPCCommands[] =
{ "invalidateblock", &invalidateblock, false, false },
{ "reconsiderblock", &reconsiderblock, false, false },
{ "recalculatesupply", &recalculatesupply, false, false },
{ "dumputxoset", &dumputxoset, false, false },
{ "reservebalance", &reservebalance, false, true},
{ "checkwallet", &checkwallet, false, true},
{ "repairwallet", &repairwallet, false, true},
+1
View File
@@ -227,6 +227,7 @@ extern json_spirit::Value getchaintips(const json_spirit::Array& params, bool fH
extern json_spirit::Value invalidateblock(const json_spirit::Array& params, bool fHelp);
extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool fHelp);
extern json_spirit::Value recalculatesupply(const json_spirit::Array& params, bool fHelp);
extern json_spirit::Value dumputxoset(const json_spirit::Array& params, bool fHelp);
extern json_spirit::Value getaddressbalance(const json_spirit::Array& params, bool fHelp);
extern json_spirit::Value getaddressutxos(const json_spirit::Array& params, bool fHelp);
+487
View File
@@ -0,0 +1,487 @@
// Copyright (c) 2024-2025 Triangles developers
// Distributed under the MIT/X11 software license
#include "utxosnapshot.h"
#include "main.h"
#include "txdb.h"
#include "checkpoints.h"
#include "util.h"
#include "ui_interface.h"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <leveldb/db.h>
#include <leveldb/write_batch.h>
#include <leveldb/cache.h>
#include <leveldb/filter_policy.h>
#include <openssl/sha.h>
#include <vector>
#include <algorithm>
#include <cstdio>
namespace fs = boost::filesystem;
namespace UtxoSnapshot {
// ---------------------------------------------------------------------------
// DumpSnapshot - create a UTXO snapshot from the current chain state
// ---------------------------------------------------------------------------
bool DumpSnapshot(const fs::path& destPath,
unsigned int nHeaders,
std::string& strError)
{
LOCK(cs_main);
if (!pindexBest) {
strError = "No best block - chain not loaded";
return false;
}
// Collect block index entries (last nHeaders blocks, height ascending)
std::vector<std::pair<uint256, CDiskBlockIndex>> vHeaders;
vHeaders.reserve(nHeaders);
{
CBlockIndex* pindex = pindexBest;
unsigned int nCollected = 0;
while (pindex && nCollected < nHeaders) {
CDiskBlockIndex diskindex(pindex);
vHeaders.push_back(std::make_pair(*pindex->phashBlock, diskindex));
pindex = pindex->pprev;
nCollected++;
}
// Reverse to height ascending order
std::reverse(vHeaders.begin(), vHeaders.end());
}
// Count UTXOs first
int nUtxoCount = 0;
{
CTxDB txdbRead("r");
txdbRead.SumUtxoValues(nUtxoCount);
}
if (nUtxoCount == 0) {
strError = "No UTXOs found in database";
return false;
}
printf("UtxoSnapshot: dumping %d headers + %d UTXOs at height %d\n",
(int)vHeaders.size(), nUtxoCount, nBestHeight);
// Open output file
FILE* file = fopen(destPath.string().c_str(), "wb");
if (!file) {
strError = "Cannot create file: " + destPath.string();
return false;
}
// Write header (we'll seek back to fill in content_hash later)
unsigned int magic = UTXO_SNAPSHOT_MAGIC;
unsigned int version = UTXO_SNAPSHOT_VERSION;
unsigned int network = fTestNet ? 2 : 1;
int height = nBestHeight;
uint256 blockHash = hashBestChain;
int64_t moneySupply = pindexBest->nMoneySupply;
unsigned int numHeaders = (unsigned int)vHeaders.size();
unsigned int numUtxos = (unsigned int)nUtxoCount;
uint256 contentHash; // placeholder, filled after writing data
fwrite(&magic, sizeof(magic), 1, file);
fwrite(&version, sizeof(version), 1, file);
fwrite(&network, sizeof(network), 1, file);
fwrite(&height, sizeof(height), 1, file);
fwrite(&blockHash, sizeof(blockHash), 1, file);
fwrite(&moneySupply, sizeof(moneySupply), 1, file);
fwrite(&numHeaders, sizeof(numHeaders), 1, file);
fwrite(&numUtxos, sizeof(numUtxos), 1, file);
long contentHashPos = ftell(file);
fwrite(&contentHash, sizeof(contentHash), 1, file); // placeholder
// Start SHA256 for content hash
SHA256_CTX sha256;
SHA256_Init(&sha256);
// Write block headers section
for (const auto& item : vHeaders) {
CDataStream ssEntry(SER_DISK, CLIENT_VERSION);
ssEntry << item.first; // block hash
ssEntry << item.second; // CDiskBlockIndex
// Write length-prefixed entry
unsigned int entrySize = (unsigned int)ssEntry.size();
std::string strEntry = ssEntry.str();
fwrite(&entrySize, sizeof(entrySize), 1, file);
fwrite(strEntry.data(), 1, entrySize, file);
SHA256_Update(&sha256, &entrySize, sizeof(entrySize));
SHA256_Update(&sha256, strEntry.data(), entrySize);
}
// Write UTXO section using LevelDB iterator (same pattern as SumUtxoValues)
{
// Access the global LevelDB directly via a CTxDB instance
// We need a raw iterator, so we re-implement the iteration pattern
extern leveldb::DB *txdb;
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << std::make_pair(std::string("u"), std::make_pair(uint256(0), (unsigned int)0));
std::string strPrefixBegin = ssKeyPrefix.str();
leveldb::Iterator* it = txdb->NewIterator(leveldb::ReadOptions());
unsigned int nWritten = 0;
for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) {
// Check key prefix is still "u"
CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION);
std::string strKeyType;
ssKey >> strKeyType;
if (strKeyType != "u")
break;
// Extract outpoint from key
uint256 txhash;
unsigned int nIndex;
ssKey >> txhash;
ssKey >> nIndex;
// Extract UTXO entry from value
CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION);
CUtxoEntry entry;
ssValue >> entry;
// Serialize the UTXO record
CDataStream ssRecord(SER_DISK, CLIENT_VERSION);
ssRecord << txhash;
ssRecord << nIndex;
ssRecord << entry;
unsigned int recordSize = (unsigned int)ssRecord.size();
std::string strRecord = ssRecord.str();
fwrite(&recordSize, sizeof(recordSize), 1, file);
fwrite(strRecord.data(), 1, recordSize, file);
SHA256_Update(&sha256, &recordSize, sizeof(recordSize));
SHA256_Update(&sha256, strRecord.data(), recordSize);
nWritten++;
if (nWritten % 10000 == 0)
printf("UtxoSnapshot: wrote %d / %d UTXOs\n", nWritten, nUtxoCount);
}
delete it;
// Update actual count (in case it changed during iteration)
if (nWritten != numUtxos) {
numUtxos = nWritten;
// Seek back and update numUtxos in header
long currentPos = ftell(file);
fseek(file, contentHashPos - sizeof(numUtxos), SEEK_SET);
fwrite(&numUtxos, sizeof(numUtxos), 1, file);
fseek(file, currentPos, SEEK_SET);
}
}
// Finalize content hash and write it to the header
SHA256_Final((unsigned char*)&contentHash, &sha256);
fseek(file, contentHashPos, SEEK_SET);
fwrite(&contentHash, sizeof(contentHash), 1, file);
fclose(file);
printf("UtxoSnapshot: wrote %s (%d headers, %d UTXOs, hash=%s)\n",
destPath.string().c_str(), numHeaders, numUtxos,
contentHash.ToString().c_str());
return true;
}
// ---------------------------------------------------------------------------
// LoadSnapshot - load a UTXO snapshot into a fresh LevelDB
// ---------------------------------------------------------------------------
bool LoadSnapshot(const fs::path& snapshotPath,
const fs::path& dataDir,
std::string& strError)
{
FILE* file = fopen(snapshotPath.string().c_str(), "rb");
if (!file) {
strError = "Cannot open snapshot file: " + snapshotPath.string();
return false;
}
// Read header
unsigned int magic, version, network;
int height;
uint256 blockHash;
int64_t moneySupply;
unsigned int numHeaders, numUtxos;
uint256 expectedContentHash;
if (fread(&magic, sizeof(magic), 1, file) != 1 ||
fread(&version, sizeof(version), 1, file) != 1 ||
fread(&network, sizeof(network), 1, file) != 1 ||
fread(&height, sizeof(height), 1, file) != 1 ||
fread(&blockHash, sizeof(blockHash), 1, file) != 1 ||
fread(&moneySupply, sizeof(moneySupply), 1, file) != 1 ||
fread(&numHeaders, sizeof(numHeaders), 1, file) != 1 ||
fread(&numUtxos, sizeof(numUtxos), 1, file) != 1 ||
fread(&expectedContentHash, sizeof(expectedContentHash), 1, file) != 1) {
fclose(file);
strError = "Truncated snapshot header";
return false;
}
// Validate header
if (magic != UTXO_SNAPSHOT_MAGIC) {
fclose(file);
strError = "Invalid snapshot magic (not a UTXO snapshot file)";
return false;
}
if (version != UTXO_SNAPSHOT_VERSION) {
fclose(file);
strError = "Unsupported snapshot version: " + std::to_string(version);
return false;
}
unsigned int expectedNetwork = fTestNet ? 2 : 1;
if (network != expectedNetwork) {
fclose(file);
strError = "Network mismatch: snapshot is " + std::string(network == 1 ? "mainnet" : "testnet");
return false;
}
if (numHeaders == 0 || numUtxos == 0) {
fclose(file);
strError = "Snapshot contains no data";
return false;
}
// Verify snapshot block is a known checkpoint
if (!Checkpoints::IsKnownCheckpoint(height, blockHash)) {
fclose(file);
strError = "Snapshot block " + blockHash.ToString() + " at height "
+ std::to_string(height) + " is not a known checkpoint";
return false;
}
printf("UtxoSnapshot: loading snapshot at height %d (%d headers, %d UTXOs)\n",
height, numHeaders, numUtxos);
// Create fresh LevelDB directory
fs::path txleveldbPath = dataDir / "txleveldb";
if (fs::exists(txleveldbPath))
fs::remove_all(txleveldbPath);
fs::create_directories(txleveldbPath);
// Open LevelDB directly (not via CTxDB - it's not initialized yet)
leveldb::Options options;
int nCacheSizeMB = GetArg("-dbcache", 2048);
options.block_cache = leveldb::NewLRUCache(nCacheSizeMB * 1048576);
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
options.write_buffer_size = 64 * 1048576;
options.max_open_files = 1000;
options.create_if_missing = true;
leveldb::DB* pdb = NULL;
leveldb::Status status = leveldb::DB::Open(options, txleveldbPath.string(), &pdb);
if (!status.ok()) {
fclose(file);
delete options.filter_policy;
delete options.block_cache;
strError = "Cannot create LevelDB: " + status.ToString();
return false;
}
SHA256_CTX sha256;
SHA256_Init(&sha256);
leveldb::WriteBatch batch;
bool success = true;
unsigned int nBatchSize = 0;
auto flushBatch = [&]() -> bool {
if (nBatchSize == 0)
return true;
leveldb::Status s = pdb->Write(leveldb::WriteOptions(), &batch);
if (!s.ok()) {
strError = "LevelDB write failed: " + s.ToString();
return false;
}
batch.Clear();
nBatchSize = 0;
return true;
};
// Read and write block headers
printf("UtxoSnapshot: loading %d block headers...\n", numHeaders);
uiInterface.InitMessage(_("Loading UTXO snapshot (headers)..."));
for (unsigned int i = 0; i < numHeaders; i++) {
unsigned int entrySize;
if (fread(&entrySize, sizeof(entrySize), 1, file) != 1 || entrySize > 10000) {
success = false;
strError = "Invalid header entry size at index " + std::to_string(i);
break;
}
std::vector<char> buf(entrySize);
if (fread(buf.data(), 1, entrySize, file) != entrySize) {
success = false;
strError = "Truncated header entry at index " + std::to_string(i);
break;
}
SHA256_Update(&sha256, &entrySize, sizeof(entrySize));
SHA256_Update(&sha256, buf.data(), entrySize);
// Parse: block_hash + CDiskBlockIndex
CDataStream ssEntry(buf.data(), buf.data() + buf.size(), SER_DISK, CLIENT_VERSION);
uint256 entryHash;
CDiskBlockIndex diskindex;
ssEntry >> entryHash;
ssEntry >> diskindex;
// Write to LevelDB as "blockindex" key
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey << std::make_pair(std::string("blockindex"), entryHash);
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue << diskindex;
batch.Put(ssKey.str(), ssValue.str());
nBatchSize++;
if (nBatchSize >= 1000) {
if (!flushBatch()) { success = false; break; }
}
}
if (success && !flushBatch())
success = false;
// Read and write UTXOs
if (success) {
printf("UtxoSnapshot: loading %d UTXOs...\n", numUtxos);
for (unsigned int i = 0; i < numUtxos; i++) {
unsigned int recordSize;
if (fread(&recordSize, sizeof(recordSize), 1, file) != 1 || recordSize > 100000) {
success = false;
strError = "Invalid UTXO record size at index " + std::to_string(i);
break;
}
std::vector<char> buf(recordSize);
if (fread(buf.data(), 1, recordSize, file) != recordSize) {
success = false;
strError = "Truncated UTXO record at index " + std::to_string(i);
break;
}
SHA256_Update(&sha256, &recordSize, sizeof(recordSize));
SHA256_Update(&sha256, buf.data(), recordSize);
// Parse: txid + output_index + CUtxoEntry
CDataStream ssRecord(buf.data(), buf.data() + buf.size(), SER_DISK, CLIENT_VERSION);
uint256 txhash;
unsigned int nIndex;
CUtxoEntry entry;
ssRecord >> txhash;
ssRecord >> nIndex;
ssRecord >> entry;
// Write to LevelDB with "u" prefix key
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey << std::make_pair(std::string("u"), std::make_pair(txhash, nIndex));
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue << entry;
batch.Put(ssKey.str(), ssValue.str());
nBatchSize++;
if (nBatchSize >= 50000) {
if (!flushBatch()) { success = false; break; }
if (i % 50000 == 0) {
std::string strMsg = strprintf(_("Loading UTXO snapshot (%d%%)..."),
i * 100 / numUtxos);
uiInterface.InitMessage(strMsg);
printf("UtxoSnapshot: loaded %d / %d UTXOs\n", i, numUtxos);
}
}
}
if (success && !flushBatch())
success = false;
}
// Verify content hash
if (success) {
uint256 actualHash;
SHA256_Final((unsigned char*)&actualHash, &sha256);
if (actualHash != expectedContentHash) {
success = false;
strError = "Content hash mismatch - snapshot may be corrupted";
}
}
// Write metadata
if (success) {
leveldb::WriteBatch metaBatch;
// hashBestChain
CDataStream ssKey1(SER_DISK, CLIENT_VERSION);
ssKey1 << std::string("hashBestChain");
CDataStream ssVal1(SER_DISK, CLIENT_VERSION);
ssVal1 << blockHash;
metaBatch.Put(ssKey1.str(), ssVal1.str());
// dbformat = 3
CDataStream ssKey2(SER_DISK, CLIENT_VERSION);
ssKey2 << std::string("dbformat");
CDataStream ssVal2(SER_DISK, CLIENT_VERSION);
ssVal2 << (int)3;
metaBatch.Put(ssKey2.str(), ssVal2.str());
// version
CDataStream ssKey3(SER_DISK, CLIENT_VERSION);
ssKey3 << std::string("version");
CDataStream ssVal3(SER_DISK, CLIENT_VERSION);
ssVal3 << DATABASE_VERSION;
metaBatch.Put(ssKey3.str(), ssVal3.str());
leveldb::Status s = pdb->Write(leveldb::WriteOptions(), &metaBatch);
if (!s.ok()) {
success = false;
strError = "Failed to write metadata: " + s.ToString();
}
}
// Clean up LevelDB
delete pdb;
delete options.filter_policy;
delete options.block_cache;
fclose(file);
if (!success) {
// Remove corrupted/incomplete database
printf("UtxoSnapshot: load failed: %s\n", strError.c_str());
if (fs::exists(txleveldbPath))
fs::remove_all(txleveldbPath);
return false;
}
printf("UtxoSnapshot: successfully loaded %d headers + %d UTXOs at height %d\n",
numHeaders, numUtxos, height);
return true;
}
} // namespace UtxoSnapshot
+39
View File
@@ -0,0 +1,39 @@
// Copyright (c) 2024-2025 Triangles developers
// Distributed under the MIT/X11 software license
#ifndef TRIANGLES_UTXOSNAPSHOT_H
#define TRIANGLES_UTXOSNAPSHOT_H
#include <string>
#include <boost/filesystem.hpp>
// UTXO snapshot file magic bytes
static const unsigned int UTXO_SNAPSHOT_MAGIC = 0x53585455; // "UTXS" little-endian
// UTXO snapshot format version
static const unsigned int UTXO_SNAPSHOT_VERSION = 1;
// Number of block index entries to include in snapshot (covers difficulty,
// median time, stake modifier, and reorg depth requirements)
static const unsigned int UTXO_SNAPSHOT_DEFAULT_HEADERS = 2000;
namespace UtxoSnapshot {
// Create a UTXO snapshot from the current chain state.
// Writes last nHeaders block index entries + all UTXOs to destPath.
// Returns true on success, sets strError on failure.
bool DumpSnapshot(const boost::filesystem::path& destPath,
unsigned int nHeaders,
std::string& strError);
// Load a UTXO snapshot from a file into a fresh LevelDB.
// Writes block index entries, UTXOs, hashBestChain, and dbformat.
// The LevelDB must NOT be open yet (call before LoadBlockIndex).
// Returns true on success, sets strError on failure.
bool LoadSnapshot(const boost::filesystem::path& snapshotPath,
const boost::filesystem::path& dataDir,
std::string& strError);
} // namespace UtxoSnapshot
#endif // TRIANGLES_UTXOSNAPSHOT_H