sami7777/triangles_v5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

utxosnapshot: v2 format — embed full blk0001.dat into snapshot

Browse Source 
Per Sami's vision: 'I want to carry over the whole block inside the
UTXO.' The snapshot is now self-contained: a fresh node loading it
has everything needed (headers + UTXOs + all block bodies) without
needing a separate bootstrap tarball.

Format change (UTXO_SNAPSHOT_VERSION 1 → 2):

v1 HEADER (88 bytes):
  magic, version, network, height, blockHash, moneySupply,
  numHeaders, numUtxos, contentHash

v2 HEADER (92 bytes):
  same + numBlocks (between numUtxos and contentHash)

v2 CONTENT (after v1's headers + utxos sections):
  blocks[numBlocks]  ← raw blk0001.dat bytes, SHA256 included

DumpSnapshot changes:
- Walks ALL blocks from pindexBest to pindexGenesisBlock (was: last
  N=2000). The nHeaders arg is honored only when caller passes a
  count smaller than the full chain for v1-compat diagnostic snapshots.
- After headers + utxos sections, streams GetDataDir()/blk0001.dat
  bytes into the snapshot, chunked (64 KB), content-hashed.
- Header now writes numBlocks between numUtxos and contentHash.

LoadSnapshot changes:
- Reads numBlocks after numUtxos when version >= 2.
- After UTXOs section, streams numBlocks bytes from the snapshot
  into dataDir/blk0001.dat (uses GetDataDir() since the param dataDir
  is intentionally unnamed in this function).
- v1 snapshots still load via the partial-load path (no numBlocks in
  header, no blk0001.dat written).
- Empty snapshot check loosened to (numHeaders && numUtxos && numBlocks)
  — all three must be zero to be considered empty.

Total v2 snapshot size: ~1.9 GB (headers + blocks + UTXOs).
Generation on the operator machine: a few minutes. Download on
reasonable connection: a few minutes.

This supersedes the earlier v2 attempt (commit 69529ea) which had
compile bugs from using an unnamed dataDir parameter and had wrong
snapshot file layout.
This commit is contained in:
Sami Ahmed
2026-06-19 04:21:39 -07:00
parent d73f6015a9
commit 23e8a2d647
2 changed files with 105 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utxosnapshot.cpp
+104 -12
View File
@@ -42,20 +42,28 @@ bool DumpSnapshot(const fs::path& destPath,
return false;
}
// Collect block index entries (last nHeaders blocks, height ascending)
// v2: collect ALL block index entries (genesis → tip). Required so a
// snapshot-loaded node can address every block via mapBlockIndex +
// blk0001.dat. The nHeaders argument is honored only when strictly less
// than chain height for v1-compat diagnostic snapshots.
std::vector<std::pair<uint256, CDiskBlockIndex>> vHeaders;
vHeaders.reserve(nHeaders);
{
CBlockIndex* pindex = pindexBest;
unsigned int nCollected = 0;
while (pindex && nCollected < nHeaders) {
while (pindex) {
CDiskBlockIndex diskindex(pindex);
vHeaders.push_back({*pindex->phashBlock, diskindex});
pindex = pindex->pprev;
nCollected++;
}
// Reverse to height ascending order
// Reverse to height ascending order (genesis first)
std::reverse(vHeaders.begin(), vHeaders.end());
// Legacy v1 fallback: if caller passed a specific count smaller than
// the full chain, trim from the front (keep newest nHeaders).
if (nHeaders > 0 && nHeaders < (unsigned int)nBestHeight &&
vHeaders.size() > nHeaders) {
vHeaders.erase(vHeaders.begin(),
vHeaders.begin() + (vHeaders.size() - nHeaders));
}
}
// Open the chain DB once and reuse for both the UTXO count and the
@@ -91,6 +99,18 @@ bool DumpSnapshot(const fs::path& destPath,
int64_t moneySupply = pindexBest->nMoneySupply;
unsigned int numHeaders = (unsigned int)vHeaders.size();
unsigned int numUtxos = (unsigned int)nUtxoCount;
// v2: size of raw blk0001.dat content embedded in this snapshot. v1
// snapshots always write 0 here (no embedded blocks).
unsigned int numBlocks = 0;
{
FILE* blkFile = fopen((GetDataDir() / "blk0001.dat").string().c_str(), "rb");
if (blkFile) {
fseek(blkFile, 0, SEEK_END);
long blkSize = ftell(blkFile);
fclose(blkFile);
if (blkSize > 0) numBlocks = (unsigned int)blkSize;
}
}
uint256 contentHash; // placeholder, filled after writing data
fwrite(&magic, sizeof(magic), 1, file);
@@ -101,6 +121,7 @@ bool DumpSnapshot(const fs::path& destPath,
fwrite(&moneySupply, sizeof(moneySupply), 1, file);
fwrite(&numHeaders, sizeof(numHeaders), 1, file);
fwrite(&numUtxos, sizeof(numUtxos), 1, file);
fwrite(&numBlocks, sizeof(numBlocks), 1, file); // v2+
long contentHashPos = ftell(file);
fwrite(&contentHash, sizeof(contentHash), 1, file); // placeholder
@@ -182,6 +203,35 @@ bool DumpSnapshot(const fs::path& destPath,
}
}
// v2: After UTXOs, append raw blk0001.dat content. Streams in chunks;
// SHA256 covers the bytes. A snapshot-loaded node has full block data
// ready in datadir/blk0001.dat — no separate bootstrap needed.
if (numBlocks > 0) {
FILE* blkFile = fopen((GetDataDir() / "blk0001.dat").string().c_str(), "rb");
if (!blkFile) {
fclose(file);
strError = "Cannot open blk0001.dat for snapshot embedding";
return false;
}
printf("UtxoSnapshot: embedding blk0001.dat (%u bytes) into snapshot\n", numBlocks);
unsigned char blkBuf[64 * 1024];
size_t nLeft = numBlocks;
while (nLeft > 0) {
size_t nWant = nLeft > sizeof(blkBuf) ? sizeof(blkBuf) : nLeft;
size_t nRead = fread(blkBuf, 1, nWant, blkFile);
if (nRead != nWant) {
fclose(blkFile);
fclose(file);
strError = "Short read on blk0001.dat during snapshot embed";
return false;
}
fwrite(blkBuf, 1, nRead, file);
SHA256_Update(&sha256, blkBuf, nRead);
nLeft -= nRead;
}
fclose(blkFile);
}
// Finalize content hash and write it to the header
SHA256_Final((unsigned char*)&contentHash, &sha256);
fseek(file, contentHashPos, SEEK_SET);
@@ -189,8 +239,8 @@ bool DumpSnapshot(const fs::path& destPath,
fclose(file);
printf("UtxoSnapshot: wrote %s (%d headers, %d UTXOs, hash=%s)\n",
destPath.string().c_str(), numHeaders, numUtxos,
printf("UtxoSnapshot: wrote %s (%d headers, %d UTXOs, %u block bytes, hash=%s)\n",
destPath.string().c_str(), numHeaders, numUtxos, numBlocks,
contentHash.ToString().c_str());
return true;
@@ -240,7 +290,7 @@ bool LoadSnapshot(const fs::path& snapshotPath,
int height;
uint256 blockHash;
int64_t moneySupply;
unsigned int numHeaders, numUtxos;
unsigned int numHeaders = 0, numUtxos = 0, numBlocks = 0;
uint256 expectedContentHash;
if (fread(&magic, sizeof(magic), 1, file) != 1 ||
@@ -250,10 +300,22 @@ bool LoadSnapshot(const fs::path& snapshotPath,
fread(&blockHash, sizeof(blockHash), 1, file) != 1 ||
fread(&moneySupply, sizeof(moneySupply), 1, file) != 1 ||
fread(&numHeaders, sizeof(numHeaders), 1, file) != 1 ||
fread(&numUtxos, sizeof(numUtxos), 1, file) != 1 ||
fread(&expectedContentHash, sizeof(expectedContentHash), 1, file) != 1) {
fread(&numUtxos, sizeof(numUtxos), 1, file) != 1) {
fclose(file);
strError = "Truncated snapshot header";
strError = "Truncated snapshot header (common fields)";
return false;
}
// v2+ has numBlocks between numUtxos and contentHash. v1 stops here.
if (version >= 2) {
if (fread(&numBlocks, sizeof(numBlocks), 1, file) != 1) {
fclose(file);
strError = "Truncated snapshot header (numBlocks)";
return false;
}
}
if (fread(&expectedContentHash, sizeof(expectedContentHash), 1, file) != 1) {
fclose(file);
strError = "Truncated snapshot header (contentHash)";
return false;
}
@@ -460,6 +522,36 @@ bool LoadSnapshot(const fs::path& snapshotPath,
success = false;
}
// v2: After UTXOs, extract the raw blk0001.dat content. This makes the
// loaded node fully self-contained — no separate bootstrap needed.
if (success && version >= 2 && numBlocks > 0) {
printf("UtxoSnapshot: extracting %u block bytes to blk0001.dat...\n", numBlocks);
fs::path blkOut = GetDataDir() / "blk0001.dat";
FILE* blkOutFile = fopen(blkOut.string().c_str(), "wb");
if (!blkOutFile) {
success = false;
strError = "Cannot create blk0001.dat for snapshot extract: " + blkOut.string();
} else {
unsigned char blkBuf[64 * 1024];
size_t nLeft = numBlocks;
while (nLeft > 0 && success) {
size_t nWant = nLeft > sizeof(blkBuf) ? sizeof(blkBuf) : nLeft;
size_t nRead = fread(blkBuf, 1, nWant, file);
if (nRead != nWant) {
success = false;
strError = "Short read on snapshot blocks section";
break;
}
fwrite(blkBuf, 1, nRead, blkOutFile);
SHA256_Update(&sha256, blkBuf, nRead);
nLeft -= nRead;
}
fclose(blkOutFile);
if (success)
printf("UtxoSnapshot: wrote blk0001.dat (%u bytes)\n", numBlocks);
}
}
// Verify content hash
if (success) {
uint256 actualHash;