M1.1: Extract CTxDBBase abstract storage interface

First step of the multi-phase chaindb modernization plan. Introduces a
backend-agnostic abstraction over the chain database:

  * CTxDBBase — abstract class owning all serialization and named
    operations (ReadTxIndex, WriteBlockIndex, ReadAddressBalance, etc.).
    Templated Read/Write/Erase/Exists dispatch to byte-level virtuals
    (ReadRaw/WriteRaw/EraseRaw/ExistsRaw) so every backend produces
    bit-identical key bytes — required for migration and dual-backend
    parity testing later.

  * CTxDBIteratorBase — abstract iterator. Backends implement Seek,
    Valid, Next, KeyStr, ValueStr.

  * CTxDB now inherits from CTxDBBase and only implements the byte-level
    I/O, batch lifecycle, NewIterator, and LoadBlockIndex (which still
    uses leveldb directly during the v3 dbformat upgrade — extracted to
    base in a later phase).

  * UTXO read-through cache moved to txdb-base.cpp under an anonymous
    namespace — backend-agnostic so RocksDB will get it for free.

  * GetAddressUtxos / GetAddressTxIds / SumUtxoValues moved to base,
    using NewIterator() instead of pdb->NewIterator().

No call-site changes — every existing CTxDB user keeps working exactly
as before. Stack allocations like `CTxDB txdb("r")` still work because
CTxDB remains a concrete, cheap-to-construct class. Behavior is
bit-identical: same key serialization, same batch semantics, same
LoadBlockIndex flow.

Sets up M1.2 (factory + caller conversion to CTxDBBase&) and M1.3
(RocksDB backend) — neither requires touching consensus paths.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-25 02:37:45 -07:00
parent b9d631e968
commit b28525057a
5 changed files with 789 additions and 658 deletions
+1
View File
@@ -73,6 +73,7 @@ set(CORE_SOURCES
rpcrawtransaction.cpp
rpcsmessage.cpp
zmqpublishnotifier.cpp
txdb-base.cpp
txdb-leveldb.cpp
utxosnapshot.cpp
lz4/lz4.c
+457
View File
@@ -0,0 +1,457 @@
// Copyright (c) 2009-2012 The Bitcoin developers.
// Copyright (c) 2026 The Triangles developers.
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "txdb-base.h"
#include "addressindex.h"
#include "main.h"
#include "sync.h"
#include <unordered_map>
using namespace std;
// ============================================================================
// Schema versioning
// ============================================================================
bool CTxDBBase::ReadVersion(int& nVersion)
{
nVersion = 0;
return Read(string("version"), nVersion);
}
bool CTxDBBase::WriteVersion(int nVersion)
{
return Write(string("version"), nVersion);
}
bool CTxDBBase::ReadDbFormat(int& nDbFormat)
{
nDbFormat = 1;
return Read(string("dbformat"), nDbFormat);
}
bool CTxDBBase::WriteDbFormat(int nDbFormat)
{
return Write(string("dbformat"), nDbFormat);
}
// ============================================================================
// Tx index
// ============================================================================
bool CTxDBBase::ReadTxIndex(uint256 hash, CTxIndex& txindex)
{
assert(!fClient);
txindex.SetNull();
return Read(make_pair(string("tx"), hash), txindex);
}
bool CTxDBBase::UpdateTxIndex(uint256 hash, const CTxIndex& txindex)
{
assert(!fClient);
return Write(make_pair(string("tx"), hash), txindex);
}
bool CTxDBBase::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight)
{
assert(!fClient);
uint256 hash = tx.GetHash();
CTxIndex txindex(pos, tx.vout.size());
return Write(make_pair(string("tx"), hash), txindex);
}
bool CTxDBBase::EraseTxIndex(const CTransaction& tx)
{
assert(!fClient);
uint256 hash = tx.GetHash();
return Erase(make_pair(string("tx"), hash));
}
bool CTxDBBase::ContainsTx(uint256 hash)
{
assert(!fClient);
return Exists(make_pair(string("tx"), hash));
}
bool CTxDBBase::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex)
{
assert(!fClient);
tx.SetNull();
if (!ReadTxIndex(hash, txindex))
return false;
return tx.ReadFromDisk(txindex.pos);
}
bool CTxDBBase::ReadDiskTx(uint256 hash, CTransaction& tx)
{
CTxIndex txindex;
return ReadDiskTx(hash, tx, txindex);
}
bool CTxDBBase::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex)
{
return ReadDiskTx(outpoint.hash, tx, txindex);
}
bool CTxDBBase::ReadDiskTx(COutPoint outpoint, CTransaction& tx)
{
CTxIndex txindex;
return ReadDiskTx(outpoint.hash, tx, txindex);
}
// ============================================================================
// Block index
// ============================================================================
bool CTxDBBase::WriteBlockIndex(const CDiskBlockIndex& blockindex)
{
return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex);
}
// ============================================================================
// Best chain / checkpoint metadata
// ============================================================================
bool CTxDBBase::ReadHashBestChain(uint256& hashBestChain)
{
return Read(string("hashBestChain"), hashBestChain);
}
bool CTxDBBase::WriteHashBestChain(uint256 hashBestChain)
{
return Write(string("hashBestChain"), hashBestChain);
}
bool CTxDBBase::ReadAddressIndexBestChain(uint256& hashBestChain)
{
return Read(string("addressIndexBestChain"), hashBestChain);
}
bool CTxDBBase::WriteAddressIndexBestChain(uint256 hashBestChain)
{
return Write(string("addressIndexBestChain"), hashBestChain);
}
bool CTxDBBase::ReadAddressIndexStartHeight(int& nHeight)
{
return Read(string("addressIndexStartHeight"), nHeight);
}
bool CTxDBBase::WriteAddressIndexStartHeight(int nHeight)
{
return Write(string("addressIndexStartHeight"), nHeight);
}
bool CTxDBBase::ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust)
{
return Read(string("bnBestInvalidTrust"), bnBestInvalidTrust);
}
bool CTxDBBase::WriteBestInvalidTrust(CBigNum bnBestInvalidTrust)
{
return Write(string("bnBestInvalidTrust"), bnBestInvalidTrust);
}
bool CTxDBBase::ReadSyncCheckpoint(uint256& hashCheckpoint)
{
return Read(string("hashSyncCheckpoint"), hashCheckpoint);
}
bool CTxDBBase::WriteSyncCheckpoint(uint256 hashCheckpoint)
{
return Write(string("hashSyncCheckpoint"), hashCheckpoint);
}
bool CTxDBBase::ReadCheckpointPubKey(string& strPubKey)
{
return Read(string("strCheckpointPubKey"), strPubKey);
}
bool CTxDBBase::WriteCheckpointPubKey(const string& strPubKey)
{
return Write(string("strCheckpointPubKey"), strPubKey);
}
// ============================================================================
// Address index
// ============================================================================
bool CTxDBBase::ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance)
{
return Read(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance);
}
bool CTxDBBase::WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance)
{
return Write(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance);
}
bool CTxDBBase::ReadAddressUtxo(int nType, const uint160& hashBytes,
const uint256& txhash, int nIndex,
int64_t& nValue, int& nHeight)
{
CAddressUtxoValue val;
if (!Read(make_pair(string("addrutxo"),
CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), val))
return false;
nValue = val.nValue;
nHeight = val.nHeight;
return true;
}
bool CTxDBBase::WriteAddressUtxo(int nType, const uint160& hashBytes,
const uint256& txhash, int nIndex,
int64_t nValue, int nHeight, const CScript& script)
{
return Write(make_pair(string("addrutxo"),
CAddressUtxoKey(nType, hashBytes, txhash, nIndex)),
CAddressUtxoValue(nValue, nHeight, script));
}
bool CTxDBBase::EraseAddressUtxo(int nType, const uint160& hashBytes,
const uint256& txhash, int nIndex)
{
return Erase(make_pair(string("addrutxo"),
CAddressUtxoKey(nType, hashBytes, txhash, nIndex)));
}
bool CTxDBBase::WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight,
int nTxIndex, const uint256& txhash)
{
return Write(make_pair(string("addrtxid"),
CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)),
(char)0);
}
bool CTxDBBase::EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight,
int nTxIndex, const uint256& txhash)
{
return Erase(make_pair(string("addrtxid"),
CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)));
}
bool CTxDBBase::GetAddressUtxos(int nType, const uint160& hashBytes,
std::vector<std::pair<COutPoint, std::pair<int64_t, int> > >& vUtxos)
{
vUtxos.clear();
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("addrutxo"),
CAddressUtxoKey(nType, hashBytes, uint256(0), 0));
string strPrefixBegin = ssKeyPrefix.str();
auto it = NewIterator();
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
const string keyStr = it->KeyStr();
CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(),
SER_DISK, CLIENT_VERSION);
string strKeyType;
CAddressUtxoKey utxoKey;
ssKey >> strKeyType;
if (strKeyType != "addrutxo")
break;
ssKey >> utxoKey;
if (utxoKey.nType != nType || utxoKey.hashBytes != hashBytes)
break;
const string valueStr = it->ValueStr();
CDataStream ssValue(valueStr.data(), valueStr.data() + valueStr.size(),
SER_DISK, CLIENT_VERSION);
CAddressUtxoValue utxoValue;
ssValue >> utxoValue;
COutPoint outpoint(utxoKey.txhash, utxoKey.nIndex);
vUtxos.push_back(make_pair(outpoint,
make_pair(utxoValue.nValue, utxoValue.nHeight)));
}
return true;
}
bool CTxDBBase::GetAddressTxIds(int nType, const uint160& hashBytes,
int nStartHeight, int nEndHeight,
std::vector<uint256>& vTxIds)
{
vTxIds.clear();
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("addrtxid"),
CAddressTxIdKey(nType, hashBytes, nStartHeight, 0, uint256(0)));
string strPrefixBegin = ssKeyPrefix.str();
auto it = NewIterator();
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
const string keyStr = it->KeyStr();
CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(),
SER_DISK, CLIENT_VERSION);
string strKeyType;
CAddressTxIdKey txIdKey;
ssKey >> strKeyType;
if (strKeyType != "addrtxid")
break;
ssKey >> txIdKey;
if (txIdKey.nType != nType || txIdKey.hashBytes != hashBytes)
break;
if (txIdKey.nHeight > nEndHeight)
break;
vTxIds.push_back(txIdKey.txhash);
}
return true;
}
// ============================================================================
// In-memory UTXO cache (read-through, backend-agnostic)
//
// Avoids hitting the underlying KV store for every FetchInputs call. On a 2M+
// block chain with millions of UTXOs, this dramatically reduces I/O during
// both IBD (ConnectBlock validation reads inputs) and steady-state (mempool
// acceptance, staking). Writes/erases update both cache and the backend.
// ============================================================================
namespace {
struct COutPointHasher {
size_t operator()(const COutPoint& op) const {
return op.hash.Get64() ^
(std::hash<unsigned int>()(op.n) * 0x9e3779b97f4a7c15ULL);
}
};
struct CUtxoCacheEntry {
CUtxoEntry utxo;
bool fPresent; // true = exists, false = known absent (negative cache)
CUtxoCacheEntry() : fPresent(false) {}
CUtxoCacheEntry(const CUtxoEntry& u, bool p) : utxo(u), fPresent(p) {}
};
std::unordered_map<COutPoint, CUtxoCacheEntry, COutPointHasher> g_mapUtxoCache;
CCriticalSection g_cs_utxoCache;
const size_t UTXO_CACHE_MAX_ENTRIES = 2000000; // ~400MB at ~200 bytes each
} // anonymous namespace
bool CTxDBBase::ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry)
{
entry.SetNull();
COutPoint outpoint(hash, n);
{
LOCK(g_cs_utxoCache);
auto it = g_mapUtxoCache.find(outpoint);
if (it != g_mapUtxoCache.end())
{
if (it->second.fPresent) {
entry = it->second.utxo;
return true;
}
return false;
}
}
bool fFound = Read(make_pair(string("u"), make_pair(hash, n)), entry);
{
LOCK(g_cs_utxoCache);
if (g_mapUtxoCache.size() < UTXO_CACHE_MAX_ENTRIES)
{
if (fFound)
g_mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true);
else
g_mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false);
}
}
return fFound;
}
bool CTxDBBase::WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry)
{
COutPoint outpoint(hash, n);
{
LOCK(g_cs_utxoCache);
g_mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true);
// Periodic eviction: clear half when over the limit. Simple but
// effective — the cache repopulates with the hot working set.
if (g_mapUtxoCache.size() > UTXO_CACHE_MAX_ENTRIES)
{
size_t nTarget = UTXO_CACHE_MAX_ENTRIES / 2;
auto it = g_mapUtxoCache.begin();
while (g_mapUtxoCache.size() > nTarget && it != g_mapUtxoCache.end())
it = g_mapUtxoCache.erase(it);
}
}
return Write(make_pair(string("u"), make_pair(hash, n)), entry);
}
bool CTxDBBase::EraseUtxo(const uint256& hash, unsigned int n)
{
COutPoint outpoint(hash, n);
{
LOCK(g_cs_utxoCache);
g_mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false);
}
return Erase(make_pair(string("u"), make_pair(hash, n)));
}
bool CTxDBBase::HaveUtxo(const uint256& hash, unsigned int n)
{
COutPoint outpoint(hash, n);
{
LOCK(g_cs_utxoCache);
auto it = g_mapUtxoCache.find(outpoint);
if (it != g_mapUtxoCache.end())
return it->second.fPresent;
}
if (Exists(make_pair(string("u"), make_pair(hash, n))))
return true;
// Lazy fallback: check old CTxIndex vSpent for databases upgrading from
// pre-UTXO format. vSpent[n] null = output not spent = UTXO exists.
CTxIndex txindex;
if (ReadTxIndex(hash, txindex))
{
if (n < txindex.vSpent.size() && txindex.vSpent[n].IsNull())
return true;
}
return false;
}
int64_t CTxDBBase::SumUtxoValues(int& nCount)
{
nCount = 0;
int64_t nTotal = 0;
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("u"), make_pair(uint256(0), (unsigned int)0));
string strPrefixBegin = ssKeyPrefix.str();
auto it = NewIterator();
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
const string keyStr = it->KeyStr();
CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(),
SER_DISK, CLIENT_VERSION);
string strKeyType;
ssKey >> strKeyType;
if (strKeyType != "u")
break;
const string valueStr = it->ValueStr();
CDataStream ssValue(valueStr.data(), valueStr.data() + valueStr.size(),
SER_DISK, CLIENT_VERSION);
CUtxoEntry entry;
ssValue >> entry;
nTotal += entry.nValue;
nCount++;
}
return nTotal;
}
+211
View File
@@ -0,0 +1,211 @@
// Copyright (c) 2009-2012 The Bitcoin developers.
// Copyright (c) 2026 The Triangles developers.
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef TRIANGLES_TXDB_BASE_H
#define TRIANGLES_TXDB_BASE_H
#include "main.h"
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
class CScript;
class CTransaction;
class CDiskTxPos;
class CTxIndex;
class CDiskBlockIndex;
class CUtxoEntry;
class CBigNum;
// ----------------------------------------------------------------------------
// Backend-agnostic key/value iterator.
//
// Each CTxDBBase backend returns a std::unique_ptr<CTxDBIteratorBase> from
// NewIterator(). Iterators yield raw serialized key/value bytes; callers
// deserialize using the same SER_DISK / CLIENT_VERSION conventions used by
// CTxDBBase's templated Read/Write paths.
//
// Iterators do NOT see uncommitted writes in an active batch. All current
// iteration sites (block-index scan, address-index range queries, UTXO sum)
// run outside transactions, so this is safe.
// ----------------------------------------------------------------------------
class CTxDBIteratorBase
{
public:
virtual ~CTxDBIteratorBase() = default;
virtual void Seek(const std::string& key) = 0;
virtual bool Valid() const = 0;
virtual void Next() = 0;
virtual std::string KeyStr() const = 0;
virtual std::string ValueStr() const = 0;
};
// ----------------------------------------------------------------------------
// Abstract chain database interface.
//
// All key/value serialization happens in this base class via CDataStream with
// SER_DISK / CLIENT_VERSION. Backends only implement byte-level I/O, so every
// backend produces bit-identical key bytes — required for migration and
// dual-backend parity testing.
//
// Named operations (ReadTxIndex, WriteBlockIndex, etc.) are implemented in
// terms of the templated Read/Write/Erase/Exists, which dispatch to the
// virtual byte-level methods. To add a new backend:
//
// 1. Subclass CTxDBBase.
// 2. Implement Close, TxnBegin/Commit/Abort.
// 3. Implement ReadRaw, WriteRaw, EraseRaw, ExistsRaw.
// 4. Implement NewIterator (return a subclass of CTxDBIteratorBase).
// 5. Implement LoadBlockIndex (still backend-specific in M1; will be
// extracted to the base in a later phase).
// ----------------------------------------------------------------------------
class CTxDBBase
{
public:
virtual ~CTxDBBase() = default;
// Destroys the underlying shared global state accessed by this DB.
virtual void Close() = 0;
// Batches (transaction-like atomic groups of writes/deletes).
virtual bool TxnBegin() = 0;
virtual bool TxnCommit() = 0;
virtual bool TxnAbort() = 0;
bool IsReadOnly() const { return fReadOnly; }
// ── Schema versioning ────────────────────────────────────────────────────
bool ReadVersion(int& nVersion);
bool WriteVersion(int nVersion);
bool ReadDbFormat(int& nDbFormat);
bool WriteDbFormat(int nDbFormat);
// ── Tx index ─────────────────────────────────────────────────────────────
bool ReadTxIndex(uint256 hash, CTxIndex& txindex);
bool UpdateTxIndex(uint256 hash, const CTxIndex& txindex);
bool AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight);
bool EraseTxIndex(const CTransaction& tx);
bool ContainsTx(uint256 hash);
bool ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex);
bool ReadDiskTx(uint256 hash, CTransaction& tx);
bool ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex);
bool ReadDiskTx(COutPoint outpoint, CTransaction& tx);
// ── Block index ──────────────────────────────────────────────────────────
bool WriteBlockIndex(const CDiskBlockIndex& blockindex);
// ── Best chain / checkpoint metadata ─────────────────────────────────────
bool ReadHashBestChain(uint256& hashBestChain);
bool WriteHashBestChain(uint256 hashBestChain);
bool ReadAddressIndexBestChain(uint256& hashBestChain);
bool WriteAddressIndexBestChain(uint256 hashBestChain);
bool ReadAddressIndexStartHeight(int& nHeight);
bool WriteAddressIndexStartHeight(int nHeight);
bool ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust);
bool WriteBestInvalidTrust(CBigNum bnBestInvalidTrust);
bool ReadSyncCheckpoint(uint256& hashCheckpoint);
bool WriteSyncCheckpoint(uint256 hashCheckpoint);
bool ReadCheckpointPubKey(std::string& strPubKey);
bool WriteCheckpointPubKey(const std::string& strPubKey);
virtual bool LoadBlockIndex() = 0;
// ── Address index ────────────────────────────────────────────────────────
bool ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance);
bool WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance);
bool ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash,
int nIndex, int64_t& nValue, int& nHeight);
bool WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash,
int nIndex, int64_t nValue, int nHeight, const CScript& script);
bool EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash,
int nIndex);
bool WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight,
int nTxIndex, const uint256& txhash);
bool EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight,
int nTxIndex, const uint256& txhash);
bool GetAddressUtxos(int nType, const uint160& hashBytes,
std::vector<std::pair<COutPoint, std::pair<int64_t, int> > >& vUtxos);
bool GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight,
int nEndHeight, std::vector<uint256>& vTxIds);
// ── UTXO set ─────────────────────────────────────────────────────────────
bool ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry);
bool WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry);
bool EraseUtxo(const uint256& hash, unsigned int n);
bool HaveUtxo(const uint256& hash, unsigned int n);
int64_t SumUtxoValues(int& nCount);
protected:
bool fReadOnly = false;
// Byte-level I/O — backends implement these.
virtual bool ReadRaw(const std::string& key, std::string& value) const = 0;
virtual bool WriteRaw(const std::string& key, const std::string& value) = 0;
virtual bool EraseRaw(const std::string& key) = 0;
virtual bool ExistsRaw(const std::string& key) const = 0;
virtual std::unique_ptr<CTxDBIteratorBase> NewIterator() const = 0;
// Templated Read/Write/Erase/Exists are non-virtual (templates can't be
// virtual in C++) — they serialize and dispatch to the byte-level virtuals.
template<typename K, typename T>
bool Read(const K& key, T& value) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
std::string strValue;
if (!ReadRaw(ssKey.str(), strValue))
return false;
try {
CDataStream ssValue(strValue.data(),
strValue.data() + strValue.size(),
SER_DISK, CLIENT_VERSION);
ssValue >> value;
} catch (std::exception&) {
return false;
}
return true;
}
template<typename K, typename T>
bool Write(const K& key, const T& value)
{
if (fReadOnly)
assert(!"Write called on database in read-only mode");
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(10000);
ssValue << value;
return WriteRaw(ssKey.str(), ssValue.str());
}
template<typename K>
bool Erase(const K& key)
{
if (fReadOnly)
assert(!"Erase called on database in read-only mode");
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
return EraseRaw(ssKey.str());
}
template<typename K>
bool Exists(const K& key) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
return ExistsRaw(ssKey.str());
}
};
#endif // TRIANGLES_TXDB_BASE_H
+87 -445
View File
@@ -4,7 +4,6 @@
// file license.txt or http://www.opensource.org/licenses/mit-license.php.
#include <map>
#include <unordered_map>
#include <boost/version.hpp>
#include <boost/filesystem.hpp>
@@ -41,29 +40,22 @@ static leveldb::Options GetOptions() {
// memtable flushes and compactions, which is a big win during IBD
// when millions of tx index entries are written sequentially.
options.write_buffer_size = 64 * 1048576;
// Allow more open files for better read performance on large chains
options.max_open_files = 1000;
return options;
}
void init_blockindex(leveldb::Options& options, bool fRemoveOld = false) {
// First time init.
fs::path directory = GetDataDir() / "txleveldb";
if (fRemoveOld) {
fs::remove_all(directory); // remove directory
fs::remove_all(directory);
unsigned int nFile = 1;
while (true)
{
fs::path strBlockFile = GetDataDir() / strprintf("blk%04u.dat", nFile);
// Break if no such file
if( !fs::exists( strBlockFile ) )
if(!fs::exists(strBlockFile))
break;
fs::remove(strBlockFile);
nFile++;
}
}
@@ -76,8 +68,6 @@ void init_blockindex(leveldb::Options& options, bool fRemoveOld = false) {
}
}
// CDB subclasses are created and destroyed VERY OFTEN. That's why
// we shouldn't treat this as a free operations.
CTxDB::CTxDB(const char* pszMode)
{
assert(pszMode);
@@ -95,7 +85,7 @@ CTxDB::CTxDB(const char* pszMode)
options.create_if_missing = fCreate;
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
init_blockindex(options); // Init directory
init_blockindex(options);
pdb = txdb;
if (Exists(string("version")))
@@ -107,18 +97,17 @@ CTxDB::CTxDB(const char* pszMode)
{
printf("Required index version is %d, removing old database\n", DATABASE_VERSION);
// Leveldb instance destruction
delete txdb;
txdb = pdb = NULL;
delete activeBatch;
activeBatch = NULL;
init_blockindex(options, true); // Remove directory and create new database
init_blockindex(options, true);
pdb = txdb;
bool fTmp = fReadOnly;
fReadOnly = false;
WriteVersion(DATABASE_VERSION); // Save transaction index version
WriteVersion(DATABASE_VERSION);
fReadOnly = fTmp;
}
}
@@ -171,6 +160,8 @@ bool CTxDB::TxnCommit()
return true;
}
namespace {
class CBatchScanner : public leveldb::WriteBatch::Handler {
public:
std::string needle;
@@ -196,16 +187,32 @@ public:
}
};
// When performing a read, if we have an active batch we need to check it first
// before reading from the database, as the rest of the code assumes that once
// a database transaction begins reads are consistent with it. It would be good
// to change that assumption in future and avoid the performance hit, though in
// practice it does not appear to be large.
bool CTxDB::ScanBatch(const CDataStream &key, string *value, bool *deleted) const {
class CLevelDBIterator final : public CTxDBIteratorBase {
public:
explicit CLevelDBIterator(leveldb::Iterator* pit) : pit(pit) {}
~CLevelDBIterator() override { delete pit; }
void Seek(const std::string& key) override { pit->Seek(key); }
bool Valid() const override { return pit->Valid(); }
void Next() override { pit->Next(); }
std::string KeyStr() const override { return pit->key().ToString(); }
std::string ValueStr() const override { return pit->value().ToString(); }
private:
leveldb::Iterator* pit;
};
} // anonymous namespace
// When performing a read with an active batch, check the batch first. The
// rest of the codebase assumes that once a batch is open, reads are
// consistent with the pending writes inside it.
bool CTxDB::ScanBatch(const std::string& key, string* value, bool* deleted) const
{
assert(activeBatch);
*deleted = false;
CBatchScanner scanner;
scanner.needle = key.str();
scanner.needle = key;
scanner.deleted = deleted;
scanner.foundValue = value;
leveldb::Status status = activeBatch->Iterate(&scanner);
@@ -215,132 +222,71 @@ bool CTxDB::ScanBatch(const CDataStream &key, string *value, bool *deleted) cons
return scanner.foundEntry;
}
bool CTxDB::ReadTxIndex(uint256 hash, CTxIndex& txindex)
bool CTxDB::ReadRaw(const std::string& key, std::string& value) const
{
assert(!fClient);
txindex.SetNull();
return Read(make_pair(string("tx"), hash), txindex);
bool readFromDb = true;
if (activeBatch) {
bool deleted = false;
readFromDb = ScanBatch(key, &value, &deleted) == false;
if (deleted)
return false;
}
if (readFromDb) {
leveldb::Status status = pdb->Get(leveldb::ReadOptions(), key, &value);
if (!status.ok()) {
if (status.IsNotFound())
return false;
printf("LevelDB read failure: %s\n", status.ToString().c_str());
return false;
}
}
return true;
}
bool CTxDB::UpdateTxIndex(uint256 hash, const CTxIndex& txindex)
bool CTxDB::WriteRaw(const std::string& key, const std::string& value)
{
assert(!fClient);
return Write(make_pair(string("tx"), hash), txindex);
}
bool CTxDB::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight)
{
assert(!fClient);
// Add to tx index
uint256 hash = tx.GetHash();
CTxIndex txindex(pos, tx.vout.size());
return Write(make_pair(string("tx"), hash), txindex);
}
bool CTxDB::EraseTxIndex(const CTransaction& tx)
{
assert(!fClient);
uint256 hash = tx.GetHash();
return Erase(make_pair(string("tx"), hash));
}
bool CTxDB::ContainsTx(uint256 hash)
{
assert(!fClient);
return Exists(make_pair(string("tx"), hash));
}
bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex)
{
assert(!fClient);
tx.SetNull();
if (!ReadTxIndex(hash, txindex))
if (activeBatch) {
activeBatch->Put(key, value);
return true;
}
leveldb::Status status = pdb->Put(leveldb::WriteOptions(), key, value);
if (!status.ok()) {
printf("LevelDB write failure: %s\n", status.ToString().c_str());
return false;
return (tx.ReadFromDisk(txindex.pos));
}
return true;
}
bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx)
bool CTxDB::EraseRaw(const std::string& key)
{
CTxIndex txindex;
return ReadDiskTx(hash, tx, txindex);
if (!pdb)
return false;
if (activeBatch) {
activeBatch->Delete(key);
return true;
}
leveldb::Status status = pdb->Delete(leveldb::WriteOptions(), key);
return (status.ok() || status.IsNotFound());
}
bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex)
bool CTxDB::ExistsRaw(const std::string& key) const
{
return ReadDiskTx(outpoint.hash, tx, txindex);
std::string unused;
if (activeBatch) {
bool deleted = false;
if (ScanBatch(key, &unused, &deleted) && !deleted)
return true;
}
leveldb::Status status = pdb->Get(leveldb::ReadOptions(), key, &unused);
return status.IsNotFound() == false;
}
bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx)
std::unique_ptr<CTxDBIteratorBase> CTxDB::NewIterator() const
{
CTxIndex txindex;
return ReadDiskTx(outpoint.hash, tx, txindex);
}
bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex)
{
return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex);
}
bool CTxDB::ReadHashBestChain(uint256& hashBestChain)
{
return Read(string("hashBestChain"), hashBestChain);
}
bool CTxDB::WriteHashBestChain(uint256 hashBestChain)
{
return Write(string("hashBestChain"), hashBestChain);
}
bool CTxDB::ReadAddressIndexBestChain(uint256& hashBestChain)
{
return Read(string("addressIndexBestChain"), hashBestChain);
}
bool CTxDB::WriteAddressIndexBestChain(uint256 hashBestChain)
{
return Write(string("addressIndexBestChain"), hashBestChain);
}
bool CTxDB::ReadAddressIndexStartHeight(int& nHeight)
{
return Read(string("addressIndexStartHeight"), nHeight);
}
bool CTxDB::WriteAddressIndexStartHeight(int nHeight)
{
return Write(string("addressIndexStartHeight"), nHeight);
}
bool CTxDB::ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust)
{
return Read(string("bnBestInvalidTrust"), bnBestInvalidTrust);
}
bool CTxDB::WriteBestInvalidTrust(CBigNum bnBestInvalidTrust)
{
return Write(string("bnBestInvalidTrust"), bnBestInvalidTrust);
}
bool CTxDB::ReadSyncCheckpoint(uint256& hashCheckpoint)
{
return Read(string("hashSyncCheckpoint"), hashCheckpoint);
}
bool CTxDB::WriteSyncCheckpoint(uint256 hashCheckpoint)
{
return Write(string("hashSyncCheckpoint"), hashCheckpoint);
}
bool CTxDB::ReadCheckpointPubKey(string& strPubKey)
{
return Read(string("strCheckpointPubKey"), strPubKey);
}
bool CTxDB::WriteCheckpointPubKey(const string& strPubKey)
{
return Write(string("strCheckpointPubKey"), strPubKey);
return std::unique_ptr<CTxDBIteratorBase>(
new CLevelDBIterator(pdb->NewIterator(leveldb::ReadOptions())));
}
static CBlockIndex *InsertBlockIndex(uint256 hash)
@@ -348,12 +294,10 @@ static CBlockIndex *InsertBlockIndex(uint256 hash)
if (hash == 0)
return NULL;
// Return existing
map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hash);
if (mi != mapBlockIndex.end())
return (*mi).second;
// Create new
CBlockIndex* pindexNew = new CBlockIndex();
if (!pindexNew)
throw runtime_error("LoadBlockIndex() : new CBlockIndex failed");
@@ -366,8 +310,7 @@ static CBlockIndex *InsertBlockIndex(uint256 hash)
bool CTxDB::LoadBlockIndex()
{
if (mapBlockIndex.size() > 0) {
// Already loaded once in this session. It can happen during migration
// from BDB.
// Already loaded once in this session. Can happen during BDB migration.
return true;
}
@@ -377,39 +320,32 @@ bool CTxDB::LoadBlockIndex()
CDiskBlockIndex::fSerializeChainTrust = (nDbFormat >= 2);
if (CDiskBlockIndex::fSerializeChainTrust)
printf("LoadBlockIndex(): DB format v%d nChainTrust persisted\n", nDbFormat);
printf("LoadBlockIndex(): DB format v%d - nChainTrust persisted\n", nDbFormat);
else
printf("LoadBlockIndex(): DB format v%d will recalculate nChainTrust (one-time upgrade)\n", nDbFormat);
printf("LoadBlockIndex(): DB format v%d - will recalculate nChainTrust (one-time upgrade)\n", nDbFormat);
// The block index is an in-memory structure that maps hashes to on-disk
// locations where the contents of the block can be found. Here, we scan it
// out of the DB and into mapBlockIndex.
// Scan the block index out of the DB into mapBlockIndex.
int64_t nPhaseStart = GetTimeMillis();
int64_t nTotalStart = nPhaseStart;
leveldb::Iterator *iterator = pdb->NewIterator(leveldb::ReadOptions());
// Seek to start key.
CDataStream ssStartKey(SER_DISK, CLIENT_VERSION);
ssStartKey << make_pair(string("blockindex"), uint256(0));
iterator->Seek(ssStartKey.str());
// Now read each entry.
int nBlocksLoaded = 0;
while (iterator->Valid())
{
// Report progress every 100k blocks
if (++nBlocksLoaded % 100000 == 0)
{
std::string strMsg = strprintf(_("Loading block index... (%d blocks)"), nBlocksLoaded);
uiInterface.InitMessage(strMsg);
}
// Unpack keys and values.
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.write(iterator->key().data(), iterator->key().size());
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.write(iterator->value().data(), iterator->value().size());
string strType;
ssKey >> strType;
// Did we reach the end of the data to read?
if (fRequestShutdown || strType != "blockindex")
break;
CDiskBlockIndex diskindex;
@@ -417,7 +353,6 @@ bool CTxDB::LoadBlockIndex()
uint256 blockHash = diskindex.GetBlockHash();
// Construct block index object
CBlockIndex* pindexNew = InsertBlockIndex(blockHash);
pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev);
pindexNew->pnext = InsertBlockIndex(diskindex.hashNext);
@@ -436,10 +371,8 @@ bool CTxDB::LoadBlockIndex()
pindexNew->nTime = diskindex.nTime;
pindexNew->nBits = diskindex.nBits;
pindexNew->nNonce = diskindex.nNonce;
// nChainTrust is populated from disk if fSerializeChainTrust, else stays 0
pindexNew->nChainTrust = diskindex.nChainTrust;
// Watch for genesis block
if (pindexGenesisBlock == NULL && blockHash == (!fTestNet ? hashGenesisBlockOfficial : hashGenesisBlockTestNet))
pindexGenesisBlock = pindexNew;
@@ -448,8 +381,6 @@ bool CTxDB::LoadBlockIndex()
return error("LoadBlockIndex() : CheckIndex failed at %d", pindexNew->nHeight);
}
// setStakeSeen is populated below for recent blocks only (Change D)
iterator->Next();
}
delete iterator;
@@ -513,7 +444,6 @@ bool CTxDB::LoadBlockIndex()
ssValue << diskindex;
batch.Put(ssKey.str(), ssValue.str());
// Flush in chunks to limit memory usage
if (++nCount % 100000 == 0)
{
pdb->Write(leveldb::WriteOptions(), &batch);
@@ -521,7 +451,6 @@ bool CTxDB::LoadBlockIndex()
printf("LoadBlockIndex(): upgraded %d / %d block index entries\n", nCount, (int)vSortedByHeight.size());
}
}
// Write remaining entries + format version
CDataStream ssFmtKey(SER_DISK, CLIENT_VERSION);
ssFmtKey << string("dbformat");
CDataStream ssFmtValue(SER_DISK, CLIENT_VERSION);
@@ -536,8 +465,6 @@ bool CTxDB::LoadBlockIndex()
}
else
{
// nChainTrust was loaded from disk. Only need stake modifier checksums
// for blocks above the last checkpoint (typically very few or zero).
int nLastCheckpointHeight = Checkpoints::GetTotalBlocksEstimate();
bool fNeedModifierCheck = false;
for (const auto& item : mapBlockIndex)
@@ -569,16 +496,12 @@ bool CTxDB::LoadBlockIndex()
printf("STARTUP-PERF: chain_trust_and_modifiers %" PRId64 "ms\n", GetTimeMillis() - nPhaseStart);
// Bump dbformat to 3 if needed (databases that already had v2 nChainTrust upgrade).
// UTXO entries are written by ConnectBlock during normal sync. For databases upgrading
// from older versions, FetchInputs has a lazy fallback to the old CTxIndex path.
if (nDbFormat < 3)
{
WriteDbFormat(3);
printf("LoadBlockIndex(): bumped dbformat to v3 (UTXO model with lazy fallback)\n");
}
// Load hashBestChain pointer to end of best chain
nPhaseStart = GetTimeMillis();
if (!ReadHashBestChain(hashBestChain))
{
@@ -594,7 +517,6 @@ bool CTxDB::LoadBlockIndex()
printf("STARTUP-PERF: best_chain %" PRId64 "ms\n", GetTimeMillis() - nPhaseStart);
// ---- setStakeSeen: only populate for recent blocks (DoS protection) ----
nPhaseStart = GetTimeMillis();
{
int nStakeSeenDepth = 500;
@@ -617,7 +539,6 @@ bool CTxDB::LoadBlockIndex()
DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str());
// Re-evaluate best chain: scan for competing tips with equal or greater trust.
// This fixes nodes stuck on the wrong fork after consensus rule changes.
{
CBlockIndex* pindexBetter = NULL;
for (const auto& item : mapBlockIndex)
@@ -660,29 +581,25 @@ bool CTxDB::LoadBlockIndex()
}
}
// triangles: load hashSyncCheckpoint (best-effort, non-fatal)
if (!ReadSyncCheckpoint(Checkpoints::hashSyncCheckpoint))
printf("LoadBlockIndex(): no sync checkpoint in DB, using default\n");
else
printf("LoadBlockIndex(): synchronized checkpoint %s\n", Checkpoints::hashSyncCheckpoint.ToString().c_str());
// If the stored checkpoint isn't in our index, reset to genesis so we don't assert-crash
if (!mapBlockIndex.count(Checkpoints::hashSyncCheckpoint))
{
printf("LoadBlockIndex(): sync checkpoint not in index, resetting to genesis\n");
Checkpoints::hashSyncCheckpoint = (!fTestNet ? hashGenesisBlockOfficial : hashGenesisBlockTestNet);
}
// Load bnBestInvalidTrust, OK if it doesn't exist
CBigNum bnBestInvalidTrust;
ReadBestInvalidTrust(bnBestInvalidTrust);
nBestInvalidTrust = bnBestInvalidTrust.getuint256();
// Verify blocks in the best chain
nPhaseStart = GetTimeMillis();
int nCheckLevel = GetArg("-checklevel", 1);
int nCheckDepth = GetArg( "-checkblocks", 50);
if (nCheckDepth == 0)
nCheckDepth = 1000000000; // suffices until the year 19000
nCheckDepth = 1000000000;
if (nCheckDepth > nBestHeight)
nCheckDepth = nBestHeight;
printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel);
@@ -695,14 +612,11 @@ bool CTxDB::LoadBlockIndex()
CBlock block;
if (!block.ReadFromDisk(pindex))
return error("LoadBlockIndex() : block.ReadFromDisk failed");
// check level 1: verify block validity
// check level 7: verify block signature too
if (nCheckLevel>0 && !block.CheckBlock(true, true, (nCheckLevel>6)))
{
printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str());
pindexFork = pindex->pprev;
}
// check level 2: verify transaction index validity
if (nCheckLevel>1)
{
pair<unsigned int, unsigned int> pos = make_pair(pindex->nFile, pindex->nBlockPos);
@@ -713,10 +627,8 @@ bool CTxDB::LoadBlockIndex()
CTxIndex txindex;
if (ReadTxIndex(hashTx, txindex))
{
// check level 3: checker transaction hashes
if (nCheckLevel>2 || pindex->nFile != txindex.pos.nFile || pindex->nBlockPos != txindex.pos.nBlockPos)
{
// either an error or a duplicate transaction
CTransaction txFound;
if (!txFound.ReadFromDisk(txindex.pos))
{
@@ -724,13 +636,12 @@ bool CTxDB::LoadBlockIndex()
pindexFork = pindex->pprev;
}
else
if (txFound.GetHash() != hashTx) // not a duplicate tx
if (txFound.GetHash() != hashTx)
{
printf("LoadBlockIndex(): *** invalid tx position for %s\n", hashTx.ToString().c_str());
pindexFork = pindex->pprev;
}
}
// check level 4: verify spent inputs were removed from UTXO set
if (nCheckLevel>3 && !tx.IsCoinBase())
{
for (const CTxIn &txin : tx.vin)
@@ -749,7 +660,6 @@ bool CTxDB::LoadBlockIndex()
}
if (pindexFork && !fRequestShutdown)
{
// Reorg back to the fork
printf("LoadBlockIndex() : *** moving best chain pointer back to block %d\n", pindexFork->nHeight);
CBlock block;
if (!block.ReadFromDisk(pindexFork))
@@ -762,271 +672,3 @@ bool CTxDB::LoadBlockIndex()
return true;
}
// ============================================================================
// Address index methods
// ============================================================================
bool CTxDB::ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance)
{
return Read(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance);
}
bool CTxDB::WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance)
{
return Write(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance);
}
bool CTxDB::ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t& nValue, int& nHeight)
{
CAddressUtxoValue val;
if (!Read(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), val))
return false;
nValue = val.nValue;
nHeight = val.nHeight;
return true;
}
bool CTxDB::WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t nValue, int nHeight, const CScript& script)
{
return Write(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex)),
CAddressUtxoValue(nValue, nHeight, script));
}
bool CTxDB::EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex)
{
return Erase(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex)));
}
bool CTxDB::WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash)
{
return Write(make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)), (char)0);
}
bool CTxDB::EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash)
{
return Erase(make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)));
}
bool CTxDB::GetAddressUtxos(int nType, const uint160& hashBytes, std::vector<std::pair<COutPoint, std::pair<int64_t, int> > >& vUtxos)
{
vUtxos.clear();
// Build the key prefix to seek to
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, uint256(0), 0));
std::string strPrefixBegin = ssKeyPrefix.str();
leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions());
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
// Deserialize the key
CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION);
std::string strKeyType;
CAddressUtxoKey utxoKey;
ssKey >> strKeyType;
if (strKeyType != "addrutxo")
break;
ssKey >> utxoKey;
if (utxoKey.nType != nType || utxoKey.hashBytes != hashBytes)
break;
// Deserialize the value
CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION);
CAddressUtxoValue utxoValue;
ssValue >> utxoValue;
COutPoint outpoint(utxoKey.txhash, utxoKey.nIndex);
vUtxos.push_back(make_pair(outpoint, make_pair(utxoValue.nValue, utxoValue.nHeight)));
}
delete it;
return true;
}
bool CTxDB::GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight, int nEndHeight, std::vector<uint256>& vTxIds)
{
vTxIds.clear();
// Build the key prefix to seek to
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nStartHeight, 0, uint256(0)));
std::string strPrefixBegin = ssKeyPrefix.str();
leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions());
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION);
std::string strKeyType;
CAddressTxIdKey txIdKey;
ssKey >> strKeyType;
if (strKeyType != "addrtxid")
break;
ssKey >> txIdKey;
if (txIdKey.nType != nType || txIdKey.hashBytes != hashBytes)
break;
if (txIdKey.nHeight > nEndHeight)
break;
vTxIds.push_back(txIdKey.txhash);
}
delete it;
return true;
}
// ---------- In-memory UTXO cache ----------
//
// Read-through cache that avoids hitting LevelDB for every FetchInputs call.
// On a 2M+ block chain with millions of UTXOs, this dramatically reduces I/O
// during both IBD (ConnectBlock validation reads inputs) and normal operation
// (mempool acceptance, staking). Writes/erases update both cache and LevelDB.
struct COutPointHasher {
size_t operator()(const COutPoint& op) const {
// Mix the lower 64 bits of the hash with the output index
return op.hash.Get64() ^ (std::hash<unsigned int>()(op.n) * 0x9e3779b97f4a7c15ULL);
}
};
// Cache entry: the UTXO data plus a flag indicating "known absent from DB"
struct CUtxoCacheEntry {
CUtxoEntry utxo;
bool fPresent; // true = UTXO exists, false = known deleted/absent
CUtxoCacheEntry() : fPresent(false) {}
CUtxoCacheEntry(const CUtxoEntry& u, bool p) : utxo(u), fPresent(p) {}
};
static std::unordered_map<COutPoint, CUtxoCacheEntry, COutPointHasher> mapUtxoCache;
static CCriticalSection cs_utxoCache;
static const size_t UTXO_CACHE_MAX_ENTRIES = 2000000; // ~400MB at ~200 bytes each
// ---------- UTXO database methods ----------
bool CTxDB::ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry)
{
entry.SetNull();
COutPoint outpoint(hash, n);
{
LOCK(cs_utxoCache);
auto it = mapUtxoCache.find(outpoint);
if (it != mapUtxoCache.end())
{
if (it->second.fPresent) {
entry = it->second.utxo;
return true;
}
return false; // cached as absent
}
}
// Cache miss — read from LevelDB
bool fFound = Read(make_pair(string("u"), make_pair(hash, n)), entry);
{
LOCK(cs_utxoCache);
// Only cache if under limit (don't evict here — eviction is periodic)
if (mapUtxoCache.size() < UTXO_CACHE_MAX_ENTRIES)
{
if (fFound)
mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true);
else
mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false);
}
}
return fFound;
}
bool CTxDB::WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry)
{
COutPoint outpoint(hash, n);
{
LOCK(cs_utxoCache);
mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true);
// Periodic eviction: if cache is over limit, clear half of it.
// This is a simple but effective strategy — the cache will quickly
// repopulate with the hot working set.
if (mapUtxoCache.size() > UTXO_CACHE_MAX_ENTRIES)
{
size_t nTarget = UTXO_CACHE_MAX_ENTRIES / 2;
auto it = mapUtxoCache.begin();
while (mapUtxoCache.size() > nTarget && it != mapUtxoCache.end())
it = mapUtxoCache.erase(it);
}
}
return Write(make_pair(string("u"), make_pair(hash, n)), entry);
}
bool CTxDB::EraseUtxo(const uint256& hash, unsigned int n)
{
COutPoint outpoint(hash, n);
{
LOCK(cs_utxoCache);
// Mark as absent in cache (negative cache) so future reads don't hit DB
mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false);
}
return Erase(make_pair(string("u"), make_pair(hash, n)));
}
bool CTxDB::HaveUtxo(const uint256& hash, unsigned int n)
{
COutPoint outpoint(hash, n);
{
LOCK(cs_utxoCache);
auto it = mapUtxoCache.find(outpoint);
if (it != mapUtxoCache.end())
return it->second.fPresent;
}
if (Exists(make_pair(string("u"), make_pair(hash, n))))
return true;
// Lazy fallback: check old CTxIndex vSpent for databases upgrading from pre-UTXO format
CTxIndex txindex;
if (ReadTxIndex(hash, txindex))
{
if (n < txindex.vSpent.size() && txindex.vSpent[n].IsNull())
return true; // vSpent[n] is null = output NOT spent = UTXO exists
}
return false;
}
int64_t CTxDB::SumUtxoValues(int& nCount)
{
nCount = 0;
int64_t nTotal = 0;
// Seek to the start of UTXO entries (key prefix "u")
CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION);
ssKeyPrefix << make_pair(string("u"), make_pair(uint256(0), (unsigned int)0));
std::string strPrefixBegin = ssKeyPrefix.str();
leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions());
for (it->Seek(strPrefixBegin); it->Valid(); it->Next())
{
// Check key prefix is still "u"
CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION);
std::string strKeyType;
ssKey >> strKeyType;
if (strKeyType != "u")
break;
// Deserialize the UTXO entry and sum the value
CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION);
CUtxoEntry entry;
ssValue >> entry;
nTotal += entry.nValue;
nCount++;
}
delete it;
return nTotal;
}
+33 -213
View File
@@ -6,241 +6,61 @@
#ifndef TRIANGLES_LEVELDB_H
#define TRIANGLES_LEVELDB_H
#include "main.h"
#include <map>
#include <string>
#include <vector>
#include "txdb-base.h"
#include <leveldb/db.h>
#include <leveldb/write_batch.h>
// Class that provides access to a LevelDB. Note that this class is frequently
// instantiated on the stack and then destroyed again, so instantiation has to
// be very cheap. Unfortunately that means, a CTxDB instance is actually just a
// wrapper around some global state.
// LevelDB backend for the chain database.
//
// A LevelDB is a key/value store that is optimized for fast usage on hard
// disks. It prefers long read/writes to seeks and is based on a series of
// sorted key/value mapping files that are stacked on top of each other, with
// newer files overriding older files. A background thread compacts them
// together when too many files stack up.
// Cheap to construct/destruct: every instance shares a single global
// leveldb::DB pointer, opened lazily on first use. Most of the codebase
// instantiates a CTxDB on the stack for short-lived operations.
//
// Learn more: http://code.google.com/p/leveldb/
class CTxDB
// The protected templated Read/Write/Erase/Exists live in CTxDBBase and
// dispatch to ReadRaw/WriteRaw/EraseRaw/ExistsRaw below, which handle the
// active-batch logic so reads-after-writes within an open batch see their
// own pending changes.
class CTxDB final : public CTxDBBase
{
public:
CTxDB(const char* pszMode="r+");
~CTxDB() {
// Note that this is not the same as Close() because it deletes only
// data scoped to this TxDB object.
CTxDB(const char* pszMode = "r+");
~CTxDB() override {
delete activeBatch;
}
// Destroys the underlying shared global state accessed by this TxDB.
void Close();
void Close() override;
private:
leveldb::DB *pdb; // Points to the global instance.
// A batch stores up writes and deletes for atomic application. When this
// field is non-NULL, writes/deletes go there instead of directly to disk.
leveldb::WriteBatch *activeBatch;
leveldb::Options options;
bool fReadOnly;
int nVersion;
protected:
// Returns true and sets (value,false) if activeBatch contains the given key
// or leaves value alone and sets deleted = true if activeBatch contains a
// delete for it.
bool ScanBatch(const CDataStream &key, std::string *value, bool *deleted) const;
template<typename K, typename T>
bool Read(const K& key, T& value)
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
std::string strValue;
bool readFromDb = true;
if (activeBatch) {
// First we must search for it in the currently pending set of
// changes to the db. If not found in the batch, go on to read disk.
bool deleted = false;
readFromDb = ScanBatch(ssKey, &strValue, &deleted) == false;
if (deleted) {
return false;
}
}
if (readFromDb) {
leveldb::Status status = pdb->Get(leveldb::ReadOptions(),
ssKey.str(), &strValue);
if (!status.ok()) {
if (status.IsNotFound())
return false;
// Some unexpected error.
printf("LevelDB read failure: %s\n", status.ToString().c_str());
return false;
}
}
// Unserialize value
try {
CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(),
SER_DISK, CLIENT_VERSION);
ssValue >> value;
}
catch (std::exception &e) {
return false;
}
return true;
}
template<typename K, typename T>
bool Write(const K& key, const T& value)
{
if (fReadOnly)
assert(!"Write called on database in read-only mode");
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(10000);
ssValue << value;
if (activeBatch) {
activeBatch->Put(ssKey.str(), ssValue.str());
return true;
}
leveldb::Status status = pdb->Put(leveldb::WriteOptions(), ssKey.str(), ssValue.str());
if (!status.ok()) {
printf("LevelDB write failure: %s\n", status.ToString().c_str());
return false;
}
return true;
}
template<typename K>
bool Erase(const K& key)
{
if (!pdb)
return false;
if (fReadOnly)
assert(!"Erase called on database in read-only mode");
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
if (activeBatch) {
activeBatch->Delete(ssKey.str());
return true;
}
leveldb::Status status = pdb->Delete(leveldb::WriteOptions(), ssKey.str());
return (status.ok() || status.IsNotFound());
}
template<typename K>
bool Exists(const K& key)
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(1000);
ssKey << key;
std::string unused;
if (activeBatch) {
bool deleted;
if (ScanBatch(ssKey, &unused, &deleted) && !deleted) {
return true;
}
}
leveldb::Status status = pdb->Get(leveldb::ReadOptions(), ssKey.str(), &unused);
return status.IsNotFound() == false;
}
public:
bool TxnBegin();
bool TxnCommit();
bool TxnAbort()
bool TxnBegin() override;
bool TxnCommit() override;
bool TxnAbort() override
{
delete activeBatch;
activeBatch = NULL;
return true;
}
bool ReadVersion(int& nVersion)
{
nVersion = 0;
return Read(std::string("version"), nVersion);
}
bool LoadBlockIndex() override;
bool WriteVersion(int nVersion)
{
return Write(std::string("version"), nVersion);
}
bool ReadDbFormat(int& nDbFormat)
{
nDbFormat = 1;
return Read(std::string("dbformat"), nDbFormat);
}
bool WriteDbFormat(int nDbFormat)
{
return Write(std::string("dbformat"), nDbFormat);
}
bool ReadTxIndex(uint256 hash, CTxIndex& txindex);
bool UpdateTxIndex(uint256 hash, const CTxIndex& txindex);
bool AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight);
bool EraseTxIndex(const CTransaction& tx);
bool ContainsTx(uint256 hash);
bool ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex);
bool ReadDiskTx(uint256 hash, CTransaction& tx);
bool ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex);
bool ReadDiskTx(COutPoint outpoint, CTransaction& tx);
bool WriteBlockIndex(const CDiskBlockIndex& blockindex);
bool ReadHashBestChain(uint256& hashBestChain);
bool WriteHashBestChain(uint256 hashBestChain);
bool ReadAddressIndexBestChain(uint256& hashBestChain);
bool WriteAddressIndexBestChain(uint256 hashBestChain);
bool ReadAddressIndexStartHeight(int& nHeight);
bool WriteAddressIndexStartHeight(int nHeight);
bool ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust);
bool WriteBestInvalidTrust(CBigNum bnBestInvalidTrust);
bool ReadSyncCheckpoint(uint256& hashCheckpoint);
bool WriteSyncCheckpoint(uint256 hashCheckpoint);
bool ReadCheckpointPubKey(std::string& strPubKey);
bool WriteCheckpointPubKey(const std::string& strPubKey);
bool LoadBlockIndex();
// Address index methods
bool ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance);
bool WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance);
bool ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t& nValue, int& nHeight);
bool WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t nValue, int nHeight, const CScript& script);
bool EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex);
bool WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash);
bool EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash);
// Address index iteration (for RPC queries)
bool GetAddressUtxos(int nType, const uint160& hashBytes, std::vector<std::pair<COutPoint, std::pair<int64_t, int> > >& vUtxos);
bool GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight, int nEndHeight, std::vector<uint256>& vTxIds);
// UTXO database methods
bool ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry);
bool WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry);
bool EraseUtxo(const uint256& hash, unsigned int n);
bool HaveUtxo(const uint256& hash, unsigned int n);
int64_t SumUtxoValues(int& nCount);
protected:
bool ReadRaw(const std::string& key, std::string& value) const override;
bool WriteRaw(const std::string& key, const std::string& value) override;
bool EraseRaw(const std::string& key) override;
bool ExistsRaw(const std::string& key) const override;
std::unique_ptr<CTxDBIteratorBase> NewIterator() const override;
private:
leveldb::DB* pdb; // Points to the global instance.
leveldb::WriteBatch* activeBatch; // When non-NULL, writes/deletes go here.
leveldb::Options options;
int nVersion;
// Returns true and sets (value,false) if activeBatch contains the given
// key, or leaves value alone and sets deleted=true if activeBatch contains
// a delete for it.
bool ScanBatch(const std::string& key, std::string* value, bool* deleted) const;
bool LoadBlockIndexGuts();
};
#endif // TRIANGLES_LEVELDB_H