From b28525057a7b8e76fd114264378a350accbb257d Mon Sep 17 00:00:00 2001 From: Sami Ahmed Date: Sat, 25 Apr 2026 02:37:45 -0700 Subject: [PATCH] M1.1: Extract CTxDBBase abstract storage interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of the multi-phase chaindb modernization plan. Introduces a backend-agnostic abstraction over the chain database: * CTxDBBase — abstract class owning all serialization and named operations (ReadTxIndex, WriteBlockIndex, ReadAddressBalance, etc.). Templated Read/Write/Erase/Exists dispatch to byte-level virtuals (ReadRaw/WriteRaw/EraseRaw/ExistsRaw) so every backend produces bit-identical key bytes — required for migration and dual-backend parity testing later. * CTxDBIteratorBase — abstract iterator. Backends implement Seek, Valid, Next, KeyStr, ValueStr. * CTxDB now inherits from CTxDBBase and only implements the byte-level I/O, batch lifecycle, NewIterator, and LoadBlockIndex (which still uses leveldb directly during the v3 dbformat upgrade — extracted to base in a later phase). * UTXO read-through cache moved to txdb-base.cpp under an anonymous namespace — backend-agnostic so RocksDB will get it for free. * GetAddressUtxos / GetAddressTxIds / SumUtxoValues moved to base, using NewIterator() instead of pdb->NewIterator(). No call-site changes — every existing CTxDB user keeps working exactly as before. Stack allocations like `CTxDB txdb("r")` still work because CTxDB remains a concrete, cheap-to-construct class. Behavior is bit-identical: same key serialization, same batch semantics, same LoadBlockIndex flow. Sets up M1.2 (factory + caller conversion to CTxDBBase&) and M1.3 (RocksDB backend) — neither requires touching consensus paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/CMakeLists.txt | 1 + src/txdb-base.cpp | 457 +++++++++++++++++++++++++++++++++++++ src/txdb-base.h | 211 +++++++++++++++++ src/txdb-leveldb.cpp | 532 +++++++------------------------------------ src/txdb-leveldb.h | 246 +++----------------- 5 files changed, 789 insertions(+), 658 deletions(-) create mode 100644 src/txdb-base.cpp create mode 100644 src/txdb-base.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 27cd6f3..26ba6be 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -73,6 +73,7 @@ set(CORE_SOURCES rpcrawtransaction.cpp rpcsmessage.cpp zmqpublishnotifier.cpp + txdb-base.cpp txdb-leveldb.cpp utxosnapshot.cpp lz4/lz4.c diff --git a/src/txdb-base.cpp b/src/txdb-base.cpp new file mode 100644 index 0000000..cbc1592 --- /dev/null +++ b/src/txdb-base.cpp @@ -0,0 +1,457 @@ +// Copyright (c) 2009-2012 The Bitcoin developers. +// Copyright (c) 2026 The Triangles developers. +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "txdb-base.h" + +#include "addressindex.h" +#include "main.h" +#include "sync.h" + +#include + +using namespace std; + +// ============================================================================ +// Schema versioning +// ============================================================================ +bool CTxDBBase::ReadVersion(int& nVersion) +{ + nVersion = 0; + return Read(string("version"), nVersion); +} + +bool CTxDBBase::WriteVersion(int nVersion) +{ + return Write(string("version"), nVersion); +} + +bool CTxDBBase::ReadDbFormat(int& nDbFormat) +{ + nDbFormat = 1; + return Read(string("dbformat"), nDbFormat); +} + +bool CTxDBBase::WriteDbFormat(int nDbFormat) +{ + return Write(string("dbformat"), nDbFormat); +} + +// ============================================================================ +// Tx index +// ============================================================================ +bool CTxDBBase::ReadTxIndex(uint256 hash, CTxIndex& txindex) +{ + assert(!fClient); + txindex.SetNull(); + return Read(make_pair(string("tx"), hash), txindex); +} + +bool CTxDBBase::UpdateTxIndex(uint256 hash, const CTxIndex& txindex) +{ + assert(!fClient); + return Write(make_pair(string("tx"), hash), txindex); +} + +bool CTxDBBase::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight) +{ + assert(!fClient); + uint256 hash = tx.GetHash(); + CTxIndex txindex(pos, tx.vout.size()); + return Write(make_pair(string("tx"), hash), txindex); +} + +bool CTxDBBase::EraseTxIndex(const CTransaction& tx) +{ + assert(!fClient); + uint256 hash = tx.GetHash(); + return Erase(make_pair(string("tx"), hash)); +} + +bool CTxDBBase::ContainsTx(uint256 hash) +{ + assert(!fClient); + return Exists(make_pair(string("tx"), hash)); +} + +bool CTxDBBase::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex) +{ + assert(!fClient); + tx.SetNull(); + if (!ReadTxIndex(hash, txindex)) + return false; + return tx.ReadFromDisk(txindex.pos); +} + +bool CTxDBBase::ReadDiskTx(uint256 hash, CTransaction& tx) +{ + CTxIndex txindex; + return ReadDiskTx(hash, tx, txindex); +} + +bool CTxDBBase::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex) +{ + return ReadDiskTx(outpoint.hash, tx, txindex); +} + +bool CTxDBBase::ReadDiskTx(COutPoint outpoint, CTransaction& tx) +{ + CTxIndex txindex; + return ReadDiskTx(outpoint.hash, tx, txindex); +} + +// ============================================================================ +// Block index +// ============================================================================ +bool CTxDBBase::WriteBlockIndex(const CDiskBlockIndex& blockindex) +{ + return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); +} + +// ============================================================================ +// Best chain / checkpoint metadata +// ============================================================================ +bool CTxDBBase::ReadHashBestChain(uint256& hashBestChain) +{ + return Read(string("hashBestChain"), hashBestChain); +} + +bool CTxDBBase::WriteHashBestChain(uint256 hashBestChain) +{ + return Write(string("hashBestChain"), hashBestChain); +} + +bool CTxDBBase::ReadAddressIndexBestChain(uint256& hashBestChain) +{ + return Read(string("addressIndexBestChain"), hashBestChain); +} + +bool CTxDBBase::WriteAddressIndexBestChain(uint256 hashBestChain) +{ + return Write(string("addressIndexBestChain"), hashBestChain); +} + +bool CTxDBBase::ReadAddressIndexStartHeight(int& nHeight) +{ + return Read(string("addressIndexStartHeight"), nHeight); +} + +bool CTxDBBase::WriteAddressIndexStartHeight(int nHeight) +{ + return Write(string("addressIndexStartHeight"), nHeight); +} + +bool CTxDBBase::ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust) +{ + return Read(string("bnBestInvalidTrust"), bnBestInvalidTrust); +} + +bool CTxDBBase::WriteBestInvalidTrust(CBigNum bnBestInvalidTrust) +{ + return Write(string("bnBestInvalidTrust"), bnBestInvalidTrust); +} + +bool CTxDBBase::ReadSyncCheckpoint(uint256& hashCheckpoint) +{ + return Read(string("hashSyncCheckpoint"), hashCheckpoint); +} + +bool CTxDBBase::WriteSyncCheckpoint(uint256 hashCheckpoint) +{ + return Write(string("hashSyncCheckpoint"), hashCheckpoint); +} + +bool CTxDBBase::ReadCheckpointPubKey(string& strPubKey) +{ + return Read(string("strCheckpointPubKey"), strPubKey); +} + +bool CTxDBBase::WriteCheckpointPubKey(const string& strPubKey) +{ + return Write(string("strCheckpointPubKey"), strPubKey); +} + +// ============================================================================ +// Address index +// ============================================================================ +bool CTxDBBase::ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance) +{ + return Read(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance); +} + +bool CTxDBBase::WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance) +{ + return Write(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance); +} + +bool CTxDBBase::ReadAddressUtxo(int nType, const uint160& hashBytes, + const uint256& txhash, int nIndex, + int64_t& nValue, int& nHeight) +{ + CAddressUtxoValue val; + if (!Read(make_pair(string("addrutxo"), + CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), val)) + return false; + nValue = val.nValue; + nHeight = val.nHeight; + return true; +} + +bool CTxDBBase::WriteAddressUtxo(int nType, const uint160& hashBytes, + const uint256& txhash, int nIndex, + int64_t nValue, int nHeight, const CScript& script) +{ + return Write(make_pair(string("addrutxo"), + CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), + CAddressUtxoValue(nValue, nHeight, script)); +} + +bool CTxDBBase::EraseAddressUtxo(int nType, const uint160& hashBytes, + const uint256& txhash, int nIndex) +{ + return Erase(make_pair(string("addrutxo"), + CAddressUtxoKey(nType, hashBytes, txhash, nIndex))); +} + +bool CTxDBBase::WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, + int nTxIndex, const uint256& txhash) +{ + return Write(make_pair(string("addrtxid"), + CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)), + (char)0); +} + +bool CTxDBBase::EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, + int nTxIndex, const uint256& txhash) +{ + return Erase(make_pair(string("addrtxid"), + CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash))); +} + +bool CTxDBBase::GetAddressUtxos(int nType, const uint160& hashBytes, + std::vector > >& vUtxos) +{ + vUtxos.clear(); + + CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); + ssKeyPrefix << make_pair(string("addrutxo"), + CAddressUtxoKey(nType, hashBytes, uint256(0), 0)); + string strPrefixBegin = ssKeyPrefix.str(); + + auto it = NewIterator(); + for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) + { + const string keyStr = it->KeyStr(); + CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(), + SER_DISK, CLIENT_VERSION); + string strKeyType; + CAddressUtxoKey utxoKey; + ssKey >> strKeyType; + if (strKeyType != "addrutxo") + break; + ssKey >> utxoKey; + if (utxoKey.nType != nType || utxoKey.hashBytes != hashBytes) + break; + + const string valueStr = it->ValueStr(); + CDataStream ssValue(valueStr.data(), valueStr.data() + valueStr.size(), + SER_DISK, CLIENT_VERSION); + CAddressUtxoValue utxoValue; + ssValue >> utxoValue; + + COutPoint outpoint(utxoKey.txhash, utxoKey.nIndex); + vUtxos.push_back(make_pair(outpoint, + make_pair(utxoValue.nValue, utxoValue.nHeight))); + } + return true; +} + +bool CTxDBBase::GetAddressTxIds(int nType, const uint160& hashBytes, + int nStartHeight, int nEndHeight, + std::vector& vTxIds) +{ + vTxIds.clear(); + + CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); + ssKeyPrefix << make_pair(string("addrtxid"), + CAddressTxIdKey(nType, hashBytes, nStartHeight, 0, uint256(0))); + string strPrefixBegin = ssKeyPrefix.str(); + + auto it = NewIterator(); + for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) + { + const string keyStr = it->KeyStr(); + CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(), + SER_DISK, CLIENT_VERSION); + string strKeyType; + CAddressTxIdKey txIdKey; + ssKey >> strKeyType; + if (strKeyType != "addrtxid") + break; + ssKey >> txIdKey; + if (txIdKey.nType != nType || txIdKey.hashBytes != hashBytes) + break; + if (txIdKey.nHeight > nEndHeight) + break; + + vTxIds.push_back(txIdKey.txhash); + } + return true; +} + +// ============================================================================ +// In-memory UTXO cache (read-through, backend-agnostic) +// +// Avoids hitting the underlying KV store for every FetchInputs call. On a 2M+ +// block chain with millions of UTXOs, this dramatically reduces I/O during +// both IBD (ConnectBlock validation reads inputs) and steady-state (mempool +// acceptance, staking). Writes/erases update both cache and the backend. +// ============================================================================ +namespace { + +struct COutPointHasher { + size_t operator()(const COutPoint& op) const { + return op.hash.Get64() ^ + (std::hash()(op.n) * 0x9e3779b97f4a7c15ULL); + } +}; + +struct CUtxoCacheEntry { + CUtxoEntry utxo; + bool fPresent; // true = exists, false = known absent (negative cache) + CUtxoCacheEntry() : fPresent(false) {} + CUtxoCacheEntry(const CUtxoEntry& u, bool p) : utxo(u), fPresent(p) {} +}; + +std::unordered_map g_mapUtxoCache; +CCriticalSection g_cs_utxoCache; +const size_t UTXO_CACHE_MAX_ENTRIES = 2000000; // ~400MB at ~200 bytes each + +} // anonymous namespace + +bool CTxDBBase::ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry) +{ + entry.SetNull(); + COutPoint outpoint(hash, n); + + { + LOCK(g_cs_utxoCache); + auto it = g_mapUtxoCache.find(outpoint); + if (it != g_mapUtxoCache.end()) + { + if (it->second.fPresent) { + entry = it->second.utxo; + return true; + } + return false; + } + } + + bool fFound = Read(make_pair(string("u"), make_pair(hash, n)), entry); + + { + LOCK(g_cs_utxoCache); + if (g_mapUtxoCache.size() < UTXO_CACHE_MAX_ENTRIES) + { + if (fFound) + g_mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true); + else + g_mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false); + } + } + + return fFound; +} + +bool CTxDBBase::WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry) +{ + COutPoint outpoint(hash, n); + + { + LOCK(g_cs_utxoCache); + g_mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true); + + // Periodic eviction: clear half when over the limit. Simple but + // effective — the cache repopulates with the hot working set. + if (g_mapUtxoCache.size() > UTXO_CACHE_MAX_ENTRIES) + { + size_t nTarget = UTXO_CACHE_MAX_ENTRIES / 2; + auto it = g_mapUtxoCache.begin(); + while (g_mapUtxoCache.size() > nTarget && it != g_mapUtxoCache.end()) + it = g_mapUtxoCache.erase(it); + } + } + + return Write(make_pair(string("u"), make_pair(hash, n)), entry); +} + +bool CTxDBBase::EraseUtxo(const uint256& hash, unsigned int n) +{ + COutPoint outpoint(hash, n); + + { + LOCK(g_cs_utxoCache); + g_mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false); + } + + return Erase(make_pair(string("u"), make_pair(hash, n))); +} + +bool CTxDBBase::HaveUtxo(const uint256& hash, unsigned int n) +{ + COutPoint outpoint(hash, n); + + { + LOCK(g_cs_utxoCache); + auto it = g_mapUtxoCache.find(outpoint); + if (it != g_mapUtxoCache.end()) + return it->second.fPresent; + } + + if (Exists(make_pair(string("u"), make_pair(hash, n)))) + return true; + + // Lazy fallback: check old CTxIndex vSpent for databases upgrading from + // pre-UTXO format. vSpent[n] null = output not spent = UTXO exists. + CTxIndex txindex; + if (ReadTxIndex(hash, txindex)) + { + if (n < txindex.vSpent.size() && txindex.vSpent[n].IsNull()) + return true; + } + + return false; +} + +int64_t CTxDBBase::SumUtxoValues(int& nCount) +{ + nCount = 0; + int64_t nTotal = 0; + + CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); + ssKeyPrefix << make_pair(string("u"), make_pair(uint256(0), (unsigned int)0)); + string strPrefixBegin = ssKeyPrefix.str(); + + auto it = NewIterator(); + for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) + { + const string keyStr = it->KeyStr(); + CDataStream ssKey(keyStr.data(), keyStr.data() + keyStr.size(), + SER_DISK, CLIENT_VERSION); + string strKeyType; + ssKey >> strKeyType; + if (strKeyType != "u") + break; + + const string valueStr = it->ValueStr(); + CDataStream ssValue(valueStr.data(), valueStr.data() + valueStr.size(), + SER_DISK, CLIENT_VERSION); + CUtxoEntry entry; + ssValue >> entry; + + nTotal += entry.nValue; + nCount++; + } + return nTotal; +} diff --git a/src/txdb-base.h b/src/txdb-base.h new file mode 100644 index 0000000..46a164d --- /dev/null +++ b/src/txdb-base.h @@ -0,0 +1,211 @@ +// Copyright (c) 2009-2012 The Bitcoin developers. +// Copyright (c) 2026 The Triangles developers. +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef TRIANGLES_TXDB_BASE_H +#define TRIANGLES_TXDB_BASE_H + +#include "main.h" + +#include +#include +#include +#include +#include + +class CScript; +class CTransaction; +class CDiskTxPos; +class CTxIndex; +class CDiskBlockIndex; +class CUtxoEntry; +class CBigNum; + +// ---------------------------------------------------------------------------- +// Backend-agnostic key/value iterator. +// +// Each CTxDBBase backend returns a std::unique_ptr from +// NewIterator(). Iterators yield raw serialized key/value bytes; callers +// deserialize using the same SER_DISK / CLIENT_VERSION conventions used by +// CTxDBBase's templated Read/Write paths. +// +// Iterators do NOT see uncommitted writes in an active batch. All current +// iteration sites (block-index scan, address-index range queries, UTXO sum) +// run outside transactions, so this is safe. +// ---------------------------------------------------------------------------- +class CTxDBIteratorBase +{ +public: + virtual ~CTxDBIteratorBase() = default; + + virtual void Seek(const std::string& key) = 0; + virtual bool Valid() const = 0; + virtual void Next() = 0; + virtual std::string KeyStr() const = 0; + virtual std::string ValueStr() const = 0; +}; + +// ---------------------------------------------------------------------------- +// Abstract chain database interface. +// +// All key/value serialization happens in this base class via CDataStream with +// SER_DISK / CLIENT_VERSION. Backends only implement byte-level I/O, so every +// backend produces bit-identical key bytes — required for migration and +// dual-backend parity testing. +// +// Named operations (ReadTxIndex, WriteBlockIndex, etc.) are implemented in +// terms of the templated Read/Write/Erase/Exists, which dispatch to the +// virtual byte-level methods. To add a new backend: +// +// 1. Subclass CTxDBBase. +// 2. Implement Close, TxnBegin/Commit/Abort. +// 3. Implement ReadRaw, WriteRaw, EraseRaw, ExistsRaw. +// 4. Implement NewIterator (return a subclass of CTxDBIteratorBase). +// 5. Implement LoadBlockIndex (still backend-specific in M1; will be +// extracted to the base in a later phase). +// ---------------------------------------------------------------------------- +class CTxDBBase +{ +public: + virtual ~CTxDBBase() = default; + + // Destroys the underlying shared global state accessed by this DB. + virtual void Close() = 0; + + // Batches (transaction-like atomic groups of writes/deletes). + virtual bool TxnBegin() = 0; + virtual bool TxnCommit() = 0; + virtual bool TxnAbort() = 0; + + bool IsReadOnly() const { return fReadOnly; } + + // ── Schema versioning ──────────────────────────────────────────────────── + bool ReadVersion(int& nVersion); + bool WriteVersion(int nVersion); + bool ReadDbFormat(int& nDbFormat); + bool WriteDbFormat(int nDbFormat); + + // ── Tx index ───────────────────────────────────────────────────────────── + bool ReadTxIndex(uint256 hash, CTxIndex& txindex); + bool UpdateTxIndex(uint256 hash, const CTxIndex& txindex); + bool AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight); + bool EraseTxIndex(const CTransaction& tx); + bool ContainsTx(uint256 hash); + bool ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex); + bool ReadDiskTx(uint256 hash, CTransaction& tx); + bool ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex); + bool ReadDiskTx(COutPoint outpoint, CTransaction& tx); + + // ── Block index ────────────────────────────────────────────────────────── + bool WriteBlockIndex(const CDiskBlockIndex& blockindex); + + // ── Best chain / checkpoint metadata ───────────────────────────────────── + bool ReadHashBestChain(uint256& hashBestChain); + bool WriteHashBestChain(uint256 hashBestChain); + bool ReadAddressIndexBestChain(uint256& hashBestChain); + bool WriteAddressIndexBestChain(uint256 hashBestChain); + bool ReadAddressIndexStartHeight(int& nHeight); + bool WriteAddressIndexStartHeight(int nHeight); + bool ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust); + bool WriteBestInvalidTrust(CBigNum bnBestInvalidTrust); + bool ReadSyncCheckpoint(uint256& hashCheckpoint); + bool WriteSyncCheckpoint(uint256 hashCheckpoint); + bool ReadCheckpointPubKey(std::string& strPubKey); + bool WriteCheckpointPubKey(const std::string& strPubKey); + + virtual bool LoadBlockIndex() = 0; + + // ── Address index ──────────────────────────────────────────────────────── + bool ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance); + bool WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance); + bool ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, + int nIndex, int64_t& nValue, int& nHeight); + bool WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, + int nIndex, int64_t nValue, int nHeight, const CScript& script); + bool EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, + int nIndex); + bool WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, + int nTxIndex, const uint256& txhash); + bool EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, + int nTxIndex, const uint256& txhash); + bool GetAddressUtxos(int nType, const uint160& hashBytes, + std::vector > >& vUtxos); + bool GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight, + int nEndHeight, std::vector& vTxIds); + + // ── UTXO set ───────────────────────────────────────────────────────────── + bool ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry); + bool WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry); + bool EraseUtxo(const uint256& hash, unsigned int n); + bool HaveUtxo(const uint256& hash, unsigned int n); + int64_t SumUtxoValues(int& nCount); + +protected: + bool fReadOnly = false; + + // Byte-level I/O — backends implement these. + virtual bool ReadRaw(const std::string& key, std::string& value) const = 0; + virtual bool WriteRaw(const std::string& key, const std::string& value) = 0; + virtual bool EraseRaw(const std::string& key) = 0; + virtual bool ExistsRaw(const std::string& key) const = 0; + virtual std::unique_ptr NewIterator() const = 0; + + // Templated Read/Write/Erase/Exists are non-virtual (templates can't be + // virtual in C++) — they serialize and dispatch to the byte-level virtuals. + template + bool Read(const K& key, T& value) const + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + std::string strValue; + if (!ReadRaw(ssKey.str(), strValue)) + return false; + try { + CDataStream ssValue(strValue.data(), + strValue.data() + strValue.size(), + SER_DISK, CLIENT_VERSION); + ssValue >> value; + } catch (std::exception&) { + return false; + } + return true; + } + + template + bool Write(const K& key, const T& value) + { + if (fReadOnly) + assert(!"Write called on database in read-only mode"); + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + ssValue.reserve(10000); + ssValue << value; + return WriteRaw(ssKey.str(), ssValue.str()); + } + + template + bool Erase(const K& key) + { + if (fReadOnly) + assert(!"Erase called on database in read-only mode"); + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + return EraseRaw(ssKey.str()); + } + + template + bool Exists(const K& key) const + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(1000); + ssKey << key; + return ExistsRaw(ssKey.str()); + } +}; + +#endif // TRIANGLES_TXDB_BASE_H diff --git a/src/txdb-leveldb.cpp b/src/txdb-leveldb.cpp index b8ac798..d1dfa87 100644 --- a/src/txdb-leveldb.cpp +++ b/src/txdb-leveldb.cpp @@ -4,7 +4,6 @@ // file license.txt or http://www.opensource.org/licenses/mit-license.php. #include -#include #include #include @@ -41,29 +40,22 @@ static leveldb::Options GetOptions() { // memtable flushes and compactions, which is a big win during IBD // when millions of tx index entries are written sequentially. options.write_buffer_size = 64 * 1048576; - // Allow more open files for better read performance on large chains options.max_open_files = 1000; return options; } void init_blockindex(leveldb::Options& options, bool fRemoveOld = false) { - // First time init. fs::path directory = GetDataDir() / "txleveldb"; if (fRemoveOld) { - fs::remove_all(directory); // remove directory + fs::remove_all(directory); unsigned int nFile = 1; - while (true) { fs::path strBlockFile = GetDataDir() / strprintf("blk%04u.dat", nFile); - - // Break if no such file - if( !fs::exists( strBlockFile ) ) + if(!fs::exists(strBlockFile)) break; - fs::remove(strBlockFile); - nFile++; } } @@ -76,8 +68,6 @@ void init_blockindex(leveldb::Options& options, bool fRemoveOld = false) { } } -// CDB subclasses are created and destroyed VERY OFTEN. That's why -// we shouldn't treat this as a free operations. CTxDB::CTxDB(const char* pszMode) { assert(pszMode); @@ -95,7 +85,7 @@ CTxDB::CTxDB(const char* pszMode) options.create_if_missing = fCreate; options.filter_policy = leveldb::NewBloomFilterPolicy(10); - init_blockindex(options); // Init directory + init_blockindex(options); pdb = txdb; if (Exists(string("version"))) @@ -107,18 +97,17 @@ CTxDB::CTxDB(const char* pszMode) { printf("Required index version is %d, removing old database\n", DATABASE_VERSION); - // Leveldb instance destruction delete txdb; txdb = pdb = NULL; delete activeBatch; activeBatch = NULL; - init_blockindex(options, true); // Remove directory and create new database + init_blockindex(options, true); pdb = txdb; bool fTmp = fReadOnly; fReadOnly = false; - WriteVersion(DATABASE_VERSION); // Save transaction index version + WriteVersion(DATABASE_VERSION); fReadOnly = fTmp; } } @@ -171,6 +160,8 @@ bool CTxDB::TxnCommit() return true; } +namespace { + class CBatchScanner : public leveldb::WriteBatch::Handler { public: std::string needle; @@ -196,16 +187,32 @@ public: } }; -// When performing a read, if we have an active batch we need to check it first -// before reading from the database, as the rest of the code assumes that once -// a database transaction begins reads are consistent with it. It would be good -// to change that assumption in future and avoid the performance hit, though in -// practice it does not appear to be large. -bool CTxDB::ScanBatch(const CDataStream &key, string *value, bool *deleted) const { +class CLevelDBIterator final : public CTxDBIteratorBase { +public: + explicit CLevelDBIterator(leveldb::Iterator* pit) : pit(pit) {} + ~CLevelDBIterator() override { delete pit; } + + void Seek(const std::string& key) override { pit->Seek(key); } + bool Valid() const override { return pit->Valid(); } + void Next() override { pit->Next(); } + std::string KeyStr() const override { return pit->key().ToString(); } + std::string ValueStr() const override { return pit->value().ToString(); } + +private: + leveldb::Iterator* pit; +}; + +} // anonymous namespace + +// When performing a read with an active batch, check the batch first. The +// rest of the codebase assumes that once a batch is open, reads are +// consistent with the pending writes inside it. +bool CTxDB::ScanBatch(const std::string& key, string* value, bool* deleted) const +{ assert(activeBatch); *deleted = false; CBatchScanner scanner; - scanner.needle = key.str(); + scanner.needle = key; scanner.deleted = deleted; scanner.foundValue = value; leveldb::Status status = activeBatch->Iterate(&scanner); @@ -215,132 +222,71 @@ bool CTxDB::ScanBatch(const CDataStream &key, string *value, bool *deleted) cons return scanner.foundEntry; } -bool CTxDB::ReadTxIndex(uint256 hash, CTxIndex& txindex) +bool CTxDB::ReadRaw(const std::string& key, std::string& value) const { - assert(!fClient); - txindex.SetNull(); - return Read(make_pair(string("tx"), hash), txindex); + bool readFromDb = true; + if (activeBatch) { + bool deleted = false; + readFromDb = ScanBatch(key, &value, &deleted) == false; + if (deleted) + return false; + } + if (readFromDb) { + leveldb::Status status = pdb->Get(leveldb::ReadOptions(), key, &value); + if (!status.ok()) { + if (status.IsNotFound()) + return false; + printf("LevelDB read failure: %s\n", status.ToString().c_str()); + return false; + } + } + return true; } -bool CTxDB::UpdateTxIndex(uint256 hash, const CTxIndex& txindex) +bool CTxDB::WriteRaw(const std::string& key, const std::string& value) { - assert(!fClient); - return Write(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight) -{ - assert(!fClient); - - // Add to tx index - uint256 hash = tx.GetHash(); - CTxIndex txindex(pos, tx.vout.size()); - return Write(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::EraseTxIndex(const CTransaction& tx) -{ - assert(!fClient); - uint256 hash = tx.GetHash(); - - return Erase(make_pair(string("tx"), hash)); -} - -bool CTxDB::ContainsTx(uint256 hash) -{ - assert(!fClient); - return Exists(make_pair(string("tx"), hash)); -} - -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex) -{ - assert(!fClient); - tx.SetNull(); - if (!ReadTxIndex(hash, txindex)) + if (activeBatch) { + activeBatch->Put(key, value); + return true; + } + leveldb::Status status = pdb->Put(leveldb::WriteOptions(), key, value); + if (!status.ok()) { + printf("LevelDB write failure: %s\n", status.ToString().c_str()); return false; - return (tx.ReadFromDisk(txindex.pos)); + } + return true; } -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx) +bool CTxDB::EraseRaw(const std::string& key) { - CTxIndex txindex; - return ReadDiskTx(hash, tx, txindex); + if (!pdb) + return false; + if (activeBatch) { + activeBatch->Delete(key); + return true; + } + leveldb::Status status = pdb->Delete(leveldb::WriteOptions(), key); + return (status.ok() || status.IsNotFound()); } -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex) +bool CTxDB::ExistsRaw(const std::string& key) const { - return ReadDiskTx(outpoint.hash, tx, txindex); + std::string unused; + + if (activeBatch) { + bool deleted = false; + if (ScanBatch(key, &unused, &deleted) && !deleted) + return true; + } + + leveldb::Status status = pdb->Get(leveldb::ReadOptions(), key, &unused); + return status.IsNotFound() == false; } -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx) +std::unique_ptr CTxDB::NewIterator() const { - CTxIndex txindex; - return ReadDiskTx(outpoint.hash, tx, txindex); -} - -bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) -{ - return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); -} - -bool CTxDB::ReadHashBestChain(uint256& hashBestChain) -{ - return Read(string("hashBestChain"), hashBestChain); -} - -bool CTxDB::WriteHashBestChain(uint256 hashBestChain) -{ - return Write(string("hashBestChain"), hashBestChain); -} - -bool CTxDB::ReadAddressIndexBestChain(uint256& hashBestChain) -{ - return Read(string("addressIndexBestChain"), hashBestChain); -} - -bool CTxDB::WriteAddressIndexBestChain(uint256 hashBestChain) -{ - return Write(string("addressIndexBestChain"), hashBestChain); -} - -bool CTxDB::ReadAddressIndexStartHeight(int& nHeight) -{ - return Read(string("addressIndexStartHeight"), nHeight); -} - -bool CTxDB::WriteAddressIndexStartHeight(int nHeight) -{ - return Write(string("addressIndexStartHeight"), nHeight); -} - -bool CTxDB::ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust) -{ - return Read(string("bnBestInvalidTrust"), bnBestInvalidTrust); -} - -bool CTxDB::WriteBestInvalidTrust(CBigNum bnBestInvalidTrust) -{ - return Write(string("bnBestInvalidTrust"), bnBestInvalidTrust); -} - -bool CTxDB::ReadSyncCheckpoint(uint256& hashCheckpoint) -{ - return Read(string("hashSyncCheckpoint"), hashCheckpoint); -} - -bool CTxDB::WriteSyncCheckpoint(uint256 hashCheckpoint) -{ - return Write(string("hashSyncCheckpoint"), hashCheckpoint); -} - -bool CTxDB::ReadCheckpointPubKey(string& strPubKey) -{ - return Read(string("strCheckpointPubKey"), strPubKey); -} - -bool CTxDB::WriteCheckpointPubKey(const string& strPubKey) -{ - return Write(string("strCheckpointPubKey"), strPubKey); + return std::unique_ptr( + new CLevelDBIterator(pdb->NewIterator(leveldb::ReadOptions()))); } static CBlockIndex *InsertBlockIndex(uint256 hash) @@ -348,12 +294,10 @@ static CBlockIndex *InsertBlockIndex(uint256 hash) if (hash == 0) return NULL; - // Return existing map::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) return (*mi).second; - // Create new CBlockIndex* pindexNew = new CBlockIndex(); if (!pindexNew) throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); @@ -366,8 +310,7 @@ static CBlockIndex *InsertBlockIndex(uint256 hash) bool CTxDB::LoadBlockIndex() { if (mapBlockIndex.size() > 0) { - // Already loaded once in this session. It can happen during migration - // from BDB. + // Already loaded once in this session. Can happen during BDB migration. return true; } @@ -377,39 +320,32 @@ bool CTxDB::LoadBlockIndex() CDiskBlockIndex::fSerializeChainTrust = (nDbFormat >= 2); if (CDiskBlockIndex::fSerializeChainTrust) - printf("LoadBlockIndex(): DB format v%d — nChainTrust persisted\n", nDbFormat); + printf("LoadBlockIndex(): DB format v%d - nChainTrust persisted\n", nDbFormat); else - printf("LoadBlockIndex(): DB format v%d — will recalculate nChainTrust (one-time upgrade)\n", nDbFormat); + printf("LoadBlockIndex(): DB format v%d - will recalculate nChainTrust (one-time upgrade)\n", nDbFormat); - // The block index is an in-memory structure that maps hashes to on-disk - // locations where the contents of the block can be found. Here, we scan it - // out of the DB and into mapBlockIndex. + // Scan the block index out of the DB into mapBlockIndex. int64_t nPhaseStart = GetTimeMillis(); int64_t nTotalStart = nPhaseStart; leveldb::Iterator *iterator = pdb->NewIterator(leveldb::ReadOptions()); - // Seek to start key. CDataStream ssStartKey(SER_DISK, CLIENT_VERSION); ssStartKey << make_pair(string("blockindex"), uint256(0)); iterator->Seek(ssStartKey.str()); - // Now read each entry. int nBlocksLoaded = 0; while (iterator->Valid()) { - // Report progress every 100k blocks if (++nBlocksLoaded % 100000 == 0) { std::string strMsg = strprintf(_("Loading block index... (%d blocks)"), nBlocksLoaded); uiInterface.InitMessage(strMsg); } - // Unpack keys and values. CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.write(iterator->key().data(), iterator->key().size()); CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.write(iterator->value().data(), iterator->value().size()); string strType; ssKey >> strType; - // Did we reach the end of the data to read? if (fRequestShutdown || strType != "blockindex") break; CDiskBlockIndex diskindex; @@ -417,7 +353,6 @@ bool CTxDB::LoadBlockIndex() uint256 blockHash = diskindex.GetBlockHash(); - // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(blockHash); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); @@ -436,10 +371,8 @@ bool CTxDB::LoadBlockIndex() pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; - // nChainTrust is populated from disk if fSerializeChainTrust, else stays 0 pindexNew->nChainTrust = diskindex.nChainTrust; - // Watch for genesis block if (pindexGenesisBlock == NULL && blockHash == (!fTestNet ? hashGenesisBlockOfficial : hashGenesisBlockTestNet)) pindexGenesisBlock = pindexNew; @@ -448,8 +381,6 @@ bool CTxDB::LoadBlockIndex() return error("LoadBlockIndex() : CheckIndex failed at %d", pindexNew->nHeight); } - // setStakeSeen is populated below for recent blocks only (Change D) - iterator->Next(); } delete iterator; @@ -513,7 +444,6 @@ bool CTxDB::LoadBlockIndex() ssValue << diskindex; batch.Put(ssKey.str(), ssValue.str()); - // Flush in chunks to limit memory usage if (++nCount % 100000 == 0) { pdb->Write(leveldb::WriteOptions(), &batch); @@ -521,7 +451,6 @@ bool CTxDB::LoadBlockIndex() printf("LoadBlockIndex(): upgraded %d / %d block index entries\n", nCount, (int)vSortedByHeight.size()); } } - // Write remaining entries + format version CDataStream ssFmtKey(SER_DISK, CLIENT_VERSION); ssFmtKey << string("dbformat"); CDataStream ssFmtValue(SER_DISK, CLIENT_VERSION); @@ -536,8 +465,6 @@ bool CTxDB::LoadBlockIndex() } else { - // nChainTrust was loaded from disk. Only need stake modifier checksums - // for blocks above the last checkpoint (typically very few or zero). int nLastCheckpointHeight = Checkpoints::GetTotalBlocksEstimate(); bool fNeedModifierCheck = false; for (const auto& item : mapBlockIndex) @@ -569,16 +496,12 @@ bool CTxDB::LoadBlockIndex() printf("STARTUP-PERF: chain_trust_and_modifiers %" PRId64 "ms\n", GetTimeMillis() - nPhaseStart); - // Bump dbformat to 3 if needed (databases that already had v2 nChainTrust upgrade). - // UTXO entries are written by ConnectBlock during normal sync. For databases upgrading - // from older versions, FetchInputs has a lazy fallback to the old CTxIndex path. if (nDbFormat < 3) { WriteDbFormat(3); printf("LoadBlockIndex(): bumped dbformat to v3 (UTXO model with lazy fallback)\n"); } - // Load hashBestChain pointer to end of best chain nPhaseStart = GetTimeMillis(); if (!ReadHashBestChain(hashBestChain)) { @@ -594,7 +517,6 @@ bool CTxDB::LoadBlockIndex() printf("STARTUP-PERF: best_chain %" PRId64 "ms\n", GetTimeMillis() - nPhaseStart); - // ---- setStakeSeen: only populate for recent blocks (DoS protection) ---- nPhaseStart = GetTimeMillis(); { int nStakeSeenDepth = 500; @@ -617,7 +539,6 @@ bool CTxDB::LoadBlockIndex() DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); // Re-evaluate best chain: scan for competing tips with equal or greater trust. - // This fixes nodes stuck on the wrong fork after consensus rule changes. { CBlockIndex* pindexBetter = NULL; for (const auto& item : mapBlockIndex) @@ -660,29 +581,25 @@ bool CTxDB::LoadBlockIndex() } } - // triangles: load hashSyncCheckpoint (best-effort, non-fatal) if (!ReadSyncCheckpoint(Checkpoints::hashSyncCheckpoint)) printf("LoadBlockIndex(): no sync checkpoint in DB, using default\n"); else printf("LoadBlockIndex(): synchronized checkpoint %s\n", Checkpoints::hashSyncCheckpoint.ToString().c_str()); - // If the stored checkpoint isn't in our index, reset to genesis so we don't assert-crash if (!mapBlockIndex.count(Checkpoints::hashSyncCheckpoint)) { printf("LoadBlockIndex(): sync checkpoint not in index, resetting to genesis\n"); Checkpoints::hashSyncCheckpoint = (!fTestNet ? hashGenesisBlockOfficial : hashGenesisBlockTestNet); } - // Load bnBestInvalidTrust, OK if it doesn't exist CBigNum bnBestInvalidTrust; ReadBestInvalidTrust(bnBestInvalidTrust); nBestInvalidTrust = bnBestInvalidTrust.getuint256(); - // Verify blocks in the best chain nPhaseStart = GetTimeMillis(); int nCheckLevel = GetArg("-checklevel", 1); int nCheckDepth = GetArg( "-checkblocks", 50); if (nCheckDepth == 0) - nCheckDepth = 1000000000; // suffices until the year 19000 + nCheckDepth = 1000000000; if (nCheckDepth > nBestHeight) nCheckDepth = nBestHeight; printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); @@ -695,14 +612,11 @@ bool CTxDB::LoadBlockIndex() CBlock block; if (!block.ReadFromDisk(pindex)) return error("LoadBlockIndex() : block.ReadFromDisk failed"); - // check level 1: verify block validity - // check level 7: verify block signature too if (nCheckLevel>0 && !block.CheckBlock(true, true, (nCheckLevel>6))) { printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); pindexFork = pindex->pprev; } - // check level 2: verify transaction index validity if (nCheckLevel>1) { pair pos = make_pair(pindex->nFile, pindex->nBlockPos); @@ -713,10 +627,8 @@ bool CTxDB::LoadBlockIndex() CTxIndex txindex; if (ReadTxIndex(hashTx, txindex)) { - // check level 3: checker transaction hashes if (nCheckLevel>2 || pindex->nFile != txindex.pos.nFile || pindex->nBlockPos != txindex.pos.nBlockPos) { - // either an error or a duplicate transaction CTransaction txFound; if (!txFound.ReadFromDisk(txindex.pos)) { @@ -724,13 +636,12 @@ bool CTxDB::LoadBlockIndex() pindexFork = pindex->pprev; } else - if (txFound.GetHash() != hashTx) // not a duplicate tx + if (txFound.GetHash() != hashTx) { printf("LoadBlockIndex(): *** invalid tx position for %s\n", hashTx.ToString().c_str()); pindexFork = pindex->pprev; } } - // check level 4: verify spent inputs were removed from UTXO set if (nCheckLevel>3 && !tx.IsCoinBase()) { for (const CTxIn &txin : tx.vin) @@ -749,7 +660,6 @@ bool CTxDB::LoadBlockIndex() } if (pindexFork && !fRequestShutdown) { - // Reorg back to the fork printf("LoadBlockIndex() : *** moving best chain pointer back to block %d\n", pindexFork->nHeight); CBlock block; if (!block.ReadFromDisk(pindexFork)) @@ -762,271 +672,3 @@ bool CTxDB::LoadBlockIndex() return true; } - -// ============================================================================ -// Address index methods -// ============================================================================ - -bool CTxDB::ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance) -{ - return Read(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance); -} - -bool CTxDB::WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance) -{ - return Write(make_pair(string("addrbal"), CAddressBalanceKey(nType, hashBytes)), nBalance); -} - -bool CTxDB::ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t& nValue, int& nHeight) -{ - CAddressUtxoValue val; - if (!Read(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), val)) - return false; - nValue = val.nValue; - nHeight = val.nHeight; - return true; -} - -bool CTxDB::WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t nValue, int nHeight, const CScript& script) -{ - return Write(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex)), - CAddressUtxoValue(nValue, nHeight, script)); -} - -bool CTxDB::EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex) -{ - return Erase(make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, txhash, nIndex))); -} - -bool CTxDB::WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash) -{ - return Write(make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash)), (char)0); -} - -bool CTxDB::EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash) -{ - return Erase(make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nHeight, nTxIndex, txhash))); -} - -bool CTxDB::GetAddressUtxos(int nType, const uint160& hashBytes, std::vector > >& vUtxos) -{ - vUtxos.clear(); - - // Build the key prefix to seek to - CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); - ssKeyPrefix << make_pair(string("addrutxo"), CAddressUtxoKey(nType, hashBytes, uint256(0), 0)); - std::string strPrefixBegin = ssKeyPrefix.str(); - - leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions()); - for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) - { - // Deserialize the key - CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION); - std::string strKeyType; - CAddressUtxoKey utxoKey; - ssKey >> strKeyType; - if (strKeyType != "addrutxo") - break; - ssKey >> utxoKey; - if (utxoKey.nType != nType || utxoKey.hashBytes != hashBytes) - break; - - // Deserialize the value - CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION); - CAddressUtxoValue utxoValue; - ssValue >> utxoValue; - - COutPoint outpoint(utxoKey.txhash, utxoKey.nIndex); - vUtxos.push_back(make_pair(outpoint, make_pair(utxoValue.nValue, utxoValue.nHeight))); - } - delete it; - return true; -} - -bool CTxDB::GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight, int nEndHeight, std::vector& vTxIds) -{ - vTxIds.clear(); - - // Build the key prefix to seek to - CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); - ssKeyPrefix << make_pair(string("addrtxid"), CAddressTxIdKey(nType, hashBytes, nStartHeight, 0, uint256(0))); - std::string strPrefixBegin = ssKeyPrefix.str(); - - leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions()); - for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) - { - CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION); - std::string strKeyType; - CAddressTxIdKey txIdKey; - ssKey >> strKeyType; - if (strKeyType != "addrtxid") - break; - ssKey >> txIdKey; - if (txIdKey.nType != nType || txIdKey.hashBytes != hashBytes) - break; - if (txIdKey.nHeight > nEndHeight) - break; - - vTxIds.push_back(txIdKey.txhash); - } - delete it; - return true; -} - -// ---------- In-memory UTXO cache ---------- -// -// Read-through cache that avoids hitting LevelDB for every FetchInputs call. -// On a 2M+ block chain with millions of UTXOs, this dramatically reduces I/O -// during both IBD (ConnectBlock validation reads inputs) and normal operation -// (mempool acceptance, staking). Writes/erases update both cache and LevelDB. - -struct COutPointHasher { - size_t operator()(const COutPoint& op) const { - // Mix the lower 64 bits of the hash with the output index - return op.hash.Get64() ^ (std::hash()(op.n) * 0x9e3779b97f4a7c15ULL); - } -}; - -// Cache entry: the UTXO data plus a flag indicating "known absent from DB" -struct CUtxoCacheEntry { - CUtxoEntry utxo; - bool fPresent; // true = UTXO exists, false = known deleted/absent - CUtxoCacheEntry() : fPresent(false) {} - CUtxoCacheEntry(const CUtxoEntry& u, bool p) : utxo(u), fPresent(p) {} -}; - -static std::unordered_map mapUtxoCache; -static CCriticalSection cs_utxoCache; -static const size_t UTXO_CACHE_MAX_ENTRIES = 2000000; // ~400MB at ~200 bytes each - -// ---------- UTXO database methods ---------- - -bool CTxDB::ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry) -{ - entry.SetNull(); - COutPoint outpoint(hash, n); - - { - LOCK(cs_utxoCache); - auto it = mapUtxoCache.find(outpoint); - if (it != mapUtxoCache.end()) - { - if (it->second.fPresent) { - entry = it->second.utxo; - return true; - } - return false; // cached as absent - } - } - - // Cache miss — read from LevelDB - bool fFound = Read(make_pair(string("u"), make_pair(hash, n)), entry); - - { - LOCK(cs_utxoCache); - // Only cache if under limit (don't evict here — eviction is periodic) - if (mapUtxoCache.size() < UTXO_CACHE_MAX_ENTRIES) - { - if (fFound) - mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true); - else - mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false); - } - } - - return fFound; -} - -bool CTxDB::WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry) -{ - COutPoint outpoint(hash, n); - - { - LOCK(cs_utxoCache); - mapUtxoCache[outpoint] = CUtxoCacheEntry(entry, true); - - // Periodic eviction: if cache is over limit, clear half of it. - // This is a simple but effective strategy — the cache will quickly - // repopulate with the hot working set. - if (mapUtxoCache.size() > UTXO_CACHE_MAX_ENTRIES) - { - size_t nTarget = UTXO_CACHE_MAX_ENTRIES / 2; - auto it = mapUtxoCache.begin(); - while (mapUtxoCache.size() > nTarget && it != mapUtxoCache.end()) - it = mapUtxoCache.erase(it); - } - } - - return Write(make_pair(string("u"), make_pair(hash, n)), entry); -} - -bool CTxDB::EraseUtxo(const uint256& hash, unsigned int n) -{ - COutPoint outpoint(hash, n); - - { - LOCK(cs_utxoCache); - // Mark as absent in cache (negative cache) so future reads don't hit DB - mapUtxoCache[outpoint] = CUtxoCacheEntry(CUtxoEntry(), false); - } - - return Erase(make_pair(string("u"), make_pair(hash, n))); -} - -bool CTxDB::HaveUtxo(const uint256& hash, unsigned int n) -{ - COutPoint outpoint(hash, n); - - { - LOCK(cs_utxoCache); - auto it = mapUtxoCache.find(outpoint); - if (it != mapUtxoCache.end()) - return it->second.fPresent; - } - - if (Exists(make_pair(string("u"), make_pair(hash, n)))) - return true; - - // Lazy fallback: check old CTxIndex vSpent for databases upgrading from pre-UTXO format - CTxIndex txindex; - if (ReadTxIndex(hash, txindex)) - { - if (n < txindex.vSpent.size() && txindex.vSpent[n].IsNull()) - return true; // vSpent[n] is null = output NOT spent = UTXO exists - } - - return false; -} - -int64_t CTxDB::SumUtxoValues(int& nCount) -{ - nCount = 0; - int64_t nTotal = 0; - - // Seek to the start of UTXO entries (key prefix "u") - CDataStream ssKeyPrefix(SER_DISK, CLIENT_VERSION); - ssKeyPrefix << make_pair(string("u"), make_pair(uint256(0), (unsigned int)0)); - std::string strPrefixBegin = ssKeyPrefix.str(); - - leveldb::Iterator* it = pdb->NewIterator(leveldb::ReadOptions()); - for (it->Seek(strPrefixBegin); it->Valid(); it->Next()) - { - // Check key prefix is still "u" - CDataStream ssKey(it->key().data(), it->key().data() + it->key().size(), SER_DISK, CLIENT_VERSION); - std::string strKeyType; - ssKey >> strKeyType; - if (strKeyType != "u") - break; - - // Deserialize the UTXO entry and sum the value - CDataStream ssValue(it->value().data(), it->value().data() + it->value().size(), SER_DISK, CLIENT_VERSION); - CUtxoEntry entry; - ssValue >> entry; - - nTotal += entry.nValue; - nCount++; - } - delete it; - return nTotal; -} - diff --git a/src/txdb-leveldb.h b/src/txdb-leveldb.h index d8a0299..ab9e594 100644 --- a/src/txdb-leveldb.h +++ b/src/txdb-leveldb.h @@ -6,241 +6,61 @@ #ifndef TRIANGLES_LEVELDB_H #define TRIANGLES_LEVELDB_H -#include "main.h" - -#include -#include -#include +#include "txdb-base.h" #include #include -// Class that provides access to a LevelDB. Note that this class is frequently -// instantiated on the stack and then destroyed again, so instantiation has to -// be very cheap. Unfortunately that means, a CTxDB instance is actually just a -// wrapper around some global state. +// LevelDB backend for the chain database. // -// A LevelDB is a key/value store that is optimized for fast usage on hard -// disks. It prefers long read/writes to seeks and is based on a series of -// sorted key/value mapping files that are stacked on top of each other, with -// newer files overriding older files. A background thread compacts them -// together when too many files stack up. +// Cheap to construct/destruct: every instance shares a single global +// leveldb::DB pointer, opened lazily on first use. Most of the codebase +// instantiates a CTxDB on the stack for short-lived operations. // -// Learn more: http://code.google.com/p/leveldb/ -class CTxDB +// The protected templated Read/Write/Erase/Exists live in CTxDBBase and +// dispatch to ReadRaw/WriteRaw/EraseRaw/ExistsRaw below, which handle the +// active-batch logic so reads-after-writes within an open batch see their +// own pending changes. +class CTxDB final : public CTxDBBase { public: - CTxDB(const char* pszMode="r+"); - ~CTxDB() { - // Note that this is not the same as Close() because it deletes only - // data scoped to this TxDB object. + CTxDB(const char* pszMode = "r+"); + ~CTxDB() override { delete activeBatch; } - // Destroys the underlying shared global state accessed by this TxDB. - void Close(); + void Close() override; -private: - leveldb::DB *pdb; // Points to the global instance. - - // A batch stores up writes and deletes for atomic application. When this - // field is non-NULL, writes/deletes go there instead of directly to disk. - leveldb::WriteBatch *activeBatch; - leveldb::Options options; - bool fReadOnly; - int nVersion; - -protected: - // Returns true and sets (value,false) if activeBatch contains the given key - // or leaves value alone and sets deleted = true if activeBatch contains a - // delete for it. - bool ScanBatch(const CDataStream &key, std::string *value, bool *deleted) const; - - template - bool Read(const K& key, T& value) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - std::string strValue; - - bool readFromDb = true; - if (activeBatch) { - // First we must search for it in the currently pending set of - // changes to the db. If not found in the batch, go on to read disk. - bool deleted = false; - readFromDb = ScanBatch(ssKey, &strValue, &deleted) == false; - if (deleted) { - return false; - } - } - if (readFromDb) { - leveldb::Status status = pdb->Get(leveldb::ReadOptions(), - ssKey.str(), &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - // Some unexpected error. - printf("LevelDB read failure: %s\n", status.ToString().c_str()); - return false; - } - } - // Unserialize value - try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), - SER_DISK, CLIENT_VERSION); - ssValue >> value; - } - catch (std::exception &e) { - return false; - } - return true; - } - - template - bool Write(const K& key, const T& value) - { - if (fReadOnly) - assert(!"Write called on database in read-only mode"); - - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(10000); - ssValue << value; - - if (activeBatch) { - activeBatch->Put(ssKey.str(), ssValue.str()); - return true; - } - leveldb::Status status = pdb->Put(leveldb::WriteOptions(), ssKey.str(), ssValue.str()); - if (!status.ok()) { - printf("LevelDB write failure: %s\n", status.ToString().c_str()); - return false; - } - return true; - } - - template - bool Erase(const K& key) - { - if (!pdb) - return false; - if (fReadOnly) - assert(!"Erase called on database in read-only mode"); - - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - if (activeBatch) { - activeBatch->Delete(ssKey.str()); - return true; - } - leveldb::Status status = pdb->Delete(leveldb::WriteOptions(), ssKey.str()); - return (status.ok() || status.IsNotFound()); - } - - template - bool Exists(const K& key) - { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - std::string unused; - - if (activeBatch) { - bool deleted; - if (ScanBatch(ssKey, &unused, &deleted) && !deleted) { - return true; - } - } - - - leveldb::Status status = pdb->Get(leveldb::ReadOptions(), ssKey.str(), &unused); - return status.IsNotFound() == false; - } - - -public: - bool TxnBegin(); - bool TxnCommit(); - bool TxnAbort() + bool TxnBegin() override; + bool TxnCommit() override; + bool TxnAbort() override { delete activeBatch; activeBatch = NULL; return true; } - bool ReadVersion(int& nVersion) - { - nVersion = 0; - return Read(std::string("version"), nVersion); - } + bool LoadBlockIndex() override; - bool WriteVersion(int nVersion) - { - return Write(std::string("version"), nVersion); - } - - bool ReadDbFormat(int& nDbFormat) - { - nDbFormat = 1; - return Read(std::string("dbformat"), nDbFormat); - } - - bool WriteDbFormat(int nDbFormat) - { - return Write(std::string("dbformat"), nDbFormat); - } - - bool ReadTxIndex(uint256 hash, CTxIndex& txindex); - bool UpdateTxIndex(uint256 hash, const CTxIndex& txindex); - bool AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight); - bool EraseTxIndex(const CTransaction& tx); - bool ContainsTx(uint256 hash); - bool ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex); - bool ReadDiskTx(uint256 hash, CTransaction& tx); - bool ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex); - bool ReadDiskTx(COutPoint outpoint, CTransaction& tx); - bool WriteBlockIndex(const CDiskBlockIndex& blockindex); - bool ReadHashBestChain(uint256& hashBestChain); - bool WriteHashBestChain(uint256 hashBestChain); - bool ReadAddressIndexBestChain(uint256& hashBestChain); - bool WriteAddressIndexBestChain(uint256 hashBestChain); - bool ReadAddressIndexStartHeight(int& nHeight); - bool WriteAddressIndexStartHeight(int nHeight); - bool ReadBestInvalidTrust(CBigNum& bnBestInvalidTrust); - bool WriteBestInvalidTrust(CBigNum bnBestInvalidTrust); - bool ReadSyncCheckpoint(uint256& hashCheckpoint); - bool WriteSyncCheckpoint(uint256 hashCheckpoint); - bool ReadCheckpointPubKey(std::string& strPubKey); - bool WriteCheckpointPubKey(const std::string& strPubKey); - bool LoadBlockIndex(); - - // Address index methods - bool ReadAddressBalance(int nType, const uint160& hashBytes, int64_t& nBalance); - bool WriteAddressBalance(int nType, const uint160& hashBytes, int64_t nBalance); - bool ReadAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t& nValue, int& nHeight); - bool WriteAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex, int64_t nValue, int nHeight, const CScript& script); - bool EraseAddressUtxo(int nType, const uint160& hashBytes, const uint256& txhash, int nIndex); - bool WriteAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash); - bool EraseAddressTxId(int nType, const uint160& hashBytes, int nHeight, int nTxIndex, const uint256& txhash); - - // Address index iteration (for RPC queries) - bool GetAddressUtxos(int nType, const uint160& hashBytes, std::vector > >& vUtxos); - bool GetAddressTxIds(int nType, const uint160& hashBytes, int nStartHeight, int nEndHeight, std::vector& vTxIds); - - // UTXO database methods - bool ReadUtxo(const uint256& hash, unsigned int n, CUtxoEntry& entry); - bool WriteUtxo(const uint256& hash, unsigned int n, const CUtxoEntry& entry); - bool EraseUtxo(const uint256& hash, unsigned int n); - bool HaveUtxo(const uint256& hash, unsigned int n); - int64_t SumUtxoValues(int& nCount); +protected: + bool ReadRaw(const std::string& key, std::string& value) const override; + bool WriteRaw(const std::string& key, const std::string& value) override; + bool EraseRaw(const std::string& key) override; + bool ExistsRaw(const std::string& key) const override; + std::unique_ptr NewIterator() const override; private: + leveldb::DB* pdb; // Points to the global instance. + leveldb::WriteBatch* activeBatch; // When non-NULL, writes/deletes go here. + leveldb::Options options; + int nVersion; + + // Returns true and sets (value,false) if activeBatch contains the given + // key, or leaves value alone and sets deleted=true if activeBatch contains + // a delete for it. + bool ScanBatch(const std::string& key, std::string* value, bool* deleted) const; + bool LoadBlockIndexGuts(); }; - #endif // TRIANGLES_LEVELDB_H