diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 7fbb1740302..d960d8836da 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -79,6 +79,74 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 } ``` +####Query UTXO set (by address/script) +`GET /rest/getutxoindex//
/
/.../
.json` + +The getutxoindex command allows querying of the UTXO set given a set of addresses (or script). + +To use this function, you must start bitcoin with the -txoutindex parameter. + +Output: +``` +[ (array of json object) + { + \"confirmations\" : n, (numeric) The number of confirmations + \"txid\" : \"txid\", (string) The transaction id + \"vout\" : n, (numeric) The vout value + \"value\" : x.xxx, (numeric) The transaction value in btc + \"scriptPubKey\" : { (json object) + \"asm\" : \"code\", (string) + \"hex\" : \"hex\", (string) + \"reqSigs\" : n, (numeric) Number of required signatures + \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash + \"addresses\" : [ (array of string) array of bitcoin addresses + \"bitcoinaddress\" (string) bitcoin address + ,... + ] + }, + \"version\" : n, (numeric) The transaction version + \"coinbase\" : true|false (boolean) Coinbase or not + \"bestblockhash\" : \"hash\", (string) The block hash of the best block + \"bestblockheight\" : n, (numeric) The block height of the best block + \"bestblocktime\" : n, (numeric) The block time of the best block + \"blockhash\" : \"hash\", (string) The block hash of the block the tx is in (only if confirmations > 0) + \"blockheight\" : n, (numeric) The block height of the block the tx is in (only if confirmations > 0) + \"blocktime\" : ttt, (numeric) The block time in seconds since 1.1.1970 GMT (only if confirmations > 0) + } + ,... +] +``` + +Example: +``` +$ curl localhost:18332/rest/getutxoindex/checkmempool/mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV.json 2>/dev/null | json_pp +[ + { + "confirmations" : 721918, + "txid" : "75bc54c673ed535db361a6e89c08bf7256d1378e2c645229d469d41042356e54", + "vout" : 0, + "value" : 0.001, + "scriptPubKey" : { + "asm" : "OP_DUP OP_HASH160 a7092d2dc8778b56d4c352697081c687b451ab6d OP_EQUALVERIFY OP_CHECKSIG", + "hex" : "76a914a7092d2dc8778b56d4c352697081c687b451ab6d88ac", + "reqSigs" : 1, + "type" : "pubkeyhash", + "addresses" : [ + "mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV" + ] + }, + "version" : 1, + "coinbase" : false, + "bestblockhash" : "00000000007872ee19923a5604d86a6c9bfa3041c417a7ecf60dc034387b173f", + "blockheight" : 244755, + "bestblocktime" : 1475309084, + "blockhash" : "000000000001c163caa76dbc16c7b383fb10257829b3617c5a1ffb91ea3824db", + "bestblockheight" : 966672, + "blocktime" : 1400786412, + } +] +``` + ####Memory pool `GET /rest/mempool/info.json` diff --git a/src/Makefile.am b/src/Makefile.am index cb88171348e..5c16be936c9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -89,6 +89,8 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + coinsbyscript.h \ + coinstats.h \ compat.h \ compat/byteswap.h \ compat/endian.h \ @@ -305,6 +307,8 @@ libbitcoin_common_a_SOURCES = \ base58.cpp \ chainparams.cpp \ coins.cpp \ + coinsbyscript.cpp \ + coinstats.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 073bbde0165..563aaed9646 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -96,7 +96,7 @@ static void MempoolEviction(benchmark::State& state) tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - CTxMemPool pool; + CTxMemPool pool(false); while (state.KeepRunning()) { AddTx(tx1, 10000LL, pool); diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp new file mode 100644 index 00000000000..78d0fe51b5b --- /dev/null +++ b/src/coinsbyscript.cpp @@ -0,0 +1,280 @@ +// Copyright (c) 2014-2016 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "coinsbyscript.h" +#include "txdb.h" +#include "hash.h" +#include "ui_interface.h" + +#include + +#include + +using namespace std; + +static const char DB_COINS_BYSCRIPT = 'd'; +static const char DB_FLAG = 'F'; +static const char DB_BEST_BLOCK = 'B'; + +CCoinsViewByScript::CCoinsViewByScript(CCoinsViewByScriptDB* viewIn) : base(viewIn) { } + +bool CCoinsViewByScript::GetCoinsByScript(const CScript &script, CCoinsByScript &coins) { + const CScriptID key = CScriptID(script); + if (cacheCoinsByScript.count(key)) { + coins = cacheCoinsByScript[key]; + return true; + } + if (base->GetCoinsByScriptID(key, coins)) { + cacheCoinsByScript[key] = coins; + return true; + } + return false; +} + +CCoinsMapByScript::iterator CCoinsViewByScript::FetchCoinsByScript(const CScript &script, bool fRequireExisting) { + const CScriptID key = CScriptID(script); + CCoinsMapByScript::iterator it = cacheCoinsByScript.find(key); + if (it != cacheCoinsByScript.end()) + return it; + + CCoinsByScript tmp; + if (!base->GetCoinsByScriptID(key, tmp)) + { + if (fRequireExisting) + return cacheCoinsByScript.end(); + } + + return cacheCoinsByScript.emplace_hint(it, key, tmp); +} + +CCoinsByScript &CCoinsViewByScript::GetCoinsByScript(const CScript &script, bool fRequireExisting) { + CCoinsMapByScript::iterator it = FetchCoinsByScript(script, fRequireExisting); + assert(it != cacheCoinsByScript.end()); + return it->second; +} + +uint256 CCoinsViewByScript::GetBestBlock() const { + return hashBlock; +} + +void CCoinsViewByScript::SetBestBlock(const uint256 &hashBlockIn) { + hashBlock = hashBlockIn; +} + +bool CCoinsViewByScript::Flush() { + bool fOk = base->BatchWrite(this, hashBlock); + return fOk; +} + +CCoinsViewByScriptDB::CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "coinsbyscript", nCacheSize, fMemory, fWipe, true) +{ +} + +bool CCoinsViewByScriptDB::GetCoinsByScriptID(const CScriptID &scriptID, CCoinsByScript &coins) const { + return db.Read(make_pair(DB_COINS_BYSCRIPT, scriptID), coins); +} + +bool CCoinsViewByScriptDB::BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock) { + CDBBatch batch(db); + size_t count = 0; + for (CCoinsMapByScript::iterator it = pcoinsViewByScriptIn->cacheCoinsByScript.begin(); it != pcoinsViewByScriptIn->cacheCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + pcoinsViewByScriptIn->cacheCoinsByScript.erase(itOld); + count++; + } + pcoinsViewByScriptIn->cacheCoinsByScript.clear(); + + if (!hashBlock.IsNull()) + batch.Write(DB_BEST_BLOCK, hashBlock); + + LogPrint(BCLog::COINDB, "Committing %zu coin address indexes to coin database...\n", (unsigned int)count); + return db.WriteBatch(batch); +} + +bool CCoinsViewByScriptDB::WriteFlag(const std::string &name, bool fValue) { + return db.Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); +} + +bool CCoinsViewByScriptDB::ReadFlag(const std::string &name, bool &fValue) { + char ch; + if (!db.Read(std::make_pair(DB_FLAG, name), ch)) + return false; + fValue = ch == '1'; + return true; +} + +CCoinsViewByScriptDBCursor *CCoinsViewByScriptDB::Cursor() const +{ + CCoinsViewByScriptDBCursor *i = new CCoinsViewByScriptDBCursor(const_cast(&db)->NewIterator()); + /* It seems that there are no "const iterators" for LevelDB. Since we + only need read operations on it, use a const-cast to get around + that restriction. */ + i->pcursor->Seek(DB_COINS_BYSCRIPT); + if (!i->pcursor->Valid()) + // If db empty then set this cursor invalid + i->keyTmp.first = 0; + else + // Cache key of first record + i->pcursor->GetKey(i->keyTmp); + return i; +} + +bool CCoinsViewByScriptDBCursor::GetKey(CScriptID &key) const +{ + // Return cached key + if (keyTmp.first == DB_COINS_BYSCRIPT) { + key = keyTmp.second; + return true; + } + return false; +} + +bool CCoinsViewByScriptDBCursor::GetValue(CCoinsByScript &coins) const +{ + return pcursor->GetValue(coins); +} + +unsigned int CCoinsViewByScriptDBCursor::GetValueSize() const +{ + return pcursor->GetValueSize(); +} + +bool CCoinsViewByScriptDBCursor::Valid() const +{ + return keyTmp.first == DB_COINS_BYSCRIPT; +} + +void CCoinsViewByScriptDBCursor::Next() +{ + pcursor->Next(); + if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false +} + +bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() +{ + std::unique_ptr pcursor(Cursor()); + + std::vector v; + int64_t i = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + CScriptID hash; + if (!pcursor->GetKey(hash)) + break; + v.push_back(hash); + if (v.size() >= 10000) + { + i += v.size(); + CDBBatch batch(db); + for(auto& av: v) + { + const CScriptID& _hash = av; + batch.Erase(make_pair(DB_COINS_BYSCRIPT, _hash)); // delete + } + db.WriteBatch(batch); + v.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!v.empty()) + { + i += v.size(); + CDBBatch batch(db); + for(auto& av: v) + { + const CScriptID& hash = av; + batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete + } + db.WriteBatch(batch); + } + if (i > 0) + LogPrintf("Address index with %d addresses successfully deleted.\n", i); + + return true; +} + +bool CCoinsViewByScriptDB::GenerateAllCoinsByScript(CCoinsViewDB* coinsIn) +{ + LogPrintf("Building address index for -txoutindex. Be patient...\n"); + int64_t nTxCount = coinsIn->CountCoins(); + + std::unique_ptr pcursor(coinsIn->Cursor()); + + CCoinsMapByScript mapCoinsByScript; + int64_t i = 0; + int64_t progress = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + if (progress % 1000 == 0 && nTxCount > 0) + uiInterface.ShowProgress(_("Building address index..."), (int)(((double)progress / (double)nTxCount) * (double)100)); + progress++; + + uint256 txhash; + CCoins coins; + if (!pcursor->GetKey(txhash) || !pcursor->GetValue(coins)) + break; + + for (unsigned int j = 0; j < coins.vout.size(); j++) + { + if (coins.vout[j].IsNull() || coins.vout[j].scriptPubKey.IsUnspendable()) + continue; + + const CScriptID key = CScriptID(coins.vout[j].scriptPubKey); + if (!mapCoinsByScript.count(key)) + { + CCoinsByScript coinsByScript; + GetCoinsByScriptID(key, coinsByScript); + mapCoinsByScript.insert(make_pair(key, coinsByScript)); + } + mapCoinsByScript[key].setCoins.insert(COutPoint(txhash, (uint32_t)j)); + i++; + } + + if (mapCoinsByScript.size() >= 10000) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + mapCoinsByScript.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!mapCoinsByScript.empty()) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + } + LogPrintf("Address index with %d outputs successfully built.\n", i); + return true; +} diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h new file mode 100644 index 00000000000..c9404e69665 --- /dev/null +++ b/src/coinsbyscript.h @@ -0,0 +1,116 @@ +// Copyright (c) 2014-2016 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COINSBYSCRIPT_H +#define BITCOIN_COINSBYSCRIPT_H + +#include "coins.h" +#include "dbwrapper.h" +#include "primitives/transaction.h" +#include "serialize.h" +#include "uint256.h" +#include "script/standard.h" + +class CCoinsViewDB; +class CCoinsViewByScriptDB; +class CScript; + +class CCoinsByScript +{ +public: + // unspent transaction outputs + std::set setCoins; + + // empty constructor + CCoinsByScript() { } + + bool IsEmpty() const { + return (setCoins.empty()); + } + + void swap(CCoinsByScript &to) { + to.setCoins.swap(setCoins); + } + + ADD_SERIALIZE_METHODS; + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(setCoins); + } +}; + +typedef std::map CCoinsMapByScript; + +/** Adds a memory cache for coins by address */ +class CCoinsViewByScript +{ +private: + CCoinsViewByScriptDB *base; + + mutable uint256 hashBlock; + +public: + CCoinsMapByScript cacheCoinsByScript; // accessed also from CCoinsViewByScriptDB + CCoinsViewByScript(CCoinsViewByScriptDB* baseIn); + + bool GetCoinsByScript(const CScript &script, CCoinsByScript &coins); + + // Return a modifiable reference to a CCoinsByScript. + CCoinsByScript &GetCoinsByScript(const CScript &script, bool fRequireExisting = true); + + void SetBestBlock(const uint256 &hashBlock); + uint256 GetBestBlock() const; + + /** + * Push the modifications applied to this cache to its base. + * Failure to call this method before destruction will cause the changes to be forgotten. + * If false is returned, the state of this cache (and its backing view) will be undefined. + */ + bool Flush(); + +private: + CCoinsMapByScript::iterator FetchCoinsByScript(const CScript &script, bool fRequireExisting); +}; + +/** Cursor for iterating over a CCoinsViewByScriptDB */ +class CCoinsViewByScriptDBCursor +{ +public: + ~CCoinsViewByScriptDBCursor() {} + + bool GetKey(CScriptID &key) const; + bool GetValue(CCoinsByScript &coins) const; + unsigned int GetValueSize() const; + + bool Valid() const; + void Next(); + +private: + CCoinsViewByScriptDBCursor(CDBIterator* pcursorIn): + pcursor(pcursorIn) {} + uint256 hashBlock; + std::unique_ptr pcursor; + std::pair keyTmp; + + friend class CCoinsViewByScriptDB; +}; + +/** coinsbyscript database (coinsbyscript/) */ +class CCoinsViewByScriptDB +{ +protected: + CDBWrapper db; +public: + CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + + bool GetCoinsByScriptID(const CScriptID &scriptID, CCoinsByScript &coins) const; + bool BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock); + bool WriteFlag(const std::string &name, bool fValue); + bool ReadFlag(const std::string &name, bool &fValue); + bool DeleteAllCoinsByScript(); // removes txoutindex + bool GenerateAllCoinsByScript(CCoinsViewDB* coinsIn); // creates txoutindex + CCoinsViewByScriptDBCursor *Cursor() const; +}; + +#endif // BITCOIN_COINSBYSCRIPT_H diff --git a/src/coinstats.cpp b/src/coinstats.cpp new file mode 100644 index 00000000000..82882bd0f84 --- /dev/null +++ b/src/coinstats.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2014-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "coinstats.h" +#include "validation.h" + +#include + +#include // boost::thread::interrupt + +using namespace std; + +//! Calculate statistics about the unspent transaction output set +bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats) +{ + std::unique_ptr pcursor(view->Cursor()); + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + stats.hashBlock = pcursor->GetBestBlock(); + { + LOCK(cs_main); + BlockMap::const_iterator iter = mapBlockIndex.find(stats.hashBlock); + if (iter == mapBlockIndex.end()) + stats.nHeight = 0; + else + stats.nHeight = iter->second->nHeight; + } + + ss << stats.hashBlock; + CAmount nTotalAmount = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + uint256 key; + CCoins coins; + if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { + stats.nTransactions++; + ss << key; + for (unsigned int i=0; iGetValueSize(); + ss << VARINT(0); + } else { + return error("%s: unable to read value", __func__); + } + pcursor->Next(); + } + stats.hashSerialized = ss.GetHash(); + stats.nTotalAmount = nTotalAmount; + + std::unique_ptr pcursordb(viewbyscriptdb->Cursor()); + while (pcursordb->Valid()) { + boost::this_thread::interruption_point(); + CScriptID hash; + CCoinsByScript coinsByScript; + if (pcursordb->GetKey(hash) && pcursordb->GetValue(coinsByScript)) { + stats.nAddresses++; + stats.nAddressesOutputs += coinsByScript.setCoins.size(); + } else { + return error("%s: unable to read value", __func__); + } + pcursordb->Next(); + } + return true; +} diff --git a/src/coinstats.h b/src/coinstats.h new file mode 100644 index 00000000000..64e8beec776 --- /dev/null +++ b/src/coinstats.h @@ -0,0 +1,29 @@ +// Copyright (c) 2014-2016 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COINSTATS_H +#define BITCOIN_COINSTATS_H + +#include "coins.h" +#include "txdb.h" +#include "coinsbyscript.h" + +struct CCoinsStats +{ + int nHeight{}; + uint256 hashBlock; + uint64_t nTransactions{}; + uint64_t nTransactionOutputs{}; + uint64_t nAddresses{}; + uint64_t nAddressesOutputs{}; // equal nTransactionOutputs (if addressindex is enabled) + uint64_t nSerializedSize{}; + uint256 hashSerialized; + CAmount nTotalAmount{}; + + CCoinsStats() {} +}; + +bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats); + +#endif // BITCOIN_COINSTATS_H diff --git a/src/init.cpp b/src/init.cpp index f06c9e11000..498ee3b75fc 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -11,6 +11,7 @@ #include "addrman.h" #include "amount.h" +#include "coinstats.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" @@ -233,6 +234,10 @@ void Shutdown() pcoinscatcher = NULL; delete pcoinsdbview; pcoinsdbview = NULL; + delete pcoinsByScript; + pcoinsByScript = NULL; + delete pcoinsByScriptDB; + pcoinsByScriptDB = NULL; delete pblocktree; pblocktree = NULL; } @@ -369,6 +374,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); + strUsage += HelpMessageOpt("-txoutindex", strprintf(_("Maintain an address to unspent outputs index (rpc: getutxoindex). The index is built on first use. (default: %u)"), 0)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); @@ -1433,11 +1439,13 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) UnloadBlockIndex(); delete pcoinsTip; delete pcoinsdbview; + delete pcoinsByScriptDB; delete pcoinscatcher; delete pblocktree; pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex || fReindexChainState); + pcoinsByScriptDB = new CCoinsViewByScriptDB(nCoinDBCache, false, false); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); @@ -1485,6 +1493,58 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // Check -txoutindex + pcoinsByScriptDB->ReadFlag("txoutindex", fTxOutIndex); + if (IsArgSet("-txoutindex")) + { + if (GetBoolArg("-txoutindex", false)) + { + // build index + if (!fTxOutIndex) + { + if (!pcoinsByScriptDB->DeleteAllCoinsByScript()) + { + strLoadError = _("Error deleting txoutindex"); + break; + } + if (!pcoinsByScriptDB->GenerateAllCoinsByScript(pcoinsdbview)) + { + strLoadError = _("Error building txoutindex"); + break; + } + CCoinsStats stats; + if (!GetUTXOStats(pcoinsTip, pcoinsByScriptDB, stats)) + { + strLoadError = _("Error GetUTXOStats for txoutindex"); + break; + } + if (stats.nTransactionOutputs != stats.nAddressesOutputs) + { + strLoadError = _("Error compare stats for txoutindex"); + break; + } + pcoinsByScriptDB->WriteFlag("txoutindex", true); + fTxOutIndex = true; + } + } + else + { + if (fTxOutIndex) + { + // remove index + pcoinsByScriptDB->DeleteAllCoinsByScript(); + pcoinsByScriptDB->WriteFlag("txoutindex", false); + fTxOutIndex = false; + } + } + } + else if (fTxOutIndex) + return InitError(_("You need to provide -txoutindex. Do -txoutindex=0 to delete the index.")); + + // Init -txoutindex + if (fTxOutIndex) + pcoinsByScript = new CCoinsViewByScript(pcoinsByScriptDB); + uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks", diff --git a/src/rest.cpp b/src/rest.cpp index 9dcaf269d64..16b89a854d6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "base58.h" #include "chain.h" #include "chainparams.h" #include "primitives/block.h" @@ -22,6 +23,7 @@ #include static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once +static const size_t MAX_GETUTXOINDEX_SCRIPTS = 15; //allow a max of 15 scripts to be queried at once enum RetFormat { RF_UNDEF, @@ -59,6 +61,7 @@ struct CCoin { /* Defined in rawtransaction.cpp */ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex); static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { @@ -598,6 +601,109 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) return true; // continue to process further HTTP reqs on this cxn } +static bool rest_getutxoindex(HTTPRequest* req, const std::string& strURIPart) +{ + if (!CheckWarmup(req)) + return false; + if (!fTxOutIndex) + return RESTERR(req, HTTP_BAD_REQUEST, "To use this function, you must start bitcoin with the -txoutindex parameter."); + + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); + + std::vector uriParts; + if (param.length() > 1) + { + std::string strUriParams = param.substr(1); + boost::split(uriParts, strUriParams, boost::is_any_of("/")); + } + + // throw exception in case of a empty request + std::string strRequestMutable = req->ReadBody(); + if (strRequestMutable.length() == 0 && uriParts.size() == 0) + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + + bool fInputParsed = false; + bool fCheckMemPool = false; + std::vector vScripts; + + // parse/deserialize input + // only json format supported + + if (uriParts.size() > 0) + { + + //inputs is sent over URI scheme (/rest/getutxoindex/checkmempool/addr1/addr2/...) + if (uriParts.size() > 0 && uriParts[0] == "checkmempool") + fCheckMemPool = true; + + for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) + { + CScript script; + CBitcoinAddress address(uriParts[i]); + if (address.IsValid()) { + script = GetScriptForDestination(address.Get()); + } else if (IsHex(uriParts[i])) { + std::vector data(ParseHex(uriParts[i])); + script = CScript(data.begin(), data.end()); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid Bitcoin address or script: " + uriParts[i]); + } + + vScripts.push_back(script); + } + + if (vScripts.size() > 0) + fInputParsed = true; + else + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + } + + switch (rf) { + case RF_JSON: { + if (!fInputParsed) + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + break; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + + // limit max scripts + if (vScripts.size() > MAX_GETUTXOINDEX_SCRIPTS) + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Error: max scripts exceeded (max: %d, tried: %d)", MAX_GETUTXOINDEX_SCRIPTS, vScripts.size())); + + int nMinDepth = fCheckMemPool ? 0 : 1; + UniValue vObjects(UniValue::VARR); + std::vector > vSort; + + for(auto& scriptObj: vScripts) + { + const CScript& script = scriptObj; + CCoinsByScript coinsByScript; + pcoinsByScript->GetCoinsByScript(script, coinsByScript); + + if (nMinDepth == 0) + mempool.GetCoinsByScript(script, coinsByScript); + + CoinsByScriptToJSON(coinsByScript, nMinDepth, vObjects, vSort, true); + } + + UniValue results(UniValue::VARR); + sort(vSort.begin(), vSort.end()); + for (unsigned int i = 0; i < vSort.size(); i++) + { + results.push_back(vObjects[vSort[i].second]); + } + + // return json string + std::string strJSON = results.write() + "\n"; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); + return true; +} + static const struct { const char* prefix; bool (*handler)(HTTPRequest* req, const std::string& strReq); @@ -610,6 +716,7 @@ static const struct { {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/getutxoindex", rest_getutxoindex}, }; bool StartREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 01066d0eb2e..443f148e3f0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -6,11 +6,13 @@ #include "rpc/blockchain.h" #include "amount.h" +#include "base58.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "coins.h" #include "consensus/validation.h" +#include "coinstats.h" #include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" @@ -21,11 +23,14 @@ #include "util.h" #include "utilstrencodings.h" #include "hash.h" +#include "txdb.h" +#include "dbwrapper.h" #include #include +#include #include // boost::thread::interrupt #include @@ -43,6 +48,7 @@ static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { @@ -760,60 +766,6 @@ UniValue getblock(const JSONRPCRequest& request) return blockToJSON(block, pblockindex); } -struct CCoinsStats -{ - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nSerializedSize; - uint256 hashSerialized; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} -}; - -//! Calculate statistics about the unspent transaction output set -static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) -{ - std::unique_ptr pcursor(view->Cursor()); - - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - stats.hashBlock = pcursor->GetBestBlock(); - { - LOCK(cs_main); - stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; - } - ss << stats.hashBlock; - CAmount nTotalAmount = 0; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; iGetValueSize(); - ss << VARINT(0); - } else { - return error("%s: unable to read value", __func__); - } - pcursor->Next(); - } - stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; - return true; -} - UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) @@ -876,6 +828,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" + " \"addresses\": n, (numeric) The number of addresses and scripts. Only if -txoutindex=1\n" + " \"utxoindex\": n, (numeric) The number of output transactions. Only if -txoutindex=1\n" " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" " \"total_amount\": x.xxx (numeric) The total amount\n" @@ -889,11 +843,13 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsTip, stats)) { + if (GetUTXOStats(pcoinsTip, pcoinsByScriptDB, stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); + ret.push_back(Pair("addresses", (int64_t)stats.nAddresses)); + ret.push_back(Pair("utxoindex", (int64_t)stats.nAddressesOutputs)); ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); @@ -983,6 +939,117 @@ UniValue gettxout(const JSONRPCRequest& request) return ret; } +UniValue getutxoindex(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) + throw std::runtime_error( + "getutxoindex ( minconf [\"address\",...] count from )\n" + "\nReturns a list of unspent transaction outputs by address (or script).\n" + "The list is ordered by confirmations in descending order.\n" + "Note that passing minconf=0 will include the mempool.\n" + "\nTo use this function, you must start bitcoin with the -txoutindex parameter.\n" + "\nArguments:\n" + "1. minconf (numeric) Minimum confirmations\n" + "2. \"addresses\" (string) A json array of bitcoin addresses (or scripts)\n" + " [\n" + " \"address\" (string) bitcoin address (or script)\n" + " ,...\n" + " ]\n" + "3. count (numeric, optional, default=999999999) The number of outputs to return\n" + "4. from (numeric, optional, default=0) The number of outputs to skip\n" + "\nResult\n" + "[ (array of json object)\n" + " {\n" + " \"confirmations\" : n, (numeric) The number of confirmations\n" + " \"txid\" : \"txid\", (string) The transaction id \n" + " \"vout\" : n, (numeric) The vout value\n" + " \"value\" : x.xxx, (numeric) The transaction value in btc\n" + " \"scriptPubKey\" : { (json object)\n" + " \"asm\" : \"code\", (string) \n" + " \"hex\" : \"hex\", (string) \n" + " \"reqSigs\" : n, (numeric) Number of required signatures\n" + " \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash\n" + " \"addresses\" : [ (array of string) array of bitcoin addresses\n" + " \"bitcoinaddress\" (string) bitcoin address\n" + " ,...\n" + " ]\n" + " },\n" + " \"version\" : n, (numeric) The transaction version\n" + " \"coinbase\" : true|false (boolean) Coinbase or not\n" + " \"bestblockhash\" : \"hash\", (string) The block hash of the best block\n" + " \"bestblockheight\" : n, (numeric) The block height of the best block\n" + " \"bestblocktime\" : n, (numeric) The block time of the best block\n" + " \"blockhash\" : \"hash\", (string) The block hash of the block the tx is in (only if confirmations > 0)\n" + " \"blockheight\" : n, (numeric) The block height of the block the tx is in (only if confirmations > 0)\n" + " \"blocktime\" : ttt, (numeric) The block time in seconds since 1.1.1970 GMT (only if confirmations > 0)\n" + " }\n" + " ,...\n" + "]\n" + "\nExamples:\n" + + HelpExampleCli("getutxoindex", "6 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + "\nAs a json rpc call\n" + + HelpExampleRpc("getutxoindex", "6, \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + ); + + if (!fTxOutIndex) + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "To use this function, you must start bitcoin with the -txoutindex parameter."); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VARR)(UniValue::VNUM)(UniValue::VNUM)); + + UniValue vObjects(UniValue::VARR); + std::vector > vSort; + int nMinDepth = request.params[0].get_int(); + UniValue inputs = request.params[1].get_array(); + + int nCount = 999999999; + if (request.params.size() > 2) + nCount = request.params[2].get_int(); + int nFrom = 0; + if (request.params.size() > 3) + nFrom = request.params[3].get_int(); + + if (nMinDepth < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative minconf"); + if (nCount < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative count"); + if (nFrom < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative from"); + + for (unsigned int idx = 0; idx < inputs.size(); idx++) { + const UniValue& input = inputs[idx]; + CScript script; + CBitcoinAddress address(input.get_str()); + if (address.IsValid()) { + script = GetScriptForDestination(address.Get()); + } else if (IsHex(input.get_str())) { + std::vector data(ParseHex(input.get_str())); + script = CScript(data.begin(), data.end()); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script: " + input.get_str()); + } + + CCoinsByScript coinsByScript; + pcoinsByScript->GetCoinsByScript(script, coinsByScript); + + if (nMinDepth == 0) + mempool.GetCoinsByScript(script, coinsByScript); + + CoinsByScriptToJSON(coinsByScript, nMinDepth, vObjects, vSort, true); + } + + UniValue results(UniValue::VARR); + sort(vSort.begin(), vSort.end()); + for (unsigned int i = (unsigned int)nFrom; i < vSort.size(); i++) + { + if (i == (unsigned int)nCount + (unsigned int)nFrom) + break; + + results.push_back(vObjects[vSort[i].second]); + } + + return results; +} + UniValue verifychain(const JSONRPCRequest& request) { int nCheckLevel = GetArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -1432,6 +1499,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, true, {} }, { "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} }, { "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} }, + { "blockchain", "getutxoindex", &getutxoindex, true, {"minconf", "addresses", "count", "from"} }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} }, { "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} }, { "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} }, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index c021441b0a7..f9f86bb6bec 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -37,4 +37,3 @@ UniValue mempoolToJSON(bool fVerbose = false); UniValue blockheaderToJSON(const CBlockIndex* blockindex); #endif - diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 941bdd93796..8c32dfb5540 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -91,6 +91,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, + { "getutxoindex", 0, "minconf" }, + { "getutxoindex", 1, "addresses" }, + { "getutxoindex", 2, "count" }, + { "getutxoindex", 3, "from" }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "importprivkey", 2, "rescan" }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 717e9d75f39..d9147837fe1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -122,6 +122,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) } } +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex) +{ + BOOST_FOREACH(const COutPoint &outpoint, coinsByScript.setCoins) + { + CCoins coins; + if (nMinDepth == 0) + { + LOCK(mempool.cs); + CCoinsViewMemPool view(pcoinsTip, mempool); + if (!view.GetCoins(outpoint.hash, coins)) + continue; + mempool.pruneSpent(outpoint.hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } + else if (!pcoinsTip->GetCoins(outpoint.hash, coins)) + continue; + + if (outpoint.n < coins.vout.size() && !coins.vout[outpoint.n].IsNull() && !coins.vout[outpoint.n].scriptPubKey.IsUnspendable()) + { + // should not happen + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT && (!chainActive[coins.nHeight] || !chainActive[coins.nHeight]->phashBlock)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal Error: !chainActive[coins.nHeight]"); + + BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); + CBlockIndex *pindex = it->second; + + int nConfirmations = 0; + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + nConfirmations = pindex->nHeight - coins.nHeight + 1; + if (nConfirmations < nMinDepth) + continue; + + UniValue oScriptPubKey(UniValue::VOBJ); + ScriptPubKeyToJSON(coins.vout[outpoint.n].scriptPubKey, oScriptPubKey, fIncludeHex); + + UniValue o(UniValue::VOBJ); + o.push_back(Pair("confirmations", nConfirmations)); + o.push_back(Pair("txid", outpoint.hash.GetHex())); + o.push_back(Pair("vout", (int)outpoint.n)); + o.push_back(Pair("value", ValueFromAmount(coins.vout[outpoint.n].nValue))); + o.push_back(Pair("scriptPubKey", oScriptPubKey)); + o.push_back(Pair("version", coins.nVersion)); + o.push_back(Pair("coinbase", coins.fCoinBase)); + o.push_back(Pair("bestblockhash", pindex->GetBlockHash().GetHex())); + o.push_back(Pair("bestblockheight", pindex->nHeight)); + o.push_back(Pair("bestblocktime", pindex->GetBlockTime())); + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + { + o.push_back(Pair("blockhash", chainActive[coins.nHeight]->GetBlockHash().GetHex())); + o.push_back(Pair("blockheight", coins.nHeight)); + o.push_back(Pair("blocktime", chainActive[coins.nHeight]->GetBlockTime())); + } + vObjects.push_back(o); + vSort.push_back(std::make_pair(coins.nHeight, (unsigned int)vObjects.size() - 1)); + } + } +} + UniValue getrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 9e4a56919d4..c5c55d1b16d 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -57,7 +57,7 @@ static CBlock BuildBlockTestCase() { BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -156,7 +156,7 @@ class TestHeaderAndShortIDs { BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -222,7 +222,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -272,7 +272,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { - CTxMemPool pool; + CTxMemPool pool(false); CMutableTransaction coinbase; coinbase.vin.resize(1); coinbase.vin[0].scriptSig.resize(10); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 51b28d09fa4..e802d89f534 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -53,8 +53,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) txGrandChild[i].vout[0].nValue = 11000LL; } - - CTxMemPool testPool; + CTxMemPool testPool(false); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); @@ -118,7 +117,7 @@ void CheckSort(CTxMemPool &pool, std::vector &sortedOrder) BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -319,7 +318,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -430,7 +429,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CMutableTransaction tx1 = CMutableTransaction(); diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index ed6782ea345..40dbaa56d5e 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -17,7 +17,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CBlockPolicyEstimator feeEst; - CTxMemPool mpool(&feeEst); + bool fTxOutIndex = false; + CTxMemPool mpool(fTxOutIndex, &feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); diff --git a/src/txdb.cpp b/src/txdb.cpp index a3889fdf79b..72698aff37b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -9,6 +9,7 @@ #include "hash.h" #include "pow.h" #include "uint256.h" +#include "ui_interface.h" #include @@ -25,7 +26,7 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } @@ -98,7 +99,8 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const that restriction. */ i->pcursor->Seek(DB_COINS); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (!i->pcursor->Valid() || !i->pcursor->GetKey(i->keyTmp)) + i->keyTmp.first = 0; return i; } @@ -134,6 +136,19 @@ void CCoinsViewDBCursor::Next() keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false } +int64_t CCoinsViewDB::CountCoins() const +{ + std::unique_ptr pcursor(Cursor()); + + int64_t i = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + i++; + pcursor->Next(); + } + return i; +} + bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index d9214ba6185..17042641741 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -9,6 +9,7 @@ #include "coins.h" #include "dbwrapper.h" #include "chain.h" +#include "coinsbyscript.h" #include #include @@ -79,6 +80,7 @@ class CCoinsViewDB : public CCoinsView bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + int64_t CountCoins() const; CCoinsViewCursor *Cursor() const; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ac842da6bf3..583326ff038 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -331,8 +331,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : - nTransactionsUpdated(0), minerPolicyEstimator(estimator) +CTxMemPool::CTxMemPool(const bool& _fTxOutIndex, CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator), fTxOutIndex(_fTxOutIndex) { _clear(); //lock free clear @@ -361,6 +361,16 @@ unsigned int CTxMemPool::GetTransactionsUpdated() const return nTransactionsUpdated; } +void CTxMemPool::GetCoinsByScript(const CScript& script, CCoinsByScript& coinsByScript) const +{ + LOCK(cs); + CCoinsMapByScript::const_iterator it = mapCoinsByScript.find(CScriptID(script)); + if (it != mapCoinsByScript.end()) + { + coinsByScript.setCoins.insert(it->second.setCoins.begin(), it->second.setCoins.end()); + } +} + void CTxMemPool::AddTransactionsUpdated(unsigned int n) { LOCK(cs); @@ -423,6 +433,11 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; + if (fTxOutIndex) + for (unsigned int i = 0; i < tx.vout.size(); i++) + if (!tx.vout[i].IsNull() && !tx.vout[i].scriptPubKey.IsUnspendable()) + mapCoinsByScript[CScriptID(tx.vout[i].scriptPubKey)].setCoins.insert(COutPoint(hash, (uint32_t)i)); + return true; } @@ -485,6 +500,24 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso // Remove transaction from memory pool { LOCK(cs); + + if (fTxOutIndex) + { + for (unsigned int i = 0; i < origTx.vout.size(); i++) + { + if (origTx.vout[i].IsNull() || origTx.vout[i].scriptPubKey.IsUnspendable()) + continue; + + CCoinsMapByScript::iterator it = mapCoinsByScript.find(CScriptID(origTx.vout[i].scriptPubKey)); + if (it != mapCoinsByScript.end()) + { + it->second.setCoins.erase(COutPoint(origTx.GetHash(), (uint32_t)i)); + if (it->second.setCoins.empty()) + mapCoinsByScript.erase(it); + } + } + } + setEntries txToRemove; txiter origit = mapTx.find(origTx.GetHash()); if (origit != mapTx.end()) { diff --git a/src/txmempool.h b/src/txmempool.h index 92c4d9f9d47..3a3ec9005fc 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "coinsbyscript.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -415,6 +416,8 @@ class CTxMemPool CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. + const bool fTxOutIndex; + CCoinsMapByScript mapCoinsByScript; // only used if -txoutindex uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; @@ -496,7 +499,7 @@ class CTxMemPool /** Create a new CTxMemPool. */ - CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); + CTxMemPool(const bool& _fTxOutIndex, CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -526,6 +529,7 @@ class CTxMemPool void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); + void GetCoinsByScript(const CScript& script, CCoinsByScript& coinsByScript) const; /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a block. diff --git a/src/validation.cpp b/src/validation.cpp index 6c60be45a1a..4dc00d3d9f1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -65,6 +65,7 @@ int nScriptCheckThreads = 0; std::atomic_bool fImporting(false); bool fReindex = false; bool fTxIndex = false; +bool fTxOutIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -82,7 +83,7 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; -CTxMemPool mempool(&feeEstimator); +CTxMemPool mempool(fTxOutIndex, &feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -175,6 +176,8 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc } CCoinsViewCache *pcoinsTip = NULL; +CCoinsViewByScriptDB *pcoinsByScriptDB = NULL; +CCoinsViewByScript *pcoinsByScript = NULL; CBlockTreeDB *pblocktree = NULL; enum FlushStateMode { @@ -1530,7 +1533,7 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean * will be true if no problems were found. Otherwise, the return value will be false in case * of problems. Note that in any case, coins may be modified. */ -static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean = NULL) +static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, CBlockUndo& blockUndo, bool* pfClean = NULL) { assert(pindex->GetBlockHash() == view.GetBestBlock()); @@ -1539,7 +1542,6 @@ static bool DisconnectBlock(const CBlock& block, CValidationState& state, const bool fClean = true; - CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); if (pos.IsNull()) return error("DisconnectBlock(): no undo data available"); @@ -1687,7 +1689,7 @@ static int64_t nTimeTotal = 0; * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) + CCoinsViewCache& view, const CChainParams& chainparams, CBlockUndo& blockundo, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -1808,8 +1810,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); - CBlockUndo blockundo; - CCheckQueueControl control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); std::vector prevheights; @@ -2030,6 +2030,10 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); + if (fTxOutIndex) { + if (!pcoinsByScript->Flush()) + return AbortNode(state, "Failed to write to coin database"); + } nLastFlush = nNow; } if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) { @@ -2054,6 +2058,62 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +void static UpdateAddressIndex(const CTxOut& txout, const COutPoint& outpoint, bool fInsert) +{ + if (!txout.IsNull() && !txout.scriptPubKey.IsUnspendable()) + { + CCoinsByScript &coinsByScript = pcoinsByScript->GetCoinsByScript(txout.scriptPubKey, !fInsert); + if (fInsert) + coinsByScript.setCoins.insert(outpoint); + else + coinsByScript.setCoins.erase(outpoint); + } +} + +void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool fConnect) +{ + if (!fTxOutIndex) + return; + + assert(block.vtx.size() > 0); + unsigned int i = 0; + if (!fConnect) + i = block.vtx.size() - 1; // iterate backwards + + while (true) + { + const CTransaction &tx = *block.vtx[i]; + + if (i > 0) + { + for (unsigned int j = 0; j < tx.vin.size(); j++) + UpdateAddressIndex(blockundo.vtxundo[i-1].vprevout[j].txout, tx.vin[j].prevout, !fConnect); + } + + for (unsigned int j = 0; j < tx.vout.size(); j++) + { + CTxOut& txout = const_cast(tx.vout[j]); + const COutPoint outpoint(tx.GetHash(),((uint32_t)j)); + UpdateAddressIndex(txout, outpoint, fConnect); + } + + if (fConnect) + { + if (i == block.vtx.size() - 1) + break; + i++; + } + else + { + if (i == 0) + break; + i--; + } + } + + pcoinsByScript->SetBestBlock(block.GetHash()); +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -2130,11 +2190,13 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { + CBlockUndo blockUndo; CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (!DisconnectBlock(block, state, pindexDelete, view, blockUndo)) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); + UpdateAddressIndex(block, blockUndo, false); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. @@ -2267,8 +2329,9 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { + CBlockUndo blockundo; CCoinsViewCache view(pcoinsTip); - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams, blockundo); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) @@ -2279,6 +2342,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); + UpdateAddressIndex(blockConnecting, blockundo, true); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); @@ -3261,6 +3325,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); CCoinsViewCache viewNew(pcoinsTip); + CBlockUndo blockundo; CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; @@ -3272,7 +3337,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, blockundo, true)) return false; assert(state.IsValid()); @@ -3657,7 +3722,8 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + CBlockUndo undo; + if (!DisconnectBlock(block, state, pindex, coins, undo, &fClean)) return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); pindexState = pindex->pprev; if (!fClean) { @@ -3680,9 +3746,10 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); pindex = chainActive.Next(pindex); CBlock block; + CBlockUndo undo; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins, chainparams)) + if (!ConnectBlock(block, state, pindex, coins, chainparams, undo)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } diff --git a/src/validation.h b/src/validation.h index c0f9b6d513a..990388af95b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -13,6 +13,7 @@ #include "amount.h" #include "coins.h" #include "fs.h" +#include "coinsbyscript.h" #include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" #include "sync.h" @@ -33,6 +34,7 @@ class CBlockIndex; class CBlockTreeDB; +class CBlockUndo; class CBloomFilter; class CChainParams; class CInv; @@ -166,6 +168,7 @@ extern std::atomic_bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; +extern bool fTxOutIndex; extern bool fIsBareMultisigStd; extern bool fRequireStandard; extern bool fCheckBlockIndex; @@ -484,6 +487,14 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Context-independent validity checks */ +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); +bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); + +/** Context-dependent validity checks */ +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex *pindexPrev); +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex *pindexPrev); + /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); @@ -525,6 +536,10 @@ extern CChain chainActive; /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip; +/** Only used if -txoutindex */ +extern CCoinsViewByScriptDB *pcoinsByScriptDB; +extern CCoinsViewByScript *pcoinsByScript; + /** Global variable that points to the active block tree (protected by cs_main) */ extern CBlockTreeDB *pblocktree; diff --git a/test/functional/rest.py b/test/functional/rest.py index 776211d301f..47e4cfcac72 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -47,14 +47,14 @@ def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-txoutindex"]] * 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) self.is_network_split=False - self.sync_all() def run_test(self): url = urllib.parse.urlparse(self.nodes[0].url) @@ -67,7 +67,8 @@ def run_test(self): assert_equal(self.nodes[0].getbalance(), 50) - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + address = self.nodes[1].getnewaddress() + txid = self.nodes[0].sendtoaddress(address, 0.1) self.sync_all() self.nodes[2].generate(1) self.sync_all() @@ -128,8 +129,6 @@ def run_test(self): assert_equal(json_obj['bitmap'], "10") #test binary response - bb_hash = self.nodes[0].getbestblockhash() - binaryRequest = b'\x01\x02' binaryRequest += hex_str_to_bytes(txid) binaryRequest += pack("i", n) @@ -152,8 +151,9 @@ def run_test(self): ############################ # do a tx and don't sync - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) - json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + address2 = self.nodes[1].getnewaddress() + txid2 = self.nodes[0].sendtoaddress(address2, 0.1) + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid2+self.FORMAT_SEPARATOR+"json") json_obj = json.loads(json_string) vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) # get n of 0.1 outpoint @@ -162,12 +162,12 @@ def run_test(self): if vout['value'] == 0.1: n = vout['n'] - json_request = '/'+txid+'-'+str(n) + json_request = '/'+txid2+'-'+str(n) json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) assert_equal(len(json_obj['utxos']), 0) #there should be a outpoint because it has just added to the mempool - json_request = '/checkmempool/'+txid+'-'+str(n) + json_request = '/checkmempool/'+txid2+'-'+str(n) json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) assert_equal(len(json_obj['utxos']), 1) #there should be a outpoint because it has just added to the mempool @@ -187,14 +187,14 @@ def run_test(self): #test limits json_request = '/checkmempool/' for x in range(0, 20): - json_request += txid+'-'+str(n)+'/' + json_request += txid2+'-'+str(n)+'/' json_request = json_request.rstrip("/") response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 400) #must be a 400 because we exceeding the limits json_request = '/checkmempool/' for x in range(0, 15): - json_request += txid+'-'+str(n)+'/' + json_request += txid2+'-'+str(n)+'/' json_request = json_request.rstrip("/") response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 200) #must be a 200 because we are within the limits @@ -202,6 +202,85 @@ def run_test(self): self.nodes[0].generate(1) #generate block to not affect upcoming tests self.sync_all() + + ######################################## + # GETUTXOINDEX: query an unspent txout # + ######################################## + json_request = '/checkmempool/'+address + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there is just one txout + assert_equal(len(json_obj), 1) + txout = json_obj[0] + + #check details + assert_equal(txout['blockhash'], bb_hash) + assert_equal(txout['txid'], txid) + assert_equal(txout['value'], 0.1) + + + ############################################### + # GETUTXOINDEX: query multiple unspent txouts # + ############################################### + json_request = '/checkmempool/'+address+'/'+address2 + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there are two txouts + assert_equal(len(json_obj), 2) + txout1 = json_obj[0] + txout2 = json_obj[1] + + #check details + assert_equal(txout1['txid'], txid) + assert_equal(txout1['value'], 0.1) + assert_equal(txout2['txid'], txid2) + assert_equal(txout2['value'], 0.1) + + + ###################################### + # GETUTXOINDEX: query unseen address # + ###################################### + + json_request = '/checkmempool/'+self.nodes[0].getnewaddress() + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there are no txouts (brand new address) + assert_equal(len(json_obj), 0) + + + ################################## + # GETUTXOINDEX: invalid requests # + ################################## + + + #do some invalid requests + json_request = '{"checkmempool' + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 400) #must be a 400 because we send a invalid json request + + json_request = '/checkmempool/xxxx' + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 400) #must be a 400 because we send a invalid json request + + #test limits + json_request = '/checkmempool/' + for x in range(0, 20): + json_request += address+'/' + json_request = json_request.rstrip("/") + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + assert_equal(response.status, 400) #must be a 400 because we exceeding the limits + + json_request = '/checkmempool/' + for x in range(0, 15): + json_request += address+'/' + json_request = json_request.rstrip("/") + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + assert_equal(response.status, 200) #must be a 200 because we are within the limits + + ################ # /rest/block/ # ################ diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 0996b1bc208..aac7a6a58a6 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -110,6 +110,7 @@ 'listsinceblock.py', 'p2p-leaktests.py', 'import-abort-rescan.py', + 'utxoindex.py', ] EXTENDED_SCRIPTS = [ diff --git a/test/functional/utxoindex.py b/test/functional/utxoindex.py new file mode 100755 index 00000000000..7f0d17e5a51 --- /dev/null +++ b/test/functional/utxoindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class UTXOIndexTest(BitcoinTestFramework): + """Tests -txoutindex""" + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + self.extra_args = [["-txoutindex"], [], []] + + def setup_network(self, split=False): + print("Setup network...") + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + connect_nodes(self.nodes[1], 0) + connect_nodes(self.nodes[2], 0) + self.is_network_split = False + + def run_test(self): + print("Generating test blockchain...") + + # Check that there's no UTXO on any of the nodes + for node in self.nodes: + assert_equal(len(node.listunspent()), 0) + + # mining + self.nodes[0].generate(101) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 50) + + # TX1: send from node0 to node1 + # - check if txout from tx1 is there + address = self.nodes[1].getnewaddress() + txid1 = self.nodes[0].sendtoaddress(address, 10) + self.nodes[0].generate(101) # node will collect its own fee + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 5090) + assert_equal(self.nodes[1].getbalance(), 10) + txouts = self.nodes[0].getutxoindex(1, (address,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid1) + + # Stop node 2. We want to restart it later and orphan a node 1 block in + # order to test txoutindex handling the reorg. In other words, node 2 is + # stopped so that it won't build on a node 1 block. + stop_node(self.nodes[2], 2) + self.nodes.pop() + + # TX2: send from node1 to node0 + # - check if txout from tx1 is gone + # - check if txout from tx2 is there + address2 = self.nodes[0].getnewaddress() + txid2 = self.nodes[1].sendtoaddress(address2, 5) + self.nodes[1].generate(1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 5145) + txouts = self.nodes[0].getutxoindex(1, (address,)) + assert_equal(txouts, []) + txouts = self.nodes[0].getutxoindex(1, (address2,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid2) + + # start node 2 + self.nodes.append(start_node(2, self.options.tmpdir, self.extra_args[2])) + + # mine 10 blocks alone to have the longest chain + self.nodes[2].generate(10) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[1], 2) + self.nodes[2].generate(1) + sync_blocks(self.nodes) + + # TX2 must be reverted + # - check if txout from tx1 is there again + # - check if txout from tx2 is gone + assert_equal(self.nodes[0].getbalance(), 5640) + txouts = self.nodes[0].getutxoindex(1, (address,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid1) + txouts = self.nodes[0].getutxoindex(1, (address2,)) + assert_equal(txouts, []) + +if __name__ == '__main__': + UTXOIndexTest().main() +