From c889147288139d2588cd5176f80cf94ac8104a4d Mon Sep 17 00:00:00 2001 From: Cozz Lovan Date: Sun, 5 Oct 2014 22:30:01 +0200 Subject: [PATCH 01/15] txoutsbyaddress index --- qa/rpc-tests/txoutsbyaddress.sh | 163 ++++++++++++++++++++++++++ src/Makefile.am | 4 + src/bench/mempool_eviction.cpp | 2 +- src/coinsbyscript.cpp | 50 ++++++++ src/coinsbyscript.h | 63 ++++++++++ src/coinstats.cpp | 68 +++++++++++ src/coinstats.h | 28 +++++ src/init.cpp | 60 +++++++++- src/rpc/blockchain.cpp | 229 ++++++++++++++++++++++++++++--------- src/rpc/client.cpp | 4 + src/test/blockencodings_tests.cpp | 8 +- src/test/mempool_tests.cpp | 10 +- src/test/policyestimator_tests.cpp | 3 +- src/txdb.cpp | 184 ++++++++++++++++++++++++++++- src/txdb.h | 11 ++ src/txmempool.cpp | 36 +++++- src/txmempool.h | 6 +- src/validation.cpp | 83 ++++++++++++-- src/validation.h | 17 +++ 19 files changed, 948 insertions(+), 81 deletions(-) create mode 100644 qa/rpc-tests/txoutsbyaddress.sh create mode 100644 src/coinsbyscript.cpp create mode 100644 src/coinsbyscript.h create mode 100644 src/coinstats.cpp create mode 100644 src/coinstats.h diff --git a/qa/rpc-tests/txoutsbyaddress.sh b/qa/rpc-tests/txoutsbyaddress.sh new file mode 100644 index 00000000000..40c21f77413 --- /dev/null +++ b/qa/rpc-tests/txoutsbyaddress.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Copyright (c) 2013-2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Test -txoutsbyaddressindex + +if [ $# -lt 1 ]; then + echo "Usage: $0 path_to_binaries" + echo "e.g. $0 ../../src" + exit 1 +fi + +set -f + +BITCOIND=${1}/bitcoind +CLI=${1}/bitcoin-cli + +DIR="${BASH_SOURCE%/*}" +SENDANDWAIT="${DIR}/send.sh" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +. "$DIR/util.sh" + +D=$(mktemp -d test.XXXXX) + +D1=${D}/node1 +CreateDataDir "$D1" port=11000 rpcport=11001 +B1ARGS="-datadir=$D1" +$BITCOIND $B1ARGS -txoutsbyaddressindex -sendfreetransactions=1 & +B1PID=$! + +D2=${D}/node2 +CreateDataDir "$D2" port=11010 rpcport=11011 +B2ARGS="-datadir=$D2" +$BITCOIND $B2ARGS -sendfreetransactions=1 & +B2PID=$! + +D3=${D}/node3 +CreateDataDir "$D3" port=11020 rpcport=11021 +B3ARGS="-datadir=$D3" +$BITCOIND $B3ARGS -sendfreetransactions=1 & +B3PID=$! + +# Wait until all three nodes are at the same block number +function WaitBlocks { + while : + do + sleep 1 + declare -i BLOCKS1=$( GetBlocks $B1ARGS ) + declare -i BLOCKS2=$( GetBlocks $B2ARGS ) + declare -i BLOCKS3=$( GetBlocks $B3ARGS ) + if (( BLOCKS1 == BLOCKS2 && BLOCKS2 == BLOCKS3 )) + then + break + fi + done +} + +function WaitBlocks2 { + while : + do + sleep 1 + declare -i BLOCKS1=$( GetBlocks $B1ARGS ) + declare -i BLOCKS2=$( GetBlocks $B2ARGS ) + if (( BLOCKS1 == BLOCKS2 )) + then + break + fi + done +} + +function CleanUp { +$CLI $B3ARGS stop > /dev/null 2>&1 +wait $B3PID +$CLI $B2ARGS stop > /dev/null 2>&1 +wait $B2PID +$CLI $B1ARGS stop > /dev/null 2>&1 +wait $B1PID + +rm -rf $D +} + +function ErrorAndExit { +echo "$@" 1>&2; +CleanUp +exit 1 +} + +echo "Generating test blockchain..." + +# mining +$CLI $B2ARGS addnode "127.0.0.1:11000" "onetry" +$CLI $B3ARGS addnode "127.0.0.1:11000" "onetry" +$CLI $B1ARGS generate 101 +WaitBlocks +CheckBalance "$B1ARGS" 50 + +# TX1: send from node1 to node2 +# - check if txout from tx1 is there +address=$($CLI $B2ARGS getnewaddress) +txid1=$($CLI $B1ARGS sendtoaddress $address 10) +$CLI $B1ARGS generate 1 +WaitBlocks +CheckBalance "$B1ARGS" 90 +CheckBalance "$B2ARGS" 10 +txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") +txid=$(ExtractKey "txid" "$txouts") +if [ -z "$txid" ] || [ $txid != $txid1 ] ; then + ErrorAndExit "wrong txid1: $txid != $txid1" +fi + +# stop node 3 +$CLI $B3ARGS stop > /dev/null 2>&1 +wait $B3PID + +# TX2: send from node2 to node1 +# - check if txout from tx1 is gone +# - check if txout from tx2 is there +address2=$($CLI $B1ARGS getnewaddress) +txid2=$($CLI $B2ARGS sendtoaddress $address2 5) +$CLI $B2ARGS generate 1 +WaitBlocks2 +CheckBalance "$B1ARGS" 145 +txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") +txid=$(ExtractKey "txid" "$txouts") +if [ ! -z "$txid" ] ; then + ErrorAndExit "txid not empty: $txid" +fi +txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address2""\"]") +txid=$(ExtractKey "txid" "$txouts") +if [ -z "$txid" ] || [ $txid != $txid2 ] ; then + ErrorAndExit "wrong txid2: $txid != $txid2" +fi + +# start node 3 +$BITCOIND $B3ARGS & +B3PID=$! + +# mine 10 blocks alone to have the longest chain +$CLI $B3ARGS generate 10 +$CLI $B1ARGS addnode "127.0.0.1:11020" "onetry" +$CLI $B2ARGS addnode "127.0.0.1:11020" "onetry" +$CLI $B3ARGS generate 1 +WaitBlocks + +# TX2 must be reverted +# - check if txout from tx1 is there again +# - check if txout from tx2 is gone +CheckBalance "$B1ARGS" 640 +txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") +txid=$(ExtractKey "txid" "$txouts") +if [ -z "$txid" ] || [ $txid != $txid1 ] ; then + ErrorAndExit "wrong txid1 : $txid != $txid1" +fi +txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address2""\"]") +txid=$(ExtractKey "txid" "$txouts") +if [ ! -z "$txid" ] ; then + ErrorAndExit "txid is not empty: $txid" +fi + +echo "Tests successful, cleaning up" +CleanUp +exit 0 diff --git a/src/Makefile.am b/src/Makefile.am index cb88171348e..5c16be936c9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -89,6 +89,8 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + coinsbyscript.h \ + coinstats.h \ compat.h \ compat/byteswap.h \ compat/endian.h \ @@ -305,6 +307,8 @@ libbitcoin_common_a_SOURCES = \ base58.cpp \ chainparams.cpp \ coins.cpp \ + coinsbyscript.cpp \ + coinstats.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 073bbde0165..563aaed9646 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -96,7 +96,7 @@ static void MempoolEviction(benchmark::State& state) tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - CTxMemPool pool; + CTxMemPool pool(false); while (state.KeepRunning()) { AddTx(tx1, 10000LL, pool); diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp new file mode 100644 index 00000000000..92ad633494d --- /dev/null +++ b/src/coinsbyscript.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2014 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "coinsbyscript.h" +#include "txdb.h" +#include "hash.h" + +#include + +CCoinsViewByScript::CCoinsViewByScript(CCoinsViewDB* viewIn) : base(viewIn) { } + +bool CCoinsViewByScript::GetCoinsByScript(const CScript &script, CCoinsByScript &coins) { + const uint160 key = CCoinsViewByScript::getKey(script); + if (cacheCoinsByScript.count(key)) { + coins = cacheCoinsByScript[key]; + return true; + } + if (base->GetCoinsByHashOfScript(key, coins)) { + cacheCoinsByScript[key] = coins; + return true; + } + return false; +} + +CCoinsMapByScript::iterator CCoinsViewByScript::FetchCoinsByScript(const CScript &script, bool fRequireExisting) { + const uint160 key = CCoinsViewByScript::getKey(script); + CCoinsMapByScript::iterator it = cacheCoinsByScript.find(key); + if (it != cacheCoinsByScript.end()) + return it; + CCoinsByScript tmp; + if (!base->GetCoinsByHashOfScript(key, tmp)) + { + if (fRequireExisting) + return cacheCoinsByScript.end(); + } + CCoinsMapByScript::iterator ret = cacheCoinsByScript.insert(it, std::make_pair(key, CCoinsByScript())); + tmp.swap(ret->second); + return ret; +} + +CCoinsByScript &CCoinsViewByScript::GetCoinsByScript(const CScript &script, bool fRequireExisting) { + CCoinsMapByScript::iterator it = FetchCoinsByScript(script, fRequireExisting); + assert(it != cacheCoinsByScript.end()); + return it->second; +} + +uint160 CCoinsViewByScript::getKey(const CScript &script) { + return Hash160(script); +} diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h new file mode 100644 index 00000000000..637ec8d99ad --- /dev/null +++ b/src/coinsbyscript.h @@ -0,0 +1,63 @@ +// Copyright (c) 2014 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COINSBYSCRIPT_H +#define BITCOIN_COINSBYSCRIPT_H + +#include "coins.h" +#include "primitives/transaction.h" +#include "serialize.h" +#include "uint256.h" + +class CCoinsViewDB; +class CScript; + +class CCoinsByScript +{ +public: + // unspent transaction outputs + std::set setCoins; + + // empty constructor + CCoinsByScript() { } + + bool IsEmpty() const { + return (setCoins.empty()); + } + + void swap(CCoinsByScript &to) { + to.setCoins.swap(setCoins); + } + + ADD_SERIALIZE_METHODS; + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(setCoins); + } +}; + +typedef std::map CCoinsMapByScript; // uint160 = hash of script + +/** Adds a memory cache for coins by address */ +class CCoinsViewByScript +{ +private: + CCoinsViewDB *base; + +public: + CCoinsMapByScript cacheCoinsByScript; // accessed also from CCoinsViewDB in txdb.cpp + CCoinsViewByScript(CCoinsViewDB* baseIn); + + bool GetCoinsByScript(const CScript &script, CCoinsByScript &coins); + + // Return a modifiable reference to a CCoinsByScript. + CCoinsByScript &GetCoinsByScript(const CScript &script, bool fRequireExisting = true); + + static uint160 getKey(const CScript &script); // we use the hash of the script as key in the database + +private: + CCoinsMapByScript::iterator FetchCoinsByScript(const CScript &script, bool fRequireExisting); +}; + +#endif // BITCOIN_COINSBYSCRIPT_H diff --git a/src/coinstats.cpp b/src/coinstats.cpp new file mode 100644 index 00000000000..c39fd035b09 --- /dev/null +++ b/src/coinstats.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2014-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "coinstats.h" +#include "validation.h" + +#include + +#include // boost::thread::interrupt + +using namespace std; + +//! Calculate statistics about the unspent transaction output set +bool GetUTXOStats(CCoinsView *view, CCoinsViewDB *viewdb, CCoinsStats &stats) +{ + boost::scoped_ptr pcursor(view->Cursor()); + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + stats.hashBlock = pcursor->GetBestBlock(); + { + LOCK(cs_main); + stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; + } + ss << stats.hashBlock; + CAmount nTotalAmount = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + uint256 key; + CCoins coins; + if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { + stats.nTransactions++; + ss << key; + for (unsigned int i=0; iGetValueSize(); + ss << VARINT(0); + } else { + return error("%s: unable to read value", __func__); + } + pcursor->Next(); + } + stats.hashSerialized = ss.GetHash(); + stats.nTotalAmount = nTotalAmount; + + boost::scoped_ptr pcursordb(viewdb->RawCursor()); + pcursordb->Seek('d'); + while (pcursordb->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + CCoinsByScript coinsByScript; + if (pcursordb->GetKey(key) && key.first == 'd' && pcursordb->GetValue(coinsByScript)) { + stats.nAddresses++; + stats.nAddressesOutputs += coinsByScript.setCoins.size(); + } else { + return error("%s: unable to read value", __func__); + } + pcursordb->Next(); + } + return true; +} diff --git a/src/coinstats.h b/src/coinstats.h new file mode 100644 index 00000000000..29faf6df640 --- /dev/null +++ b/src/coinstats.h @@ -0,0 +1,28 @@ +// Copyright (c) 2014-2016 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COINSTATS_H +#define BITCOIN_COINSTATS_H + +#include "coins.h" +#include "txdb.h" + +struct CCoinsStats +{ + int nHeight; + uint256 hashBlock; + uint64_t nTransactions; + uint64_t nTransactionOutputs; + uint64_t nAddresses; + uint64_t nAddressesOutputs; // equal nTransactionOutputs (if addressindex is enabled) + uint64_t nSerializedSize; + uint256 hashSerialized; + CAmount nTotalAmount; + + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nAddresses(0), nAddressesOutputs(0), nSerializedSize(0), nTotalAmount(0) {} +}; + +bool GetUTXOStats(CCoinsView *view, CCoinsViewDB *viewdb, CCoinsStats &stats); + +#endif // BITCOIN_COINSTATS_H diff --git a/src/init.cpp b/src/init.cpp index f06c9e11000..6966cbe1f0f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -11,6 +11,7 @@ #include "addrman.h" #include "amount.h" +#include "coinstats.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" @@ -162,7 +163,6 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked // Writes do not need similar protection, as failure to write is handled by the caller. }; -static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; static std::unique_ptr globalVerifyHandle; @@ -233,6 +233,8 @@ void Shutdown() pcoinscatcher = NULL; delete pcoinsdbview; pcoinsdbview = NULL; + delete pcoinsByScript; + pcoinsByScript = NULL; delete pblocktree; pblocktree = NULL; } @@ -369,6 +371,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); + strUsage += HelpMessageOpt("-txoutsbyaddressindex", strprintf(_("Maintain an address to unspent outputs index (rpc: gettxoutsbyaddress). The index is built on first use. (default: %u)"), 0)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); @@ -1485,6 +1488,61 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // Check -txoutsbyaddressindex + pcoinsdbview->ReadFlag("txoutsbyaddressindex", fTxOutsByAddressIndex); + if (IsArgSet("-txoutsbyaddressindex")) + { + if (GetBoolArg("-txoutsbyaddressindex", false)) + { + // build index + if (!fTxOutsByAddressIndex) + { + if (!pcoinsdbview->DeleteAllCoinsByScript()) + { + strLoadError = _("Error deleting txoutsbyaddressindex"); + break; + } + if (!pcoinsdbview->GenerateAllCoinsByScript()) + { + strLoadError = _("Error building txoutsbyaddressindex"); + break; + } + CCoinsStats stats; + if (!GetUTXOStats(pcoinsTip, pcoinsdbview, stats)) + { + strLoadError = _("Error GetUTXOStats for txoutsbyaddressindex"); + break; + } + if (stats.nTransactionOutputs != stats.nAddressesOutputs) + { + strLoadError = _("Error compare stats for txoutsbyaddressindex"); + break; + } + pcoinsdbview->WriteFlag("txoutsbyaddressindex", true); + fTxOutsByAddressIndex = true; + } + } + else + { + if (fTxOutsByAddressIndex) + { + // remove index + pcoinsdbview->DeleteAllCoinsByScript(); + pcoinsdbview->WriteFlag("txoutsbyaddressindex", false); + fTxOutsByAddressIndex = false; + } + } + } + else if (fTxOutsByAddressIndex) + return InitError(_("You need to provide -txoutsbyaddressindex. Do -txoutsbyaddressindex=0 to delete the index.")); + + // Init -txoutsbyaddressindex + if (fTxOutsByAddressIndex) + { + pcoinsByScript = new CCoinsViewByScript(pcoinsdbview); + pcoinsdbview->SetCoinsViewByScript(pcoinsByScript); + } + uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks", diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 01066d0eb2e..18cda164ae1 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -6,11 +6,13 @@ #include "rpc/blockchain.h" #include "amount.h" +#include "base58.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "coins.h" #include "consensus/validation.h" +#include "coinstats.h" #include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" @@ -21,11 +23,14 @@ #include "util.h" #include "utilstrencodings.h" #include "hash.h" +#include "txdb.h" +#include "dbwrapper.h" #include #include +#include #include // boost::thread::interrupt #include @@ -760,60 +765,6 @@ UniValue getblock(const JSONRPCRequest& request) return blockToJSON(block, pblockindex); } -struct CCoinsStats -{ - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nSerializedSize; - uint256 hashSerialized; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} -}; - -//! Calculate statistics about the unspent transaction output set -static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) -{ - std::unique_ptr pcursor(view->Cursor()); - - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - stats.hashBlock = pcursor->GetBestBlock(); - { - LOCK(cs_main); - stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; - } - ss << stats.hashBlock; - CAmount nTotalAmount = 0; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; iGetValueSize(); - ss << VARINT(0); - } else { - return error("%s: unable to read value", __func__); - } - pcursor->Next(); - } - stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; - return true; -} - UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) @@ -876,6 +827,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" + " \"addresses\": n, (numeric) The number of addresses and scripts. Only if -txoutsbyaddressindex=1\n" + " \"txoutsbyaddress\": n, (numeric) The number of output transactions. Only if -txoutsbyaddressindex=1\n" " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" " \"total_amount\": x.xxx (numeric) The total amount\n" @@ -889,11 +842,13 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsTip, stats)) { + if (GetUTXOStats(pcoinsTip, pcoinsdbview, stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); + ret.push_back(Pair("addresses", (int64_t)stats.nAddresses)); + ret.push_back(Pair("txoutsbyaddress", (int64_t)stats.nAddressesOutputs)); ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); @@ -983,6 +938,169 @@ UniValue gettxout(const JSONRPCRequest& request) return ret; } +UniValue gettxoutsbyaddress(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) + throw runtime_error( + "gettxoutsbyaddress ( minconf [\"address\",...] count from )\n" + "\nReturns a list of unspent transaction outputs by address (or script).\n" + "The list is ordered by confirmations in descending order.\n" + "Note that passing minconf=0 will include the mempool.\n" + "\nTo use this function, you must start bitcoin with the -txoutsbyaddressindex parameter.\n" + "\nArguments:\n" + "1. minconf (numeric) Minimum confirmations\n" + "2. \"addresses\" (string) A json array of bitcoin addresses (or scripts)\n" + " [\n" + " \"address\" (string) bitcoin address (or script)\n" + " ,...\n" + " ]\n" + "3. count (numeric, optional, default=999999999) The number of outputs to return\n" + "4. from (numeric, optional, default=0) The number of outputs to skip\n" + "\nResult\n" + "[ (array of json object)\n" + " {\n" + " \"confirmations\" : n, (numeric) The number of confirmations\n" + " \"txid\" : \"txid\", (string) The transaction id \n" + " \"vout\" : n, (numeric) The vout value\n" + " \"value\" : x.xxx, (numeric) The transaction value in btc\n" + " \"scriptPubKey\" : { (json object)\n" + " \"asm\" : \"code\", (string) \n" + " \"hex\" : \"hex\", (string) \n" + " \"reqSigs\" : n, (numeric) Number of required signatures\n" + " \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash\n" + " \"addresses\" : [ (array of string) array of bitcoin addresses\n" + " \"bitcoinaddress\" (string) bitcoin address\n" + " ,...\n" + " ]\n" + " },\n" + " \"version\" : n, (numeric) The transaction version\n" + " \"coinbase\" : true|false (boolean) Coinbase or not\n" + " \"bestblockhash\" : \"hash\", (string) The block hash of the best block\n" + " \"bestblockheight\" : n, (numeric) The block height of the best block\n" + " \"bestblocktime\" : n, (numeric) The block time of the best block\n" + " \"blockhash\" : \"hash\", (string) The block hash of the block the tx is in (only if confirmations > 0)\n" + " \"blockheight\" : n, (numeric) The block height of the block the tx is in (only if confirmations > 0)\n" + " \"blocktime\" : ttt, (numeric) The block time in seconds since 1.1.1970 GMT (only if confirmations > 0)\n" + " }\n" + " ,...\n" + "]\n" + "\nExamples:\n" + + HelpExampleCli("gettxoutsbyaddress", "6 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + "\nAs a json rpc call\n" + + HelpExampleRpc("gettxoutsbyaddress", "6, \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + ); + + if (!fTxOutsByAddressIndex) + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "To use this function, you must start bitcoin with the -txoutsbyaddressindex parameter."); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VARR)(UniValue::VNUM)(UniValue::VNUM)); + + UniValue vObjects(UniValue::VARR); + vector > vSort; + int nMinDepth = request.params[0].get_int(); + UniValue inputs = request.params[1].get_array(); + + int nCount = 999999999; + if (request.params.size() > 2) + nCount = request.params[2].get_int(); + int nFrom = 0; + if (request.params.size() > 3) + nFrom = request.params[3].get_int(); + + if (nMinDepth < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative minconf"); + if (nCount < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative count"); + if (nFrom < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative from"); + + for (unsigned int idx = 0; idx < inputs.size(); idx++) { + const UniValue& input = inputs[idx]; + CScript script; + CBitcoinAddress address(input.get_str()); + if (address.IsValid()) { + script = GetScriptForDestination(address.Get()); + } else if (IsHex(input.get_str())) { + std::vector data(ParseHex(input.get_str())); + script = CScript(data.begin(), data.end()); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script: " + input.get_str()); + } + + CCoinsByScript coinsByScript; + pcoinsByScript->GetCoinsByScript(script, coinsByScript); + + if (nMinDepth == 0) + mempool.GetCoinsByScript(script, coinsByScript); + + BOOST_FOREACH(const COutPoint &outpoint, coinsByScript.setCoins) + { + CCoins coins; + if (nMinDepth == 0) + { + LOCK(mempool.cs); + CCoinsViewMemPool view(pcoinsTip, mempool); + if (!view.GetCoins(outpoint.hash, coins)) + continue; + mempool.pruneSpent(outpoint.hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } + else if (!pcoinsTip->GetCoins(outpoint.hash, coins)) + continue; + + if (outpoint.n < coins.vout.size() && !coins.vout[outpoint.n].IsNull() && !coins.vout[outpoint.n].scriptPubKey.IsUnspendable()) + { + // should not happen + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT && (!chainActive[coins.nHeight] || !chainActive[coins.nHeight]->phashBlock)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal Error: !chainActive[coins.nHeight]"); + + BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); + CBlockIndex *pindex = it->second; + + int nConfirmations = 0; + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + nConfirmations = pindex->nHeight - coins.nHeight + 1; + if (nConfirmations < nMinDepth) + continue; + + UniValue oScriptPubKey(UniValue::VOBJ); + ScriptPubKeyToJSON(coins.vout[outpoint.n].scriptPubKey, oScriptPubKey, true); + + UniValue o(UniValue::VOBJ); + o.push_back(Pair("confirmations", nConfirmations)); + o.push_back(Pair("txid", outpoint.hash.GetHex())); + o.push_back(Pair("vout", (int)outpoint.n)); + o.push_back(Pair("value", ValueFromAmount(coins.vout[outpoint.n].nValue))); + o.push_back(Pair("scriptPubKey", oScriptPubKey)); + o.push_back(Pair("version", coins.nVersion)); + o.push_back(Pair("coinbase", coins.fCoinBase)); + o.push_back(Pair("bestblockhash", pindex->GetBlockHash().GetHex())); + o.push_back(Pair("bestblockheight", pindex->nHeight)); + o.push_back(Pair("bestblocktime", pindex->GetBlockTime())); + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + { + o.push_back(Pair("blockhash", chainActive[coins.nHeight]->GetBlockHash().GetHex())); + o.push_back(Pair("blockheight", coins.nHeight)); + o.push_back(Pair("blocktime", chainActive[coins.nHeight]->GetBlockTime())); + } + vObjects.push_back(o); + vSort.push_back(make_pair(coins.nHeight, (unsigned int)vObjects.size() - 1)); + } + } + } + + UniValue results(UniValue::VARR); + sort(vSort.begin(), vSort.end()); + for (unsigned int i = (unsigned int)nFrom; i < vSort.size(); i++) + { + if (i == (unsigned int)nCount + (unsigned int)nFrom) + break; + + results.push_back(vObjects[vSort[i].second]); + } + + return results; +} + UniValue verifychain(const JSONRPCRequest& request) { int nCheckLevel = GetArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -1432,6 +1550,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, true, {} }, { "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} }, { "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} }, + { "blockchain", "gettxoutsbyaddress", &gettxoutsbyaddress, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} }, { "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} }, { "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 941bdd93796..be0715eae3c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -91,6 +91,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, + { "gettxoutsbyaddress", 0 }, + { "gettxoutsbyaddress", 1 }, + { "gettxoutsbyaddress", 2 }, + { "gettxoutsbyaddress", 3 }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "importprivkey", 2, "rescan" }, diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 9e4a56919d4..c5c55d1b16d 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -57,7 +57,7 @@ static CBlock BuildBlockTestCase() { BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -156,7 +156,7 @@ class TestHeaderAndShortIDs { BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -222,7 +222,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); @@ -272,7 +272,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { - CTxMemPool pool; + CTxMemPool pool(false); CMutableTransaction coinbase; coinbase.vin.resize(1); coinbase.vin[0].scriptSig.resize(10); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 51b28d09fa4..73802fb3db5 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -53,8 +53,8 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) txGrandChild[i].vout[0].nValue = 11000LL; } - - CTxMemPool testPool; + bool fTxOutsByAddressIndex = false; + CTxMemPool testPool(fTxOutsByAddressIndex); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); @@ -118,7 +118,7 @@ void CheckSort(CTxMemPool &pool, std::vector &sortedOrder) BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -319,7 +319,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; /* 3rd highest fee */ @@ -430,7 +430,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { - CTxMemPool pool; + CTxMemPool pool(false); TestMemPoolEntryHelper entry; CMutableTransaction tx1 = CMutableTransaction(); diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index ed6782ea345..f8b24f00443 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -17,7 +17,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CBlockPolicyEstimator feeEst; - CTxMemPool mpool(&feeEst); + bool fTxOutsByAddressIndex = false; + CTxMemPool mpool(fTxOutsByAddressIndex, &feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); diff --git a/src/txdb.cpp b/src/txdb.cpp index a3889fdf79b..834b0f34f8a 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -9,6 +9,7 @@ #include "hash.h" #include "pow.h" #include "uint256.h" +#include "ui_interface.h" #include @@ -25,14 +26,23 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { + pcoinsViewByScript = NULL; +} + +void CCoinsViewDB::SetCoinsViewByScript(CCoinsViewByScript* pcoinsViewByScriptIn) { + pcoinsViewByScript = pcoinsViewByScriptIn; } bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { return db.Read(std::make_pair(DB_COINS, txid), coins); } +bool CCoinsViewDB::GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const { + return db.Read(make_pair('d', hash), coins); +} + bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { return db.Exists(std::make_pair(DB_COINS, txid)); } @@ -60,6 +70,18 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } + if (pcoinsViewByScript) // only if -txoutsbyaddressindex + { + for (CCoinsMapByScript::iterator it = pcoinsViewByScript->cacheCoinsByScript.begin(); it != pcoinsViewByScript->cacheCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair('d', it->first)); + else + batch.Write(make_pair('d', it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + pcoinsViewByScript->cacheCoinsByScript.erase(itOld); + } + pcoinsViewByScript->cacheCoinsByScript.clear(); + } if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); @@ -67,6 +89,18 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } +bool CCoinsViewDB::WriteFlag(const std::string &name, bool fValue) { + return db.Write(std::make_pair('F', name), fValue ? '1' : '0'); +} + +bool CCoinsViewDB::ReadFlag(const std::string &name, bool &fValue) { + char ch; + if (!db.Read(std::make_pair('F', name), ch)) + return false; + fValue = ch == '1'; + return true; +} + CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } @@ -90,6 +124,11 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read(DB_LAST_BLOCK, nFile); } +CDBIterator *CCoinsViewDB::RawCursor() const +{ + return const_cast(&db)->NewIterator(); +} + CCoinsViewCursor *CCoinsViewDB::Cursor() const { CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(&db)->NewIterator(), GetBestBlock()); @@ -134,6 +173,149 @@ void CCoinsViewDBCursor::Next() keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false } +int64_t CCoinsViewDB::GetPrefixCount(char prefix) const +{ + boost::scoped_ptr pcursor(RawCursor()); + pcursor->Seek(prefix); + + int64_t i = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + char key; + if (!pcursor->GetKey(key) || key != prefix) + break; + i++; + pcursor->Next(); + } catch (std::exception &e) { + return 0; + } + } + return i; +} + +bool CCoinsViewDB::DeleteAllCoinsByScript() +{ + boost::scoped_ptr pcursor(RawCursor()); + pcursor->Seek('d'); + + std::vector v; + int64_t i = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + std::pair key; + if (!pcursor->GetKey(key) || key.first != 'd') + break; + v.push_back(key.second); + if (v.size() >= 10000) + { + i += v.size(); + CDBBatch batch(db); + BOOST_FOREACH(const uint160& hash, v) + batch.Erase(make_pair('d', hash)); // delete + db.WriteBatch(batch); + v.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!v.empty()) + { + i += v.size(); + CDBBatch batch(db); + BOOST_FOREACH(const uint160& hash, v) + batch.Erase(make_pair('d', hash)); // delete + db.WriteBatch(batch); + } + if (i > 0) + LogPrintf("Address index with %d addresses successfully deleted.\n", i); + + return true; +} + +bool CCoinsViewDB::GenerateAllCoinsByScript() +{ + LogPrintf("Building address index for -txoutsbyaddressindex. Be patient...\n"); + int64_t nTxCount = GetPrefixCount('c'); + + boost::scoped_ptr pcursor(RawCursor()); + pcursor->Seek('c'); + + CCoinsMapByScript mapCoinsByScript; + int64_t i = 0; + int64_t progress = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + if (progress % 1000 == 0 && nTxCount > 0) + uiInterface.ShowProgress(_("Building address index..."), (int)(((double)progress / (double)nTxCount) * (double)100)); + progress++; + + std::pair key; + CCoins coins; + if (!pcursor->GetKey(key) || key.first != 'c') + break; + uint256 txhash = key.second; + if (!pcursor->GetValue(coins)) + break; + + for (unsigned int j = 0; j < coins.vout.size(); j++) + { + if (coins.vout[j].IsNull() || coins.vout[j].scriptPubKey.IsUnspendable()) + continue; + + const uint160 key = CCoinsViewByScript::getKey(coins.vout[j].scriptPubKey); + if (!mapCoinsByScript.count(key)) + { + CCoinsByScript coinsByScript; + GetCoinsByHashOfScript(key, coinsByScript); + mapCoinsByScript.insert(make_pair(key, coinsByScript)); + } + mapCoinsByScript[key].setCoins.insert(COutPoint(txhash, (uint32_t)j)); + i++; + } + + if (mapCoinsByScript.size() >= 10000) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair('d', it->first)); + else + batch.Write(make_pair('d', it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + mapCoinsByScript.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!mapCoinsByScript.empty()) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair('d', it->first)); + else + batch.Write(make_pair('d', it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + } + LogPrintf("Address index with %d outputs successfully built.\n", i); + return true; +} + bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index d9214ba6185..a2a27c1db4f 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -9,6 +9,7 @@ #include "coins.h" #include "dbwrapper.h" #include "chain.h" +#include "coinsbyscript.h" #include #include @@ -70,15 +71,25 @@ struct CDiskTxPos : public CDiskBlockPos /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView { +private: + CCoinsViewByScript* pcoinsViewByScript; protected: CDBWrapper db; public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); bool GetCoins(const uint256 &txid, CCoins &coins) const; + bool GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const; bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + bool WriteFlag(const std::string &name, bool fValue); + bool ReadFlag(const std::string &name, bool &fValue); + int64_t GetPrefixCount(char prefix) const; + bool DeleteAllCoinsByScript(); // removes txoutsbyaddressindex + bool GenerateAllCoinsByScript(); // creates txoutsbyaddressindex + void SetCoinsViewByScript(CCoinsViewByScript* pcoinsViewByScriptIn); + CDBIterator *RawCursor() const; CCoinsViewCursor *Cursor() const; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ac842da6bf3..3361edb210d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -331,7 +331,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : +CTxMemPool::CTxMemPool(const bool& _fTxOutsByAddressIndex, CBlockPolicyEstimator* estimator) : nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -361,6 +361,17 @@ unsigned int CTxMemPool::GetTransactionsUpdated() const return nTransactionsUpdated; } +void CTxMemPool::GetCoinsByScript(const CScript& script, CCoinsByScript& coinsByScript) const +{ + LOCK(cs); + CCoinsMapByScript::const_iterator it = mapCoinsByScript.find(CCoinsViewByScript::getKey(script)); + if (it != mapCoinsByScript.end()) + { + BOOST_FOREACH(const COutPoint &outpoint, it->second.setCoins) + coinsByScript.setCoins.insert(outpoint); + } +} + void CTxMemPool::AddTransactionsUpdated(unsigned int n) { LOCK(cs); @@ -423,6 +434,11 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; + if (fTxOutsByAddressIndex) + for (unsigned int i = 0; i < tx.vout.size(); i++) + if (!tx.vout[i].IsNull() && !tx.vout[i].scriptPubKey.IsUnspendable()) + mapCoinsByScript[CCoinsViewByScript::getKey(tx.vout[i].scriptPubKey)].setCoins.insert(COutPoint(hash, (uint32_t)i)); + return true; } @@ -485,6 +501,24 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso // Remove transaction from memory pool { LOCK(cs); + + if (fTxOutsByAddressIndex) + { + for (unsigned int i = 0; i < origTx.vout.size(); i++) + { + if (origTx.vout[i].IsNull() || origTx.vout[i].scriptPubKey.IsUnspendable()) + continue; + + CCoinsMapByScript::iterator it = mapCoinsByScript.find(CCoinsViewByScript::getKey(origTx.vout[i].scriptPubKey)); + if (it != mapCoinsByScript.end()) + { + it->second.setCoins.erase(COutPoint(origTx.GetHash(), (uint32_t)i)); + if (it->second.setCoins.empty()) + mapCoinsByScript.erase(it); + } + } + } + setEntries txToRemove; txiter origit = mapTx.find(origTx.GetHash()); if (origit != mapTx.end()) { diff --git a/src/txmempool.h b/src/txmempool.h index 92c4d9f9d47..b2297f67e21 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "coinsbyscript.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -415,6 +416,8 @@ class CTxMemPool CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. + const bool& fTxOutsByAddressIndex; + CCoinsMapByScript mapCoinsByScript; // only used if -txoutsbyaddressindex uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; @@ -496,7 +499,7 @@ class CTxMemPool /** Create a new CTxMemPool. */ - CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); + CTxMemPool(const bool& _fTxOutsByAddressIndex, CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -526,6 +529,7 @@ class CTxMemPool void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); + void GetCoinsByScript(const CScript& script, CCoinsByScript& coinsByScript) const; /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a block. diff --git a/src/validation.cpp b/src/validation.cpp index 6c60be45a1a..e43cbfc6774 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -65,6 +65,7 @@ int nScriptCheckThreads = 0; std::atomic_bool fImporting(false); bool fReindex = false; bool fTxIndex = false; +bool fTxOutsByAddressIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -82,7 +83,7 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; -CTxMemPool mempool(&feeEstimator); +CTxMemPool mempool(fTxOutsByAddressIndex, &feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -175,6 +176,8 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc } CCoinsViewCache *pcoinsTip = NULL; +CCoinsViewDB *pcoinsdbview = NULL; +CCoinsViewByScript *pcoinsByScript = NULL; CBlockTreeDB *pblocktree = NULL; enum FlushStateMode { @@ -1530,7 +1533,7 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean * will be true if no problems were found. Otherwise, the return value will be false in case * of problems. Note that in any case, coins may be modified. */ -static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean = NULL) +static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, CBlockUndo& blockUndo, bool* pfClean = NULL) { assert(pindex->GetBlockHash() == view.GetBestBlock()); @@ -1539,7 +1542,6 @@ static bool DisconnectBlock(const CBlock& block, CValidationState& state, const bool fClean = true; - CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); if (pos.IsNull()) return error("DisconnectBlock(): no undo data available"); @@ -1687,7 +1689,7 @@ static int64_t nTimeTotal = 0; * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) + CCoinsViewCache& view, const CChainParams& chainparams, CBlockUndo& blockundo, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -1808,8 +1810,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); - CBlockUndo blockundo; - CCheckQueueControl control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); std::vector prevheights; @@ -2054,6 +2054,60 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +void static UpdateAddressIndex(const CTxOut& txout, const COutPoint& outpoint, bool fInsert) +{ + if (!txout.IsNull() && !txout.scriptPubKey.IsUnspendable()) + { + CCoinsByScript &coinsByScript = pcoinsByScript->GetCoinsByScript(txout.scriptPubKey, !fInsert); + if (fInsert) + coinsByScript.setCoins.insert(outpoint); + else + coinsByScript.setCoins.erase(outpoint); + } +} + +void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool fConnect) +{ + if (!fTxOutsByAddressIndex) + return; + + assert(block.vtx.size() > 0); + unsigned int i = 0; + if (!fConnect) + i = block.vtx.size() - 1; // iterate backwards + + while (true) + { + const CTransaction &tx = *block.vtx[i]; + + if (i > 0) + { + for (unsigned int j = 0; j < tx.vin.size(); j++) + UpdateAddressIndex(blockundo.vtxundo[i-1].vprevout[j].txout, tx.vin[j].prevout, !fConnect); + } + + for (unsigned int j = 0; j < tx.vout.size(); j++) + { + CTxOut& txout = const_cast(tx.vout[j]); + const COutPoint outpoint(tx.GetHash(),((uint32_t)j)); + UpdateAddressIndex(txout, outpoint, fConnect); + } + + if (fConnect) + { + if (i == block.vtx.size() - 1) + break; + i++; + } + else + { + if (i == 0) + break; + i--; + } + } +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -2130,11 +2184,13 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { + CBlockUndo blockUndo; CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (!DisconnectBlock(block, state, pindexDelete, view, blockUndo)) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); + UpdateAddressIndex(block, blockUndo, false); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. @@ -2267,8 +2323,9 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { + CBlockUndo blockundo; CCoinsViewCache view(pcoinsTip); - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams, blockundo); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) @@ -2279,6 +2336,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); + UpdateAddressIndex(*pblock, blockundo, true); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); @@ -3261,6 +3319,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); CCoinsViewCache viewNew(pcoinsTip); + CBlockUndo blockundo; CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; @@ -3272,7 +3331,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, blockundo, true)) return false; assert(state.IsValid()); @@ -3657,7 +3716,8 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + CBlockUndo undo; + if (!DisconnectBlock(block, state, pindex, coins, undo, &fClean)) return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); pindexState = pindex->pprev; if (!fClean) { @@ -3680,9 +3740,10 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); pindex = chainActive.Next(pindex); CBlock block; + CBlockUndo undo; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins, chainparams)) + if (!ConnectBlock(block, state, pindex, coins, chainparams, undo)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } diff --git a/src/validation.h b/src/validation.h index c0f9b6d513a..274adffd1ae 100644 --- a/src/validation.h +++ b/src/validation.h @@ -13,6 +13,7 @@ #include "amount.h" #include "coins.h" #include "fs.h" +#include "coinsbyscript.h" #include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" #include "sync.h" @@ -33,6 +34,7 @@ class CBlockIndex; class CBlockTreeDB; +class CBlockUndo; class CBloomFilter; class CChainParams; class CInv; @@ -166,6 +168,7 @@ extern std::atomic_bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; +extern bool fTxOutsByAddressIndex; extern bool fIsBareMultisigStd; extern bool fRequireStandard; extern bool fCheckBlockIndex; @@ -484,6 +487,14 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Context-independent validity checks */ +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); +bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); + +/** Context-dependent validity checks */ +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex *pindexPrev); +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex *pindexPrev); + /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); @@ -525,6 +536,12 @@ extern CChain chainActive; /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip; +/** Global variable that points to the active CCoinsView (protected by cs_main) */ +extern CCoinsViewDB *pcoinsdbview; + +/** Only used if -txoutsbyaddressindex */ +extern CCoinsViewByScript *pcoinsByScript; + /** Global variable that points to the active block tree (protected by cs_main) */ extern CBlockTreeDB *pblocktree; From 536eb705b78dde70916138f1888a8af9384c0181 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 3 Sep 2016 21:11:34 +1200 Subject: [PATCH 02/15] move txoutsbyaddress index to its own database --- src/coinsbyscript.cpp | 232 ++++++++++++++++++++++++++++++++++++++++++++++++- src/coinsbyscript.h | 62 ++++++++++++- src/coinstats.cpp | 9 +- src/coinstats.h | 3 +- src/init.cpp | 23 ++--- src/rpc/blockchain.cpp | 2 +- src/txdb.cpp | 168 +---------------------------------- src/txdb.h | 11 +-- src/validation.cpp | 8 +- src/validation.h | 4 +- 10 files changed, 318 insertions(+), 204 deletions(-) diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index 92ad633494d..8462cc2800d 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -1,14 +1,23 @@ -// Copyright (c) 2014 The Bitcoin developers +// Copyright (c) 2014-2016 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "coinsbyscript.h" #include "txdb.h" #include "hash.h" +#include "ui_interface.h" #include -CCoinsViewByScript::CCoinsViewByScript(CCoinsViewDB* viewIn) : base(viewIn) { } +#include + +using namespace std; + +static const char DB_COINS_BYSCRIPT = 'd'; +static const char DB_FLAG = 'F'; +static const char DB_BEST_BLOCK = 'B'; + +CCoinsViewByScript::CCoinsViewByScript(CCoinsViewByScriptDB* viewIn) : base(viewIn) { } bool CCoinsViewByScript::GetCoinsByScript(const CScript &script, CCoinsByScript &coins) { const uint160 key = CCoinsViewByScript::getKey(script); @@ -48,3 +57,222 @@ CCoinsByScript &CCoinsViewByScript::GetCoinsByScript(const CScript &script, bool uint160 CCoinsViewByScript::getKey(const CScript &script) { return Hash160(script); } + +uint256 CCoinsViewByScript::GetBestBlock() const { + return hashBlock; +} + +void CCoinsViewByScript::SetBestBlock(const uint256 &hashBlockIn) { + hashBlock = hashBlockIn; +} + +bool CCoinsViewByScript::Flush() { + bool fOk = base->BatchWrite(this, hashBlock); + return fOk; +} + +CCoinsViewByScriptDB::CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "coinsbyscript", nCacheSize, fMemory, fWipe, true) +{ +} + +bool CCoinsViewByScriptDB::GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const { + return db.Read(make_pair(DB_COINS_BYSCRIPT, hash), coins); +} + +bool CCoinsViewByScriptDB::BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock) { + CDBBatch batch(db); + size_t count = 0; + for (CCoinsMapByScript::iterator it = pcoinsViewByScriptIn->cacheCoinsByScript.begin(); it != pcoinsViewByScriptIn->cacheCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + pcoinsViewByScriptIn->cacheCoinsByScript.erase(itOld); + count++; + } + pcoinsViewByScriptIn->cacheCoinsByScript.clear(); + + if (!hashBlock.IsNull()) + batch.Write(DB_BEST_BLOCK, hashBlock); + + LogPrint("coindb", "Committing %u coin address indexes to coin database...\n", (unsigned int)count); + return db.WriteBatch(batch); +} + +bool CCoinsViewByScriptDB::WriteFlag(const std::string &name, bool fValue) { + return db.Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); +} + +bool CCoinsViewByScriptDB::ReadFlag(const std::string &name, bool &fValue) { + char ch; + if (!db.Read(std::make_pair(DB_FLAG, name), ch)) + return false; + fValue = ch == '1'; + return true; +} + +CCoinsViewByScriptDBCursor *CCoinsViewByScriptDB::Cursor() const +{ + CCoinsViewByScriptDBCursor *i = new CCoinsViewByScriptDBCursor(const_cast(&db)->NewIterator()); + /* It seems that there are no "const iterators" for LevelDB. Since we + only need read operations on it, use a const-cast to get around + that restriction. */ + i->pcursor->Seek(DB_COINS_BYSCRIPT); + if (!i->pcursor->Valid()) + // If db empty then set this cursor invalid + i->keyTmp.first = 0; + else + // Cache key of first record + i->pcursor->GetKey(i->keyTmp); + return i; +} + +bool CCoinsViewByScriptDBCursor::GetKey(uint160 &key) const +{ + // Return cached key + if (keyTmp.first == DB_COINS_BYSCRIPT) { + key = keyTmp.second; + return true; + } + return false; +} + +bool CCoinsViewByScriptDBCursor::GetValue(CCoinsByScript &coins) const +{ + return pcursor->GetValue(coins); +} + +unsigned int CCoinsViewByScriptDBCursor::GetValueSize() const +{ + return pcursor->GetValueSize(); +} + +bool CCoinsViewByScriptDBCursor::Valid() const +{ + return keyTmp.first == DB_COINS_BYSCRIPT; +} + +void CCoinsViewByScriptDBCursor::Next() +{ + pcursor->Next(); + if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false +} + +bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() +{ + boost::scoped_ptr pcursor(Cursor()); + + std::vector v; + int64_t i = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + uint160 hash; + if (!pcursor->GetKey(hash)) + break; + v.push_back(hash); + if (v.size() >= 10000) + { + i += v.size(); + CDBBatch batch(db); + BOOST_FOREACH(const uint160& hash, v) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete + db.WriteBatch(batch); + v.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!v.empty()) + { + i += v.size(); + CDBBatch batch(db); + BOOST_FOREACH(const uint160& hash, v) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete + db.WriteBatch(batch); + } + if (i > 0) + LogPrintf("Address index with %d addresses successfully deleted.\n", i); + + return true; +} + +bool CCoinsViewByScriptDB::GenerateAllCoinsByScript(CCoinsViewDB* coinsIn) +{ + LogPrintf("Building address index for -txoutsbyaddressindex. Be patient...\n"); + int64_t nTxCount = coinsIn->CountCoins(); + + boost::scoped_ptr pcursor(coinsIn->Cursor()); + + CCoinsMapByScript mapCoinsByScript; + int64_t i = 0; + int64_t progress = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + try { + if (progress % 1000 == 0 && nTxCount > 0) + uiInterface.ShowProgress(_("Building address index..."), (int)(((double)progress / (double)nTxCount) * (double)100)); + progress++; + + uint256 txhash; + CCoins coins; + if (!pcursor->GetKey(txhash) || !pcursor->GetValue(coins)) + break; + + for (unsigned int j = 0; j < coins.vout.size(); j++) + { + if (coins.vout[j].IsNull() || coins.vout[j].scriptPubKey.IsUnspendable()) + continue; + + const uint160 key = CCoinsViewByScript::getKey(coins.vout[j].scriptPubKey); + if (!mapCoinsByScript.count(key)) + { + CCoinsByScript coinsByScript; + GetCoinsByHashOfScript(key, coinsByScript); + mapCoinsByScript.insert(make_pair(key, coinsByScript)); + } + mapCoinsByScript[key].setCoins.insert(COutPoint(txhash, (uint32_t)j)); + i++; + } + + if (mapCoinsByScript.size() >= 10000) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + mapCoinsByScript.clear(); + } + + pcursor->Next(); + } catch (std::exception &e) { + return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + } + } + if (!mapCoinsByScript.empty()) + { + CDBBatch batch(db); + for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { + if (it->second.IsEmpty()) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, it->first)); + else + batch.Write(make_pair(DB_COINS_BYSCRIPT, it->first), it->second); + CCoinsMapByScript::iterator itOld = it++; + mapCoinsByScript.erase(itOld); + } + db.WriteBatch(batch); + } + LogPrintf("Address index with %d outputs successfully built.\n", i); + return true; +} diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h index 637ec8d99ad..5a2d12943ea 100644 --- a/src/coinsbyscript.h +++ b/src/coinsbyscript.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin developers +// Copyright (c) 2014-2016 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,11 +6,13 @@ #define BITCOIN_COINSBYSCRIPT_H #include "coins.h" +#include "dbwrapper.h" #include "primitives/transaction.h" #include "serialize.h" #include "uint256.h" class CCoinsViewDB; +class CCoinsViewByScriptDB; class CScript; class CCoinsByScript @@ -43,11 +45,13 @@ typedef std::map CCoinsMapByScript; // uint160 = hash o class CCoinsViewByScript { private: - CCoinsViewDB *base; + CCoinsViewByScriptDB *base; + + mutable uint256 hashBlock; public: - CCoinsMapByScript cacheCoinsByScript; // accessed also from CCoinsViewDB in txdb.cpp - CCoinsViewByScript(CCoinsViewDB* baseIn); + CCoinsMapByScript cacheCoinsByScript; // accessed also from CCoinsViewByScriptDB + CCoinsViewByScript(CCoinsViewByScriptDB* baseIn); bool GetCoinsByScript(const CScript &script, CCoinsByScript &coins); @@ -56,8 +60,58 @@ class CCoinsViewByScript static uint160 getKey(const CScript &script); // we use the hash of the script as key in the database + void SetBestBlock(const uint256 &hashBlock); + uint256 GetBestBlock() const; + + /** + * Push the modifications applied to this cache to its base. + * Failure to call this method before destruction will cause the changes to be forgotten. + * If false is returned, the state of this cache (and its backing view) will be undefined. + */ + bool Flush(); + private: CCoinsMapByScript::iterator FetchCoinsByScript(const CScript &script, bool fRequireExisting); }; +/** Cursor for iterating over a CCoinsViewByScriptDB */ +class CCoinsViewByScriptDBCursor +{ +public: + ~CCoinsViewByScriptDBCursor() {} + + bool GetKey(uint160 &key) const; + bool GetValue(CCoinsByScript &coins) const; + unsigned int GetValueSize() const; + + bool Valid() const; + void Next(); + +private: + CCoinsViewByScriptDBCursor(CDBIterator* pcursorIn): + pcursor(pcursorIn) {} + uint256 hashBlock; + boost::scoped_ptr pcursor; + std::pair keyTmp; + + friend class CCoinsViewByScriptDB; +}; + +/** coinsbyscript database (coinsbyscript/) */ +class CCoinsViewByScriptDB +{ +protected: + CDBWrapper db; +public: + CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + + bool GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const; + bool BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock); + bool WriteFlag(const std::string &name, bool fValue); + bool ReadFlag(const std::string &name, bool &fValue); + bool DeleteAllCoinsByScript(); // removes txoutsbyaddressindex + bool GenerateAllCoinsByScript(CCoinsViewDB* coinsIn); // creates txoutsbyaddressindex + CCoinsViewByScriptDBCursor *Cursor() const; +}; + #endif // BITCOIN_COINSBYSCRIPT_H diff --git a/src/coinstats.cpp b/src/coinstats.cpp index c39fd035b09..34328955b45 100644 --- a/src/coinstats.cpp +++ b/src/coinstats.cpp @@ -12,7 +12,7 @@ using namespace std; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView *view, CCoinsViewDB *viewdb, CCoinsStats &stats) +bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats) { boost::scoped_ptr pcursor(view->Cursor()); @@ -50,13 +50,12 @@ bool GetUTXOStats(CCoinsView *view, CCoinsViewDB *viewdb, CCoinsStats &stats) stats.hashSerialized = ss.GetHash(); stats.nTotalAmount = nTotalAmount; - boost::scoped_ptr pcursordb(viewdb->RawCursor()); - pcursordb->Seek('d'); + boost::scoped_ptr pcursordb(viewbyscriptdb->Cursor()); while (pcursordb->Valid()) { boost::this_thread::interruption_point(); - std::pair key; + uint160 hash; CCoinsByScript coinsByScript; - if (pcursordb->GetKey(key) && key.first == 'd' && pcursordb->GetValue(coinsByScript)) { + if (pcursordb->GetKey(hash) && pcursordb->GetValue(coinsByScript)) { stats.nAddresses++; stats.nAddressesOutputs += coinsByScript.setCoins.size(); } else { diff --git a/src/coinstats.h b/src/coinstats.h index 29faf6df640..bac4f3db34e 100644 --- a/src/coinstats.h +++ b/src/coinstats.h @@ -7,6 +7,7 @@ #include "coins.h" #include "txdb.h" +#include "coinsbyscript.h" struct CCoinsStats { @@ -23,6 +24,6 @@ struct CCoinsStats CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nAddresses(0), nAddressesOutputs(0), nSerializedSize(0), nTotalAmount(0) {} }; -bool GetUTXOStats(CCoinsView *view, CCoinsViewDB *viewdb, CCoinsStats &stats); +bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats); #endif // BITCOIN_COINSTATS_H diff --git a/src/init.cpp b/src/init.cpp index 6966cbe1f0f..e71ba48e0e2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -163,6 +163,7 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked // Writes do not need similar protection, as failure to write is handled by the caller. }; +static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; static std::unique_ptr globalVerifyHandle; @@ -235,6 +236,8 @@ void Shutdown() pcoinsdbview = NULL; delete pcoinsByScript; pcoinsByScript = NULL; + delete pcoinsByScriptDB; + pcoinsByScriptDB = NULL; delete pblocktree; pblocktree = NULL; } @@ -1441,6 +1444,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex || fReindexChainState); + pcoinsByScriptDB = new CCoinsViewByScriptDB(nCoinDBCache, false, false); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); @@ -1489,7 +1493,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } // Check -txoutsbyaddressindex - pcoinsdbview->ReadFlag("txoutsbyaddressindex", fTxOutsByAddressIndex); + pcoinsByScriptDB->ReadFlag("txoutsbyaddressindex", fTxOutsByAddressIndex); if (IsArgSet("-txoutsbyaddressindex")) { if (GetBoolArg("-txoutsbyaddressindex", false)) @@ -1497,18 +1501,18 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // build index if (!fTxOutsByAddressIndex) { - if (!pcoinsdbview->DeleteAllCoinsByScript()) + if (!pcoinsByScriptDB->DeleteAllCoinsByScript()) { strLoadError = _("Error deleting txoutsbyaddressindex"); break; } - if (!pcoinsdbview->GenerateAllCoinsByScript()) + if (!pcoinsByScriptDB->GenerateAllCoinsByScript(pcoinsdbview)) { strLoadError = _("Error building txoutsbyaddressindex"); break; } CCoinsStats stats; - if (!GetUTXOStats(pcoinsTip, pcoinsdbview, stats)) + if (!GetUTXOStats(pcoinsTip, pcoinsByScriptDB, stats)) { strLoadError = _("Error GetUTXOStats for txoutsbyaddressindex"); break; @@ -1518,7 +1522,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) strLoadError = _("Error compare stats for txoutsbyaddressindex"); break; } - pcoinsdbview->WriteFlag("txoutsbyaddressindex", true); + pcoinsByScriptDB->WriteFlag("txoutsbyaddressindex", true); fTxOutsByAddressIndex = true; } } @@ -1527,8 +1531,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (fTxOutsByAddressIndex) { // remove index - pcoinsdbview->DeleteAllCoinsByScript(); - pcoinsdbview->WriteFlag("txoutsbyaddressindex", false); + pcoinsByScriptDB->DeleteAllCoinsByScript(); + pcoinsByScriptDB->WriteFlag("txoutsbyaddressindex", false); fTxOutsByAddressIndex = false; } } @@ -1538,10 +1542,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // Init -txoutsbyaddressindex if (fTxOutsByAddressIndex) - { - pcoinsByScript = new CCoinsViewByScript(pcoinsdbview); - pcoinsdbview->SetCoinsViewByScript(pcoinsByScript); - } + pcoinsByScript = new CCoinsViewByScript(pcoinsByScriptDB); uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 18cda164ae1..fddf7955ea2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -842,7 +842,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsTip, pcoinsdbview, stats)) { + if (GetUTXOStats(pcoinsTip, pcoinsByScriptDB, stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); diff --git a/src/txdb.cpp b/src/txdb.cpp index 834b0f34f8a..90f48b346b9 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -28,21 +28,12 @@ static const char DB_LAST_BLOCK = 'l'; CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { - pcoinsViewByScript = NULL; -} - -void CCoinsViewDB::SetCoinsViewByScript(CCoinsViewByScript* pcoinsViewByScriptIn) { - pcoinsViewByScript = pcoinsViewByScriptIn; } bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { return db.Read(std::make_pair(DB_COINS, txid), coins); } -bool CCoinsViewDB::GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const { - return db.Read(make_pair('d', hash), coins); -} - bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { return db.Exists(std::make_pair(DB_COINS, txid)); } @@ -70,18 +61,6 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } - if (pcoinsViewByScript) // only if -txoutsbyaddressindex - { - for (CCoinsMapByScript::iterator it = pcoinsViewByScript->cacheCoinsByScript.begin(); it != pcoinsViewByScript->cacheCoinsByScript.end();) { - if (it->second.IsEmpty()) - batch.Erase(make_pair('d', it->first)); - else - batch.Write(make_pair('d', it->first), it->second); - CCoinsMapByScript::iterator itOld = it++; - pcoinsViewByScript->cacheCoinsByScript.erase(itOld); - } - pcoinsViewByScript->cacheCoinsByScript.clear(); - } if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); @@ -89,18 +68,6 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -bool CCoinsViewDB::WriteFlag(const std::string &name, bool fValue) { - return db.Write(std::make_pair('F', name), fValue ? '1' : '0'); -} - -bool CCoinsViewDB::ReadFlag(const std::string &name, bool &fValue) { - char ch; - if (!db.Read(std::make_pair('F', name), ch)) - return false; - fValue = ch == '1'; - return true; -} - CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } @@ -124,11 +91,6 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read(DB_LAST_BLOCK, nFile); } -CDBIterator *CCoinsViewDB::RawCursor() const -{ - return const_cast(&db)->NewIterator(); -} - CCoinsViewCursor *CCoinsViewDB::Cursor() const { CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(&db)->NewIterator(), GetBestBlock()); @@ -173,18 +135,14 @@ void CCoinsViewDBCursor::Next() keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false } -int64_t CCoinsViewDB::GetPrefixCount(char prefix) const +int64_t CCoinsViewDB::CountCoins() const { - boost::scoped_ptr pcursor(RawCursor()); - pcursor->Seek(prefix); + boost::scoped_ptr pcursor(Cursor()); int64_t i = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); try { - char key; - if (!pcursor->GetKey(key) || key != prefix) - break; i++; pcursor->Next(); } catch (std::exception &e) { @@ -194,128 +152,6 @@ int64_t CCoinsViewDB::GetPrefixCount(char prefix) const return i; } -bool CCoinsViewDB::DeleteAllCoinsByScript() -{ - boost::scoped_ptr pcursor(RawCursor()); - pcursor->Seek('d'); - - std::vector v; - int64_t i = 0; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - try { - std::pair key; - if (!pcursor->GetKey(key) || key.first != 'd') - break; - v.push_back(key.second); - if (v.size() >= 10000) - { - i += v.size(); - CDBBatch batch(db); - BOOST_FOREACH(const uint160& hash, v) - batch.Erase(make_pair('d', hash)); // delete - db.WriteBatch(batch); - v.clear(); - } - - pcursor->Next(); - } catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __func__, e.what()); - } - } - if (!v.empty()) - { - i += v.size(); - CDBBatch batch(db); - BOOST_FOREACH(const uint160& hash, v) - batch.Erase(make_pair('d', hash)); // delete - db.WriteBatch(batch); - } - if (i > 0) - LogPrintf("Address index with %d addresses successfully deleted.\n", i); - - return true; -} - -bool CCoinsViewDB::GenerateAllCoinsByScript() -{ - LogPrintf("Building address index for -txoutsbyaddressindex. Be patient...\n"); - int64_t nTxCount = GetPrefixCount('c'); - - boost::scoped_ptr pcursor(RawCursor()); - pcursor->Seek('c'); - - CCoinsMapByScript mapCoinsByScript; - int64_t i = 0; - int64_t progress = 0; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - try { - if (progress % 1000 == 0 && nTxCount > 0) - uiInterface.ShowProgress(_("Building address index..."), (int)(((double)progress / (double)nTxCount) * (double)100)); - progress++; - - std::pair key; - CCoins coins; - if (!pcursor->GetKey(key) || key.first != 'c') - break; - uint256 txhash = key.second; - if (!pcursor->GetValue(coins)) - break; - - for (unsigned int j = 0; j < coins.vout.size(); j++) - { - if (coins.vout[j].IsNull() || coins.vout[j].scriptPubKey.IsUnspendable()) - continue; - - const uint160 key = CCoinsViewByScript::getKey(coins.vout[j].scriptPubKey); - if (!mapCoinsByScript.count(key)) - { - CCoinsByScript coinsByScript; - GetCoinsByHashOfScript(key, coinsByScript); - mapCoinsByScript.insert(make_pair(key, coinsByScript)); - } - mapCoinsByScript[key].setCoins.insert(COutPoint(txhash, (uint32_t)j)); - i++; - } - - if (mapCoinsByScript.size() >= 10000) - { - CDBBatch batch(db); - for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { - if (it->second.IsEmpty()) - batch.Erase(make_pair('d', it->first)); - else - batch.Write(make_pair('d', it->first), it->second); - CCoinsMapByScript::iterator itOld = it++; - mapCoinsByScript.erase(itOld); - } - db.WriteBatch(batch); - mapCoinsByScript.clear(); - } - - pcursor->Next(); - } catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __func__, e.what()); - } - } - if (!mapCoinsByScript.empty()) - { - CDBBatch batch(db); - for (CCoinsMapByScript::iterator it = mapCoinsByScript.begin(); it != mapCoinsByScript.end();) { - if (it->second.IsEmpty()) - batch.Erase(make_pair('d', it->first)); - else - batch.Write(make_pair('d', it->first), it->second); - CCoinsMapByScript::iterator itOld = it++; - mapCoinsByScript.erase(itOld); - } - db.WriteBatch(batch); - } - LogPrintf("Address index with %d outputs successfully built.\n", i); - return true; -} - bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index a2a27c1db4f..17042641741 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -71,25 +71,16 @@ struct CDiskTxPos : public CDiskBlockPos /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView { -private: - CCoinsViewByScript* pcoinsViewByScript; protected: CDBWrapper db; public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const; bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - bool WriteFlag(const std::string &name, bool fValue); - bool ReadFlag(const std::string &name, bool &fValue); - int64_t GetPrefixCount(char prefix) const; - bool DeleteAllCoinsByScript(); // removes txoutsbyaddressindex - bool GenerateAllCoinsByScript(); // creates txoutsbyaddressindex - void SetCoinsViewByScript(CCoinsViewByScript* pcoinsViewByScriptIn); - CDBIterator *RawCursor() const; + int64_t CountCoins() const; CCoinsViewCursor *Cursor() const; }; diff --git a/src/validation.cpp b/src/validation.cpp index e43cbfc6774..e59ea0ba8f6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -176,7 +176,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc } CCoinsViewCache *pcoinsTip = NULL; -CCoinsViewDB *pcoinsdbview = NULL; +CCoinsViewByScriptDB *pcoinsByScriptDB = NULL; CCoinsViewByScript *pcoinsByScript = NULL; CBlockTreeDB *pblocktree = NULL; @@ -2030,6 +2030,10 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); + if (fTxOutsByAddressIndex) { + if (!pcoinsByScript->Flush()) + return AbortNode(state, "Failed to write to coin database"); + } nLastFlush = nNow; } if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) { @@ -2106,6 +2110,8 @@ void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool i--; } } + + pcoinsByScript->SetBestBlock(block.GetHash()); } /** Update chainActive and related internal data structures. */ diff --git a/src/validation.h b/src/validation.h index 274adffd1ae..3d871edc7de 100644 --- a/src/validation.h +++ b/src/validation.h @@ -536,10 +536,8 @@ extern CChain chainActive; /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip; -/** Global variable that points to the active CCoinsView (protected by cs_main) */ -extern CCoinsViewDB *pcoinsdbview; - /** Only used if -txoutsbyaddressindex */ +extern CCoinsViewByScriptDB *pcoinsByScriptDB; extern CCoinsViewByScript *pcoinsByScript; /** Global variable that points to the active block tree (protected by cs_main) */ From 3ae8aa098dfdfd0bf454d6cafddcb2ce6b577715 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 3 Sep 2016 21:16:28 +1200 Subject: [PATCH 03/15] add txoutsbyaddress querying via rest interface --- src/coinsbyscript.cpp | 2 +- src/rest.cpp | 135 +++++++++++++++++++++++++++++++++++++++++++++ src/rpc/blockchain.cpp | 59 ++------------------ src/rpc/blockchain.h | 1 - src/rpc/rawtransaction.cpp | 57 +++++++++++++++++++ src/txmempool.cpp | 2 +- 6 files changed, 198 insertions(+), 58 deletions(-) diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index 8462cc2800d..c3c73becb18 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -96,7 +96,7 @@ bool CCoinsViewByScriptDB::BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint("coindb", "Committing %u coin address indexes to coin database...\n", (unsigned int)count); + LogPrint(BCLog::COINDB, "Committing %u coin address indexes to coin database...\n", (unsigned int)count); return db.WriteBatch(batch); } diff --git a/src/rest.cpp b/src/rest.cpp index 9dcaf269d64..47849d24aea 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "base58.h" #include "chain.h" #include "chainparams.h" #include "primitives/block.h" @@ -22,6 +23,7 @@ #include static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once +static const size_t MAX_GETTXOUTSBYADDRESS_SCRIPTS = 15; //allow a max of 15 scripts to be queried at once enum RetFormat { RF_UNDEF, @@ -59,6 +61,7 @@ struct CCoin { /* Defined in rawtransaction.cpp */ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex); static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { @@ -598,6 +601,137 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) return true; // continue to process further HTTP reqs on this cxn } +static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIPart) +{ + if (!CheckWarmup(req)) + return false; + if (!fTxOutsByAddressIndex) + return RESTERR(req, HTTP_BAD_REQUEST, "To use this function, you must start bitcoin with the -txoutsbyaddressindex parameter."); + + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); + + std::vector uriParts; + if (param.length() > 1) + { + std::string strUriParams = param.substr(1); + boost::split(uriParts, strUriParams, boost::is_any_of("/")); + } + + // throw exception in case of a empty request + std::string strRequestMutable = req->ReadBody(); + if (strRequestMutable.length() == 0 && uriParts.size() == 0) + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + + bool fInputParsed = false; + bool fCheckMemPool = false; + std::vector vScripts; + + // parse/deserialize input + // input-format = output-format, rest/gettxoutsbyaddress/bin requires binary input, gives binary output, ... + + if (uriParts.size() > 0) + { + + //inputs is sent over URI scheme (/rest/gettxoutsbyaddress/checkmempool/addr1/addr2/...) + if (uriParts.size() > 0 && uriParts[0] == "checkmempool") + fCheckMemPool = true; + + for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) + { + CScript script; + CBitcoinAddress address(uriParts[i]); + if (address.IsValid()) { + script = GetScriptForDestination(address.Get()); + } else if (IsHex(uriParts[i])) { + std::vector data(ParseHex(uriParts[i])); + script = CScript(data.begin(), data.end()); + } else { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid Bitcoin address or script: " + uriParts[i]); + } + + vScripts.push_back(script); + } + + if (vScripts.size() > 0) + fInputParsed = true; + else + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + } + + switch (rf) { + case RF_HEX: { + // convert hex to bin, continue then with bin part + std::vector strRequestV = ParseHex(strRequestMutable); + strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + } + + case RF_BINARY: { + try { + //deserialize only if user sent a request + if (strRequestMutable.size() > 0) + { + if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA + return RESTERR(req, HTTP_BAD_REQUEST, "Combination of URI scheme inputs and raw post data is not allowed"); + + CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); + oss << strRequestMutable; + oss >> fCheckMemPool; + //DN: TODO: fix serialization + //oss >> vScripts; + } + } catch (const std::ios_base::failure& e) { + // abort in case of unreadable binary data + return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); + } + break; + } + + case RF_JSON: { + if (!fInputParsed) + return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); + break; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } + + // limit max scriptis + if (vScripts.size() > MAX_GETTXOUTSBYADDRESS_SCRIPTS) + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Error: max scripts exceeded (max: %d, tried: %d)", MAX_GETTXOUTSBYADDRESS_SCRIPTS, vScripts.size())); + + int nMinDepth = fCheckMemPool ? 0 : 1; + UniValue vObjects(UniValue::VARR); + std::vector > vSort; + + BOOST_FOREACH(const CScript &script, vScripts) + { + CCoinsByScript coinsByScript; + pcoinsByScript->GetCoinsByScript(script, coinsByScript); + + if (nMinDepth == 0) + mempool.GetCoinsByScript(script, coinsByScript); + + CoinsByScriptToJSON(coinsByScript, nMinDepth, vObjects, vSort, true); + } + + //DN: TODO: BIN and HEX + + UniValue results(UniValue::VARR); + sort(vSort.begin(), vSort.end()); + for (unsigned int i = 0; i < vSort.size(); i++) + { + results.push_back(vObjects[vSort[i].second]); + } + + // return json string + std::string strJSON = results.write() + "\n"; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); + return true; +} + static const struct { const char* prefix; bool (*handler)(HTTPRequest* req, const std::string& strReq); @@ -610,6 +744,7 @@ static const struct { {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/gettxoutsbyaddress", rest_gettxoutsbyaddress}, }; bool StartREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index fddf7955ea2..982f4520f8e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -48,6 +48,7 @@ static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { @@ -941,7 +942,7 @@ UniValue gettxout(const JSONRPCRequest& request) UniValue gettxoutsbyaddress(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) - throw runtime_error( + throw std::runtime_error( "gettxoutsbyaddress ( minconf [\"address\",...] count from )\n" "\nReturns a list of unspent transaction outputs by address (or script).\n" "The list is ordered by confirmations in descending order.\n" @@ -996,7 +997,7 @@ UniValue gettxoutsbyaddress(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VARR)(UniValue::VNUM)(UniValue::VNUM)); UniValue vObjects(UniValue::VARR); - vector > vSort; + std::vector > vSort; int nMinDepth = request.params[0].get_int(); UniValue inputs = request.params[1].get_array(); @@ -1033,59 +1034,7 @@ UniValue gettxoutsbyaddress(const JSONRPCRequest& request) if (nMinDepth == 0) mempool.GetCoinsByScript(script, coinsByScript); - BOOST_FOREACH(const COutPoint &outpoint, coinsByScript.setCoins) - { - CCoins coins; - if (nMinDepth == 0) - { - LOCK(mempool.cs); - CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(outpoint.hash, coins)) - continue; - mempool.pruneSpent(outpoint.hash, coins); // TODO: this should be done by the CCoinsViewMemPool - } - else if (!pcoinsTip->GetCoins(outpoint.hash, coins)) - continue; - - if (outpoint.n < coins.vout.size() && !coins.vout[outpoint.n].IsNull() && !coins.vout[outpoint.n].scriptPubKey.IsUnspendable()) - { - // should not happen - if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT && (!chainActive[coins.nHeight] || !chainActive[coins.nHeight]->phashBlock)) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal Error: !chainActive[coins.nHeight]"); - - BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); - CBlockIndex *pindex = it->second; - - int nConfirmations = 0; - if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) - nConfirmations = pindex->nHeight - coins.nHeight + 1; - if (nConfirmations < nMinDepth) - continue; - - UniValue oScriptPubKey(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[outpoint.n].scriptPubKey, oScriptPubKey, true); - - UniValue o(UniValue::VOBJ); - o.push_back(Pair("confirmations", nConfirmations)); - o.push_back(Pair("txid", outpoint.hash.GetHex())); - o.push_back(Pair("vout", (int)outpoint.n)); - o.push_back(Pair("value", ValueFromAmount(coins.vout[outpoint.n].nValue))); - o.push_back(Pair("scriptPubKey", oScriptPubKey)); - o.push_back(Pair("version", coins.nVersion)); - o.push_back(Pair("coinbase", coins.fCoinBase)); - o.push_back(Pair("bestblockhash", pindex->GetBlockHash().GetHex())); - o.push_back(Pair("bestblockheight", pindex->nHeight)); - o.push_back(Pair("bestblocktime", pindex->GetBlockTime())); - if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) - { - o.push_back(Pair("blockhash", chainActive[coins.nHeight]->GetBlockHash().GetHex())); - o.push_back(Pair("blockheight", coins.nHeight)); - o.push_back(Pair("blocktime", chainActive[coins.nHeight]->GetBlockTime())); - } - vObjects.push_back(o); - vSort.push_back(make_pair(coins.nHeight, (unsigned int)vObjects.size() - 1)); - } - } + CoinsByScriptToJSON(coinsByScript, nMinDepth, vObjects, vSort, true); } UniValue results(UniValue::VARR); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index c021441b0a7..f9f86bb6bec 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -37,4 +37,3 @@ UniValue mempoolToJSON(bool fVerbose = false); UniValue blockheaderToJSON(const CBlockIndex* blockindex); #endif - diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 717e9d75f39..d9147837fe1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -122,6 +122,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) } } +void CoinsByScriptToJSON(const CCoinsByScript& coinsByScript, int nMinDepth, UniValue& vObjects, std::vector>& vSort, bool fIncludeHex) +{ + BOOST_FOREACH(const COutPoint &outpoint, coinsByScript.setCoins) + { + CCoins coins; + if (nMinDepth == 0) + { + LOCK(mempool.cs); + CCoinsViewMemPool view(pcoinsTip, mempool); + if (!view.GetCoins(outpoint.hash, coins)) + continue; + mempool.pruneSpent(outpoint.hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } + else if (!pcoinsTip->GetCoins(outpoint.hash, coins)) + continue; + + if (outpoint.n < coins.vout.size() && !coins.vout[outpoint.n].IsNull() && !coins.vout[outpoint.n].scriptPubKey.IsUnspendable()) + { + // should not happen + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT && (!chainActive[coins.nHeight] || !chainActive[coins.nHeight]->phashBlock)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal Error: !chainActive[coins.nHeight]"); + + BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); + CBlockIndex *pindex = it->second; + + int nConfirmations = 0; + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + nConfirmations = pindex->nHeight - coins.nHeight + 1; + if (nConfirmations < nMinDepth) + continue; + + UniValue oScriptPubKey(UniValue::VOBJ); + ScriptPubKeyToJSON(coins.vout[outpoint.n].scriptPubKey, oScriptPubKey, fIncludeHex); + + UniValue o(UniValue::VOBJ); + o.push_back(Pair("confirmations", nConfirmations)); + o.push_back(Pair("txid", outpoint.hash.GetHex())); + o.push_back(Pair("vout", (int)outpoint.n)); + o.push_back(Pair("value", ValueFromAmount(coins.vout[outpoint.n].nValue))); + o.push_back(Pair("scriptPubKey", oScriptPubKey)); + o.push_back(Pair("version", coins.nVersion)); + o.push_back(Pair("coinbase", coins.fCoinBase)); + o.push_back(Pair("bestblockhash", pindex->GetBlockHash().GetHex())); + o.push_back(Pair("bestblockheight", pindex->nHeight)); + o.push_back(Pair("bestblocktime", pindex->GetBlockTime())); + if ((unsigned int)coins.nHeight != MEMPOOL_HEIGHT) + { + o.push_back(Pair("blockhash", chainActive[coins.nHeight]->GetBlockHash().GetHex())); + o.push_back(Pair("blockheight", coins.nHeight)); + o.push_back(Pair("blocktime", chainActive[coins.nHeight]->GetBlockTime())); + } + vObjects.push_back(o); + vSort.push_back(std::make_pair(coins.nHeight, (unsigned int)vObjects.size() - 1)); + } + } +} + UniValue getrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 3361edb210d..f962c828970 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -332,7 +332,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, } CTxMemPool::CTxMemPool(const bool& _fTxOutsByAddressIndex, CBlockPolicyEstimator* estimator) : - nTransactionsUpdated(0), minerPolicyEstimator(estimator) + nTransactionsUpdated(0), fTxOutsByAddressIndex(_fTxOutsByAddressIndex), minerPolicyEstimator(estimator) { _clear(); //lock free clear From 44f9b4a168b2e006b45b3ff6d34cb5689984ef61 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 3 Sep 2016 23:08:19 +1200 Subject: [PATCH 04/15] C++11: s/boost::scoped_ptr/std::unique_ptr/ --- src/coinsbyscript.cpp | 4 ++-- src/coinsbyscript.h | 2 +- src/coinstats.cpp | 4 ++-- src/txdb.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index c3c73becb18..bd842147db2 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -162,7 +162,7 @@ void CCoinsViewByScriptDBCursor::Next() bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() { - boost::scoped_ptr pcursor(Cursor()); + std::unique_ptr pcursor(Cursor()); std::vector v; int64_t i = 0; @@ -207,7 +207,7 @@ bool CCoinsViewByScriptDB::GenerateAllCoinsByScript(CCoinsViewDB* coinsIn) LogPrintf("Building address index for -txoutsbyaddressindex. Be patient...\n"); int64_t nTxCount = coinsIn->CountCoins(); - boost::scoped_ptr pcursor(coinsIn->Cursor()); + std::unique_ptr pcursor(coinsIn->Cursor()); CCoinsMapByScript mapCoinsByScript; int64_t i = 0; diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h index 5a2d12943ea..124e601d284 100644 --- a/src/coinsbyscript.h +++ b/src/coinsbyscript.h @@ -91,7 +91,7 @@ class CCoinsViewByScriptDBCursor CCoinsViewByScriptDBCursor(CDBIterator* pcursorIn): pcursor(pcursorIn) {} uint256 hashBlock; - boost::scoped_ptr pcursor; + std::unique_ptr pcursor; std::pair keyTmp; friend class CCoinsViewByScriptDB; diff --git a/src/coinstats.cpp b/src/coinstats.cpp index 34328955b45..b8d3d3206c8 100644 --- a/src/coinstats.cpp +++ b/src/coinstats.cpp @@ -14,7 +14,7 @@ using namespace std; //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats) { - boost::scoped_ptr pcursor(view->Cursor()); + std::unique_ptr pcursor(view->Cursor()); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); @@ -50,7 +50,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoins stats.hashSerialized = ss.GetHash(); stats.nTotalAmount = nTotalAmount; - boost::scoped_ptr pcursordb(viewbyscriptdb->Cursor()); + std::unique_ptr pcursordb(viewbyscriptdb->Cursor()); while (pcursordb->Valid()) { boost::this_thread::interruption_point(); uint160 hash; diff --git a/src/txdb.cpp b/src/txdb.cpp index 90f48b346b9..e5dd8456ded 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -137,7 +137,7 @@ void CCoinsViewDBCursor::Next() int64_t CCoinsViewDB::CountCoins() const { - boost::scoped_ptr pcursor(Cursor()); + std::unique_ptr pcursor(Cursor()); int64_t i = 0; while (pcursor->Valid()) { From a2873c261c565b00e9ad8827d8d8eb9b5e55733e Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Thu, 15 Sep 2016 18:24:22 +1200 Subject: [PATCH 05/15] convert txoutsbyaddress test to python --- qa/rpc-tests/txoutsbyaddress.py | 96 +++++++++++++++++++++++ qa/rpc-tests/txoutsbyaddress.sh | 163 ---------------------------------------- src/coinstats.cpp | 7 +- src/txdb.cpp | 3 +- test/functional/test_runner.py | 1 + 5 files changed, 105 insertions(+), 165 deletions(-) create mode 100644 qa/rpc-tests/txoutsbyaddress.py delete mode 100644 qa/rpc-tests/txoutsbyaddress.sh diff --git a/qa/rpc-tests/txoutsbyaddress.py b/qa/rpc-tests/txoutsbyaddress.py new file mode 100644 index 00000000000..2213b1d6477 --- /dev/null +++ b/qa/rpc-tests/txoutsbyaddress.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TxOutsByAddressTest(BitcoinTestFramework): + """Tests -txoutsbyaddressindex""" + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + + def setup_network(self, split=False): + print("Setup network...") + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, ["-txoutsbyaddressindex"])) + self.nodes.append(start_node(1, self.options.tmpdir)) + self.nodes.append(start_node(2, self.options.tmpdir)) + self.is_network_split = False + self.sync_all() + + def run_test(self): + print("Generating test blockchain...") + + # Check that there's no UTXO on any of the nodes + for node in self.nodes: + assert_equal(len(node.listunspent()), 0) + + # mining + connect_nodes(self.nodes[1], 0) + connect_nodes(self.nodes[2], 0) + self.nodes[0].generate(101) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 50) + + # TX1: send from node0 to node1 + # - check if txout from tx1 is there + address = self.nodes[1].getnewaddress() + txid1 = self.nodes[0].sendtoaddress(address, 10) + self.nodes[0].generate(101) # node will collect its own fee + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 5090) + assert_equal(self.nodes[1].getbalance(), 10) + txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid1) + + # stop node 2 + stop_node(self.nodes[2], 2) + self.nodes.pop() + + # TX2: send from node1 to node0 + # - check if txout from tx1 is gone + # - check if txout from tx2 is there + address2 = self.nodes[0].getnewaddress() + txid2 = self.nodes[1].sendtoaddress(address2, 5) + self.nodes[1].generate(1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), 5145) + txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + assert_equal(txouts, []) + txouts = self.nodes[0].gettxoutsbyaddress(1, (address2,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid2) + + # start node 2 + self.nodes.append(start_node(2, self.options.tmpdir, self.args)) + + # mine 10 blocks alone to have the longest chain + self.nodes[2].generate(10) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[1], 2) + self.nodes[2].generate(1) + sync_blocks(self.nodes) + + # TX2 must be reverted + # - check if txout from tx1 is there again + # - check if txout from tx2 is gone + assert_equal(self.nodes[0].getbalance(), 5640) + txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + txid = txouts[0]["txid"] + assert_is_hash_string(txid) + assert_equal(txid, txid1) + txouts = self.nodes[0].gettxoutsbyaddress(1, (address2,)) + assert_equal(txouts, []) + +if __name__ == '__main__': + TxOutsByAddressTest().main() + diff --git a/qa/rpc-tests/txoutsbyaddress.sh b/qa/rpc-tests/txoutsbyaddress.sh deleted file mode 100644 index 40c21f77413..00000000000 --- a/qa/rpc-tests/txoutsbyaddress.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2013-2014 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Test -txoutsbyaddressindex - -if [ $# -lt 1 ]; then - echo "Usage: $0 path_to_binaries" - echo "e.g. $0 ../../src" - exit 1 -fi - -set -f - -BITCOIND=${1}/bitcoind -CLI=${1}/bitcoin-cli - -DIR="${BASH_SOURCE%/*}" -SENDANDWAIT="${DIR}/send.sh" -if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi -. "$DIR/util.sh" - -D=$(mktemp -d test.XXXXX) - -D1=${D}/node1 -CreateDataDir "$D1" port=11000 rpcport=11001 -B1ARGS="-datadir=$D1" -$BITCOIND $B1ARGS -txoutsbyaddressindex -sendfreetransactions=1 & -B1PID=$! - -D2=${D}/node2 -CreateDataDir "$D2" port=11010 rpcport=11011 -B2ARGS="-datadir=$D2" -$BITCOIND $B2ARGS -sendfreetransactions=1 & -B2PID=$! - -D3=${D}/node3 -CreateDataDir "$D3" port=11020 rpcport=11021 -B3ARGS="-datadir=$D3" -$BITCOIND $B3ARGS -sendfreetransactions=1 & -B3PID=$! - -# Wait until all three nodes are at the same block number -function WaitBlocks { - while : - do - sleep 1 - declare -i BLOCKS1=$( GetBlocks $B1ARGS ) - declare -i BLOCKS2=$( GetBlocks $B2ARGS ) - declare -i BLOCKS3=$( GetBlocks $B3ARGS ) - if (( BLOCKS1 == BLOCKS2 && BLOCKS2 == BLOCKS3 )) - then - break - fi - done -} - -function WaitBlocks2 { - while : - do - sleep 1 - declare -i BLOCKS1=$( GetBlocks $B1ARGS ) - declare -i BLOCKS2=$( GetBlocks $B2ARGS ) - if (( BLOCKS1 == BLOCKS2 )) - then - break - fi - done -} - -function CleanUp { -$CLI $B3ARGS stop > /dev/null 2>&1 -wait $B3PID -$CLI $B2ARGS stop > /dev/null 2>&1 -wait $B2PID -$CLI $B1ARGS stop > /dev/null 2>&1 -wait $B1PID - -rm -rf $D -} - -function ErrorAndExit { -echo "$@" 1>&2; -CleanUp -exit 1 -} - -echo "Generating test blockchain..." - -# mining -$CLI $B2ARGS addnode "127.0.0.1:11000" "onetry" -$CLI $B3ARGS addnode "127.0.0.1:11000" "onetry" -$CLI $B1ARGS generate 101 -WaitBlocks -CheckBalance "$B1ARGS" 50 - -# TX1: send from node1 to node2 -# - check if txout from tx1 is there -address=$($CLI $B2ARGS getnewaddress) -txid1=$($CLI $B1ARGS sendtoaddress $address 10) -$CLI $B1ARGS generate 1 -WaitBlocks -CheckBalance "$B1ARGS" 90 -CheckBalance "$B2ARGS" 10 -txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") -txid=$(ExtractKey "txid" "$txouts") -if [ -z "$txid" ] || [ $txid != $txid1 ] ; then - ErrorAndExit "wrong txid1: $txid != $txid1" -fi - -# stop node 3 -$CLI $B3ARGS stop > /dev/null 2>&1 -wait $B3PID - -# TX2: send from node2 to node1 -# - check if txout from tx1 is gone -# - check if txout from tx2 is there -address2=$($CLI $B1ARGS getnewaddress) -txid2=$($CLI $B2ARGS sendtoaddress $address2 5) -$CLI $B2ARGS generate 1 -WaitBlocks2 -CheckBalance "$B1ARGS" 145 -txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") -txid=$(ExtractKey "txid" "$txouts") -if [ ! -z "$txid" ] ; then - ErrorAndExit "txid not empty: $txid" -fi -txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address2""\"]") -txid=$(ExtractKey "txid" "$txouts") -if [ -z "$txid" ] || [ $txid != $txid2 ] ; then - ErrorAndExit "wrong txid2: $txid != $txid2" -fi - -# start node 3 -$BITCOIND $B3ARGS & -B3PID=$! - -# mine 10 blocks alone to have the longest chain -$CLI $B3ARGS generate 10 -$CLI $B1ARGS addnode "127.0.0.1:11020" "onetry" -$CLI $B2ARGS addnode "127.0.0.1:11020" "onetry" -$CLI $B3ARGS generate 1 -WaitBlocks - -# TX2 must be reverted -# - check if txout from tx1 is there again -# - check if txout from tx2 is gone -CheckBalance "$B1ARGS" 640 -txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address""\"]") -txid=$(ExtractKey "txid" "$txouts") -if [ -z "$txid" ] || [ $txid != $txid1 ] ; then - ErrorAndExit "wrong txid1 : $txid != $txid1" -fi -txouts=$($CLI $B1ARGS gettxoutsbyaddress 1 "[\"""$address2""\"]") -txid=$(ExtractKey "txid" "$txouts") -if [ ! -z "$txid" ] ; then - ErrorAndExit "txid is not empty: $txid" -fi - -echo "Tests successful, cleaning up" -CleanUp -exit 0 diff --git a/src/coinstats.cpp b/src/coinstats.cpp index b8d3d3206c8..21c533da14b 100644 --- a/src/coinstats.cpp +++ b/src/coinstats.cpp @@ -20,8 +20,13 @@ bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoins stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; + BlockMap::const_iterator iter = mapBlockIndex.find(stats.hashBlock); + if (iter == mapBlockIndex.end()) + stats.nHeight = 0; + else + stats.nHeight = iter->second->nHeight; } + ss << stats.hashBlock; CAmount nTotalAmount = 0; while (pcursor->Valid()) { diff --git a/src/txdb.cpp b/src/txdb.cpp index e5dd8456ded..a36b4c052a6 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -99,7 +99,8 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const that restriction. */ i->pcursor->Seek(DB_COINS); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (!i->pcursor->Valid() || !i->pcursor->GetKey(i->keyTmp)) + i->keyTmp.first = 0; return i; } diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 0996b1bc208..9dd21e32521 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -110,6 +110,7 @@ 'listsinceblock.py', 'p2p-leaktests.py', 'import-abort-rescan.py', + 'txoutsbyaddress.py', ] EXTENDED_SCRIPTS = [ From ddddc03455e45d1d0b9ce5e46d650b3448719a8c Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Thu, 15 Sep 2016 22:40:16 +1200 Subject: [PATCH 06/15] - add txoutsbyaddress REST interface tests - remove skeleton HEX/BIN format support from REST interface for gettxoutsbyaddress endpoint --- src/rest.cpp | 35 ++--------------- test/functional/rest.py | 100 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 42 deletions(-) diff --git a/src/rest.cpp b/src/rest.cpp index 47849d24aea..83810909d67 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -628,7 +628,7 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP std::vector vScripts; // parse/deserialize input - // input-format = output-format, rest/gettxoutsbyaddress/bin requires binary input, gives binary output, ... + // only json format supported if (uriParts.size() > 0) { @@ -660,44 +660,17 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP } switch (rf) { - case RF_HEX: { - // convert hex to bin, continue then with bin part - std::vector strRequestV = ParseHex(strRequestMutable); - strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); - } - - case RF_BINARY: { - try { - //deserialize only if user sent a request - if (strRequestMutable.size() > 0) - { - if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA - return RESTERR(req, HTTP_BAD_REQUEST, "Combination of URI scheme inputs and raw post data is not allowed"); - - CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); - oss << strRequestMutable; - oss >> fCheckMemPool; - //DN: TODO: fix serialization - //oss >> vScripts; - } - } catch (const std::ios_base::failure& e) { - // abort in case of unreadable binary data - return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); - } - break; - } - case RF_JSON: { if (!fInputParsed) return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); break; } default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } - // limit max scriptis + // limit max scripts if (vScripts.size() > MAX_GETTXOUTSBYADDRESS_SCRIPTS) return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Error: max scripts exceeded (max: %d, tried: %d)", MAX_GETTXOUTSBYADDRESS_SCRIPTS, vScripts.size())); @@ -716,8 +689,6 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP CoinsByScriptToJSON(coinsByScript, nMinDepth, vObjects, vSort, true); } - //DN: TODO: BIN and HEX - UniValue results(UniValue::VARR); sort(vSort.begin(), vSort.end()); for (unsigned int i = 0; i < vSort.size(); i++) diff --git a/test/functional/rest.py b/test/functional/rest.py index 776211d301f..5f131f688cc 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -49,7 +49,8 @@ def __init__(self): self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + args = ["-txoutsbyaddressindex"] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [args, args, args]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -67,7 +68,8 @@ def run_test(self): assert_equal(self.nodes[0].getbalance(), 50) - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + address = self.nodes[1].getnewaddress() + txid = self.nodes[0].sendtoaddress(address, 0.1) self.sync_all() self.nodes[2].generate(1) self.sync_all() @@ -128,8 +130,6 @@ def run_test(self): assert_equal(json_obj['bitmap'], "10") #test binary response - bb_hash = self.nodes[0].getbestblockhash() - binaryRequest = b'\x01\x02' binaryRequest += hex_str_to_bytes(txid) binaryRequest += pack("i", n) @@ -152,8 +152,9 @@ def run_test(self): ############################ # do a tx and don't sync - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) - json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json") + address2 = self.nodes[1].getnewaddress() + txid2 = self.nodes[0].sendtoaddress(address2, 0.1) + json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid2+self.FORMAT_SEPARATOR+"json") json_obj = json.loads(json_string) vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then) # get n of 0.1 outpoint @@ -162,12 +163,12 @@ def run_test(self): if vout['value'] == 0.1: n = vout['n'] - json_request = '/'+txid+'-'+str(n) + json_request = '/'+txid2+'-'+str(n) json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) assert_equal(len(json_obj['utxos']), 0) #there should be a outpoint because it has just added to the mempool - json_request = '/checkmempool/'+txid+'-'+str(n) + json_request = '/checkmempool/'+txid2+'-'+str(n) json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) assert_equal(len(json_obj['utxos']), 1) #there should be a outpoint because it has just added to the mempool @@ -187,14 +188,14 @@ def run_test(self): #test limits json_request = '/checkmempool/' for x in range(0, 20): - json_request += txid+'-'+str(n)+'/' + json_request += txid2+'-'+str(n)+'/' json_request = json_request.rstrip("/") response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 400) #must be a 400 because we exceeding the limits json_request = '/checkmempool/' for x in range(0, 15): - json_request += txid+'-'+str(n)+'/' + json_request += txid2+'-'+str(n)+'/' json_request = json_request.rstrip("/") response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 200) #must be a 200 because we are within the limits @@ -202,6 +203,85 @@ def run_test(self): self.nodes[0].generate(1) #generate block to not affect upcoming tests self.sync_all() + + ############################################## + # GETTXOUTSBYADDRESS: query an unspent txout # + ############################################## + json_request = '/checkmempool/'+address + json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there is just one txout + assert_equal(len(json_obj), 1) + txout = json_obj[0] + + #check details + assert_equal(txout['blockhash'], bb_hash) + assert_equal(txout['txid'], txid) + assert_equal(txout['value'], 0.1) + + + ##################################################### + # GETTXOUTSBYADDRESS: query multiple unspent txouts # + ##################################################### + json_request = '/checkmempool/'+address+'/'+address2 + json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there are two txouts + assert_equal(len(json_obj), 2) + txout1 = json_obj[0] + txout2 = json_obj[1] + + #check details + assert_equal(txout1['txid'], txid) + assert_equal(txout1['value'], 0.1) + assert_equal(txout2['txid'], txid2) + assert_equal(txout2['value'], 0.1) + + + ############################################ + # GETTXOUTSBYADDRESS: query unseen address # + ############################################ + + json_request = '/checkmempool/'+self.nodes[0].getnewaddress() + json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_obj = json.loads(json_string) + + #make sure there are no txouts (brand new address) + assert_equal(len(json_obj), 0) + + + ######################################## + # GETTXOUTSBYADDRESS: invalid requests # + ######################################## + + + #do some invalid requests + json_request = '{"checkmempool' + response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 400) #must be a 400 because we send a invalid json request + + json_request = '/checkmempool/xxxx' + response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+self.FORMAT_SEPARATOR+'json', json_request, True) + assert_equal(response.status, 400) #must be a 400 because we send a invalid json request + + #test limits + json_request = '/checkmempool/' + for x in range(0, 20): + json_request += address+'/' + json_request = json_request.rstrip("/") + response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + assert_equal(response.status, 400) #must be a 400 because we exceeding the limits + + json_request = '/checkmempool/' + for x in range(0, 15): + json_request += address+'/' + json_request = json_request.rstrip("/") + response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + assert_equal(response.status, 200) #must be a 200 because we are within the limits + + ################ # /rest/block/ # ################ From 113402255c0c938f49d499e8d2e44cd9d981359d Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 17 Sep 2016 16:55:52 +1200 Subject: [PATCH 07/15] address test review nits --- qa/rpc-tests/txoutsbyaddress.py | 13 +++++-------- test/functional/rest.py | 5 ++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/qa/rpc-tests/txoutsbyaddress.py b/qa/rpc-tests/txoutsbyaddress.py index 2213b1d6477..a84224e2c51 100644 --- a/qa/rpc-tests/txoutsbyaddress.py +++ b/qa/rpc-tests/txoutsbyaddress.py @@ -14,15 +14,14 @@ def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-txoutsbyaddressindex"], [], []] def setup_network(self, split=False): print("Setup network...") - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-txoutsbyaddressindex"])) - self.nodes.append(start_node(1, self.options.tmpdir)) - self.nodes.append(start_node(2, self.options.tmpdir)) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + connect_nodes(self.nodes[1], 0) + connect_nodes(self.nodes[2], 0) self.is_network_split = False - self.sync_all() def run_test(self): print("Generating test blockchain...") @@ -32,8 +31,6 @@ def run_test(self): assert_equal(len(node.listunspent()), 0) # mining - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) self.nodes[0].generate(101) self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -71,7 +68,7 @@ def run_test(self): assert_equal(txid, txid2) # start node 2 - self.nodes.append(start_node(2, self.options.tmpdir, self.args)) + self.nodes.append(start_node(2, self.options.tmpdir, self.extra_args[2])) # mine 10 blocks alone to have the longest chain self.nodes[2].generate(10) diff --git a/test/functional/rest.py b/test/functional/rest.py index 5f131f688cc..e00d76efe27 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -47,15 +47,14 @@ def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-txoutsbyaddressindex"]] * 3 def setup_network(self, split=False): - args = ["-txoutsbyaddressindex"] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [args, args, args]) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) self.is_network_split=False - self.sync_all() def run_test(self): url = urllib.parse.urlparse(self.nodes[0].url) From efa276b55d958f642429bb3fe55115aedde770e7 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 1 Oct 2016 13:10:46 +1300 Subject: [PATCH 08/15] rename -txoutsbyaddress to -txoutindex --- qa/rpc-tests/txoutsbyaddress.py | 4 ++-- src/coinsbyscript.cpp | 2 +- src/coinsbyscript.h | 4 ++-- src/init.cpp | 38 +++++++++++++++++++------------------- src/rest.cpp | 4 ++-- src/rpc/blockchain.cpp | 10 +++++----- src/test/mempool_tests.cpp | 4 ++-- src/test/policyestimator_tests.cpp | 4 ++-- src/txmempool.cpp | 8 ++++---- src/txmempool.h | 6 +++--- src/validation.cpp | 8 ++++---- src/validation.h | 4 ++-- test/functional/rest.py | 2 +- 13 files changed, 49 insertions(+), 49 deletions(-) diff --git a/qa/rpc-tests/txoutsbyaddress.py b/qa/rpc-tests/txoutsbyaddress.py index a84224e2c51..44969a81296 100644 --- a/qa/rpc-tests/txoutsbyaddress.py +++ b/qa/rpc-tests/txoutsbyaddress.py @@ -8,13 +8,13 @@ class TxOutsByAddressTest(BitcoinTestFramework): - """Tests -txoutsbyaddressindex""" + """Tests -txoutindex""" def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-txoutsbyaddressindex"], [], []] + self.extra_args = [["-txoutindex"], [], []] def setup_network(self, split=False): print("Setup network...") diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index bd842147db2..dc61351f459 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -204,7 +204,7 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() bool CCoinsViewByScriptDB::GenerateAllCoinsByScript(CCoinsViewDB* coinsIn) { - LogPrintf("Building address index for -txoutsbyaddressindex. Be patient...\n"); + LogPrintf("Building address index for -txoutindex. Be patient...\n"); int64_t nTxCount = coinsIn->CountCoins(); std::unique_ptr pcursor(coinsIn->Cursor()); diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h index 124e601d284..236edb41895 100644 --- a/src/coinsbyscript.h +++ b/src/coinsbyscript.h @@ -109,8 +109,8 @@ class CCoinsViewByScriptDB bool BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool DeleteAllCoinsByScript(); // removes txoutsbyaddressindex - bool GenerateAllCoinsByScript(CCoinsViewDB* coinsIn); // creates txoutsbyaddressindex + bool DeleteAllCoinsByScript(); // removes txoutindex + bool GenerateAllCoinsByScript(CCoinsViewDB* coinsIn); // creates txoutindex CCoinsViewByScriptDBCursor *Cursor() const; }; diff --git a/src/init.cpp b/src/init.cpp index e71ba48e0e2..df23c753410 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -374,7 +374,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); - strUsage += HelpMessageOpt("-txoutsbyaddressindex", strprintf(_("Maintain an address to unspent outputs index (rpc: gettxoutsbyaddress). The index is built on first use. (default: %u)"), 0)); + strUsage += HelpMessageOpt("-txoutindex", strprintf(_("Maintain an address to unspent outputs index (rpc: gettxoutsbyaddress). The index is built on first use. (default: %u)"), 0)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); @@ -1492,56 +1492,56 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - // Check -txoutsbyaddressindex - pcoinsByScriptDB->ReadFlag("txoutsbyaddressindex", fTxOutsByAddressIndex); - if (IsArgSet("-txoutsbyaddressindex")) + // Check -txoutindex + pcoinsByScriptDB->ReadFlag("txoutindex", fTxOutIndex); + if (IsArgSet("-txoutindex")) { - if (GetBoolArg("-txoutsbyaddressindex", false)) + if (GetBoolArg("-txoutindex", false)) { // build index - if (!fTxOutsByAddressIndex) + if (!fTxOutIndex) { if (!pcoinsByScriptDB->DeleteAllCoinsByScript()) { - strLoadError = _("Error deleting txoutsbyaddressindex"); + strLoadError = _("Error deleting txoutindex"); break; } if (!pcoinsByScriptDB->GenerateAllCoinsByScript(pcoinsdbview)) { - strLoadError = _("Error building txoutsbyaddressindex"); + strLoadError = _("Error building txoutindex"); break; } CCoinsStats stats; if (!GetUTXOStats(pcoinsTip, pcoinsByScriptDB, stats)) { - strLoadError = _("Error GetUTXOStats for txoutsbyaddressindex"); + strLoadError = _("Error GetUTXOStats for txoutindex"); break; } if (stats.nTransactionOutputs != stats.nAddressesOutputs) { - strLoadError = _("Error compare stats for txoutsbyaddressindex"); + strLoadError = _("Error compare stats for txoutindex"); break; } - pcoinsByScriptDB->WriteFlag("txoutsbyaddressindex", true); - fTxOutsByAddressIndex = true; + pcoinsByScriptDB->WriteFlag("txoutindex", true); + fTxOutIndex = true; } } else { - if (fTxOutsByAddressIndex) + if (fTxOutIndex) { // remove index pcoinsByScriptDB->DeleteAllCoinsByScript(); - pcoinsByScriptDB->WriteFlag("txoutsbyaddressindex", false); - fTxOutsByAddressIndex = false; + pcoinsByScriptDB->WriteFlag("txoutindex", false); + fTxOutIndex = false; } } } - else if (fTxOutsByAddressIndex) - return InitError(_("You need to provide -txoutsbyaddressindex. Do -txoutsbyaddressindex=0 to delete the index.")); + else if (fTxOutIndex) + return InitError(_("You need to provide -txoutindex. Do -txoutindex=0 to delete the index.")); - // Init -txoutsbyaddressindex - if (fTxOutsByAddressIndex) + // Init -txoutindex + if (fTxOutIndex) pcoinsByScript = new CCoinsViewByScript(pcoinsByScriptDB); uiInterface.InitMessage(_("Verifying blocks...")); diff --git a/src/rest.cpp b/src/rest.cpp index 83810909d67..c3dafe129d5 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -605,8 +605,8 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP { if (!CheckWarmup(req)) return false; - if (!fTxOutsByAddressIndex) - return RESTERR(req, HTTP_BAD_REQUEST, "To use this function, you must start bitcoin with the -txoutsbyaddressindex parameter."); + if (!fTxOutIndex) + return RESTERR(req, HTTP_BAD_REQUEST, "To use this function, you must start bitcoin with the -txoutindex parameter."); std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 982f4520f8e..dc18f5569cc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -828,8 +828,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"addresses\": n, (numeric) The number of addresses and scripts. Only if -txoutsbyaddressindex=1\n" - " \"txoutsbyaddress\": n, (numeric) The number of output transactions. Only if -txoutsbyaddressindex=1\n" + " \"addresses\": n, (numeric) The number of addresses and scripts. Only if -txoutindex=1\n" + " \"txoutsbyaddress\": n, (numeric) The number of output transactions. Only if -txoutindex=1\n" " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" " \"total_amount\": x.xxx (numeric) The total amount\n" @@ -947,7 +947,7 @@ UniValue gettxoutsbyaddress(const JSONRPCRequest& request) "\nReturns a list of unspent transaction outputs by address (or script).\n" "The list is ordered by confirmations in descending order.\n" "Note that passing minconf=0 will include the mempool.\n" - "\nTo use this function, you must start bitcoin with the -txoutsbyaddressindex parameter.\n" + "\nTo use this function, you must start bitcoin with the -txoutindex parameter.\n" "\nArguments:\n" "1. minconf (numeric) Minimum confirmations\n" "2. \"addresses\" (string) A json array of bitcoin addresses (or scripts)\n" @@ -991,8 +991,8 @@ UniValue gettxoutsbyaddress(const JSONRPCRequest& request) + HelpExampleRpc("gettxoutsbyaddress", "6, \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") ); - if (!fTxOutsByAddressIndex) - throw JSONRPCError(RPC_METHOD_NOT_FOUND, "To use this function, you must start bitcoin with the -txoutsbyaddressindex parameter."); + if (!fTxOutIndex) + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "To use this function, you must start bitcoin with the -txoutindex parameter."); RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VARR)(UniValue::VNUM)(UniValue::VNUM)); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 73802fb3db5..5d1a3028dde 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -53,8 +53,8 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) txGrandChild[i].vout[0].nValue = 11000LL; } - bool fTxOutsByAddressIndex = false; - CTxMemPool testPool(fTxOutsByAddressIndex); + bool fTxOutIndex = false; + CTxMemPool testPool(fTxOutIndex); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index f8b24f00443..40dbaa56d5e 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -17,8 +17,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CBlockPolicyEstimator feeEst; - bool fTxOutsByAddressIndex = false; - CTxMemPool mpool(fTxOutsByAddressIndex, &feeEst); + bool fTxOutIndex = false; + CTxMemPool mpool(fTxOutIndex, &feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f962c828970..f00c5bd3e7d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -331,8 +331,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const bool& _fTxOutsByAddressIndex, CBlockPolicyEstimator* estimator) : - nTransactionsUpdated(0), fTxOutsByAddressIndex(_fTxOutsByAddressIndex), minerPolicyEstimator(estimator) +CTxMemPool::CTxMemPool(const bool& _fTxOutIndex, CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), fTxOutIndex(_fTxOutIndex), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -434,7 +434,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; - if (fTxOutsByAddressIndex) + if (fTxOutIndex) for (unsigned int i = 0; i < tx.vout.size(); i++) if (!tx.vout[i].IsNull() && !tx.vout[i].scriptPubKey.IsUnspendable()) mapCoinsByScript[CCoinsViewByScript::getKey(tx.vout[i].scriptPubKey)].setCoins.insert(COutPoint(hash, (uint32_t)i)); @@ -502,7 +502,7 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso { LOCK(cs); - if (fTxOutsByAddressIndex) + if (fTxOutIndex) { for (unsigned int i = 0; i < origTx.vout.size(); i++) { diff --git a/src/txmempool.h b/src/txmempool.h index b2297f67e21..507c72967e5 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -416,8 +416,8 @@ class CTxMemPool CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. - const bool& fTxOutsByAddressIndex; - CCoinsMapByScript mapCoinsByScript; // only used if -txoutsbyaddressindex + const bool& fTxOutIndex; + CCoinsMapByScript mapCoinsByScript; // only used if -txoutindex uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; @@ -499,7 +499,7 @@ class CTxMemPool /** Create a new CTxMemPool. */ - CTxMemPool(const bool& _fTxOutsByAddressIndex, CBlockPolicyEstimator* estimator = nullptr); + CTxMemPool(const bool& _fTxOutIndex, CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is diff --git a/src/validation.cpp b/src/validation.cpp index e59ea0ba8f6..c07b1869c41 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -65,7 +65,7 @@ int nScriptCheckThreads = 0; std::atomic_bool fImporting(false); bool fReindex = false; bool fTxIndex = false; -bool fTxOutsByAddressIndex = false; +bool fTxOutIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -83,7 +83,7 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; -CTxMemPool mempool(fTxOutsByAddressIndex, &feeEstimator); +CTxMemPool mempool(fTxOutIndex, &feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -2030,7 +2030,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); - if (fTxOutsByAddressIndex) { + if (fTxOutIndex) { if (!pcoinsByScript->Flush()) return AbortNode(state, "Failed to write to coin database"); } @@ -2072,7 +2072,7 @@ void static UpdateAddressIndex(const CTxOut& txout, const COutPoint& outpoint, b void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool fConnect) { - if (!fTxOutsByAddressIndex) + if (!fTxOutIndex) return; assert(block.vtx.size() > 0); diff --git a/src/validation.h b/src/validation.h index 3d871edc7de..990388af95b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -168,7 +168,7 @@ extern std::atomic_bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; -extern bool fTxOutsByAddressIndex; +extern bool fTxOutIndex; extern bool fIsBareMultisigStd; extern bool fRequireStandard; extern bool fCheckBlockIndex; @@ -536,7 +536,7 @@ extern CChain chainActive; /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip; -/** Only used if -txoutsbyaddressindex */ +/** Only used if -txoutindex */ extern CCoinsViewByScriptDB *pcoinsByScriptDB; extern CCoinsViewByScript *pcoinsByScript; diff --git a/test/functional/rest.py b/test/functional/rest.py index e00d76efe27..a714db5d928 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -47,7 +47,7 @@ def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-txoutsbyaddressindex"]] * 3 + self.extra_args = [["-txoutindex"]] * 3 def setup_network(self, split=False): self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) From 30460d5847910edd4134294121afeb7571e3da9d Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 1 Oct 2016 13:26:57 +1300 Subject: [PATCH 09/15] set executable bit --- qa/rpc-tests/txoutsbyaddress.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 qa/rpc-tests/txoutsbyaddress.py diff --git a/qa/rpc-tests/txoutsbyaddress.py b/qa/rpc-tests/txoutsbyaddress.py old mode 100644 new mode 100755 From 6caf075e7e207997d9073db4f7829b0983b1b099 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Sat, 1 Oct 2016 19:14:03 +1300 Subject: [PATCH 10/15] add doc for gettxoutsbyaddress REST endpoint --- doc/REST-interface.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 7fbb1740302..71415171877 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -79,6 +79,74 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 } ``` +####Query UTXO set (by address/script) +`GET /rest/gettxoutsbyaddress//
/
/.../
.json` + +The gettxoutsbyaddress command allows querying of the UTXO set given a set of addresses (or script). + +To use this function, you must start bitcoin with the -txoutindex parameter. + +Output: +``` +[ (array of json object) + { + \"confirmations\" : n, (numeric) The number of confirmations + \"txid\" : \"txid\", (string) The transaction id + \"vout\" : n, (numeric) The vout value + \"value\" : x.xxx, (numeric) The transaction value in btc + \"scriptPubKey\" : { (json object) + \"asm\" : \"code\", (string) + \"hex\" : \"hex\", (string) + \"reqSigs\" : n, (numeric) Number of required signatures + \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash + \"addresses\" : [ (array of string) array of bitcoin addresses + \"bitcoinaddress\" (string) bitcoin address + ,... + ] + }, + \"version\" : n, (numeric) The transaction version + \"coinbase\" : true|false (boolean) Coinbase or not + \"bestblockhash\" : \"hash\", (string) The block hash of the best block + \"bestblockheight\" : n, (numeric) The block height of the best block + \"bestblocktime\" : n, (numeric) The block time of the best block + \"blockhash\" : \"hash\", (string) The block hash of the block the tx is in (only if confirmations > 0) + \"blockheight\" : n, (numeric) The block height of the block the tx is in (only if confirmations > 0) + \"blocktime\" : ttt, (numeric) The block time in seconds since 1.1.1970 GMT (only if confirmations > 0) + } + ,... +] +``` + +Example: +``` +$ curl localhost:18332/rest/gettxoutsbyaddress/checkmempool/mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV.json 2>/dev/null | json_pp +[ + { + "confirmations" : 721918, + "txid" : "75bc54c673ed535db361a6e89c08bf7256d1378e2c645229d469d41042356e54", + "vout" : 0, + "value" : 0.001, + "scriptPubKey" : { + "asm" : "OP_DUP OP_HASH160 a7092d2dc8778b56d4c352697081c687b451ab6d OP_EQUALVERIFY OP_CHECKSIG", + "hex" : "76a914a7092d2dc8778b56d4c352697081c687b451ab6d88ac", + "reqSigs" : 1, + "type" : "pubkeyhash", + "addresses" : [ + "mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV" + ] + }, + "version" : 1, + "coinbase" : false, + "bestblockhash" : "00000000007872ee19923a5604d86a6c9bfa3041c417a7ecf60dc034387b173f", + "blockheight" : 244755, + "bestblocktime" : 1475309084, + "blockhash" : "000000000001c163caa76dbc16c7b383fb10257829b3617c5a1ffb91ea3824db", + "bestblockheight" : 966672, + "blocktime" : 1400786412, + } +] +``` + ####Memory pool `GET /rest/mempool/info.json` From cb43d8a924448c4097cbbc3242ca91134f884d47 Mon Sep 17 00:00:00 2001 From: Daniel Newton Date: Fri, 6 Jan 2017 11:31:02 +1300 Subject: [PATCH 11/15] fix up a change missed by the rebase --- src/init.cpp | 1 + src/validation.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index df23c753410..781f4e1e25c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1439,6 +1439,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) UnloadBlockIndex(); delete pcoinsTip; delete pcoinsdbview; + delete pcoinsByScriptDB; delete pcoinscatcher; delete pblocktree; diff --git a/src/validation.cpp b/src/validation.cpp index c07b1869c41..0c49ecc2670 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2075,6 +2075,7 @@ void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool if (!fTxOutIndex) return; + assert(&block != nullptr); assert(block.vtx.size() > 0); unsigned int i = 0; if (!fConnect) @@ -2342,7 +2343,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); - UpdateAddressIndex(*pblock, blockundo, true); + UpdateAddressIndex(blockConnecting, blockundo, true); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); From 80bf01465aeba768e04410289f0985e098742992 Mon Sep 17 00:00:00 2001 From: Douglas Roark Date: Sun, 19 Feb 2017 17:45:41 -0800 Subject: [PATCH 12/15] Fixup to remove errors and warnings --- src/coinsbyscript.cpp | 4 ++-- src/rpc/blockchain.cpp | 2 +- src/rpc/client.cpp | 8 ++++---- src/validation.cpp | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index dc61351f459..ac82a0ef775 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -177,8 +177,8 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() { i += v.size(); CDBBatch batch(db); - BOOST_FOREACH(const uint160& hash, v) - batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete + BOOST_FOREACH(const uint160& _hash, v) + batch.Erase(make_pair(DB_COINS_BYSCRIPT, _hash)); // delete db.WriteBatch(batch); v.clear(); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index dc18f5569cc..a34d5f93731 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1499,7 +1499,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, true, {} }, { "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} }, { "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsbyaddress", &gettxoutsbyaddress, true }, + { "blockchain", "gettxoutsbyaddress", &gettxoutsbyaddress, true, {"minconf", "addresses", "count", "from"} }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} }, { "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} }, { "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index be0715eae3c..28e06c3b289 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -91,10 +91,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, - { "gettxoutsbyaddress", 0 }, - { "gettxoutsbyaddress", 1 }, - { "gettxoutsbyaddress", 2 }, - { "gettxoutsbyaddress", 3 }, + { "gettxoutsbyaddress", 0, "minconf" }, + { "gettxoutsbyaddress", 1, "addresses" }, + { "gettxoutsbyaddress", 2, "count" }, + { "gettxoutsbyaddress", 3, "from" }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "importprivkey", 2, "rescan" }, diff --git a/src/validation.cpp b/src/validation.cpp index 0c49ecc2670..4dc00d3d9f1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2075,7 +2075,6 @@ void static UpdateAddressIndex(const CBlock& block, CBlockUndo& blockundo, bool if (!fTxOutIndex) return; - assert(&block != nullptr); assert(block.vtx.size() > 0); unsigned int i = 0; if (!fConnect) From 82a9ea7c5bc8e22a3137bd39f663a82ac645d70d Mon Sep 17 00:00:00 2001 From: Douglas Roark Date: Sat, 18 Mar 2017 11:43:57 -0700 Subject: [PATCH 13/15] =?UTF-8?q?Replace=20PR=E2=80=99s=20remaining=20Boos?= =?UTF-8?q?t=20calls=20with=20C++11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/coinsbyscript.cpp | 10 ++++++++-- src/rest.cpp | 3 ++- src/txmempool.cpp | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index ac82a0ef775..cc4062faf05 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -177,8 +177,11 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() { i += v.size(); CDBBatch batch(db); - BOOST_FOREACH(const uint160& _hash, v) + for(auto& av: v) + { + const uint160& _hash = av; batch.Erase(make_pair(DB_COINS_BYSCRIPT, _hash)); // delete + } db.WriteBatch(batch); v.clear(); } @@ -192,8 +195,11 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() { i += v.size(); CDBBatch batch(db); - BOOST_FOREACH(const uint160& hash, v) + for(auto& av: v) + { + const uint160& hash = av; batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete + } db.WriteBatch(batch); } if (i > 0) diff --git a/src/rest.cpp b/src/rest.cpp index c3dafe129d5..5b375e5adf2 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -678,8 +678,9 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP UniValue vObjects(UniValue::VARR); std::vector > vSort; - BOOST_FOREACH(const CScript &script, vScripts) + for(auto& scriptObj: vScripts) { + const CScript& script = scriptObj; CCoinsByScript coinsByScript; pcoinsByScript->GetCoinsByScript(script, coinsByScript); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f00c5bd3e7d..9bd9292ca26 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -367,8 +367,7 @@ void CTxMemPool::GetCoinsByScript(const CScript& script, CCoinsByScript& coinsBy CCoinsMapByScript::const_iterator it = mapCoinsByScript.find(CCoinsViewByScript::getKey(script)); if (it != mapCoinsByScript.end()) { - BOOST_FOREACH(const COutPoint &outpoint, it->second.setCoins) - coinsByScript.setCoins.insert(outpoint); + coinsByScript.setCoins.insert(it->second.setCoins.begin(), it->second.setCoins.end()); } } From bd5c62976d06a91ff2db009eae43d12ece0f963b Mon Sep 17 00:00:00 2001 From: Douglas Roark Date: Tue, 21 Mar 2017 18:39:23 -0700 Subject: [PATCH 14/15] Fixup - Move file missed by a rebase --- {qa/rpc-tests => test/functional}/txoutsbyaddress.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {qa/rpc-tests => test/functional}/txoutsbyaddress.py (100%) diff --git a/qa/rpc-tests/txoutsbyaddress.py b/test/functional/txoutsbyaddress.py similarity index 100% rename from qa/rpc-tests/txoutsbyaddress.py rename to test/functional/txoutsbyaddress.py From 77aa7e091dfc2bb5d2d7f2e95df38b637cabf6ae Mon Sep 17 00:00:00 2001 From: Douglas Roark Date: Wed, 22 Mar 2017 17:40:49 -0700 Subject: [PATCH 15/15] Code feedback cleanup - Switched getutxosbyaddress to getutxoindex, and altered the files as needed to reflect this change. - Switched CCoinsMapByScript key from uint160 to CScriptID. - Various C++11-related changes. - Removed a try/catch case from CCoinsViewDB::CountCoins(). - Changed a const ref in CTxMemPool to a const copy to prevent a possible subtle error. - Added a default value (false) for UTXO indexing under CTxMemPool. --- doc/REST-interface.md | 6 ++-- src/coinsbyscript.cpp | 38 ++++++++++------------ src/coinsbyscript.h | 11 +++---- src/coinstats.cpp | 2 +- src/coinstats.h | 16 ++++----- src/init.cpp | 2 +- src/rest.cpp | 12 +++---- src/rpc/blockchain.cpp | 14 ++++---- src/rpc/client.cpp | 8 ++--- src/test/mempool_tests.cpp | 3 +- src/txdb.cpp | 8 ++--- src/txmempool.cpp | 8 ++--- src/txmempool.h | 2 +- test/functional/rest.py | 38 +++++++++++----------- test/functional/test_runner.py | 2 +- .../{txoutsbyaddress.py => utxoindex.py} | 18 +++++----- 16 files changed, 90 insertions(+), 98 deletions(-) rename test/functional/{txoutsbyaddress.py => utxoindex.py} (82%) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 71415171877..d960d8836da 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -80,9 +80,9 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 ``` ####Query UTXO set (by address/script) -`GET /rest/gettxoutsbyaddress//
/
/.../
.json` +`GET /rest/getutxoindex//
/
/.../
.json` -The gettxoutsbyaddress command allows querying of the UTXO set given a set of addresses (or script). +The getutxoindex command allows querying of the UTXO set given a set of addresses (or script). To use this function, you must start bitcoin with the -txoutindex parameter. @@ -119,7 +119,7 @@ Output: Example: ``` -$ curl localhost:18332/rest/gettxoutsbyaddress/checkmempool/mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV.json 2>/dev/null | json_pp +$ curl localhost:18332/rest/getutxoindex/checkmempool/mvkA8gYrKUmXFiuFpoxNGjMjYcV9oCkwGV.json 2>/dev/null | json_pp [ { "confirmations" : 721918, diff --git a/src/coinsbyscript.cpp b/src/coinsbyscript.cpp index cc4062faf05..78d0fe51b5b 100644 --- a/src/coinsbyscript.cpp +++ b/src/coinsbyscript.cpp @@ -20,12 +20,12 @@ static const char DB_BEST_BLOCK = 'B'; CCoinsViewByScript::CCoinsViewByScript(CCoinsViewByScriptDB* viewIn) : base(viewIn) { } bool CCoinsViewByScript::GetCoinsByScript(const CScript &script, CCoinsByScript &coins) { - const uint160 key = CCoinsViewByScript::getKey(script); + const CScriptID key = CScriptID(script); if (cacheCoinsByScript.count(key)) { coins = cacheCoinsByScript[key]; return true; } - if (base->GetCoinsByHashOfScript(key, coins)) { + if (base->GetCoinsByScriptID(key, coins)) { cacheCoinsByScript[key] = coins; return true; } @@ -33,19 +33,19 @@ bool CCoinsViewByScript::GetCoinsByScript(const CScript &script, CCoinsByScript } CCoinsMapByScript::iterator CCoinsViewByScript::FetchCoinsByScript(const CScript &script, bool fRequireExisting) { - const uint160 key = CCoinsViewByScript::getKey(script); + const CScriptID key = CScriptID(script); CCoinsMapByScript::iterator it = cacheCoinsByScript.find(key); if (it != cacheCoinsByScript.end()) return it; + CCoinsByScript tmp; - if (!base->GetCoinsByHashOfScript(key, tmp)) + if (!base->GetCoinsByScriptID(key, tmp)) { if (fRequireExisting) return cacheCoinsByScript.end(); } - CCoinsMapByScript::iterator ret = cacheCoinsByScript.insert(it, std::make_pair(key, CCoinsByScript())); - tmp.swap(ret->second); - return ret; + + return cacheCoinsByScript.emplace_hint(it, key, tmp); } CCoinsByScript &CCoinsViewByScript::GetCoinsByScript(const CScript &script, bool fRequireExisting) { @@ -54,10 +54,6 @@ CCoinsByScript &CCoinsViewByScript::GetCoinsByScript(const CScript &script, bool return it->second; } -uint160 CCoinsViewByScript::getKey(const CScript &script) { - return Hash160(script); -} - uint256 CCoinsViewByScript::GetBestBlock() const { return hashBlock; } @@ -75,8 +71,8 @@ CCoinsViewByScriptDB::CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory, bool { } -bool CCoinsViewByScriptDB::GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const { - return db.Read(make_pair(DB_COINS_BYSCRIPT, hash), coins); +bool CCoinsViewByScriptDB::GetCoinsByScriptID(const CScriptID &scriptID, CCoinsByScript &coins) const { + return db.Read(make_pair(DB_COINS_BYSCRIPT, scriptID), coins); } bool CCoinsViewByScriptDB::BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock) { @@ -96,7 +92,7 @@ bool CCoinsViewByScriptDB::BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint(BCLog::COINDB, "Committing %u coin address indexes to coin database...\n", (unsigned int)count); + LogPrint(BCLog::COINDB, "Committing %zu coin address indexes to coin database...\n", (unsigned int)count); return db.WriteBatch(batch); } @@ -128,7 +124,7 @@ CCoinsViewByScriptDBCursor *CCoinsViewByScriptDB::Cursor() const return i; } -bool CCoinsViewByScriptDBCursor::GetKey(uint160 &key) const +bool CCoinsViewByScriptDBCursor::GetKey(CScriptID &key) const { // Return cached key if (keyTmp.first == DB_COINS_BYSCRIPT) { @@ -164,12 +160,12 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() { std::unique_ptr pcursor(Cursor()); - std::vector v; + std::vector v; int64_t i = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); try { - uint160 hash; + CScriptID hash; if (!pcursor->GetKey(hash)) break; v.push_back(hash); @@ -179,7 +175,7 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() CDBBatch batch(db); for(auto& av: v) { - const uint160& _hash = av; + const CScriptID& _hash = av; batch.Erase(make_pair(DB_COINS_BYSCRIPT, _hash)); // delete } db.WriteBatch(batch); @@ -197,7 +193,7 @@ bool CCoinsViewByScriptDB::DeleteAllCoinsByScript() CDBBatch batch(db); for(auto& av: v) { - const uint160& hash = av; + const CScriptID& hash = av; batch.Erase(make_pair(DB_COINS_BYSCRIPT, hash)); // delete } db.WriteBatch(batch); @@ -235,11 +231,11 @@ bool CCoinsViewByScriptDB::GenerateAllCoinsByScript(CCoinsViewDB* coinsIn) if (coins.vout[j].IsNull() || coins.vout[j].scriptPubKey.IsUnspendable()) continue; - const uint160 key = CCoinsViewByScript::getKey(coins.vout[j].scriptPubKey); + const CScriptID key = CScriptID(coins.vout[j].scriptPubKey); if (!mapCoinsByScript.count(key)) { CCoinsByScript coinsByScript; - GetCoinsByHashOfScript(key, coinsByScript); + GetCoinsByScriptID(key, coinsByScript); mapCoinsByScript.insert(make_pair(key, coinsByScript)); } mapCoinsByScript[key].setCoins.insert(COutPoint(txhash, (uint32_t)j)); diff --git a/src/coinsbyscript.h b/src/coinsbyscript.h index 236edb41895..c9404e69665 100644 --- a/src/coinsbyscript.h +++ b/src/coinsbyscript.h @@ -10,6 +10,7 @@ #include "primitives/transaction.h" #include "serialize.h" #include "uint256.h" +#include "script/standard.h" class CCoinsViewDB; class CCoinsViewByScriptDB; @@ -39,7 +40,7 @@ class CCoinsByScript } }; -typedef std::map CCoinsMapByScript; // uint160 = hash of script +typedef std::map CCoinsMapByScript; /** Adds a memory cache for coins by address */ class CCoinsViewByScript @@ -58,8 +59,6 @@ class CCoinsViewByScript // Return a modifiable reference to a CCoinsByScript. CCoinsByScript &GetCoinsByScript(const CScript &script, bool fRequireExisting = true); - static uint160 getKey(const CScript &script); // we use the hash of the script as key in the database - void SetBestBlock(const uint256 &hashBlock); uint256 GetBestBlock() const; @@ -80,7 +79,7 @@ class CCoinsViewByScriptDBCursor public: ~CCoinsViewByScriptDBCursor() {} - bool GetKey(uint160 &key) const; + bool GetKey(CScriptID &key) const; bool GetValue(CCoinsByScript &coins) const; unsigned int GetValueSize() const; @@ -92,7 +91,7 @@ class CCoinsViewByScriptDBCursor pcursor(pcursorIn) {} uint256 hashBlock; std::unique_ptr pcursor; - std::pair keyTmp; + std::pair keyTmp; friend class CCoinsViewByScriptDB; }; @@ -105,7 +104,7 @@ class CCoinsViewByScriptDB public: CCoinsViewByScriptDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoinsByHashOfScript(const uint160 &hash, CCoinsByScript &coins) const; + bool GetCoinsByScriptID(const CScriptID &scriptID, CCoinsByScript &coins) const; bool BatchWrite(CCoinsViewByScript* pcoinsViewByScriptIn, const uint256 &hashBlock); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); diff --git a/src/coinstats.cpp b/src/coinstats.cpp index 21c533da14b..82882bd0f84 100644 --- a/src/coinstats.cpp +++ b/src/coinstats.cpp @@ -58,7 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoins std::unique_ptr pcursordb(viewbyscriptdb->Cursor()); while (pcursordb->Valid()) { boost::this_thread::interruption_point(); - uint160 hash; + CScriptID hash; CCoinsByScript coinsByScript; if (pcursordb->GetKey(hash) && pcursordb->GetValue(coinsByScript)) { stats.nAddresses++; diff --git a/src/coinstats.h b/src/coinstats.h index bac4f3db34e..64e8beec776 100644 --- a/src/coinstats.h +++ b/src/coinstats.h @@ -11,17 +11,17 @@ struct CCoinsStats { - int nHeight; + int nHeight{}; uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nAddresses; - uint64_t nAddressesOutputs; // equal nTransactionOutputs (if addressindex is enabled) - uint64_t nSerializedSize; + uint64_t nTransactions{}; + uint64_t nTransactionOutputs{}; + uint64_t nAddresses{}; + uint64_t nAddressesOutputs{}; // equal nTransactionOutputs (if addressindex is enabled) + uint64_t nSerializedSize{}; uint256 hashSerialized; - CAmount nTotalAmount; + CAmount nTotalAmount{}; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nAddresses(0), nAddressesOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() {} }; bool GetUTXOStats(CCoinsView *view, CCoinsViewByScriptDB *viewbyscriptdb, CCoinsStats &stats); diff --git a/src/init.cpp b/src/init.cpp index 781f4e1e25c..498ee3b75fc 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -374,7 +374,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); - strUsage += HelpMessageOpt("-txoutindex", strprintf(_("Maintain an address to unspent outputs index (rpc: gettxoutsbyaddress). The index is built on first use. (default: %u)"), 0)); + strUsage += HelpMessageOpt("-txoutindex", strprintf(_("Maintain an address to unspent outputs index (rpc: getutxoindex). The index is built on first use. (default: %u)"), 0)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); diff --git a/src/rest.cpp b/src/rest.cpp index 5b375e5adf2..16b89a854d6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -23,7 +23,7 @@ #include static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once -static const size_t MAX_GETTXOUTSBYADDRESS_SCRIPTS = 15; //allow a max of 15 scripts to be queried at once +static const size_t MAX_GETUTXOINDEX_SCRIPTS = 15; //allow a max of 15 scripts to be queried at once enum RetFormat { RF_UNDEF, @@ -601,7 +601,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) return true; // continue to process further HTTP reqs on this cxn } -static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIPart) +static bool rest_getutxoindex(HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -633,7 +633,7 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP if (uriParts.size() > 0) { - //inputs is sent over URI scheme (/rest/gettxoutsbyaddress/checkmempool/addr1/addr2/...) + //inputs is sent over URI scheme (/rest/getutxoindex/checkmempool/addr1/addr2/...) if (uriParts.size() > 0 && uriParts[0] == "checkmempool") fCheckMemPool = true; @@ -671,8 +671,8 @@ static bool rest_gettxoutsbyaddress(HTTPRequest* req, const std::string& strURIP } // limit max scripts - if (vScripts.size() > MAX_GETTXOUTSBYADDRESS_SCRIPTS) - return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Error: max scripts exceeded (max: %d, tried: %d)", MAX_GETTXOUTSBYADDRESS_SCRIPTS, vScripts.size())); + if (vScripts.size() > MAX_GETUTXOINDEX_SCRIPTS) + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Error: max scripts exceeded (max: %d, tried: %d)", MAX_GETUTXOINDEX_SCRIPTS, vScripts.size())); int nMinDepth = fCheckMemPool ? 0 : 1; UniValue vObjects(UniValue::VARR); @@ -716,7 +716,7 @@ static const struct { {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, - {"/rest/gettxoutsbyaddress", rest_gettxoutsbyaddress}, + {"/rest/getutxoindex", rest_getutxoindex}, }; bool StartREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a34d5f93731..443f148e3f0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -829,7 +829,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" " \"addresses\": n, (numeric) The number of addresses and scripts. Only if -txoutindex=1\n" - " \"txoutsbyaddress\": n, (numeric) The number of output transactions. Only if -txoutindex=1\n" + " \"utxoindex\": n, (numeric) The number of output transactions. Only if -txoutindex=1\n" " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" " \"total_amount\": x.xxx (numeric) The total amount\n" @@ -849,7 +849,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); ret.push_back(Pair("addresses", (int64_t)stats.nAddresses)); - ret.push_back(Pair("txoutsbyaddress", (int64_t)stats.nAddressesOutputs)); + ret.push_back(Pair("utxoindex", (int64_t)stats.nAddressesOutputs)); ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); @@ -939,11 +939,11 @@ UniValue gettxout(const JSONRPCRequest& request) return ret; } -UniValue gettxoutsbyaddress(const JSONRPCRequest& request) +UniValue getutxoindex(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) throw std::runtime_error( - "gettxoutsbyaddress ( minconf [\"address\",...] count from )\n" + "getutxoindex ( minconf [\"address\",...] count from )\n" "\nReturns a list of unspent transaction outputs by address (or script).\n" "The list is ordered by confirmations in descending order.\n" "Note that passing minconf=0 will include the mempool.\n" @@ -986,9 +986,9 @@ UniValue gettxoutsbyaddress(const JSONRPCRequest& request) " ,...\n" "]\n" "\nExamples:\n" - + HelpExampleCli("gettxoutsbyaddress", "6 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleCli("getutxoindex", "6 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + "\nAs a json rpc call\n" - + HelpExampleRpc("gettxoutsbyaddress", "6, \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleRpc("getutxoindex", "6, \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") ); if (!fTxOutIndex) @@ -1499,7 +1499,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, true, {} }, { "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} }, { "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsbyaddress", &gettxoutsbyaddress, true, {"minconf", "addresses", "count", "from"} }, + { "blockchain", "getutxoindex", &getutxoindex, true, {"minconf", "addresses", "count", "from"} }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} }, { "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} }, { "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 28e06c3b289..8c32dfb5540 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -91,10 +91,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, - { "gettxoutsbyaddress", 0, "minconf" }, - { "gettxoutsbyaddress", 1, "addresses" }, - { "gettxoutsbyaddress", 2, "count" }, - { "gettxoutsbyaddress", 3, "from" }, + { "getutxoindex", 0, "minconf" }, + { "getutxoindex", 1, "addresses" }, + { "getutxoindex", 2, "count" }, + { "getutxoindex", 3, "from" }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "importprivkey", 2, "rescan" }, diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 5d1a3028dde..e802d89f534 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -53,8 +53,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) txGrandChild[i].vout[0].nValue = 11000LL; } - bool fTxOutIndex = false; - CTxMemPool testPool(fTxOutIndex); + CTxMemPool testPool(false); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); diff --git a/src/txdb.cpp b/src/txdb.cpp index a36b4c052a6..72698aff37b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -143,12 +143,8 @@ int64_t CCoinsViewDB::CountCoins() const int64_t i = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - i++; - pcursor->Next(); - } catch (std::exception &e) { - return 0; - } + i++; + pcursor->Next(); } return i; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 9bd9292ca26..583326ff038 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -332,7 +332,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, } CTxMemPool::CTxMemPool(const bool& _fTxOutIndex, CBlockPolicyEstimator* estimator) : - nTransactionsUpdated(0), fTxOutIndex(_fTxOutIndex), minerPolicyEstimator(estimator) + nTransactionsUpdated(0), minerPolicyEstimator(estimator), fTxOutIndex(_fTxOutIndex) { _clear(); //lock free clear @@ -364,7 +364,7 @@ unsigned int CTxMemPool::GetTransactionsUpdated() const void CTxMemPool::GetCoinsByScript(const CScript& script, CCoinsByScript& coinsByScript) const { LOCK(cs); - CCoinsMapByScript::const_iterator it = mapCoinsByScript.find(CCoinsViewByScript::getKey(script)); + CCoinsMapByScript::const_iterator it = mapCoinsByScript.find(CScriptID(script)); if (it != mapCoinsByScript.end()) { coinsByScript.setCoins.insert(it->second.setCoins.begin(), it->second.setCoins.end()); @@ -436,7 +436,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, if (fTxOutIndex) for (unsigned int i = 0; i < tx.vout.size(); i++) if (!tx.vout[i].IsNull() && !tx.vout[i].scriptPubKey.IsUnspendable()) - mapCoinsByScript[CCoinsViewByScript::getKey(tx.vout[i].scriptPubKey)].setCoins.insert(COutPoint(hash, (uint32_t)i)); + mapCoinsByScript[CScriptID(tx.vout[i].scriptPubKey)].setCoins.insert(COutPoint(hash, (uint32_t)i)); return true; } @@ -508,7 +508,7 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso if (origTx.vout[i].IsNull() || origTx.vout[i].scriptPubKey.IsUnspendable()) continue; - CCoinsMapByScript::iterator it = mapCoinsByScript.find(CCoinsViewByScript::getKey(origTx.vout[i].scriptPubKey)); + CCoinsMapByScript::iterator it = mapCoinsByScript.find(CScriptID(origTx.vout[i].scriptPubKey)); if (it != mapCoinsByScript.end()) { it->second.setCoins.erase(COutPoint(origTx.GetHash(), (uint32_t)i)); diff --git a/src/txmempool.h b/src/txmempool.h index 507c72967e5..3a3ec9005fc 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -416,7 +416,7 @@ class CTxMemPool CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. - const bool& fTxOutIndex; + const bool fTxOutIndex; CCoinsMapByScript mapCoinsByScript; // only used if -txoutindex uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) diff --git a/test/functional/rest.py b/test/functional/rest.py index a714db5d928..47e4cfcac72 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -203,11 +203,11 @@ def run_test(self): self.sync_all() - ############################################## - # GETTXOUTSBYADDRESS: query an unspent txout # - ############################################## + ######################################## + # GETUTXOINDEX: query an unspent txout # + ######################################## json_request = '/checkmempool/'+address - json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) #make sure there is just one txout @@ -220,11 +220,11 @@ def run_test(self): assert_equal(txout['value'], 0.1) - ##################################################### - # GETTXOUTSBYADDRESS: query multiple unspent txouts # - ##################################################### + ############################################### + # GETUTXOINDEX: query multiple unspent txouts # + ############################################### json_request = '/checkmempool/'+address+'/'+address2 - json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) #make sure there are two txouts @@ -239,30 +239,30 @@ def run_test(self): assert_equal(txout2['value'], 0.1) - ############################################ - # GETTXOUTSBYADDRESS: query unseen address # - ############################################ + ###################################### + # GETUTXOINDEX: query unseen address # + ###################################### json_request = '/checkmempool/'+self.nodes[0].getnewaddress() - json_string = http_get_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json') + json_string = http_get_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json') json_obj = json.loads(json_string) #make sure there are no txouts (brand new address) assert_equal(len(json_obj), 0) - ######################################## - # GETTXOUTSBYADDRESS: invalid requests # - ######################################## + ################################## + # GETUTXOINDEX: invalid requests # + ################################## #do some invalid requests json_request = '{"checkmempool' - response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+self.FORMAT_SEPARATOR+'json', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+self.FORMAT_SEPARATOR+'json', json_request, True) assert_equal(response.status, 400) #must be a 400 because we send a invalid json request json_request = '/checkmempool/xxxx' - response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+self.FORMAT_SEPARATOR+'json', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+self.FORMAT_SEPARATOR+'json', json_request, True) assert_equal(response.status, 400) #must be a 400 because we send a invalid json request #test limits @@ -270,14 +270,14 @@ def run_test(self): for x in range(0, 20): json_request += address+'/' json_request = json_request.rstrip("/") - response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 400) #must be a 400 because we exceeding the limits json_request = '/checkmempool/' for x in range(0, 15): json_request += address+'/' json_request = json_request.rstrip("/") - response = http_post_call(url.hostname, url.port, '/rest/gettxoutsbyaddress'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxoindex'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 200) #must be a 200 because we are within the limits diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9dd21e32521..aac7a6a58a6 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -110,7 +110,7 @@ 'listsinceblock.py', 'p2p-leaktests.py', 'import-abort-rescan.py', - 'txoutsbyaddress.py', + 'utxoindex.py', ] EXTENDED_SCRIPTS = [ diff --git a/test/functional/txoutsbyaddress.py b/test/functional/utxoindex.py similarity index 82% rename from test/functional/txoutsbyaddress.py rename to test/functional/utxoindex.py index 44969a81296..7f0d17e5a51 100755 --- a/test/functional/txoutsbyaddress.py +++ b/test/functional/utxoindex.py @@ -7,7 +7,7 @@ from test_framework.util import * -class TxOutsByAddressTest(BitcoinTestFramework): +class UTXOIndexTest(BitcoinTestFramework): """Tests -txoutindex""" def __init__(self): @@ -43,12 +43,14 @@ def run_test(self): self.sync_all() assert_equal(self.nodes[0].getbalance(), 5090) assert_equal(self.nodes[1].getbalance(), 10) - txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + txouts = self.nodes[0].getutxoindex(1, (address,)) txid = txouts[0]["txid"] assert_is_hash_string(txid) assert_equal(txid, txid1) - # stop node 2 + # Stop node 2. We want to restart it later and orphan a node 1 block in + # order to test txoutindex handling the reorg. In other words, node 2 is + # stopped so that it won't build on a node 1 block. stop_node(self.nodes[2], 2) self.nodes.pop() @@ -60,9 +62,9 @@ def run_test(self): self.nodes[1].generate(1) self.sync_all() assert_equal(self.nodes[0].getbalance(), 5145) - txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + txouts = self.nodes[0].getutxoindex(1, (address,)) assert_equal(txouts, []) - txouts = self.nodes[0].gettxoutsbyaddress(1, (address2,)) + txouts = self.nodes[0].getutxoindex(1, (address2,)) txid = txouts[0]["txid"] assert_is_hash_string(txid) assert_equal(txid, txid2) @@ -81,13 +83,13 @@ def run_test(self): # - check if txout from tx1 is there again # - check if txout from tx2 is gone assert_equal(self.nodes[0].getbalance(), 5640) - txouts = self.nodes[0].gettxoutsbyaddress(1, (address,)) + txouts = self.nodes[0].getutxoindex(1, (address,)) txid = txouts[0]["txid"] assert_is_hash_string(txid) assert_equal(txid, txid1) - txouts = self.nodes[0].gettxoutsbyaddress(1, (address2,)) + txouts = self.nodes[0].getutxoindex(1, (address2,)) assert_equal(txouts, []) if __name__ == '__main__': - TxOutsByAddressTest().main() + UTXOIndexTest().main()