From d8eb9456fce0bc44ba8599851d02be8c06d1a12b Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Fri, 12 Aug 2016 13:04:44 +0200 Subject: [PATCH 1/2] Add mempool statistics collector --- src/Makefile.am | 5 ++ src/init.cpp | 8 +++ src/memusage.h | 1 + src/rpc/register.h | 3 + src/stats/rpc_stats.cpp | 82 ++++++++++++++++++++++++++++ src/stats/stats.cpp | 81 +++++++++++++++++++++++++++ src/stats/stats.h | 61 +++++++++++++++++++++ src/stats/stats_mempool.cpp | 130 ++++++++++++++++++++++++++++++++++++++++++++ src/stats/stats_mempool.h | 73 +++++++++++++++++++++++++ src/validation.cpp | 15 ++++- 10 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 src/stats/rpc_stats.cpp create mode 100644 src/stats/stats.cpp create mode 100644 src/stats/stats.h create mode 100644 src/stats/stats_mempool.cpp create mode 100644 src/stats/stats_mempool.h diff --git a/src/Makefile.am b/src/Makefile.am index c4f933dae1e..0daf460230c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,6 +133,8 @@ BITCOIN_CORE_H = \ script/sign.h \ script/standard.h \ script/ismine.h \ + stats/stats.h \ + stats/stats_mempool.h \ streams.h \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ @@ -205,6 +207,9 @@ libbitcoin_server_a_SOURCES = \ rpc/server.cpp \ script/sigcache.cpp \ script/ismine.cpp \ + stats/rpc_stats.cpp \ + stats/stats.cpp \ + stats/stats_mempool.cpp \ timedata.cpp \ torcontrol.cpp \ txdb.cpp \ diff --git a/src/init.cpp b/src/init.cpp index 266e1731eb4..98ac03281e3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -33,6 +33,7 @@ #include "script/standard.h" #include "script/sigcache.h" #include "scheduler.h" +#include "stats/stats.h" #include "timedata.h" #include "txdb.h" #include "txmempool.h" @@ -509,6 +510,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcservertimeout=", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT)); } + strUsage += CStats::getHelpString(showDebug); return strUsage; } @@ -1134,6 +1136,10 @@ bool AppInitParameterInteraction() } } } + + if (!CStats::parameterInteraction()) + return false; + return true; } @@ -1669,5 +1675,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) pwalletMain->postInitProcess(scheduler); #endif + CStats::DefaultStats()->startCollecting(scheduler); + return !fRequestShutdown; } diff --git a/src/memusage.h b/src/memusage.h index b69acafffd1..04243197f26 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -6,6 +6,7 @@ #define BITCOIN_MEMUSAGE_H #include "indirectmap.h" +#include "prevector.h" #include diff --git a/src/rpc/register.h b/src/rpc/register.h index 49aee2365f0..55b0eed9dbc 100644 --- a/src/rpc/register.h +++ b/src/rpc/register.h @@ -19,6 +19,8 @@ void RegisterMiscRPCCommands(CRPCTable &tableRPC); void RegisterMiningRPCCommands(CRPCTable &tableRPC); /** Register raw transaction RPC commands */ void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC); +/** Register stats RPC commands */ +void RegisterStatsRPCCommands(CRPCTable &tableRPC); static inline void RegisterAllCoreRPCCommands(CRPCTable &t) { @@ -27,6 +29,7 @@ static inline void RegisterAllCoreRPCCommands(CRPCTable &t) RegisterMiscRPCCommands(t); RegisterMiningRPCCommands(t); RegisterRawTransactionRPCCommands(t); + RegisterStatsRPCCommands(t); } #endif diff --git a/src/stats/rpc_stats.cpp b/src/stats/rpc_stats.cpp new file mode 100644 index 00000000000..89402e2d417 --- /dev/null +++ b/src/stats/rpc_stats.cpp @@ -0,0 +1,82 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "rpc/server.h" +#include "stats/stats.h" +#include "util.h" +#include "utilstrencodings.h" + +#include + +#include + +UniValue getmempoolstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 0) + throw std::runtime_error( + "getmempoolstats\n" + "\nReturns the collected mempool statistics.\n" + "\nThe samples are segregated in multiple precision groups.\n" + "\nSamples time interval is not guaranteed to be constant, hence there\n" + "is a time delta in each sample relative to the last sample.\n" + "\nResult:\n" + " [\n" + " {\n" + " \"sample_interval\" : \"interval\", (numeric) Interval target in seconds\n" + " \"time_from\" : \"timestamp\", (numeric) Timestamp, first sample\n" + " \"samples\" : [\n" + " [,,,],\n" + " [,,,],\n" + " ...\n" + " ]\n" + " }\n" + " ,...\n" + " ]\n" + "\nExamples:\n" + + HelpExampleCli("getmempoolstats", "") + HelpExampleRpc("getmempoolstats", "")); + + // get stats from the core stats model + uint64_t timeFrom = 0; + std::vector groups = CStats::DefaultStats()->mempoolCollector->getPrecisionGroupsAndIntervals(); + + UniValue groupsUni(UniValue::VARR); + for (unsigned int i = 0; i < groups.size(); i++) { + + MempoolSamplesVector samples = CStats::DefaultStats()->mempoolCollector->getSamplesForPrecision(i, timeFrom); + + // use "flat" json encoding for performance reasons + UniValue samplesObj(UniValue::VARR); + for (const struct CStatsMempoolSample& sample : samples) { + UniValue singleSample(UniValue::VARR); + singleSample.push_back(UniValue((uint64_t)sample.timeDelta)); + singleSample.push_back(UniValue((uint64_t)sample.txCount)); + singleSample.push_back(UniValue((uint64_t)sample.dynMemUsage)); + singleSample.push_back(UniValue(sample.minFeePerK)); + samplesObj.push_back(singleSample); + } + + UniValue sampleGroup(UniValue::VOBJ); + sampleGroup.push_back(Pair("sample_interval", (int)groups[i])); + sampleGroup.push_back(Pair("time_from", timeFrom)); + sampleGroup.push_back(Pair("samples", samplesObj)); + + groupsUni.push_back(sampleGroup); + } + + return groupsUni; +} + +static const CRPCCommand commands[] = +{ + // category name actor (function) okSafe argNames + // --------------------- ------------------------ ----------------------- ------ ---------- + {"stats", "getmempoolstats", &getmempoolstats, true, {}}, + +}; + +void RegisterStatsRPCCommands(CRPCTable &t) +{ + for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) + t.appendCommand(commands[vcidx].name, &commands[vcidx]); +} diff --git a/src/stats/stats.cpp b/src/stats/stats.cpp new file mode 100644 index 00000000000..2ad5c380cc0 --- /dev/null +++ b/src/stats/stats.cpp @@ -0,0 +1,81 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "stats/stats.h" + +#include "memusage.h" +#include "utiltime.h" + +#include "util.h" + + +const size_t DEFAULT_MAX_STATS_MEMORY = 10 * 1024 * 1024; // 10 MB +const bool DEFAULT_STATISTICS_ENABLED = false; +const static unsigned int STATS_COLLECT_INTERVAL = 2000; // 2 secs + +CStats* CStats::sharedInstance = NULL; + +CStats* CStats::DefaultStats() +{ + if (!sharedInstance) + sharedInstance = new CStats(); + + return sharedInstance; +} + +CStats::CStats() : statsEnabled(false), maxStatsMemory(0) +{ + /* initialize the mempool stats collector */ + mempoolCollector = std::unique_ptr(new CStatsMempool(STATS_COLLECT_INTERVAL)); +} + +CStats::~CStats() +{ + +} + +std::string CStats::getHelpString(bool showDebug) +{ + std::string strUsage = HelpMessageGroup(_("Statistic options:")); + strUsage += HelpMessageOpt("-statsenable=", strprintf("Enable statistics (default: %u)", DEFAULT_STATISTICS_ENABLED)); + strUsage += HelpMessageOpt("-statsmaxmemorytarget=", strprintf(_("Set the memory limit target for statistics in bytes (default: %u)"), DEFAULT_MAX_STATS_MEMORY)); + + return strUsage; +} + +bool CStats::parameterInteraction() +{ + if (GetBoolArg("-statsenable", DEFAULT_STATISTICS_ENABLED)) + DefaultStats()->setMaxMemoryUsageTarget(GetArg("-statsmaxmemorytarget", DEFAULT_MAX_STATS_MEMORY)); + + return true; +} + +void CStats::startCollecting(CScheduler& scheduler) +{ + if (statsEnabled) { + // dispatch the scheduler task + scheduler.scheduleEvery(std::bind(&CStats::collectCallback, this), STATS_COLLECT_INTERVAL); + } +} + +void CStats::collectCallback() +{ + if (!statsEnabled) + return; + + if (mempoolCollector->addMempoolSamples(maxStatsMemory)) { + // fire the signal if the stats did change + MempoolStatsDidChange(); + } +} + +void CStats::setMaxMemoryUsageTarget(size_t maxMem) +{ + statsEnabled = (maxMem > 0); + maxStatsMemory = maxMem; + + // for now: give 100% to mempool stats + mempoolCollector->setMaxMemoryUsageTarget(maxMem); +} diff --git a/src/stats/stats.h b/src/stats/stats.h new file mode 100644 index 00000000000..390f3d5eede --- /dev/null +++ b/src/stats/stats.h @@ -0,0 +1,61 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_STATS_H +#define BITCOIN_STATS_H + +#include "amount.h" +#include "scheduler.h" +#include "stats/stats_mempool.h" + +#include +#include + +#include + +// Class that manages various types of statistics and its memory consumption +class CStats +{ +private: + static CStats* sharedInstance; //!< singleton instance + std::atomic statsEnabled; + + //maximum amount of memory to use for the stats + std::atomic maxStatsMemory; + + /* collect callback through the scheduler task */ + void collectCallback(); + +public: + + static CStats* DefaultStats(); //shared instance + + CStats(); + ~CStats(); + + /* get the statistics module help strings */ + static std::string getHelpString(bool showDebug); + + /* access the parameters and map it to the internal model */ + static bool parameterInteraction(); + + /* dispatch the stats collector repetitive scheduler task */ + void startCollecting(CScheduler& scheduler); + + /* set the target for the maximum memory consumption (in bytes) */ + void setMaxMemoryUsageTarget(size_t maxMem); + + + + /* COLLECTOR INSTANCES + * =================== */ + std::unique_ptr mempoolCollector; + + + /* SIGNALS + * ======= */ + boost::signals2::signal MempoolStatsDidChange; //mempool signal +}; + +#endif // BITCOIN_STATS_H diff --git a/src/stats/stats_mempool.cpp b/src/stats/stats_mempool.cpp new file mode 100644 index 00000000000..0d5c51681d8 --- /dev/null +++ b/src/stats/stats_mempool.cpp @@ -0,0 +1,130 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "stats/stats_mempool.h" + +#include "memusage.h" +#include "utiltime.h" + +#include "util.h" + + +const static unsigned int precisionIntervals[] = { + 2, // == every 2 secs == 1800 samples per hour + 60, // == every minute = 1440 samples per day + 1800 // == every half-hour = ~2'160 per Month +}; + +const static unsigned int MIN_SAMPLES = 5; +const static unsigned int MAX_SAMPLES = 5000; + + +const static unsigned int fallbackMaxSamplesPerPrecision = 1000; + +std::atomic CStatsMempool::cacheMempoolSize; +std::atomic CStatsMempool::cacheMempoolDynamicMemoryUsage; +std::atomic CStatsMempool::cacheMempoolMinRelayFee; + +CStatsMempool::CStatsMempool(unsigned int collectIntervalIn) : collectInterval(collectIntervalIn) +{ + startTime = 0; + intervalCounter = 0; + + // setup the samples per precision vector + for (unsigned int interval : precisionIntervals) { + (void)(interval); + vSamplesPerPrecision.emplace_back(); + + // use the fallback max in case max memory will not be set + vMaxSamplesPerPrecision.push_back(fallbackMaxSamplesPerPrecision); + + // add starttime 0 to each level + vTimeLastSample.push_back(0); + } +} + +std::vector CStatsMempool::getPrecisionGroupsAndIntervals() { + return {std::begin(precisionIntervals), std::end(precisionIntervals)}; +} + +bool CStatsMempool::addMempoolSamples(const size_t maxStatsMemory) +{ + bool statsChanged = false; + uint64_t now = GetTime(); + { + LOCK(cs_mempool_stats); + + // set the mempool stats start time if this is the first sample + if (startTime == 0) + startTime = now; + + unsigned int biggestInterval = 0; + for (unsigned int i = 0; i < sizeof(precisionIntervals) / sizeof(precisionIntervals[0]); i++) { + // check if it's time to collect a samples for the given precision level + uint16_t timeDelta = 0; + if (intervalCounter % (precisionIntervals[i] / (collectInterval / 1000)) == 0) { + if (vTimeLastSample[i] == 0) { + // first sample, calc delta to starttime + timeDelta = now - startTime; + } else { + timeDelta = now - vTimeLastSample[i]; + } + vSamplesPerPrecision[i].push_back({timeDelta, CStatsMempool::cacheMempoolSize, CStatsMempool::cacheMempoolDynamicMemoryUsage, CStatsMempool::cacheMempoolMinRelayFee}); + statsChanged = true; + + // check if we need to remove items at the beginning + if (vSamplesPerPrecision[i].size() > vMaxSamplesPerPrecision[i]) { + // increase starttime by the removed deltas + for (unsigned int j = (vSamplesPerPrecision[i].size() - vMaxSamplesPerPrecision[i]); j > 0; j--) { + startTime += vSamplesPerPrecision[i][j].timeDelta; + } + // remove element(s) at vector front + vSamplesPerPrecision[i].erase(vSamplesPerPrecision[i].begin(), vSamplesPerPrecision[i].begin() + (vSamplesPerPrecision[i].size() - vMaxSamplesPerPrecision[i])); + + // release memory + vSamplesPerPrecision[i].shrink_to_fit(); + } + + vTimeLastSample[i] = now; + } + biggestInterval = precisionIntervals[i]; + } + + intervalCounter++; + + if (intervalCounter > biggestInterval) { + intervalCounter = 1; + } + } + return statsChanged; +} + +void CStatsMempool::setMaxMemoryUsageTarget(size_t maxMem) +{ + // calculate the memory requirement of a single sample + size_t sampleSize = memusage::MallocUsage(sizeof(CStatsMempoolSample)); + + // calculate how many samples would fit in the target + size_t maxAmountOfSamples = maxMem / sampleSize; + + // distribute the max samples equal between precision levels + unsigned int samplesPerPrecision = maxAmountOfSamples / sizeof(precisionIntervals) / sizeof(precisionIntervals[0]); + samplesPerPrecision = std::max(MIN_SAMPLES, samplesPerPrecision); + samplesPerPrecision = std::min(MAX_SAMPLES, samplesPerPrecision); + for (unsigned int i = 0; i < sizeof(precisionIntervals) / sizeof(precisionIntervals[0]); i++) { + vMaxSamplesPerPrecision[i] = samplesPerPrecision; + } +} + +MempoolSamplesVector CStatsMempool::getSamplesForPrecision(unsigned int precision, uint64_t& fromTime) +{ + LOCK(cs_mempool_stats); + + if (precision >= vSamplesPerPrecision.size()) { + return MempoolSamplesVector(); + } + + fromTime = startTime; + return vSamplesPerPrecision[precision]; +} diff --git a/src/stats/stats_mempool.h b/src/stats/stats_mempool.h new file mode 100644 index 00000000000..095b29b10f0 --- /dev/null +++ b/src/stats/stats_mempool.h @@ -0,0 +1,73 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_STATS_MEMPOOL_H +#define BITCOIN_STATS_MEMPOOL_H + +#include +#include "scheduler.h" +#include + +#include +#include +#include + +struct CStatsMempoolSample { + + /* time delta to last item as 16bit unsigned integer + * allows a max delta of ~18h */ + uint16_t timeDelta; + + size_t txCount; //! MempoolSamplesVector; + +class CStatsMempool +{ +private: + + /* caches */ + static std::atomic cacheMempoolSize; + static std::atomic cacheMempoolDynamicMemoryUsage; + static std::atomic cacheMempoolMinRelayFee; + mutable CCriticalSection cs_mempool_stats; //! startTime; //start time + std::vector vSamplesPerPrecision; //! vMaxSamplesPerPrecision; //! vTimeLastSample; //! getPrecisionGroupsAndIntervals(); + + /* get mempool samples from a given precision group index(!) + * this returns a COPY */ + MempoolSamplesVector getSamplesForPrecision(unsigned int precision, uint64_t& fromTime); +}; + +#endif // BITCOIN_STATS_MEMPOOL_H diff --git a/src/validation.cpp b/src/validation.cpp index 75a35756d4c..b21a398b0e0 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -25,6 +25,7 @@ #include "script/script.h" #include "script/sigcache.h" #include "script/standard.h" +#include "stats/stats.h" #include "timedata.h" #include "tinyformat.h" #include "txdb.h" @@ -545,6 +546,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); + CFeeRate poolMinFeeRate; AssertLockHeld(cs_main); if (pfMissingInputs) *pfMissingInputs = false; @@ -715,7 +717,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, strprintf("%d", nSigOpsCost)); - CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); + poolMinFeeRate = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + CAmount mempoolRejectFee = poolMinFeeRate.GetFee(nSize); if (mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); } @@ -953,6 +956,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C GetMainSignals().TransactionAddedToMempool(ptx); + // update mempool stats cache + CStatsMempool::setCache(pool.size(), pool.DynamicMemoryUsage(), poolMinFeeRate.GetFeePerK()); + return true; } @@ -2169,6 +2175,10 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock); + + // update mempool stats cache + CStatsMempool::setCache(mempool.size(), mempool.DynamicMemoryUsage(), mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK()); + return true; } @@ -2294,6 +2304,9 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); + // update mempool stats cache + CStatsMempool::setCache(mempool.size(), mempool.DynamicMemoryUsage(), mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK()); + int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); From c412d0a66ed8dec02ecb99c16bde1377370b8aab Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 8 May 2017 15:59:10 +0200 Subject: [PATCH 2/2] [QA] Add basic mempool stats test --- test/functional/{mempool_limit.py => mempool_limit_stats.py} | 10 +++++++++- test/functional/test_runner.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) rename test/functional/{mempool_limit.py => mempool_limit_stats.py} (79%) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit_stats.py similarity index 79% rename from test/functional/mempool_limit.py rename to test/functional/mempool_limit_stats.py index 2777291dd0e..e6b4c09bd5e 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit_stats.py @@ -13,7 +13,9 @@ def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [["-maxmempool=5", "-spendzeroconfchange=0"]] + + # set a very low (100bytes) statsmaxmemorytarget to ensure we use the min amount of samples + self.extra_args = [["-maxmempool=5", "--statsenable", "--statsmaxmemorytarget=100""-spendzeroconfchange=0"]] def run_test(self): txouts = gen_return_txouts() @@ -43,6 +45,12 @@ def run_test(self): assert(txid not in self.nodes[0].getrawmempool()) txdata = self.nodes[0].gettransaction(txid) assert(txdata['confirmations'] == 0) #confirmation should still be 0 + + # test mempool stats + time.sleep(15) + mps = self.nodes[0].getmempoolstats() + assert_equal(len(mps), 3) #fixed amount of percision levels + assert_equal(len(mps[0]['samples']), 5) #make sure we only have 5 samples (minimum history) if __name__ == '__main__': MempoolLimitTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index c87010b0f45..5e23e30d5b1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -69,7 +69,7 @@ 'sendheaders.py', 'zapwallettxes.py', 'importmulti.py', - 'mempool_limit.py', + 'mempool_limit_stats.py', 'merkle_blocks.py', 'receivedby.py', 'abandonconflict.py',