From c1451b6185dfc8d6470d75c5a34f3d1f97a2bf1b Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Sat, 27 Jun 2026 19:32:13 +0000 Subject: [PATCH 001/303] fix(eth): prevent panic parsing non-32-byte-aligned input data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ParseInputData panicked with "index out of range [N] with length N" when a transaction's input data was not 32-byte aligned. Standard ABI-encoded calldata is always word-aligned, but a malformed/non-standard tx is not: while marking the words of a dynamic field (bytes/string/slice), tryParseParams could step one slot past `processed`, which is sized len(data)/64. The panic was caught by ParseInputData's deferred recover, so it did not crash the process, but it returned nil and logged a full stack trace (debug.PrintStack) for every offending transaction — heavy log spam when such an address is rendered or crawled. Bound the dynamic-field marking loop: an index past `processed` means the declared layout runs past the data, so reject the signature (return false) instead of writing out of range. Add a regression test built from a deliberately misaligned dynamic field. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/dataparser.go | 7 +++++++ bchain/coins/eth/dataparser_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/bchain/coins/eth/dataparser.go b/bchain/coins/eth/dataparser.go index b3f641622b..b8b0b525ab 100644 --- a/bchain/coins/eth/dataparser.go +++ b/bchain/coins/eth/dataparser.go @@ -188,6 +188,13 @@ func processParam(data string, index int, dataOffset int, t *abi.Type, processed } count = ((count - 1) >> 5) + 1 for i := 0; i < count; i++ { + // A dynamic field whose declared length runs past the data + // (malformed or non-32-byte-aligned input) would index beyond + // processed, which is sized len(data)/64; treat it as a + // non-matching signature instead of panicking. + if dynIndex >= len(processed) { + return nil, 0, false + } processed[dynIndex] = true dynIndex++ } diff --git a/bchain/coins/eth/dataparser_test.go b/bchain/coins/eth/dataparser_test.go index 34cbda8ea1..12a3147aa9 100644 --- a/bchain/coins/eth/dataparser_test.go +++ b/bchain/coins/eth/dataparser_test.go @@ -430,6 +430,33 @@ func TestParseInputData(t *testing.T) { } } +// TestParseInputDataMisalignedDynamicData is a regression test for a panic +// ("index out of range [N] with length N") when a transaction's input data is +// not 32-byte aligned. Standard ABI-encoded data is always word-aligned, but a +// malformed/non-standard tx is not, and marking the words of a dynamic field +// could step one slot past `processed` (sized len(data)/64). ParseInputData must +// instead reject the signature and fall back to just the method id, without +// panicking (and without the per-tx panic-recover log spam that masked it). +func TestParseInputDataMisalignedDynamicData(t *testing.T) { + parser := NewEthereumParser(1, false) + signatures := []bchain.FourByteSignature{ + {Name: "store", Parameters: []string{"bytes"}}, + } + // 0x + 4-byte selector, then a "bytes" field: offset word (0x20), length + // word (0x21 = 33 bytes), and 66 hex chars of content. The post-selector + // data is 194 hex chars — not a multiple of 64 — so `processed` has only 3 + // entries while the 33-byte field rounds up to a word at index 3. + data := "0xaabbccdd" + + "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000000000000000000021" + + "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021" + got := parser.ParseInputData(&signatures, data) + want := &bchain.EthereumParsedInputData{MethodId: "0xaabbccdd"} + if !reflect.DeepEqual(got, want) { + t.Errorf("ParseInputData() = %#v, want %#v", got, want) + } +} + func Test_getEnsRecord(t *testing.T) { tests := []struct { name string From 58e1e6f00eee37d5d76afe0c61ab1a4ba0709df3 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Sat, 27 Jun 2026 20:05:13 +0000 Subject: [PATCH 002/303] fix(eth): demote benign erc20 balanceOf parse-failure logs to V(2) A balanceOf eth_call that succeeds but returns empty/non-32-byte data (typical of dead/self-destructed or non-ERC20-conforming tokens that linger in holders' contract lists) was logged at Warning level on every holder request. Since these warnings were added in e8558f11 they have dominated the logs - tens of thousands of lines per minute for a single widely-airdropped dead token - because glog.Warningf prints regardless of the -v verbosity. Demote all five log sites of this benign event to glog.V(2).Infof and add a sentinel eth.ErrInvalidErc20Balance so the single-contract path in api/worker.go can tell it apart from genuine RPC errors, which stay at Warning. The observeEthCallError(..., "invalid") metric is unchanged, so the rate is still observable via Prometheus. Co-Authored-By: Claude Opus 4.8 --- api/worker.go | 14 ++++++++++++-- bchain/coins/eth/contract.go | 20 ++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/api/worker.go b/api/worker.go index 9e930606ed..4959e1bc51 100644 --- a/api/worker.go +++ b/api/worker.go @@ -1088,7 +1088,13 @@ func (w *Worker) getEthereumContractBalance(addrDesc bchain.AddressDescriptor, i b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract) if err != nil { // return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract) - glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + if errors.Cause(err) == eth.ErrInvalidErc20Balance { + // Benign and high-volume: unparseable/empty balanceOf from a dead or non-ERC20-conforming + // token. Tracked via the "invalid" metric; logged at V(2) to avoid flooding logs. + glog.V(2).Infof("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + } else { + glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + } } } if b != nil { @@ -1156,7 +1162,11 @@ func (w *Worker) getEthereumContractBalanceFromBlockchain(addrDesc, contract bch b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, contract) if err != nil { // return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract) - glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, contract, err) + if errors.Cause(err) == eth.ErrInvalidErc20Balance { + glog.V(2).Infof("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, contract, err) + } else { + glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, contract, err) + } } } else { b = nil diff --git a/bchain/coins/eth/contract.go b/bchain/coins/eth/contract.go index 7f53f84963..401f1d0813 100644 --- a/bchain/coins/eth/contract.go +++ b/bchain/coins/eth/contract.go @@ -425,6 +425,13 @@ func (b *EthereumRPC) GetContractInfo(contractDesc bchain.AddressDescriptor) (*b return b.fetchContractInfo(address) } +// ErrInvalidErc20Balance is returned when a balanceOf eth_call succeeds but returns data that +// cannot be parsed as a 32-byte integer (empty "0x" or non-conforming output). It is benign and +// common for dead/self-destructed or non-ERC20-conforming tokens that linger in holders' contract +// lists; callers should treat it as "no balance" and must not log it at warning level (it is already +// tracked via the observeEthCallError "invalid" metric). +var ErrInvalidErc20Balance = errors.New("Invalid balance") + // EthereumTypeGetErc20ContractBalance returns balance of ERC20 contract for given address func (b *EthereumRPC) EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc bchain.AddressDescriptor) (*big.Int, error) { return b.EthereumTypeGetErc20ContractBalanceAtBlock(addrDesc, contractDesc, nil) @@ -441,7 +448,7 @@ func (b *EthereumRPC) EthereumTypeGetErc20ContractBalanceAtBlock(addrDesc, contr r := parseSimpleNumericProperty(data) if r == nil { b.observeEthCallError("single", "invalid") - return nil, errors.New("Invalid balance") + return nil, ErrInvalidErc20Balance } return r, nil } @@ -534,7 +541,10 @@ func (b *EthereumRPC) erc20BalancesBatchAtBlock(batcher batchCaller, callData st balances[i] = parseSimpleNumericProperty(data) if balances[i] == nil { b.observeEthCallError("single", "invalid") - glog.Warningf("erc20 single eth_call invalid result for %s: %q", hexutil.Encode(contractDesc), data) + // Benign and high-volume: a successful eth_call returning empty/non-32-byte data, typical of + // dead (self-destructed) or non-ERC20-conforming tokens that linger in holders' contract lists. + // Tracked via the "invalid" metric; logged at V(2) to avoid flooding (one line per holder request). + glog.V(2).Infof("erc20 single eth_call invalid result for %s: %q", hexutil.Encode(contractDesc), data) } } return balances, nil @@ -556,7 +566,7 @@ func (b *EthereumRPC) erc20BalancesBatchAtBlock(batcher batchCaller, callData st balances[i] = parseSimpleNumericProperty(data) if balances[i] == nil { b.observeEthCallError("single", "invalid") - glog.Warningf("erc20 single eth_call invalid result for %s: %q", hexutil.Encode(contractDescs[i]), data) + glog.V(2).Infof("erc20 single eth_call invalid result for %s: %q", hexutil.Encode(contractDescs[i]), data) } continue } @@ -565,7 +575,9 @@ func (b *EthereumRPC) erc20BalancesBatchAtBlock(batcher batchCaller, callData st balances[i] = parseSimpleNumericProperty(results[i]) if balances[i] == nil { b.observeEthCallError("batch", "invalid") - glog.Warningf("erc20 batch eth_call invalid result for %s: %q", hexutil.Encode(contractDescs[i]), results[i]) + // Benign and high-volume: see the single-call note above. Same event on the batch success path, + // dominated by widely-airdropped dead/non-conforming tokens. Tracked via the "invalid" metric. + glog.V(2).Infof("erc20 batch eth_call invalid result for %s: %q", hexutil.Encode(contractDescs[i]), results[i]) } } return balances, nil From 21ac04491e5070b9fbd3ac77a03bee06c5065ea5 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 30 Jun 2026 09:45:10 +0000 Subject: [PATCH 003/303] fix(server): lower REST/UI rate-limit defaults for individual user usage The public HTTP surface (explorer UI + REST API) is intended for individual human use -- Trezor Suite uses the WebSocket interface, not REST/UI -- so the previous defaults (180 req/min, burst 40) were far more generous than a person browsing the explorer needs. That headroom let a distributed crawler sustain heavy aggregate traffic while every per-IP key stayed comfortably under the limit, so the limiter never rejected anything. Lower the per-client defaults: REST_UI_RATE_LIMIT 180 -> 20 and REST_UI_BURST 40 -> 20. The explorer renders each page as a single request, so 20 requests/min comfortably covers real browsing while the tighter sustained rate forces a crawler onto far more source IPs for the same aggregate rate; a burst of 20 still absorbs a short flurry of page loads. Both remain overridable via the _REST_UI_* env vars. Updates the matching defaults in docs/env.md and the config test that locks the values. Co-Authored-By: Claude Opus 4.8 --- docs/env.md | 4 ++-- server/rest_rate_limiter.go | 7 +++++-- server/rest_rate_limiter_test.go | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/env.md b/docs/env.md index d8b8ccb3d0..f0b01df7c9 100644 --- a/docs/env.md +++ b/docs/env.md @@ -53,7 +53,7 @@ Blockbook reads these from its process environment. When installed from the Debi The blocklist is kept in memory only: a restart clears all active blocks. A blocked client key still blocks every other client sharing that exact key — the same IPv4 address behind CGNAT — but IPv6 blocks now key on the individual `/128`, so unrelated neighbors sharing a `/64` are no longer caught by one address's block. -- `_REST_UI_RATE_LIMIT` - Maximum number of public HTTP requests a single client key may start within `_REST_UI_RATE_WINDOW`. Accepts a non-negative integer; default `180`, `0` disables request-rate limiting. The client key uses the shared REST/WebSocket attribution rules: an IPv4 address, or an IPv6 `/64` prefix. +- `_REST_UI_RATE_LIMIT` - Maximum number of public HTTP requests a single client key may start within `_REST_UI_RATE_WINDOW`. Accepts a non-negative integer; default `20`, `0` disables request-rate limiting. The client key uses the shared REST/WebSocket attribution rules: an IPv4 address, or an IPv6 `/64` prefix. The default is tight because this surface is for individual human use (Suite uses the WebSocket interface). Although it is configured through `REST_UI_*` variables, this limiter governs **all dynamic public routes under one shared per-client budget** — both the explorer UI pages (`/address/`, `/xpub/`, `/tx/`, `/search/`, `/block/`, …) and the REST API (`/api/...`). Only static assets (`/static/`, `/favicon.ico`, `/test-websocket.html`), the API docs (`/api-docs`), the OpenAPI spec (`/openapi.yaml`), and the WebSocket endpoint (`/websocket`, which has its own limiter — see `_WS_MESSAGE_RATE_LIMIT`) are exempt. New routes are covered automatically. @@ -61,7 +61,7 @@ Blockbook reads these from its process environment. When installed from the Debi - `_REST_UI_RATE_WINDOW` - Token-bucket refill window for `_REST_UI_RATE_LIMIT`, as a Go duration string (e.g. `1m`, `60s`). Default `1m`. -- `_REST_UI_BURST` - Token-bucket burst size for one client key. Default `40`; must be positive when request-rate limiting is enabled. +- `_REST_UI_BURST` - Token-bucket burst size for one client key. Default `20`; must be positive when request-rate limiting is enabled. Allows a short flurry of page loads (the explorer renders each page as a single request) while `_REST_UI_RATE_LIMIT` caps the sustained rate. - `_REST_UI_MAX_CONCURRENT` - Maximum number of in-flight public HTTP requests (explorer UI page or REST API call) accepted from one client key. Default `12`; `0` disables the per-client concurrency limit. This protects slow or expensive handlers held open concurrently from one source. diff --git a/server/rest_rate_limiter.go b/server/rest_rate_limiter.go index 4c75deba8b..61ba54d77e 100644 --- a/server/rest_rate_limiter.go +++ b/server/rest_rate_limiter.go @@ -16,9 +16,12 @@ import ( ) const ( - defaultRestUIRateLimit = 180 + // Public HTTP (explorer UI + REST API) is for individual human use; Suite uses + // the WebSocket interface. Keep per-client rate/burst tight so a distributed + // crawler needs many more source IPs to sustain a given aggregate rate. + defaultRestUIRateLimit = 20 defaultRestUIRateWindow = time.Minute - defaultRestUIBurst = 40 + defaultRestUIBurst = 20 defaultRestUIMaxConcurrent = 12 defaultRestUIStateTTL = 10 * time.Minute defaultRestUIBlockDuration = 0 diff --git a/server/rest_rate_limiter_test.go b/server/rest_rate_limiter_test.go index 026ec3d6c1..f0fb137dbd 100644 --- a/server/rest_rate_limiter_test.go +++ b/server/rest_rate_limiter_test.go @@ -311,11 +311,11 @@ func TestReadRestUILimiterConfig(t *testing.T) { } // Lock the lowered defaults so a regression in either the constants or the // env parsing is caught here, not only in the dashboard/docs. - if cfg.rateLimit != 180 { - t.Fatalf("default rateLimit = %d, want 180", cfg.rateLimit) + if cfg.rateLimit != 20 { + t.Fatalf("default rateLimit = %d, want 20", cfg.rateLimit) } - if cfg.burst != 40 { - t.Fatalf("default burst = %d, want 40", cfg.burst) + if cfg.burst != 20 { + t.Fatalf("default burst = %d, want 20", cfg.burst) } if cfg.maxConcurrent != 12 { t.Fatalf("default maxConcurrent = %d, want 12", cfg.maxConcurrent) From 30fec6ff2486ed45098d570441e33affb9f4d565 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 30 Jun 2026 09:45:23 +0000 Subject: [PATCH 004/303] fix(api): lower default REST balance-history transaction cap to 100000 DefaultBalanceHistoryMaxTxsREST bounds how many transactions a single REST balance-history request (address or xpub) may aggregate. Each aggregated transaction costs one DB read, so an unbounded range over a heavy account is a cheap-to-send, expensive-to-serve request on the open, unauthenticated REST surface. The cap is enforced against a bounded index scan (txids are collected up to cap+1 via early StopIteration) and the request is rejected before any of the per-transaction reads run, so lowering it directly shrinks the worst-case work an anonymous request can trigger. 100000 stays well above any realistic single-account history a human would inspect via REST while tightening the ceiling from 250000. WS keeps its generous 1000000 default (Trezor Suite requests full account history over WS for its balance graph). Both remain overridable via _{WS,REST}_BALANCE_HISTORY_MAX_TXS (or the shared _BALANCE_HISTORY_MAX_TXS). Co-Authored-By: Claude Opus 4.8 --- api/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/worker.go b/api/worker.go index 4959e1bc51..2403a3db8c 100644 --- a/api/worker.go +++ b/api/worker.go @@ -2028,7 +2028,7 @@ func (w *Worker) balanceHistoryForTxid(addrDesc bchain.AddressDescriptor, txid s // (0) via _{WS,REST}_BALANCE_HISTORY_MAX_TXS (_BALANCE_HISTORY_MAX_TXS // sets both). const ( - DefaultBalanceHistoryMaxTxsREST = 250000 + DefaultBalanceHistoryMaxTxsREST = 100000 DefaultBalanceHistoryMaxTxsWS = 1000000 ) From dde4f74200801dbdc36b14e04f301b3470971fec Mon Sep 17 00:00:00 2001 From: Sketchy Stale Mermaid <278466100+53gur0@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:27:55 +0200 Subject: [PATCH 005/303] feat(eth): return evm-specific gas data --- api/types.go | 2 ++ bchain/coins/eth/ethparser.go | 17 +++++++------ bchain/types_ethereum_type.go | 7 ++++++ blockbook-api.ts | 20 ++++++++++++++++ build/tools/typescriptify/typescriptify.go | 1 + server/websocket.go | 28 ++++++++++++++-------- server/ws_types.go | 15 ++++++++++++ 7 files changed, 73 insertions(+), 17 deletions(-) diff --git a/api/types.go b/api/types.go index 08c7e34814..8bdca20b55 100644 --- a/api/types.go +++ b/api/types.go @@ -669,6 +669,8 @@ type Eip1559Fee struct { // Eip1559Fees type Eip1559Fees struct { BaseFeePerGas *Amount `json:"baseFeePerGas,omitempty"` + BlockGasUsed *Amount `json:"blockGasUsed,omitempty" ts_doc:"Total gas used by the latest block (decimal)."` + BlockGasLimit *Amount `json:"blockGasLimit,omitempty" ts_doc:"Gas limit of the latest block (decimal)."` Low *Eip1559Fee `json:"low,omitempty"` Medium *Eip1559Fee `json:"medium,omitempty"` High *Eip1559Fee `json:"high,omitempty"` diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index 30085cd8c0..27e8ef740c 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -90,13 +90,16 @@ func (p *EthereumParser) AddressContractsCacheConfig() AddressContractsCacheConf } type rpcHeader struct { - Hash string `json:"hash"` - ParentHash string `json:"parentHash"` - Difficulty string `json:"difficulty"` - Number string `json:"number"` - Time string `json:"timestamp"` - Size string `json:"size"` - Nonce string `json:"nonce"` + Hash string `json:"hash"` + ParentHash string `json:"parentHash"` + Difficulty string `json:"difficulty"` + Number string `json:"number"` + Time string `json:"timestamp"` + Size string `json:"size"` + Nonce string `json:"nonce"` + GasUsed string `json:"gasUsed"` + GasLimit string `json:"gasLimit"` + BaseFeePerGas string `json:"baseFeePerGas"` } type rpcLogWithTxHash struct { diff --git a/bchain/types_ethereum_type.go b/bchain/types_ethereum_type.go index ceec2121f8..65495bf1de 100644 --- a/bchain/types_ethereum_type.go +++ b/bchain/types_ethereum_type.go @@ -231,6 +231,11 @@ type EthereumBlockSpecificData struct { InternalDataError string `ts_doc:"Error message for processing block internal data, if any."` AddressAliasRecords []AddressAliasRecord `ts_doc:"List of address-to-alias mappings discovered in this block."` Contracts []ContractInfo `ts_doc:"List of contracts created or updated in this block."` + // GasUsed/GasLimit/BaseFeePerGas are transient block-level EIP-1559 figures used for the new-block + // notification; they are not persisted (storeBlockSpecificDataEthereumType ignores them). + GasUsed *big.Int `json:"-"` + GasLimit *big.Int `json:"-"` + BaseFeePerGas *big.Int `json:"-"` } // StakingPoolData holds data about address participation in a staking pool contract @@ -257,6 +262,8 @@ type Eip1559Fee struct { // Eip1559Fees type Eip1559Fees struct { BaseFeePerGas *big.Int `json:"baseFeePerGas,omitempty"` + BlockGasUsed *big.Int `json:"blockGasUsed,omitempty"` + BlockGasLimit *big.Int `json:"blockGasLimit,omitempty"` Low *Eip1559Fee `json:"low,omitempty"` Medium *Eip1559Fee `json:"medium,omitempty"` High *Eip1559Fee `json:"high,omitempty"` diff --git a/blockbook-api.ts b/blockbook-api.ts index 84287c7c4a..8aaea4f6b0 100644 --- a/blockbook-api.ts +++ b/blockbook-api.ts @@ -820,6 +820,10 @@ export interface Eip1559Fee { } export interface Eip1559Fees { baseFeePerGas?: string; + /** Total gas used by the latest block (decimal). */ + blockGasUsed?: string; + /** Gas limit of the latest block (decimal). */ + blockGasLimit?: string; low?: Eip1559Fee; medium?: Eip1559Fee; high?: Eip1559Fee; @@ -846,6 +850,22 @@ export interface WsLongTermFeeRateRes { /** Amount of blocks used for the long term fee rate estimation. */ blocks: number; } +export interface EthereumGasData { + /** Base fee per gas of the new block (post-London). */ + baseFeePerGas?: string; + /** Total gas used by the new block (decimal). */ + blockGasUsed?: string; + /** Gas limit of the new block (decimal). */ + blockGasLimit?: string; +} +export interface WsNewBlock { + /** Height of the new block. */ + height: number; + /** Hash of the new block. */ + hash: string; + /** EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains. */ + evm_data: EthereumGasData | null; +} export interface WsSendTransactionReq { /** Hex-encoded transaction data to broadcast (string format). */ hex?: string; diff --git a/build/tools/typescriptify/typescriptify.go b/build/tools/typescriptify/typescriptify.go index 076b06cd2e..0bd6788de6 100644 --- a/build/tools/typescriptify/typescriptify.go +++ b/build/tools/typescriptify/typescriptify.go @@ -59,6 +59,7 @@ func main() { t.Add(server.WsEstimateFeeReq{}) t.Add(server.WsEstimateFeeRes{}) t.Add(server.WsLongTermFeeRateRes{}) + t.Add(server.WsNewBlock{}) t.Add(server.WsSendTransactionReq{}) t.Add(server.WsSubscribeAddressesReq{}) t.Add(server.WsSubscribeFiatRatesReq{}) diff --git a/server/websocket.go b/server/websocket.go index a2cf911fae..0c6c5c7d99 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1067,6 +1067,8 @@ func (s *WebsocketServer) estimateFee(params []byte) (interface{}, error) { if eip1559 != nil { eip1559Api = &api.Eip1559Fees{} eip1559Api.BaseFeePerGas = (*api.Amount)(eip1559.BaseFeePerGas) + eip1559Api.BlockGasUsed = (*api.Amount)(eip1559.BlockGasUsed) + eip1559Api.BlockGasLimit = (*api.Amount)(eip1559.BlockGasLimit) eip1559Api.Instant = eip1559FeesToApi(eip1559.Instant) eip1559Api.High = eip1559FeesToApi(eip1559.High) eip1559Api.Medium = eip1559FeesToApi(eip1559.Medium) @@ -1411,24 +1413,30 @@ func (s *WebsocketServer) unsubscribeFiatRates(c *websocketChannel) (res interfa return &subscriptionResponse{false}, nil } -func (s *WebsocketServer) onNewBlockAsync(hash string, height uint32) { +func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { s.newBlockSubscriptionsLock.Lock() defer s.newBlockSubscriptionsLock.Unlock() - data := struct { - Height uint32 `json:"height"` - Hash string `json:"hash"` - }{ - Height: height, - Hash: hash, + data := &WsNewBlock{ + Height: block.Height, + Hash: block.Hash, + } + // for EVM chains attach block-level gas data so subscribers can project the next EIP-1559 base fee; + // stays nil (evm_data: null) for non-EVM chains and pre-London blocks + if bsd, ok := block.CoinSpecificData.(*bchain.EthereumBlockSpecificData); ok && bsd != nil && bsd.BaseFeePerGas != nil { + data.EVMData = &EthereumGasData{ + BaseFeePerGas: (*api.Amount)(bsd.BaseFeePerGas), + BlockGasUsed: (*api.Amount)(bsd.GasUsed), + BlockGasLimit: (*api.Amount)(bsd.GasLimit), + } } for c, id := range s.newBlockSubscriptions { c.DataOut(&WsRes{ ID: id, - Data: &data, + Data: data, }) } s.metrics.WebsocketNewBlockNotifications.Add(float64(len(s.newBlockSubscriptions))) - glog.V(2).Info("broadcasting new block ", height, " ", hash, " to ", len(s.newBlockSubscriptions), " channels") + glog.V(2).Info("broadcasting new block ", block.Height, " ", block.Hash, " to ", len(s.newBlockSubscriptions), " channels") } // setConfirmedBlockTxMetadata normalizes parsed block transactions. @@ -1584,7 +1592,7 @@ func (s *WebsocketServer) publishNewBlockTxsByAddr(block *bchain.Block) { func (s *WebsocketServer) OnNewBlock(block *bchain.Block) { s.addressSubscriptionsLock.Lock() defer s.addressSubscriptionsLock.Unlock() - go s.onNewBlockAsync(block.Hash, block.Height) + go s.onNewBlockAsync(block) if s.newBlockTxsSubscriptionCount > 0 { // Skip per-tx address matching when nobody opted into newBlockTxs. if ok, _ := s.trackWork(); ok { diff --git a/server/ws_types.go b/server/ws_types.go index feb2097108..bb95031617 100644 --- a/server/ws_types.go +++ b/server/ws_types.go @@ -148,6 +148,21 @@ type WsEstimateFeeRes struct { Eip1559 *api.Eip1559Fees `json:"eip1559,omitempty"` } +// EthereumGasData carries EVM block-level gas figures used by the frontend to deterministically +// project the next block's EIP-1559 base fee. Decimal strings, omitted for pre-London blocks. +type EthereumGasData struct { + BaseFeePerGas *api.Amount `json:"baseFeePerGas,omitempty" ts_doc:"Base fee per gas of the new block (post-London)."` + BlockGasUsed *api.Amount `json:"blockGasUsed,omitempty" ts_doc:"Total gas used by the new block (decimal)."` + BlockGasLimit *api.Amount `json:"blockGasLimit,omitempty" ts_doc:"Gas limit of the new block (decimal)."` +} + +// WsNewBlock is pushed to subscribeNewBlock subscribers when a new block is connected. +type WsNewBlock struct { + Height uint32 `json:"height" ts_doc:"Height of the new block."` + Hash string `json:"hash" ts_doc:"Hash of the new block."` + EVMData *EthereumGasData `json:"evm_data" ts_doc:"EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains."` +} + // WsLongTermFeeRateRes is returned in response to a long term fee rate request. type WsLongTermFeeRateRes struct { FeePerUnit string `json:"feePerUnit" ts_doc:"Long term fee rate (in sat/kByte)."` From 34454d036203d162dfdf37eccde6598e82ff156e Mon Sep 17 00:00:00 2001 From: Sketchy Stale Mermaid <278466100+53gur0@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:30:39 +0200 Subject: [PATCH 006/303] test(eth): `WsNewBlock` parsing sanity test --- server/ws_newblock_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 server/ws_newblock_test.go diff --git a/server/ws_newblock_test.go b/server/ws_newblock_test.go new file mode 100644 index 0000000000..577313770d --- /dev/null +++ b/server/ws_newblock_test.go @@ -0,0 +1,51 @@ +//go:build unittest + +package server + +import ( + "encoding/json" + "math/big" + "testing" + + "github.com/trezor/blockbook/api" +) + +// TestWsNewBlockJSON pins the subscribeNewBlock wire shape: non-EVM chains serialize evm_data as +// null (height/hash unchanged), while EVM chains carry decimal-string gas figures for the EIP-1559 projection. +func TestWsNewBlockJSON(t *testing.T) { + tests := []struct { + name string + in WsNewBlock + want string + }{ + { + name: "non-EVM emits evm_data null", + in: WsNewBlock{Height: 5, Hash: "0xabc"}, + want: `{"height":5,"hash":"0xabc","evm_data":null}`, + }, + { + name: "EVM emits decimal gas figures", + in: WsNewBlock{ + Height: 5, + Hash: "0xabc", + EVMData: &EthereumGasData{ + BaseFeePerGas: (*api.Amount)(big.NewInt(7)), + BlockGasUsed: (*api.Amount)(big.NewInt(21000)), + BlockGasLimit: (*api.Amount)(big.NewInt(30000000)), + }, + }, + want: `{"height":5,"hash":"0xabc","evm_data":{"baseFeePerGas":"7","blockGasUsed":"21000","blockGasLimit":"30000000"}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := json.Marshal(&tt.in) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(b) != tt.want { + t.Errorf("got %s\nwant %s", b, tt.want) + } + }) + } +} From fe79873bed463382802a654b68afcba587cd4b52 Mon Sep 17 00:00:00 2001 From: Sketchy Stale Mermaid <278466100+53gur0@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:55:53 +0200 Subject: [PATCH 007/303] feat(eth): methods to set 1559 specific fields on a new block --- bchain/coins/eth/ethrpc.go | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index eac2a5dabe..722faab90c 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1286,6 +1286,27 @@ func (b *EthereumRPC) GetBlockHash(height uint32) (string, error) { return h.Hash(), nil } +// returns early for pre-London blocks, populates EthereumBlockSpecificData +func attachBlockGas(h *rpcHeader, existing *bchain.EthereumBlockSpecificData) *bchain.EthereumBlockSpecificData { + if h.BaseFeePerGas == "" { + return existing + } + bsd := existing + if bsd == nil { + bsd = &bchain.EthereumBlockSpecificData{} + } + if baseFee, err := hexutil.DecodeUint64(h.BaseFeePerGas); err == nil { + bsd.BaseFeePerGas = new(big.Int).SetUint64(baseFee) + } + if gasUsed, err := hexutil.DecodeUint64(h.GasUsed); err == nil { + bsd.GasUsed = new(big.Int).SetUint64(gasUsed) + } + if gasLimit, err := hexutil.DecodeUint64(h.GasLimit); err == nil { + bsd.GasLimit = new(big.Int).SetUint64(gasLimit) + } + return bsd +} + func (b *EthereumRPC) ethHeaderToBlockHeader(h *rpcHeader) (*bchain.BlockHeader, error) { height, err := ethNumber(h.Number) if err != nil { @@ -1620,6 +1641,8 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } } + blockSpecificData = attachBlockGas(&head, blockSpecificData) + btxs := make([]bchain.Tx, len(body.Transactions)) for i := range body.Transactions { tx := &body.Transactions[i] @@ -1847,6 +1870,47 @@ func (b *EthereumRPC) EthereumTypeEstimateGas(params map[string]interface{}) (ui return b.Client.EstimateGas(ctx, msg) } +// getLatestBlockGas fetches the latest block's gasUsed and gasLimit, used by the frontend to +// deterministically project the next block's EIP-1559 base fee. Returns nil values (without error) +// for pre-London blocks where baseFeePerGas is absent, so the fields are omitted for non-EIP-1559 blocks. +func (b *EthereumRPC) getLatestBlockGas() (gasUsed, gasLimit *big.Int, err error) { + ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) + defer cancel() + var blk struct { + GasUsed string `json:"gasUsed"` + GasLimit string `json:"gasLimit"` + BaseFeePerGas string `json:"baseFeePerGas"` + } + if err = b.RPC.CallContext(ctx, &blk, "eth_getBlockByNumber", "latest", false); err != nil { + return nil, nil, err + } + if blk.BaseFeePerGas == "" { + // pre-London block, no base fee projection possible + return nil, nil, nil + } + gu, err := hexutil.DecodeUint64(blk.GasUsed) + if err != nil { + return nil, nil, err + } + gl, err := hexutil.DecodeUint64(blk.GasLimit) + if err != nil { + return nil, nil, err + } + return new(big.Int).SetUint64(gu), new(big.Int).SetUint64(gl), nil +} + +// setLatestBlockGas populates fees.BlockGasUsed/BlockGasLimit from the latest block. +// A failure is logged but not fatal - the priority fees and baseFeePerGas remain usable. +func (b *EthereumRPC) setLatestBlockGas(fees *bchain.Eip1559Fees) { + gasUsed, gasLimit, err := b.getLatestBlockGas() + if err != nil { + glog.Error("eth_getBlockByNumber latest for eip1559 block gas: ", err) + return + } + fees.BlockGasUsed = gasUsed + fees.BlockGasLimit = gasLimit +} + // EthereumTypeGetEip1559Fees retrieves Eip1559Fees, if supported func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) { if !b.ChainConfig.Eip1559Fees { @@ -1859,6 +1923,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) return nil, err } if fees != nil { + b.setLatestBlockGas(fees) return fees, nil } // Fall back to on-chain estimation when the alternative provider is unsupported/stale/unready, @@ -1929,6 +1994,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) fees.Instant = &f } } + b.setLatestBlockGas(&fees) return &fees, err } From c50dcf136670253ade60ee142db4f8c61e15de7f Mon Sep 17 00:00:00 2001 From: Sketchy Stale Mermaid <278466100+53gur0@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:56:34 +0200 Subject: [PATCH 008/303] test(eth): unit tests for 1559-specific fields --- bchain/coins/eth/eip1559_blockgas_test.go | 179 ++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 bchain/coins/eth/eip1559_blockgas_test.go diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go new file mode 100644 index 0000000000..80d49e37f9 --- /dev/null +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -0,0 +1,179 @@ +//go:build unittest + +package eth + +import ( + "context" + "encoding/json" + "errors" + "math/big" + "testing" + "time" + + "github.com/trezor/blockbook/bchain" +) + +// blockGasRPCStub serves a single CallContext response (raw JSON) or a transport error, +// recording the method and args so tests can assert the "latest" block tag is requested. +type blockGasRPCStub struct { + raw string + err error + method string + gotArgs []interface{} +} + +func (s *blockGasRPCStub) EthSubscribe(context.Context, interface{}, ...interface{}) (bchain.EVMClientSubscription, error) { + return nil, errors.New("not implemented") +} + +func (s *blockGasRPCStub) Close() {} + +func (s *blockGasRPCStub) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + s.method = method + s.gotArgs = args + if s.err != nil { + return s.err + } + return json.Unmarshal([]byte(s.raw), result) +} + +func equalBigInt(a, b *big.Int) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Cmp(b) == 0 +} + +func TestGetLatestBlockGas(t *testing.T) { + tests := []struct { + name string + raw string + err error + wantUsed *big.Int + wantLimit *big.Int + wantErr bool + }{ + { + name: "post-London populates gas", + raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380","baseFeePerGas":"0x7"}`, + wantUsed: big.NewInt(21000), + wantLimit: big.NewInt(30000000), + }, + { + name: "pre-London (no baseFee) omits gas", + raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380"}`, + }, + { + name: "rpc error propagates", + err: errors.New("boom"), + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := &blockGasRPCStub{raw: tt.raw, err: tt.err} + b := &EthereumRPC{RPC: stub, Timeout: time.Second} + gu, gl, err := b.getLatestBlockGas() + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !equalBigInt(gu, tt.wantUsed) { + t.Errorf("gasUsed = %v, want %v", gu, tt.wantUsed) + } + if !equalBigInt(gl, tt.wantLimit) { + t.Errorf("gasLimit = %v, want %v", gl, tt.wantLimit) + } + if stub.method != "eth_getBlockByNumber" { + t.Errorf("method = %q, want eth_getBlockByNumber", stub.method) + } + if len(stub.gotArgs) < 1 || stub.gotArgs[0] != "latest" { + t.Errorf("args = %v, want first arg \"latest\"", stub.gotArgs) + } + }) + } +} + +func TestSetLatestBlockGas(t *testing.T) { + t.Run("success sets fields", func(t *testing.T) { + b := &EthereumRPC{RPC: &blockGasRPCStub{raw: `{"gasUsed":"0x5208","gasLimit":"0x3938700","baseFeePerGas":"0x7"}`}, Timeout: time.Second} + fees := &bchain.Eip1559Fees{} + b.setLatestBlockGas(fees) + if !equalBigInt(fees.BlockGasUsed, big.NewInt(21000)) { + t.Errorf("BlockGasUsed = %v, want 21000", fees.BlockGasUsed) + } + if !equalBigInt(fees.BlockGasLimit, big.NewInt(60000000)) { + t.Errorf("BlockGasLimit = %v, want 60000000", fees.BlockGasLimit) + } + }) + + t.Run("rpc error leaves fields nil (non-fatal)", func(t *testing.T) { + b := &EthereumRPC{RPC: &blockGasRPCStub{err: errors.New("boom")}, Timeout: time.Second} + fees := &bchain.Eip1559Fees{} + b.setLatestBlockGas(fees) + if fees.BlockGasUsed != nil || fees.BlockGasLimit != nil { + t.Errorf("expected nil gas fields on error, got used=%v limit=%v", fees.BlockGasUsed, fees.BlockGasLimit) + } + }) + + t.Run("pre-London leaves fields nil", func(t *testing.T) { + b := &EthereumRPC{RPC: &blockGasRPCStub{raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380"}`}, Timeout: time.Second} + fees := &bchain.Eip1559Fees{} + b.setLatestBlockGas(fees) + if fees.BlockGasUsed != nil || fees.BlockGasLimit != nil { + t.Errorf("expected nil gas fields pre-London, got used=%v limit=%v", fees.BlockGasUsed, fees.BlockGasLimit) + } + }) +} + +func TestAttachBlockGas(t *testing.T) { + t.Run("post-London allocates and sets", func(t *testing.T) { + bsd := attachBlockGas(&rpcHeader{GasUsed: "0x5208", GasLimit: "0x3938700", BaseFeePerGas: "0x7"}, nil) + if bsd == nil { + t.Fatal("expected allocated EthereumBlockSpecificData, got nil") + } + if !equalBigInt(bsd.BaseFeePerGas, big.NewInt(7)) { + t.Errorf("BaseFeePerGas = %v, want 7", bsd.BaseFeePerGas) + } + if !equalBigInt(bsd.GasUsed, big.NewInt(21000)) { + t.Errorf("GasUsed = %v, want 21000", bsd.GasUsed) + } + if !equalBigInt(bsd.GasLimit, big.NewInt(60000000)) { + t.Errorf("GasLimit = %v, want 60000000", bsd.GasLimit) + } + }) + + t.Run("pre-London with nil existing returns nil", func(t *testing.T) { + if got := attachBlockGas(&rpcHeader{GasUsed: "0x5208", GasLimit: "0x1c9c380"}, nil); got != nil { + t.Errorf("expected nil, got %+v", got) + } + }) + + t.Run("pre-London preserves existing struct unchanged", func(t *testing.T) { + existing := &bchain.EthereumBlockSpecificData{InternalDataError: "boom"} + got := attachBlockGas(&rpcHeader{}, existing) + if got != existing { + t.Fatal("expected existing struct returned unchanged") + } + + if got.BaseFeePerGas != nil { + t.Errorf("expected no gas set pre-London, got %v", got.BaseFeePerGas) + } + }) + + t.Run("post-London preserves existing fields and adds gas", func(t *testing.T) { + existing := &bchain.EthereumBlockSpecificData{InternalDataError: "err"} + got := attachBlockGas(&rpcHeader{GasUsed: "0x1", GasLimit: "0x2", BaseFeePerGas: "0x7"}, existing) + if got.InternalDataError != "err" { + t.Errorf("InternalDataError = %q, want preserved \"err\"", got.InternalDataError) + } + if !equalBigInt(got.BaseFeePerGas, big.NewInt(7)) { + t.Errorf("BaseFeePerGas = %v, want 7", got.BaseFeePerGas) + } + }) +} From 004e1c370d3ab845830e7c169fc890a6b48e6ec7 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 19:23:43 +0200 Subject: [PATCH 009/303] fix(eth): drop per-estimate eth_getBlockByNumber("latest") for block gas EthereumTypeGetEip1559Fees called setLatestBlockGas on every request, which issued an extra synchronous eth_getBlockByNumber("latest") RPC to populate Eip1559Fees.BlockGasUsed/BlockGasLimit. This had three problems: - Extra overhead: a third blocking RPC was added to every websocket estimateFee call (on top of eth_maxPriorityFeePerGas + eth_feeHistory), and it also fired on the alternative-fee-provider fast path that was deliberately RPC-free (served from an in-memory cache). On fast L2s with many subscribers polling fees this scales linearly with load. - Data race: the alternative provider returns the same cached *Eip1559Fees pointer to every caller; setLatestBlockGas then mutated BlockGasUsed/ BlockGasLimit on that shared struct with no lock, racing concurrent estimateFee goroutines (and the field reads in server/websocket.go). - Incoherent data: baseFeePerGas came from eth_feeHistory (newest=pending) or an off-chain oracle, while gasUsed/gasLimit came from a separate "latest" block, so the three values described different blocks and could not be combined into a correct EIP-1559 next-base-fee projection. Remove getLatestBlockGas/setLatestBlockGas and the now-unpopulated BlockGasUsed/BlockGasLimit fields from the pull-side Eip1559Fees (bchain, api, blockbook-api.ts) and the websocket mapping. The push path (subscribeNewBlock evm_data) is unaffected: it still carries the block's baseFeePerGas/gasUsed/gasLimit from a single header, with no extra RPC. A follow-up commit exposes the next-block base fee from the eth_feeHistory response already fetched, so the pull path keeps a (consistent, free) projection input. Co-Authored-By: Claude Opus 4.8 --- api/types.go | 2 - bchain/coins/eth/eip1559_blockgas_test.go | 121 ++-------------------- bchain/coins/eth/ethrpc.go | 43 -------- bchain/types_ethereum_type.go | 2 - blockbook-api.ts | 4 - server/websocket.go | 2 - 6 files changed, 6 insertions(+), 168 deletions(-) diff --git a/api/types.go b/api/types.go index 8bdca20b55..08c7e34814 100644 --- a/api/types.go +++ b/api/types.go @@ -669,8 +669,6 @@ type Eip1559Fee struct { // Eip1559Fees type Eip1559Fees struct { BaseFeePerGas *Amount `json:"baseFeePerGas,omitempty"` - BlockGasUsed *Amount `json:"blockGasUsed,omitempty" ts_doc:"Total gas used by the latest block (decimal)."` - BlockGasLimit *Amount `json:"blockGasLimit,omitempty" ts_doc:"Gas limit of the latest block (decimal)."` Low *Eip1559Fee `json:"low,omitempty"` Medium *Eip1559Fee `json:"medium,omitempty"` High *Eip1559Fee `json:"high,omitempty"` diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index 80d49e37f9..d895942929 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -3,40 +3,12 @@ package eth import ( - "context" - "encoding/json" - "errors" "math/big" "testing" - "time" "github.com/trezor/blockbook/bchain" ) -// blockGasRPCStub serves a single CallContext response (raw JSON) or a transport error, -// recording the method and args so tests can assert the "latest" block tag is requested. -type blockGasRPCStub struct { - raw string - err error - method string - gotArgs []interface{} -} - -func (s *blockGasRPCStub) EthSubscribe(context.Context, interface{}, ...interface{}) (bchain.EVMClientSubscription, error) { - return nil, errors.New("not implemented") -} - -func (s *blockGasRPCStub) Close() {} - -func (s *blockGasRPCStub) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { - s.method = method - s.gotArgs = args - if s.err != nil { - return s.err - } - return json.Unmarshal([]byte(s.raw), result) -} - func equalBigInt(a, b *big.Int) bool { if a == nil || b == nil { return a == nil && b == nil @@ -44,93 +16,6 @@ func equalBigInt(a, b *big.Int) bool { return a.Cmp(b) == 0 } -func TestGetLatestBlockGas(t *testing.T) { - tests := []struct { - name string - raw string - err error - wantUsed *big.Int - wantLimit *big.Int - wantErr bool - }{ - { - name: "post-London populates gas", - raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380","baseFeePerGas":"0x7"}`, - wantUsed: big.NewInt(21000), - wantLimit: big.NewInt(30000000), - }, - { - name: "pre-London (no baseFee) omits gas", - raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380"}`, - }, - { - name: "rpc error propagates", - err: errors.New("boom"), - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stub := &blockGasRPCStub{raw: tt.raw, err: tt.err} - b := &EthereumRPC{RPC: stub, Timeout: time.Second} - gu, gl, err := b.getLatestBlockGas() - if tt.wantErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !equalBigInt(gu, tt.wantUsed) { - t.Errorf("gasUsed = %v, want %v", gu, tt.wantUsed) - } - if !equalBigInt(gl, tt.wantLimit) { - t.Errorf("gasLimit = %v, want %v", gl, tt.wantLimit) - } - if stub.method != "eth_getBlockByNumber" { - t.Errorf("method = %q, want eth_getBlockByNumber", stub.method) - } - if len(stub.gotArgs) < 1 || stub.gotArgs[0] != "latest" { - t.Errorf("args = %v, want first arg \"latest\"", stub.gotArgs) - } - }) - } -} - -func TestSetLatestBlockGas(t *testing.T) { - t.Run("success sets fields", func(t *testing.T) { - b := &EthereumRPC{RPC: &blockGasRPCStub{raw: `{"gasUsed":"0x5208","gasLimit":"0x3938700","baseFeePerGas":"0x7"}`}, Timeout: time.Second} - fees := &bchain.Eip1559Fees{} - b.setLatestBlockGas(fees) - if !equalBigInt(fees.BlockGasUsed, big.NewInt(21000)) { - t.Errorf("BlockGasUsed = %v, want 21000", fees.BlockGasUsed) - } - if !equalBigInt(fees.BlockGasLimit, big.NewInt(60000000)) { - t.Errorf("BlockGasLimit = %v, want 60000000", fees.BlockGasLimit) - } - }) - - t.Run("rpc error leaves fields nil (non-fatal)", func(t *testing.T) { - b := &EthereumRPC{RPC: &blockGasRPCStub{err: errors.New("boom")}, Timeout: time.Second} - fees := &bchain.Eip1559Fees{} - b.setLatestBlockGas(fees) - if fees.BlockGasUsed != nil || fees.BlockGasLimit != nil { - t.Errorf("expected nil gas fields on error, got used=%v limit=%v", fees.BlockGasUsed, fees.BlockGasLimit) - } - }) - - t.Run("pre-London leaves fields nil", func(t *testing.T) { - b := &EthereumRPC{RPC: &blockGasRPCStub{raw: `{"gasUsed":"0x5208","gasLimit":"0x1c9c380"}`}, Timeout: time.Second} - fees := &bchain.Eip1559Fees{} - b.setLatestBlockGas(fees) - if fees.BlockGasUsed != nil || fees.BlockGasLimit != nil { - t.Errorf("expected nil gas fields pre-London, got used=%v limit=%v", fees.BlockGasUsed, fees.BlockGasLimit) - } - }) -} - func TestAttachBlockGas(t *testing.T) { t.Run("post-London allocates and sets", func(t *testing.T) { bsd := attachBlockGas(&rpcHeader{GasUsed: "0x5208", GasLimit: "0x3938700", BaseFeePerGas: "0x7"}, nil) @@ -175,5 +60,11 @@ func TestAttachBlockGas(t *testing.T) { if !equalBigInt(got.BaseFeePerGas, big.NewInt(7)) { t.Errorf("BaseFeePerGas = %v, want 7", got.BaseFeePerGas) } + if !equalBigInt(got.GasUsed, big.NewInt(1)) { + t.Errorf("GasUsed = %v, want 1", got.GasUsed) + } + if !equalBigInt(got.GasLimit, big.NewInt(2)) { + t.Errorf("GasLimit = %v, want 2", got.GasLimit) + } }) } diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 722faab90c..8deb482379 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1870,47 +1870,6 @@ func (b *EthereumRPC) EthereumTypeEstimateGas(params map[string]interface{}) (ui return b.Client.EstimateGas(ctx, msg) } -// getLatestBlockGas fetches the latest block's gasUsed and gasLimit, used by the frontend to -// deterministically project the next block's EIP-1559 base fee. Returns nil values (without error) -// for pre-London blocks where baseFeePerGas is absent, so the fields are omitted for non-EIP-1559 blocks. -func (b *EthereumRPC) getLatestBlockGas() (gasUsed, gasLimit *big.Int, err error) { - ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) - defer cancel() - var blk struct { - GasUsed string `json:"gasUsed"` - GasLimit string `json:"gasLimit"` - BaseFeePerGas string `json:"baseFeePerGas"` - } - if err = b.RPC.CallContext(ctx, &blk, "eth_getBlockByNumber", "latest", false); err != nil { - return nil, nil, err - } - if blk.BaseFeePerGas == "" { - // pre-London block, no base fee projection possible - return nil, nil, nil - } - gu, err := hexutil.DecodeUint64(blk.GasUsed) - if err != nil { - return nil, nil, err - } - gl, err := hexutil.DecodeUint64(blk.GasLimit) - if err != nil { - return nil, nil, err - } - return new(big.Int).SetUint64(gu), new(big.Int).SetUint64(gl), nil -} - -// setLatestBlockGas populates fees.BlockGasUsed/BlockGasLimit from the latest block. -// A failure is logged but not fatal - the priority fees and baseFeePerGas remain usable. -func (b *EthereumRPC) setLatestBlockGas(fees *bchain.Eip1559Fees) { - gasUsed, gasLimit, err := b.getLatestBlockGas() - if err != nil { - glog.Error("eth_getBlockByNumber latest for eip1559 block gas: ", err) - return - } - fees.BlockGasUsed = gasUsed - fees.BlockGasLimit = gasLimit -} - // EthereumTypeGetEip1559Fees retrieves Eip1559Fees, if supported func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) { if !b.ChainConfig.Eip1559Fees { @@ -1923,7 +1882,6 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) return nil, err } if fees != nil { - b.setLatestBlockGas(fees) return fees, nil } // Fall back to on-chain estimation when the alternative provider is unsupported/stale/unready, @@ -1994,7 +1952,6 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) fees.Instant = &f } } - b.setLatestBlockGas(&fees) return &fees, err } diff --git a/bchain/types_ethereum_type.go b/bchain/types_ethereum_type.go index 65495bf1de..cbe85e6f68 100644 --- a/bchain/types_ethereum_type.go +++ b/bchain/types_ethereum_type.go @@ -262,8 +262,6 @@ type Eip1559Fee struct { // Eip1559Fees type Eip1559Fees struct { BaseFeePerGas *big.Int `json:"baseFeePerGas,omitempty"` - BlockGasUsed *big.Int `json:"blockGasUsed,omitempty"` - BlockGasLimit *big.Int `json:"blockGasLimit,omitempty"` Low *Eip1559Fee `json:"low,omitempty"` Medium *Eip1559Fee `json:"medium,omitempty"` High *Eip1559Fee `json:"high,omitempty"` diff --git a/blockbook-api.ts b/blockbook-api.ts index 8aaea4f6b0..f3cc0a2e5d 100644 --- a/blockbook-api.ts +++ b/blockbook-api.ts @@ -820,10 +820,6 @@ export interface Eip1559Fee { } export interface Eip1559Fees { baseFeePerGas?: string; - /** Total gas used by the latest block (decimal). */ - blockGasUsed?: string; - /** Gas limit of the latest block (decimal). */ - blockGasLimit?: string; low?: Eip1559Fee; medium?: Eip1559Fee; high?: Eip1559Fee; diff --git a/server/websocket.go b/server/websocket.go index 0c6c5c7d99..d0954f8370 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1067,8 +1067,6 @@ func (s *WebsocketServer) estimateFee(params []byte) (interface{}, error) { if eip1559 != nil { eip1559Api = &api.Eip1559Fees{} eip1559Api.BaseFeePerGas = (*api.Amount)(eip1559.BaseFeePerGas) - eip1559Api.BlockGasUsed = (*api.Amount)(eip1559.BlockGasUsed) - eip1559Api.BlockGasLimit = (*api.Amount)(eip1559.BlockGasLimit) eip1559Api.Instant = eip1559FeesToApi(eip1559.Instant) eip1559Api.High = eip1559FeesToApi(eip1559.High) eip1559Api.Medium = eip1559FeesToApi(eip1559.Medium) From 32f32a4d0138a73fef089037f8fbddae3c952d76 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 19:26:56 +0200 Subject: [PATCH 010/303] fix(api): rename subscribeNewBlock evm_data field to evmData The new-block notification payload used a snake_case json tag (evm_data), the only such tag among the websocket/JSON wire types, which otherwise use lowerCamelCase (height, hash, baseFeePerGas, feePerUnit, ...). Rename it to evmData for consistency before the field ships, since changing a wire-level key after release would be a breaking change. Updates the Go json tag, the JSON shape test, blockbook-api.ts and a code comment. The field is documented in openapi.yaml in the next commit. Co-Authored-By: Claude Opus 4.8 --- blockbook-api.ts | 2 +- server/websocket.go | 2 +- server/ws_newblock_test.go | 8 ++++---- server/ws_types.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/blockbook-api.ts b/blockbook-api.ts index f3cc0a2e5d..4bc7f72fc7 100644 --- a/blockbook-api.ts +++ b/blockbook-api.ts @@ -860,7 +860,7 @@ export interface WsNewBlock { /** Hash of the new block. */ hash: string; /** EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains. */ - evm_data: EthereumGasData | null; + evmData: EthereumGasData | null; } export interface WsSendTransactionReq { /** Hex-encoded transaction data to broadcast (string format). */ diff --git a/server/websocket.go b/server/websocket.go index d0954f8370..caaea7d98b 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1419,7 +1419,7 @@ func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { Hash: block.Hash, } // for EVM chains attach block-level gas data so subscribers can project the next EIP-1559 base fee; - // stays nil (evm_data: null) for non-EVM chains and pre-London blocks + // stays nil (evmData: null) for non-EVM chains and pre-London blocks if bsd, ok := block.CoinSpecificData.(*bchain.EthereumBlockSpecificData); ok && bsd != nil && bsd.BaseFeePerGas != nil { data.EVMData = &EthereumGasData{ BaseFeePerGas: (*api.Amount)(bsd.BaseFeePerGas), diff --git a/server/ws_newblock_test.go b/server/ws_newblock_test.go index 577313770d..c4fb3c740c 100644 --- a/server/ws_newblock_test.go +++ b/server/ws_newblock_test.go @@ -10,7 +10,7 @@ import ( "github.com/trezor/blockbook/api" ) -// TestWsNewBlockJSON pins the subscribeNewBlock wire shape: non-EVM chains serialize evm_data as +// TestWsNewBlockJSON pins the subscribeNewBlock wire shape: non-EVM chains serialize evmData as // null (height/hash unchanged), while EVM chains carry decimal-string gas figures for the EIP-1559 projection. func TestWsNewBlockJSON(t *testing.T) { tests := []struct { @@ -19,9 +19,9 @@ func TestWsNewBlockJSON(t *testing.T) { want string }{ { - name: "non-EVM emits evm_data null", + name: "non-EVM emits evmData null", in: WsNewBlock{Height: 5, Hash: "0xabc"}, - want: `{"height":5,"hash":"0xabc","evm_data":null}`, + want: `{"height":5,"hash":"0xabc","evmData":null}`, }, { name: "EVM emits decimal gas figures", @@ -34,7 +34,7 @@ func TestWsNewBlockJSON(t *testing.T) { BlockGasLimit: (*api.Amount)(big.NewInt(30000000)), }, }, - want: `{"height":5,"hash":"0xabc","evm_data":{"baseFeePerGas":"7","blockGasUsed":"21000","blockGasLimit":"30000000"}}`, + want: `{"height":5,"hash":"0xabc","evmData":{"baseFeePerGas":"7","blockGasUsed":"21000","blockGasLimit":"30000000"}}`, }, } for _, tt := range tests { diff --git a/server/ws_types.go b/server/ws_types.go index bb95031617..4792af7dcf 100644 --- a/server/ws_types.go +++ b/server/ws_types.go @@ -160,7 +160,7 @@ type EthereumGasData struct { type WsNewBlock struct { Height uint32 `json:"height" ts_doc:"Height of the new block."` Hash string `json:"hash" ts_doc:"Hash of the new block."` - EVMData *EthereumGasData `json:"evm_data" ts_doc:"EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains."` + EVMData *EthereumGasData `json:"evmData" ts_doc:"EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains."` } // WsLongTermFeeRateRes is returned in response to a long term fee rate request. From 062401feb7ea1fb49237d49a9d3e44735ee54270 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 19:28:32 +0200 Subject: [PATCH 011/303] docs(openapi): document subscribeNewBlock EVM gas payload The subscribeNewBlock notification grew an evmData field (WsNewBlock / EthereumGasData) that was undocumented in openapi.yaml. Because these types are not in the parity allow-list they did not break CI, but the new websocket payload was unspecified. Add EthereumGasData and WsNewBlock schemas (evmData modeled as nullable via the 3.1 oneOf/"null" pattern, matching the Go *EthereumGasData json tag without omitempty), and add both to the parity check so the generated blockbook-api.ts and openapi.yaml stay in sync. Parity typecheck passes. Co-Authored-By: Claude Opus 4.8 --- openapi.yaml | 28 ++++++++++++++++++++++++++++ tests/openapi/src/parity.ts | 3 +++ 2 files changed, 31 insertions(+) diff --git a/openapi.yaml b/openapi.yaml index 1d8e04fe87..e352863add 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2965,6 +2965,34 @@ components: type: string enum: [up, down] + EthereumGasData: + type: object + description: > + EVM block-level gas figures pushed with subscribeNewBlock notifications, used by the + frontend to deterministically project the next block's EIP-1559 base fee. Decimal strings, + omitted for pre-London blocks. + properties: + baseFeePerGas: + $ref: "#/components/schemas/AmountString" + blockGasUsed: + $ref: "#/components/schemas/AmountString" + blockGasLimit: + $ref: "#/components/schemas/AmountString" + + WsNewBlock: + type: object + description: Pushed to subscribeNewBlock subscribers when a new block is connected. + properties: + height: + type: number + hash: + type: string + evmData: + description: EVM gas data for the EIP-1559 base-fee projection; null on non-EVM chains. + oneOf: + - $ref: "#/components/schemas/EthereumGasData" + - type: "null" + WsEstimateFeeRes: type: object properties: diff --git a/tests/openapi/src/parity.ts b/tests/openapi/src/parity.ts index 0e5a9a6137..467b3f58c8 100644 --- a/tests/openapi/src/parity.ts +++ b/tests/openapi/src/parity.ts @@ -113,6 +113,8 @@ const _WsEstimateFeeReq: Compat = true; const _Eip1559Fees: Compat = true; const _WsEstimateFeeRes: Compat = true; +const _EthereumGasData: Compat = true; +const _WsNewBlock: Compat = true; const _WsSendTransactionReq: Compat = true; const _WsSubscribeAddressesReq: Compat = true; const _WsSubscribeFiatRatesReq: Compat = true; @@ -152,6 +154,7 @@ void [ _WsBlockHashReq, _WsBlockHashRes, _WsBlockReq, _WsBlockFilterReq, _WsBlockFiltersBatchReq, _WsAccountUtxoReq, _WsBalanceHistoryReq, _WsTransactionReq, _WsTransactionSpecificReq, _WsEstimateFeeReq, _Eip1559Fee, _Eip1559Fees, _WsEstimateFeeRes, + _EthereumGasData, _WsNewBlock, _WsSendTransactionReq, _WsSubscribeAddressesReq, _WsSubscribeFiatRatesReq, _WsCurrentFiatRatesReq, _WsFiatRatesForTimestampsReq, _WsFiatRatesTickersListReq, _WsMempoolFiltersReq, _WsRpcCallReq, _WsRpcCallRes, From 5bad524f531ecec6bc45a0e7bc275ff509d68241 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 19:30:14 +0200 Subject: [PATCH 012/303] test(server): cover subscribeNewBlock evmData mapping The push path (subscribeNewBlock -> evmData) is the consistent, RPC-free delivery mechanism for the EIP-1559 projection inputs, but only its JSON wire shape was tested, not the block -> payload mapping. Extract that mapping out of onNewBlockAsync into a pure newBlockNotification helper (behavior unchanged) and unit-test it: non-EVM blocks and EVM pre-London blocks leave evmData nil, while EVM post-London blocks carry baseFeePerGas/gasUsed/gasLimit taken from the connected block header. Co-Authored-By: Claude Opus 4.8 --- server/websocket.go | 16 ++++++++---- server/ws_newblock_test.go | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/server/websocket.go b/server/websocket.go index caaea7d98b..8ab15a3e9d 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1411,15 +1411,14 @@ func (s *WebsocketServer) unsubscribeFiatRates(c *websocketChannel) (res interfa return &subscriptionResponse{false}, nil } -func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { - s.newBlockSubscriptionsLock.Lock() - defer s.newBlockSubscriptionsLock.Unlock() +// newBlockNotification builds the subscribeNewBlock payload for a connected block. For EVM chains it +// attaches block-level gas data so subscribers can project the next EIP-1559 base fee; EVMData stays +// nil (evmData: null) for non-EVM chains and pre-London blocks (BaseFeePerGas absent). +func newBlockNotification(block *bchain.Block) *WsNewBlock { data := &WsNewBlock{ Height: block.Height, Hash: block.Hash, } - // for EVM chains attach block-level gas data so subscribers can project the next EIP-1559 base fee; - // stays nil (evmData: null) for non-EVM chains and pre-London blocks if bsd, ok := block.CoinSpecificData.(*bchain.EthereumBlockSpecificData); ok && bsd != nil && bsd.BaseFeePerGas != nil { data.EVMData = &EthereumGasData{ BaseFeePerGas: (*api.Amount)(bsd.BaseFeePerGas), @@ -1427,6 +1426,13 @@ func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { BlockGasLimit: (*api.Amount)(bsd.GasLimit), } } + return data +} + +func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { + s.newBlockSubscriptionsLock.Lock() + defer s.newBlockSubscriptionsLock.Unlock() + data := newBlockNotification(block) for c, id := range s.newBlockSubscriptions { c.DataOut(&WsRes{ ID: id, diff --git a/server/ws_newblock_test.go b/server/ws_newblock_test.go index c4fb3c740c..09cf5b566f 100644 --- a/server/ws_newblock_test.go +++ b/server/ws_newblock_test.go @@ -8,8 +8,58 @@ import ( "testing" "github.com/trezor/blockbook/api" + "github.com/trezor/blockbook/bchain" ) +// TestNewBlockNotification covers the block -> subscribeNewBlock payload mapping: only EVM +// post-London blocks (EthereumBlockSpecificData with BaseFeePerGas set) carry evmData; non-EVM, +// pre-London, and missing coin-specific data leave it nil. +func TestNewBlockNotification(t *testing.T) { + t.Run("non-EVM block has nil EVMData", func(t *testing.T) { + got := newBlockNotification(&bchain.Block{BlockHeader: bchain.BlockHeader{Height: 7, Hash: "0xh"}}) + if got.Height != 7 || got.Hash != "0xh" { + t.Errorf("height/hash = %d/%q, want 7/0xh", got.Height, got.Hash) + } + if got.EVMData != nil { + t.Errorf("expected nil EVMData for non-EVM block, got %+v", got.EVMData) + } + }) + + t.Run("EVM post-London block carries gas data from the block header", func(t *testing.T) { + block := &bchain.Block{ + BlockHeader: bchain.BlockHeader{Height: 7, Hash: "0xh"}, + CoinSpecificData: &bchain.EthereumBlockSpecificData{ + BaseFeePerGas: big.NewInt(7), + GasUsed: big.NewInt(21000), + GasLimit: big.NewInt(30000000), + }, + } + got := newBlockNotification(block) + if got.EVMData == nil { + t.Fatal("expected EVMData, got nil") + } + if (*big.Int)(got.EVMData.BaseFeePerGas).Cmp(big.NewInt(7)) != 0 { + t.Errorf("BaseFeePerGas = %v, want 7", got.EVMData.BaseFeePerGas) + } + if (*big.Int)(got.EVMData.BlockGasUsed).Cmp(big.NewInt(21000)) != 0 { + t.Errorf("BlockGasUsed = %v, want 21000", got.EVMData.BlockGasUsed) + } + if (*big.Int)(got.EVMData.BlockGasLimit).Cmp(big.NewInt(30000000)) != 0 { + t.Errorf("BlockGasLimit = %v, want 30000000", got.EVMData.BlockGasLimit) + } + }) + + t.Run("EVM pre-London block (no base fee) has nil EVMData", func(t *testing.T) { + block := &bchain.Block{ + BlockHeader: bchain.BlockHeader{Height: 7, Hash: "0xh"}, + CoinSpecificData: &bchain.EthereumBlockSpecificData{GasUsed: big.NewInt(21000), GasLimit: big.NewInt(30000000)}, + } + if got := newBlockNotification(block); got.EVMData != nil { + t.Errorf("expected nil EVMData pre-London, got %+v", got.EVMData) + } + }) +} + // TestWsNewBlockJSON pins the subscribeNewBlock wire shape: non-EVM chains serialize evmData as // null (height/hash unchanged), while EVM chains carry decimal-string gas figures for the EIP-1559 projection. func TestWsNewBlockJSON(t *testing.T) { From 574dec343e250442f4a980f5ab61747b029d713c Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 20:01:43 +0200 Subject: [PATCH 013/303] docs(openapi): mark WsNewBlock fields required WsNewBlock's height, hash and evmData have no omitempty in Go, so they are always present on the wire (evmData serializes as null for non-EVM/pre-London blocks). Mark them required in the schema so the spec reflects the actual always-present (evmData nullable) shape. Parity typecheck still passes. Co-Authored-By: Claude Opus 4.8 --- openapi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/openapi.yaml b/openapi.yaml index e352863add..ac02712b81 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2981,6 +2981,7 @@ components: WsNewBlock: type: object + required: [height, hash, evmData] description: Pushed to subscribeNewBlock subscribers when a new block is connected. properties: height: From 942c2c45c311e241f477a76c78a99473a09d92c9 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 23 Jun 2026 20:11:32 +0200 Subject: [PATCH 014/303] docs(eth): explain why estimateFee exposes no separate next-block base fee Records the conclusion of the fee-history investigation inline: eth_feeHistory's extra projected element has backend-dependent semantics (dropped by no-pending nodes like Erigon so baseFeePerGas is already the next-block fee; kept but index-shifted to N+1 or a noisy N+2 by some L2s), so a single "next block" field cannot be defined consistently. Clients needing an exact next-block base fee use the subscribeNewBlock evmData push instead. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/ethrpc.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 8deb482379..17a97fd439 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1926,6 +1926,14 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) hs, _ := json.Marshal(h) baseFee, _ := hexutil.DecodeUint64(h.BaseFeePerGas[blocks-1]) fees.BaseFeePerGas = big.NewInt(int64(baseFee)) + // We expose only baseFeePerGas here and deliberately do NOT add a separate "next block" base-fee + // field. eth_feeHistory returns one extra projected element beyond the requested range, but its + // meaning is backend-dependent: nodes with no distinct pending block (e.g. Erigon, which ethereum + // mainnet uses) drop the extra element, so baseFeePerGas[blocks-1] is already the next block's + // projected fee; other backends (some L2s) keep it and shift the indices, making the extra element + // either N+1 or an N+2 estimate computed off an incomplete pending block. No single field can + // describe all of these. Clients that need an exact next-block base fee should use the + // subscribeNewBlock evmData push, which carries the real previous-block header. maxBasePriorityFee := maxPriorityFeePerGas.ToInt().Int64() glog.Info("eth_maxPriorityFeePerGas ", maxPriorityFeePerGas) glog.Info("eth_feeHistory ", string(hs)) From 376aa924a4f9eba0412e1e25cacdc9869749b80b Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 07:12:55 +0200 Subject: [PATCH 015/303] fix(eth): make on-chain EIP-1559 maxFeePerGas cover the base fee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on-chain estimate (used when no alternative fee provider is configured — e.g. ethereum non-archive and the ethereum testnets) set each tier's maxFeePerGas to the priority fee alone, omitting the base fee entirely. That value is below the block's base fee and therefore not mineable. Trezor Suite consumes maxFeePerGas directly (connect EthereumFeeLevels: it only clamps to a floor, it does not add the base fee), so the on-chain path produced unusable EIP-1559 fees. Compute each tier correctly from the eth_feeHistory response: - maxPriorityFeePerGas = the tier's reward percentile (20/70/90/99), i.e. the actual tip distribution, instead of the node's single suggestion doubled. - maxFeePerGas = eip1559BaseFeeMultiplier*baseFee + tip, with the multiplier (2) documented as the EIP-1559-standard buffer (survives ~6 full blocks of +12.5%/block base-fee growth). Tunable via the named constant. This also drops the now-redundant eth_maxPriorityFeePerGas call (Suite floors the tip with coinInfo.minPriorityFee), so the on-chain path makes one RPC (eth_feeHistory) instead of two. Adds a unit test asserting the per-tier values and the maxFeePerGas > baseFee invariant. Note: this does not touch the alternative-fee (Infura) path, where the "fees too high" reports originate — that is a fee-policy decision pending coordination with the Suite team. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/eip1559_blockgas_test.go | 76 +++++++++++++++++++++++ bchain/coins/eth/ethrpc.go | 34 ++++++---- 2 files changed, 97 insertions(+), 13 deletions(-) diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index d895942929..6339146121 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -3,12 +3,88 @@ package eth import ( + "context" + "encoding/json" + "errors" "math/big" "testing" + "time" "github.com/trezor/blockbook/bchain" ) +// feeHistoryRPCStub serves a canned eth_feeHistory response; any other method errors so the test +// also asserts the on-chain path no longer issues eth_maxPriorityFeePerGas. +type feeHistoryRPCStub struct { + raw string +} + +func (s *feeHistoryRPCStub) EthSubscribe(context.Context, interface{}, ...interface{}) (bchain.EVMClientSubscription, error) { + return nil, errors.New("not implemented") +} + +func (s *feeHistoryRPCStub) Close() {} + +func (s *feeHistoryRPCStub) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + if method != "eth_feeHistory" { + return errors.New("unexpected RPC method: " + method) + } + return json.Unmarshal([]byte(s.raw), result) +} + +func TestEthereumTypeGetEip1559FeesOnChain(t *testing.T) { + // baseFeePerGas[blocks-1]=[3]=0x64=100 is the projected next-block base fee (4-element array, as a + // no-distinct-pending-block backend returns). Per-tier reward percentiles over 2 blocks. + raw := `{"oldestBlock":"0x1",` + + `"reward":[["0x1","0x2","0x3","0x4"],["0x3","0x4","0x5","0x6"]],` + + `"baseFeePerGas":["0x10","0x20","0x30","0x64"],` + + `"gasUsedRatio":[0.5,0.5,0.5]}` + b := &EthereumRPC{ + RPC: &feeHistoryRPCStub{raw: raw}, + Timeout: time.Second, + ChainConfig: &Configuration{Eip1559Fees: true}, + } + fees, err := b.EthereumTypeGetEip1559Fees() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fees == nil || fees.BaseFeePerGas == nil { + t.Fatal("expected fees with baseFeePerGas") + } + if fees.BaseFeePerGas.Int64() != 100 { + t.Errorf("BaseFeePerGas = %v, want 100", fees.BaseFeePerGas) + } + // tip = average of the tier's reward percentile; maxFeePerGas = 2*baseFee + tip. + cases := []struct { + name string + fee *bchain.Eip1559Fee + wantTip int64 + }{ + {"low", fees.Low, 2}, // avg(1,3) + {"medium", fees.Medium, 3}, // avg(2,4) + {"high", fees.High, 4}, // avg(3,5) + {"instant", fees.Instant, 5}, // avg(4,6) + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if c.fee == nil { + t.Fatal("nil tier") + } + if c.fee.MaxPriorityFeePerGas.Int64() != c.wantTip { + t.Errorf("MaxPriorityFeePerGas = %v, want %d", c.fee.MaxPriorityFeePerGas, c.wantTip) + } + wantMax := int64(eip1559BaseFeeMultiplier)*100 + c.wantTip + if c.fee.MaxFeePerGas.Int64() != wantMax { + t.Errorf("MaxFeePerGas = %v, want %d", c.fee.MaxFeePerGas, wantMax) + } + // core invariant the previous code violated: maxFeePerGas must cover the base fee. + if c.fee.MaxFeePerGas.Cmp(fees.BaseFeePerGas) <= 0 { + t.Errorf("MaxFeePerGas %v must exceed baseFee %v", c.fee.MaxFeePerGas, fees.BaseFeePerGas) + } + }) + } +} + func equalBigInt(a, b *big.Int) bool { if a == nil || b == nil { return a == nil && b == nil diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 17a97fd439..c84ed1f5c5 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1870,6 +1870,12 @@ func (b *EthereumRPC) EthereumTypeEstimateGas(params map[string]interface{}) (ui return b.Client.EstimateGas(ctx, msg) } +// eip1559BaseFeeMultiplier is the headroom applied to the projected base fee when deriving +// maxFeePerGas for the on-chain EIP-1559 estimate (maxFeePerGas = multiplier*baseFee + tip). +// 2x is the EIP-1559-standard buffer: it keeps a transaction mineable across ~6 consecutive full +// blocks, since the base fee can rise at most 12.5% per block (1.125^6 ≈ 2). Tunable. +const eip1559BaseFeeMultiplier = 2 + // EthereumTypeGetEip1559Fees retrieves Eip1559Fees, if supported func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) { if !b.ChainConfig.Eip1559Fees { @@ -1892,12 +1898,6 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) defer cancel() - var maxPriorityFeePerGas hexutil.Big - err := b.RPC.CallContext(ctx, &maxPriorityFeePerGas, "eth_maxPriorityFeePerGas") - if err != nil { - return nil, err - } - var fees bchain.Eip1559Fees type history struct { @@ -1915,7 +1915,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) } blocks := 4 - err = b.RPC.CallContext(ctx, &h, "eth_feeHistory", blocks, "pending", percentiles) + err := b.RPC.CallContext(ctx, &h, "eth_feeHistory", blocks, "pending", percentiles) if err != nil { return nil, err } @@ -1934,21 +1934,29 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) // either N+1 or an N+2 estimate computed off an incomplete pending block. No single field can // describe all of these. Clients that need an exact next-block base fee should use the // subscribeNewBlock evmData push, which carries the real previous-block header. - maxBasePriorityFee := maxPriorityFeePerGas.ToInt().Int64() - glog.Info("eth_maxPriorityFeePerGas ", maxPriorityFeePerGas) glog.Info("eth_feeHistory ", string(hs)) for i := 0; i < 4; i++ { var f bchain.Eip1559Fee + // Per-tier tip: average of the requested reward percentile (low=20th .. instant=99th) over the window. priorityFee := int64(0) for j := 0; j < len(h.Reward); j++ { p, _ := hexutil.DecodeUint64(h.Reward[j][i]) priorityFee += int64(p) } - priorityFee = priorityFee / int64(len(h.Reward)) - f.MaxFeePerGas = big.NewInt(priorityFee) - f.MaxPriorityFeePerGas = big.NewInt(maxBasePriorityFee) - maxBasePriorityFee *= 2 + if len(h.Reward) > 0 { + priorityFee /= int64(len(h.Reward)) + } + tip := big.NewInt(priorityFee) + f.MaxPriorityFeePerGas = tip + // maxFeePerGas must cover the next-block base fee plus the tip, with headroom for base-fee + // growth while the tx waits: maxFeePerGas = eip1559BaseFeeMultiplier*baseFee + tip. The previous + // code put only the tip here (omitting the base fee), which is below the base fee and therefore + // not mineable; clients such as Trezor Suite use maxFeePerGas directly. + f.MaxFeePerGas = new(big.Int).Add( + new(big.Int).Mul(fees.BaseFeePerGas, big.NewInt(eip1559BaseFeeMultiplier)), + tip, + ) switch i { case 0: fees.Low = &f From fd974ce1645206905803126b3ddde140e112e053 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 11:02:48 +0200 Subject: [PATCH 016/303] docs(eth): add fees.md explaining the EIP-1559 fee algorithm Document, with mermaid diagrams, how EIP-1559 fees are produced across Blockbook and Trezor Suite and which component owns which decision: blockbook provides ground-truth inputs (chain base fee, priority-fee tiers, congestion/trend, block gas), while the wallet owns the fee policy (base-fee source, 2x head-room buffer, maxFeePerGas composition, per-coin clamps). Covers the pull path (EthereumTypeGetEip1559Fees), the push path (subscribeNewBlock evmData), Suite's EthereumFeeLevels, the end-to-end flow, and the maxFeePerGas formula. Co-Authored-By: Claude Opus 4.8 --- docs/fees.md | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 docs/fees.md diff --git a/docs/fees.md b/docs/fees.md new file mode 100644 index 0000000000..9d6eabb230 --- /dev/null +++ b/docs/fees.md @@ -0,0 +1,156 @@ +# EVM EIP-1559 fee estimation + +This document describes how EIP-1559 fees are produced for EVM coins across **Blockbook** +(the indexer) and **Trezor Suite** (the wallet), and which component owns which decision. + +## Responsibilities + +The split is deliberate: Blockbook supplies facts, the wallet decides policy. + +**Blockbook — provides ground-truth inputs:** + +- base fee — the next-block projection, taken from the chain +- priority-fee tiers (tips) +- congestion and trend signals (via Infura, when configured) +- per-block gas — `baseFee`, `gasUsed`, `gasLimit` + +**Trezor Suite — owns the fee policy:** + +- chooses the base-fee source +- applies the head-room buffer (currently `2×`) +- composes `maxFeePerGas = 2 × baseFee + tip` +- clamps to per-coin `minFee` / `minPriorityFee` / `maxFee` limits +- displays the fee options and signs the transaction + +This is why the wallet computes `maxFeePerGas` itself instead of trusting a provider's pre-padded +value (Infura's `suggestedMaxFeePerGas`, which is the source of the "fees too high" reports). + +--- + +## Blockbook — pull path (`estimateFee` / `EthereumTypeGetEip1559Fees`) + +On-demand estimate. Returns `Eip1559Fees`: a `baseFeePerGas` plus `low/medium/high` (and `instant` +on the on-chain path) tiers, and — on the Infura path — congestion/trend ranges. + +```mermaid +flowchart TD + A["estimateFee"] --> B{"EIP-1559 supported?"} + B -->|no| Z["legacy feePerUnit only"] + B -->|yes| C{"fresh Infura fees available?"} + C -->|yes| D["use cached Infura fees"] + C -->|no| E["estimate on-chain from base fee + tips"] + D --> R["return Eip1559Fees
baseFee + fee tiers"] + E --> R +``` + +How each source is built: + +- **On-chain** (coins with no provider, or a stale one — e.g. `ethereum` non-archive, ETH testnets): + one `eth_feeHistory` call (4 blocks, newest = `pending`) yields the next-block `baseFeePerGas` + (array index `blocks-1`) and per-tier reward percentiles (20/70/90/99) used as tips. Blockbook + builds `maxFeePerGas = eip1559BaseFeeMultiplier(2) × baseFee + tip`. (Previously this field held the + tip alone — below the base fee, so not mineable; that was fixed.) +- **Infura** (archive coins, served from an in-memory cache, no node RPC): blockbook returns Infura's + `suggestedMaxFeePerGas` **unchanged**, which is padded well above the base fee (~2.5× for the high + tier). Blockbook does not rewrite it — the wallet overrides it (see the Suite section). + +--- + +## Blockbook — push path (`subscribeNewBlock` → `evmData`) + +Every connected block is pushed to subscribers with its block-level gas, so a wallet can keep its +fee projection fresh without polling. No extra RPC — the data comes from the block header already +fetched during sync. + +```mermaid +sequenceDiagram + participant N as EVM node + participant SY as Sync (connectBlocks) + participant GB as GetBlock + attachBlockGas + participant WS as WebsocketServer + participant C as Subscribers + + N->>SY: new block N + SY->>GB: GetBlock(hash, height) + GB->>GB: attachBlockGas: read baseFee, gasUsed, gasLimit + GB-->>SY: Block + EthereumBlockSpecificData (transient, not persisted) + SY->>WS: onNewBlock(block) + WS->>WS: newBlockNotification(block) + WS->>C: WsNewBlock height, hash, evmData + Note over C: evmData = baseFeePerGas, blockGasUsed, blockGasLimit
null on non-EVM chains and pre-London blocks +``` + +--- + +## Trezor Suite — `EthereumFeeLevels` (the fee policy) + +Suite computes its own fees from the chain's base fee plus a `2×` buffer, preferring that block-based +estimate and falling back to the backend's tiers when block data is unavailable (so nothing regresses). + +```mermaid +flowchart TD + A["EthereumFeeLevels.load()"] --> B["fetch backend fees + latest block"] + B --> C{"block-based estimate available?"} + C -->|yes| D["compute from chain base fee
maxFeePerGas = 2x baseFee + tip"] + C -->|no| E["fall back to backend fees"] + D --> F["clamp to coin min/max limits"] + E --> F + F --> G["fee levels: low / normal / high"] +``` + +Details and current coverage: + +- The block-based estimate derives the next-block base fee from the latest block (EIP-1559 formula on + `gasUsed`/`gasLimit`), takes tips from the block's transaction `maxPriorityFeePerGas` percentiles, + and sets `maxFeePerGas = 2 × nextBaseFee + tip`. The fallback uses the backend's `eip1559` tiers. +- **evm-rpc-backed networks**: `getBlock('latest')` returns block gas → block-based fees apply, so the + Infura padding is gone. +- **blockbook-backed networks**: `getBlock('latest')` is unsupported → block-based estimate is empty → + Suite falls back to the backend (Infura) tiers. Wiring the `subscribeNewBlock` `evmData` push as the + block-data source for these networks is the remaining step. + +--- + +## End-to-end flow + +```mermaid +sequenceDiagram + actor U as User + participant SU as Suite send form + participant CN as connect — EthereumFeeLevels + participant BB as Blockbook / node + + U->>SU: open send form + SU->>CN: blockchainEstimateFee + par estimateFee + CN->>BB: estimateFee + and latest block + CN->>BB: getBlock latest + end + BB-->>CN: eip1559 tiers (+ block, if evm-rpc) + CN->>CN: compute block-based fees (2x buffer), else fall back + CN->>CN: clamp to coin min/max + CN-->>SU: FeeLevel[] maxFeePerGas, maxPriorityFeePerGas, baseFee + SU-->>U: show fee options + U->>SU: confirm + sign + SU->>BB: sendTransaction (EIP-1559) +``` + +--- + +## The `maxFeePerGas` formula + +Both blockbook's on-chain path and Suite's block-based path compute the same value: + +``` +maxFeePerGas = 2 × baseFee + tip +``` + +- **base fee** — consensus-determined for the next block (deterministic from the parent header), so it + is taken from the chain, never from an oracle estimate. +- **tip** (`maxPriorityFeePerGas`) — the only estimated part; from fee-history reward percentiles, or + the provider's per-tier suggestion. +- **`2×` buffer** — the wallet's policy knob. It absorbs roughly 6 consecutive full blocks of base-fee + growth (the base fee can rise at most 12.5% per block) before a transaction stalls, and EIP-1559 + refunds the unused part. In blockbook it is the `eip1559BaseFeeMultiplier` constant; in Suite it is + applied in `EthereumFeeLevels`. From 318f2fa4842b99b14cfaf641b71d548d276c7108 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 12:53:47 +0200 Subject: [PATCH 017/303] feat(grafana): add a dedicated Fees dashboard section Move the two existing fee panels (estimated fee rate, alternative fee provider requests) out of General into a new top-level "Fees" section, so fee observability has one home as EVM EIP-1559 fee metrics are added. Pure relocation: panels, queries and the metrics they read are unchanged; only their x-panel-keys move general.* -> fees.* and a row.fees section is introduced. grafana.json is the git-ignored artifact rendered from these two sources by contrib/scripts/render_grafana.py. Co-Authored-By: Claude Opus 4.8 --- configs/grafana/panels.yaml | 48 ++++++++-------- configs/grafana/template.json | 105 ++++++++++++++++++---------------- 2 files changed, 81 insertions(+), 72 deletions(-) diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 9315604ccf..862f21d459 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -590,22 +590,6 @@ panels: explorer: promql: sum({{name:explorer_pending_requests}}{job="blockbook", coin="$coin"}) by (coin, method) legend: Explorer {{method}} - {{coin}} - general.estimated_fee: - title: Estimated fee rate - description: Fee estimates for 1/3/5/8 block targets; units are chain-specific. - queries: - blocks_1: - promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="1"}/1000) - legend: '{{coin}} - {{instance}} - {{blocks}} blocks' - blocks_3: - promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="3"}/1000) - legend: '{{coin}} - {{instance}} - {{blocks}} blocks' - blocks_5: - promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="5"}/1000) - legend: '{{coin}} - {{instance}} - {{blocks}} blocks' - blocks_8: - promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="8"}/1000) - legend: '{{coin}} - {{instance}} - {{blocks}} blocks' general.coingecko_requests: title: CoinGecko requests description: Status separates successful calls, throttling and hard errors from the same endpoint. @@ -626,13 +610,6 @@ panels: total: promql: sum({{name:coingecko_range_requests}}{job="blockbook", coin="$coin"}) legend: '{{coin}} total across servers' - general.alt_fee_provider_requests: - title: Alternative fee provider requests - description: External fee-source health by provider; status distinguishes HTTP, network and decode failures. - queries: - requests: - promql: rate({{name:alternative_fee_provider_requests}}{job="blockbook", coin="$coin"}[$__rate_interval]) - legend: '{{instance}} - {{provider}} - {{status}}' general.balance_history_fiat_duration_p95: title: 95% quantile of balance history fiat stage duration description: Separates batch lookup from per-point fallback, so slow fallback behavior is visible in address and xpub paths. @@ -661,6 +638,31 @@ panels: p95: promql: histogram_quantile(0.95, sum(increase({{name:fiat_rates_update_duration_seconds}}_bucket{job="blockbook", coin="$coin"}[1h])) by (coin, stage, status, le)) legend: '{{coin}} - {{stage}} - {{status}}' + row.fees: + title: Fees + fees.estimated_fee: + title: Estimated fee rate + description: Fee estimates for 1/3/5/8 block targets; units are chain-specific. + queries: + blocks_1: + promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="1"}/1000) + legend: '{{coin}} - {{instance}} - {{blocks}} blocks' + blocks_3: + promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="3"}/1000) + legend: '{{coin}} - {{instance}} - {{blocks}} blocks' + blocks_5: + promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="5"}/1000) + legend: '{{coin}} - {{instance}} - {{blocks}} blocks' + blocks_8: + promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="8"}/1000) + legend: '{{coin}} - {{instance}} - {{blocks}} blocks' + fees.alt_fee_provider_requests: + title: Alternative fee provider requests + description: External fee-source health by provider; status distinguishes HTTP, network and decode failures. + queries: + requests: + promql: rate({{name:alternative_fee_provider_requests}}{job="blockbook", coin="$coin"}[$__rate_interval]) + legend: '{{instance}} - {{provider}} - {{status}}' row.fiat_rates: title: Fiat Rates fiatrates.tip_fetches: diff --git a/configs/grafana/template.json b/configs/grafana/template.json index bfc9fe1a89..a50c79de62 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -4805,7 +4805,7 @@ }, "overrides": [] }, - "id": 45, + "id": 39, "options": { "legend": { "calcs": [ @@ -4828,29 +4828,17 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "blocks_1" + "x-query-key": "per_server" }, { "editorMode": "code", "range": true, "refId": "B", - "x-query-key": "blocks_3" - }, - { - "editorMode": "code", - "range": true, - "refId": "C", - "x-query-key": "blocks_5" - }, - { - "editorMode": "code", - "range": true, - "refId": "D", - "x-query-key": "blocks_8" + "x-query-key": "total" } ], "type": "timeseries", - "x-panel-key": "general.estimated_fee" + "x-panel-key": "general.coingecko_requests" }, { "fieldConfig": { @@ -4883,7 +4871,7 @@ }, "overrides": [] }, - "id": 39, + "id": 56, "options": { "legend": { "calcs": [ @@ -4916,7 +4904,7 @@ } ], "type": "timeseries", - "x-panel-key": "general.coingecko_requests" + "x-panel-key": "general.coingecko_range_requests" }, { "fieldConfig": { @@ -4926,7 +4914,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -4945,11 +4933,11 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, - "id": 56, + "id": 51, "options": { "legend": { "calcs": [ @@ -4972,17 +4960,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "per_server" - }, - { - "editorMode": "code", - "range": true, - "refId": "B", - "x-query-key": "total" + "x-query-key": "p95" } ], "type": "timeseries", - "x-panel-key": "general.coingecko_range_requests" + "x-panel-key": "general.balance_history_fiat_duration_p95" }, { "fieldConfig": { @@ -5015,7 +4997,7 @@ }, "overrides": [] }, - "id": 57, + "id": 52, "options": { "legend": { "calcs": [ @@ -5028,7 +5010,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -5038,11 +5020,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "requests" + "x-query-key": "fallbacks" } ], "type": "timeseries", - "x-panel-key": "general.alt_fee_provider_requests" + "x-panel-key": "general.balance_history_fiat_fallbacks" }, { "fieldConfig": { @@ -5071,11 +5053,11 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, - "id": 51, + "id": 53, "options": { "legend": { "calcs": [ @@ -5102,7 +5084,7 @@ } ], "type": "timeseries", - "x-panel-key": "general.balance_history_fiat_duration_p95" + "x-panel-key": "general.balance_history_points_p95" }, { "fieldConfig": { @@ -5131,11 +5113,11 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, - "id": 52, + "id": 55, "options": { "legend": { "calcs": [ @@ -5158,11 +5140,18 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "fallbacks" + "x-query-key": "p95" } ], "type": "timeseries", - "x-panel-key": "general.balance_history_fiat_fallbacks" + "x-panel-key": "general.fiat_update_duration_p95" + }, + { + "collapsed": false, + "id": 340, + "panels": [], + "type": "row", + "x-panel-key": "row.fees" }, { "fieldConfig": { @@ -5172,7 +5161,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5195,7 +5184,7 @@ }, "overrides": [] }, - "id": 53, + "id": 45, "options": { "legend": { "calcs": [ @@ -5218,11 +5207,29 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "p95" + "x-query-key": "blocks_1" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "blocks_3" + }, + { + "editorMode": "code", + "range": true, + "refId": "C", + "x-query-key": "blocks_5" + }, + { + "editorMode": "code", + "range": true, + "refId": "D", + "x-query-key": "blocks_8" } ], "type": "timeseries", - "x-panel-key": "general.balance_history_points_p95" + "x-panel-key": "fees.estimated_fee" }, { "fieldConfig": { @@ -5251,11 +5258,11 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, - "id": 55, + "id": 57, "options": { "legend": { "calcs": [ @@ -5268,7 +5275,7 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -5278,11 +5285,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "p95" + "x-query-key": "requests" } ], "type": "timeseries", - "x-panel-key": "general.fiat_update_duration_p95" + "x-panel-key": "fees.alt_fee_provider_requests" }, { "collapsed": false, From cb109adde7bc517a4109a657e32307efa2b1f6c5 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:25:59 +0200 Subject: [PATCH 018/303] feat(eth): expose pull-path EIP-1559 fee estimates as metrics EVM EIP-1559 fees were invisible in metrics (blockbook_estimated_fee is UTXO-only). Add two gauges populated on the successful estimateFee / EthereumTypeGetEip1559Fees paths (provider cache hit and on-chain estimate): - blockbook_eth_eip1559_fee{tier,kind}: per-tier maxFeePerGas / priority fee - blockbook_eth_eip1559_base_fee: the next-block base fee underlying them Values are raw wei (base units, like estimated_fee); Grafana divides by 1e9 to show Gwei. Emitted only on the two successful returns so error/disabled paths never write zeros, and nil tiers are skipped so no empty series appear. Dashboard (Fees section): maxFeePerGas by tier, priority tip by tier (own axis - tips are ~1000x smaller), and the maxFee/baseFee buffer ratio with a dashed 2x reference, which separates a congested chain from an estimator regression - the "fees too high" diagnostic. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/eip1559_blockgas_test.go | 60 ++++++ bchain/coins/eth/ethrpc.go | 46 +++++ common/metrics.go | 2 + configs/grafana/panels.yaml | 24 +++ configs/grafana/template.json | 212 ++++++++++++++++++++++ configs/metrics.yaml | 9 + 6 files changed, 353 insertions(+) diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index 6339146121..978c81ace3 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/common" ) // feeHistoryRPCStub serves a canned eth_feeHistory response; any other method errors so the test @@ -85,6 +87,64 @@ func TestEthereumTypeGetEip1559FeesOnChain(t *testing.T) { } } +// gaugeVecSeriesCount reports how many label series a GaugeVec currently holds, using a throwaway +// registry (same approach as gaugeValue, no prometheus/testutil dependency). +func gaugeVecSeriesCount(t *testing.T, gv *prometheus.GaugeVec) int { + t.Helper() + reg := prometheus.NewRegistry() + if err := reg.Register(gv); err != nil { + t.Fatalf("register gauge vec: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather gauge vec: %v", err) + } + n := 0 + for _, mf := range families { + n += len(mf.GetMetric()) + } + return n +} + +func TestObserveEip1559Fees(t *testing.T) { + m := &common.Metrics{ + EthEip1559Fee: prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "test_eth_eip1559_fee"}, []string{"tier", "kind"}), + EthEip1559BaseFee: prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_eth_eip1559_base_fee"}), + } + b := &EthereumRPC{} + b.metrics = m + // only low and high tiers populated; medium/instant left nil + b.observeEip1559Fees(&bchain.Eip1559Fees{ + BaseFeePerGas: big.NewInt(100), + Low: &bchain.Eip1559Fee{MaxFeePerGas: big.NewInt(250), MaxPriorityFeePerGas: big.NewInt(2)}, + High: &bchain.Eip1559Fee{MaxFeePerGas: big.NewInt(260), MaxPriorityFeePerGas: big.NewInt(4)}, + }) + + if got := gaugeValue(t, m.EthEip1559BaseFee); got != 100 { + t.Errorf("base fee gauge = %v, want 100", got) + } + for _, c := range []struct { + tier, kind string + want float64 + }{ + {"low", "max_fee", 250}, {"low", "priority_fee", 2}, + {"high", "max_fee", 260}, {"high", "priority_fee", 4}, + } { + got := gaugeValue(t, m.EthEip1559Fee.With(common.Labels{"tier": c.tier, "kind": c.kind})) + if got != c.want { + t.Errorf("eip1559_fee{tier=%s,kind=%s} = %v, want %v", c.tier, c.kind, got, c.want) + } + } + // nil tiers (medium, instant) must not create series: exactly 4 (=2 tiers x 2 kinds) expected + if n := gaugeVecSeriesCount(t, m.EthEip1559Fee); n != 4 { + t.Errorf("expected 4 tier series, got %d (nil tiers must be skipped)", n) + } + + // nil metrics must be a no-op (matches the unit-test path with no metrics wired) + (&EthereumRPC{}).observeEip1559Fees(&bchain.Eip1559Fees{BaseFeePerGas: big.NewInt(1)}) +} + func equalBigInt(a, b *big.Int) bool { if a == nil || b == nil { return a == nil && b == nil diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index c84ed1f5c5..506975ddd5 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1870,6 +1870,50 @@ func (b *EthereumRPC) EthereumTypeEstimateGas(params map[string]interface{}) (ui return b.Client.EstimateGas(ctx, msg) } +// bigIntToFloat converts a wei amount to float64 for gauge export. float64 holds integers +// exactly up to 2^53 (~9e15 wei), far above any realistic gas price, so no precision is lost; +// keeping the metric in raw wei (base units) matches the repo convention and Grafana divides +// by 1e9 to display Gwei. +func bigIntToFloat(v *big.Int) float64 { + if v == nil { + return 0 + } + f, _ := new(big.Float).SetInt(v).Float64() + return f +} + +// observeEip1559Fees records the EIP-1559 fees the pull path just produced: per-tier +// maxFeePerGas/maxPriorityFeePerGas and the underlying next-block base fee. Called only on the +// two successful return paths (provider cache hit and on-chain estimate) so the gauges never +// carry zeros from the error/disabled returns. Nil-guards mirror observeRequest. +func (b *EthereumRPC) observeEip1559Fees(fees *bchain.Eip1559Fees) { + if b.metrics == nil || fees == nil { + return + } + if b.metrics.EthEip1559BaseFee != nil && fees.BaseFeePerGas != nil { + b.metrics.EthEip1559BaseFee.Set(bigIntToFloat(fees.BaseFeePerGas)) + } + if b.metrics.EthEip1559Fee == nil { + return + } + for _, t := range []struct { + tier string + fee *bchain.Eip1559Fee + }{ + {"low", fees.Low}, {"medium", fees.Medium}, {"high", fees.High}, {"instant", fees.Instant}, + } { + if t.fee == nil { + continue + } + if t.fee.MaxFeePerGas != nil { + b.metrics.EthEip1559Fee.With(common.Labels{"tier": t.tier, "kind": "max_fee"}).Set(bigIntToFloat(t.fee.MaxFeePerGas)) + } + if t.fee.MaxPriorityFeePerGas != nil { + b.metrics.EthEip1559Fee.With(common.Labels{"tier": t.tier, "kind": "priority_fee"}).Set(bigIntToFloat(t.fee.MaxPriorityFeePerGas)) + } + } +} + // eip1559BaseFeeMultiplier is the headroom applied to the projected base fee when deriving // maxFeePerGas for the on-chain EIP-1559 estimate (maxFeePerGas = multiplier*baseFee + tip). // 2x is the EIP-1559-standard buffer: it keeps a transaction mineable across ~6 consecutive full @@ -1888,6 +1932,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) return nil, err } if fees != nil { + b.observeEip1559Fees(fees) return fees, nil } // Fall back to on-chain estimation when the alternative provider is unsupported/stale/unready, @@ -1968,6 +2013,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) fees.Instant = &f } } + b.observeEip1559Fees(&fees) return &fees, err } diff --git a/common/metrics.go b/common/metrics.go index 37bd524558..11c70808bf 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -96,6 +96,8 @@ type Metrics struct { FiatRatesFetchedTokens *prometheus.CounterVec `metric:"fiat_rates_fetched_tokens_total"` FiatRatesUnable *prometheus.CounterVec `metric:"fiat_rates_unable_total"` AlternativeFeeProviderRequests *prometheus.CounterVec `metric:"alternative_fee_provider_requests"` + EthEip1559Fee *prometheus.GaugeVec `metric:"eth_eip1559_fee"` + EthEip1559BaseFee prometheus.Gauge `metric:"eth_eip1559_base_fee"` EthSyncRpcErrors *prometheus.CounterVec `metric:"eth_sync_rpc_errors"` } diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 862f21d459..ad086569eb 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -663,6 +663,30 @@ panels: requests: promql: rate({{name:alternative_fee_provider_requests}}{job="blockbook", coin="$coin"}[$__rate_interval]) legend: '{{instance}} - {{provider}} - {{status}}' + fees.eip1559_max_fee: + title: EIP-1559 maxFeePerGas by tier (Gwei) + description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On Infura-backed coins these are the provider's padded suggestions; on on-chain coins they are 2x baseFee + tip. Compare with the base fee and the buffer-ratio panel to tell a busy chain (high but ~2x base) from an estimator regression. + queries: + by_tier: + promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / 1e9) + legend: '{{coin}} - {{instance}} - {{tier}}' + fees.eip1559_priority_fee: + title: EIP-1559 priority tip by tier (Gwei) + description: The maxPriorityFeePerGas (miner tip) served per tier, in Gwei (raw wei / 1e9), on its own axis because tips are an order of magnitude smaller than maxFeePerGas. On the on-chain path tiers map to eth_feeHistory reward percentiles 20/70/90/99; a near-zero flat tip is normal on quiet or L2 chains. Read exact values from the legend table. + queries: + by_tier: + promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="priority_fee"} / 1e9) + legend: '{{coin}} - {{instance}} - {{tier}}' + fees.eip1559_buffer_ratio: + title: EIP-1559 buffer ratio (maxFeePerGas / baseFee) + description: 'Per-tier maxFeePerGas divided by the next-block base fee - the headroom over the base fee. The on-chain estimator targets 2x by design (eip1559BaseFeeMultiplier), shown as the dashed target line. How to read it: ratio flat near its target while absolute fees are high means the chain is congested (nothing to fix); ratio drifting far above target means the estimator (or the provider) is over-buffering - the "fees too high" signature.' + queries: + ratio: + promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / on(job, coin, instance) group_left {{name:eth_eip1559_base_fee}}{job="blockbook", coin="$coin"}) + legend: '{{coin}} - {{instance}} - {{tier}}' + threshold: + promql: vector(2) + legend: target buffer (2x base fee) row.fiat_rates: title: Fiat Rates fiatrates.tip_fetches: diff --git a/configs/grafana/template.json b/configs/grafana/template.json index a50c79de62..206ba8f5db 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5291,6 +5291,218 @@ "type": "timeseries", "x-panel-key": "fees.alt_fee_provider_requests" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 341, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "by_tier" + } + ], + "type": "timeseries", + "x-panel-key": "fees.eip1559_max_fee" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 342, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "by_tier" + } + ], + "type": "timeseries", + "x-panel-key": "fees.eip1559_priority_fee" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*target.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "id": 343, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "ratio" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "threshold" + } + ], + "type": "timeseries", + "x-panel-key": "fees.eip1559_buffer_ratio" + }, { "collapsed": false, "id": 333, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 2293480acd..8fc006f1f7 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -373,6 +373,15 @@ metrics: type: counter_vec help: Requests to external fee providers, labeled by provider and outcome status labels: [provider, status] + eth_eip1559_fee: + name: blockbook_eth_eip1559_fee + type: gauge_vec + help: Most recent EIP-1559 fee tier served by the pull path (estimateFee / EthereumTypeGetEip1559Fees), in wei - divide by 1e9 for Gwei; labeled by tier (low/medium/high/instant) and kind (max_fee = maxFeePerGas, priority_fee = maxPriorityFeePerGas). EVM fees are otherwise invisible (estimated_fee is UTXO-only). Last-writer-wins per estimate, like estimated_fee + labels: [tier, kind] + eth_eip1559_base_fee: + name: blockbook_eth_eip1559_base_fee + type: gauge + help: Next-block base fee underlying the most recent pull-path EIP-1559 estimate, in wei - divide by 1e9 for Gwei. The alternative provider's estimatedBaseFee when served from the provider, otherwise eth_feeHistory's projected next-block base fee; pair with eth_block_base_fee to see projection vs realized eth_sync_rpc_errors: name: blockbook_eth_sync_rpc_errors type: counter_vec From b050aa0e3ec135d3c95003af0c5ac475612a1e3a Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:28:06 +0200 Subject: [PATCH 019/303] feat(eth): count EIP-1559 pull-path estimates by source Add blockbook_eth_eip1559_fee_source_total{source} incremented on each served estimate at the serve boundary: provider (alternative provider cache hit), onchain_fallback (provider configured but cache stale/unready, so eth_feeHistory was used) or onchain (no provider configured). This is the one signal neither alternative_fee_provider_requests (background fetch outcome) nor the provider cache-age gauge captures: how often the wallet- facing estimate actually bypassed the provider. Emitted only on the two successful returns, alongside the fee gauges. Dashboard: a non-stacked "EIP-1559 fee source" panel so a provider->fallback transition shows as a visible crossover. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/eip1559_blockgas_test.go | 53 ++++++++++++++++++++ bchain/coins/eth/ethrpc.go | 18 +++++++ common/metrics.go | 1 + configs/grafana/panels.yaml | 7 +++ configs/grafana/template.json | 60 +++++++++++++++++++++++ configs/metrics.yaml | 5 ++ 6 files changed, 144 insertions(+) diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index 978c81ace3..6be84cc2f4 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -145,6 +145,59 @@ func TestObserveEip1559Fees(t *testing.T) { (&EthereumRPC{}).observeEip1559Fees(&bchain.Eip1559Fees{BaseFeePerGas: big.NewInt(1)}) } +// counterVecValue reads the counter series carrying label=value, via a throwaway registry. +func counterVecValue(t *testing.T, cv *prometheus.CounterVec, label, value string) float64 { + t.Helper() + reg := prometheus.NewRegistry() + if err := reg.Register(cv); err != nil { + t.Fatalf("register counter vec: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather counter vec: %v", err) + } + for _, mf := range families { + for _, mm := range mf.GetMetric() { + for _, lp := range mm.GetLabel() { + if lp.GetName() == label && lp.GetValue() == value { + return mm.GetCounter().GetValue() + } + } + } + } + return 0 +} + +func TestEip1559FeeSourceMetric(t *testing.T) { + m := &common.Metrics{ + EthEip1559FeeSource: prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "test_eth_eip1559_fee_source_total"}, []string{"source"}), + EthEip1559Fee: prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "test_eth_eip1559_fee_src"}, []string{"tier", "kind"}), + EthEip1559BaseFee: prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_eth_eip1559_base_fee_src"}), + } + raw := `{"oldestBlock":"0x1",` + + `"reward":[["0x1","0x2","0x3","0x4"]],` + + `"baseFeePerGas":["0x10","0x20","0x30","0x64"],` + + `"gasUsedRatio":[0.5,0.5,0.5]}` + b := &EthereumRPC{ + RPC: &feeHistoryRPCStub{raw: raw}, + Timeout: time.Second, + ChainConfig: &Configuration{Eip1559Fees: true}, + } + b.metrics = m + if _, err := b.EthereumTypeGetEip1559Fees(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + // no alternative provider configured -> the on-chain estimate is source=onchain (not a fallback) + if got := counterVecValue(t, m.EthEip1559FeeSource, "source", "onchain"); got != 1 { + t.Errorf("source=onchain counter = %v, want 1", got) + } + if got := counterVecValue(t, m.EthEip1559FeeSource, "source", "onchain_fallback"); got != 0 { + t.Errorf("source=onchain_fallback counter = %v, want 0", got) + } +} + func equalBigInt(a, b *big.Int) bool { if a == nil || b == nil { return a == nil && b == nil diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 506975ddd5..726a3b3b60 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1914,6 +1914,16 @@ func (b *EthereumRPC) observeEip1559Fees(fees *bchain.Eip1559Fees) { } } +// observeEip1559FeeSource records which source served a pull-path estimate, observed at the serve +// boundary: the alternative provider cache (provider), the on-chain estimate after a stale/unready +// provider (onchain_fallback), or the on-chain estimate with no provider configured (onchain). +func (b *EthereumRPC) observeEip1559FeeSource(source string) { + if b.metrics == nil || b.metrics.EthEip1559FeeSource == nil { + return + } + b.metrics.EthEip1559FeeSource.With(common.Labels{"source": source}).Inc() +} + // eip1559BaseFeeMultiplier is the headroom applied to the projected base fee when deriving // maxFeePerGas for the on-chain EIP-1559 estimate (maxFeePerGas = multiplier*baseFee + tip). // 2x is the EIP-1559-standard buffer: it keeps a transaction mineable across ~6 consecutive full @@ -1932,6 +1942,7 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) return nil, err } if fees != nil { + b.observeEip1559FeeSource("provider") b.observeEip1559Fees(fees) return fees, nil } @@ -2013,6 +2024,13 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) fees.Instant = &f } } + // Reaching here with a provider configured means its cache was stale/unready (a hit would have + // returned above), so this on-chain estimate is a fallback. + source := "onchain" + if b.alternativeFeeProvider != nil { + source = "onchain_fallback" + } + b.observeEip1559FeeSource(source) b.observeEip1559Fees(&fees) return &fees, err } diff --git a/common/metrics.go b/common/metrics.go index 11c70808bf..0b9b9cfcae 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -98,6 +98,7 @@ type Metrics struct { AlternativeFeeProviderRequests *prometheus.CounterVec `metric:"alternative_fee_provider_requests"` EthEip1559Fee *prometheus.GaugeVec `metric:"eth_eip1559_fee"` EthEip1559BaseFee prometheus.Gauge `metric:"eth_eip1559_base_fee"` + EthEip1559FeeSource *prometheus.CounterVec `metric:"eth_eip1559_fee_source_total"` EthSyncRpcErrors *prometheus.CounterVec `metric:"eth_sync_rpc_errors"` } diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index ad086569eb..ea247ff05b 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -687,6 +687,13 @@ panels: threshold: promql: vector(2) legend: target buffer (2x base fee) + fees.eip1559_fee_source: + title: EIP-1559 fee source (served estimates/s) + description: 'Where each served estimate came from, at the serve boundary: provider = alternative provider (Infura/1inch) cache hit; onchain_fallback = provider configured but its cache was stale/unready, so eth_feeHistory was used; onchain = no provider configured. Healthy state is provider dominating on provider-backed coins; a burst of onchain_fallback at startup is cold-cache warmup, not an incident (the cache-age panel disambiguates). Sustained onchain_fallback means the provider is wedged.' + queries: + by_source: + promql: sum by (coin, source) (rate({{name:eth_eip1559_fee_source_total}}{job="blockbook", coin="$coin"}[$__rate_interval])) + legend: '{{coin}} - {{source}}' row.fiat_rates: title: Fiat Rates fiatrates.tip_fetches: diff --git a/configs/grafana/template.json b/configs/grafana/template.json index 206ba8f5db..ce71452308 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5503,6 +5503,66 @@ "type": "timeseries", "x-panel-key": "fees.eip1559_buffer_ratio" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 344, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "by_source" + } + ], + "type": "timeseries", + "x-panel-key": "fees.eip1559_fee_source" + }, { "collapsed": false, "id": 333, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 8fc006f1f7..4de41f8785 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -382,6 +382,11 @@ metrics: name: blockbook_eth_eip1559_base_fee type: gauge help: Next-block base fee underlying the most recent pull-path EIP-1559 estimate, in wei - divide by 1e9 for Gwei. The alternative provider's estimatedBaseFee when served from the provider, otherwise eth_feeHistory's projected next-block base fee; pair with eth_block_base_fee to see projection vs realized + eth_eip1559_fee_source_total: + name: blockbook_eth_eip1559_fee_source_total + type: counter_vec + help: Served EIP-1559 pull-path estimates by source observed at the serve boundary - provider (alternative provider cache hit), onchain_fallback (provider configured but cache stale/unready, served from eth_feeHistory) or onchain (no provider configured). Complements alternative_fee_provider_requests (background fetch) and alternative_fee_provider_last_sync_timestamp_seconds (cache age), which together explain why a fallback happened + labels: [source] eth_sync_rpc_errors: name: blockbook_eth_sync_rpc_errors type: counter_vec From 85c1f7c311846d6be697119314562e9378d4be20 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:30:20 +0200 Subject: [PATCH 020/303] feat(eth): export alternative fee provider cache freshness Add blockbook_eth..._provider_last_sync_timestamp_seconds{provider}, the unix time of the last successful refresh of the cached EVM provider (infura/1inch) fees, set via a shared observeSync() called from both providers' processData (replacing the bare p.lastSync = time.Now()) so lastSync and the metric always share one instant. Cache age is plotted as time() - metric. A timestamp gauge, not a computed *_age_seconds one (the repo's usual form): it is written only on a successful refresh, so the plotted age keeps climbing when a provider wedges and survives restarts. This is the leading indicator that explains the onchain_fallback counted by eth_eip1559_fee_source_total - once age crosses the stale window (30x the poll period) the pull path falls back to on-chain. Dashboard: a cache-age panel with a dashed default-cutoff (30m) reference. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/alternativefeeprovider.go | 13 +++ bchain/coins/eth/eip1559_blockgas_test.go | 25 ++++++ bchain/coins/eth/infurafees.go | 2 +- bchain/coins/eth/oneinchfees.go | 2 +- common/metrics.go | 1 + configs/grafana/panels.yaml | 10 +++ configs/grafana/template.json | 92 ++++++++++++++++++++++ configs/metrics.yaml | 5 ++ 8 files changed, 148 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/alternativefeeprovider.go b/bchain/coins/eth/alternativefeeprovider.go index 42cc257e6f..70f2a1a31d 100644 --- a/bchain/coins/eth/alternativefeeprovider.go +++ b/bchain/coins/eth/alternativefeeprovider.go @@ -25,6 +25,19 @@ func (p *alternativeFeeProvider) observeRequest(status string) { p.metrics.AlternativeFeeProviderRequests.With(common.Labels{"provider": p.name, "status": status}).Inc() } +// observeSync records a successful refresh of the cached fees: it advances lastSync (which the read +// path uses for its staleness check) and exports the timestamp so cache age can be plotted as +// time() - metric. Both must use the same instant, so the caller passes it in. Exporting a timestamp +// rather than an age keeps the plotted age rising when a provider wedges, since it is only written +// here on success. Callers hold p.mux. +func (p *alternativeFeeProvider) observeSync(t time.Time) { + p.lastSync = t + if p.metrics == nil || p.metrics.AlternativeFeeProviderLastSync == nil { + return + } + p.metrics.AlternativeFeeProviderLastSync.With(common.Labels{"provider": p.name}).Set(float64(t.Unix())) +} + type alternativeFeeProviderInterface interface { GetEip1559Fees() (*bchain.Eip1559Fees, error) } diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index 6be84cc2f4..bc21815ba9 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -168,6 +168,31 @@ func counterVecValue(t *testing.T, cv *prometheus.CounterVec, label, value strin return 0 } +func TestObserveSyncMetric(t *testing.T) { + gv := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "test_alt_fee_provider_last_sync"}, []string{"provider"}) + p := &alternativeFeeProvider{ + metrics: &common.Metrics{AlternativeFeeProviderLastSync: gv}, + name: "infura", + } + ts := time.Unix(1700000000, 0) + p.observeSync(ts) + + if !p.lastSync.Equal(ts) { + t.Errorf("lastSync = %v, want %v (must use the same instant as the metric)", p.lastSync, ts) + } + if got := gaugeValue(t, gv.With(common.Labels{"provider": "infura"})); got != 1700000000 { + t.Errorf("last_sync gauge = %v, want 1700000000", got) + } + + // nil metrics must still advance lastSync without panicking + p2 := &alternativeFeeProvider{name: "1inch"} + p2.observeSync(ts) + if !p2.lastSync.Equal(ts) { + t.Errorf("nil-metrics observeSync must still set lastSync") + } +} + func TestEip1559FeeSourceMetric(t *testing.T) { m := &common.Metrics{ EthEip1559FeeSource: prometheus.NewCounterVec( diff --git a/bchain/coins/eth/infurafees.go b/bchain/coins/eth/infurafees.go index a81b5940aa..98291608ce 100644 --- a/bchain/coins/eth/infurafees.go +++ b/bchain/coins/eth/infurafees.go @@ -173,7 +173,7 @@ func (p *infuraFeeProvider) processData(data *infuraFeesResult) bool { fees.BaseFeeTrend = data.BaseFeeTrend p.mux.Lock() defer p.mux.Unlock() - p.lastSync = time.Now() + p.observeSync(time.Now()) p.eip1559Fees = &fees return true } diff --git a/bchain/coins/eth/oneinchfees.go b/bchain/coins/eth/oneinchfees.go index e42f5a6652..cb3c7485c2 100644 --- a/bchain/coins/eth/oneinchfees.go +++ b/bchain/coins/eth/oneinchfees.go @@ -117,7 +117,7 @@ func (p *oneInchFeeProvider) processData(data *oneInchFeeFeesResult) bool { fees.Low = oneInchFeesFromResult(&data.Low) p.mux.Lock() defer p.mux.Unlock() - p.lastSync = time.Now() + p.observeSync(time.Now()) p.eip1559Fees = &fees return true } diff --git a/common/metrics.go b/common/metrics.go index 0b9b9cfcae..4837641e36 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -96,6 +96,7 @@ type Metrics struct { FiatRatesFetchedTokens *prometheus.CounterVec `metric:"fiat_rates_fetched_tokens_total"` FiatRatesUnable *prometheus.CounterVec `metric:"fiat_rates_unable_total"` AlternativeFeeProviderRequests *prometheus.CounterVec `metric:"alternative_fee_provider_requests"` + AlternativeFeeProviderLastSync *prometheus.GaugeVec `metric:"alternative_fee_provider_last_sync_timestamp_seconds"` EthEip1559Fee *prometheus.GaugeVec `metric:"eth_eip1559_fee"` EthEip1559BaseFee prometheus.Gauge `metric:"eth_eip1559_base_fee"` EthEip1559FeeSource *prometheus.CounterVec `metric:"eth_eip1559_fee_source_total"` diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index ea247ff05b..0a725b112a 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -687,6 +687,16 @@ panels: threshold: promql: vector(2) legend: target buffer (2x base fee) + fees.alt_fee_provider_cache_age: + title: Alternative fee provider cache age + description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the stale window - 30x the poll period - after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). The dashed line marks the default 30-minute cutoff (30x a 60s poll); adjust if a coin polls at a different period. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' + queries: + age: + promql: (time() - {{name:alternative_fee_provider_last_sync_timestamp_seconds}}{job="blockbook", coin="$coin"}) + legend: '{{coin}} - {{instance}} - {{provider}}' + cutoff: + promql: vector(1800) + legend: default stale cutoff (30x 60s = 30m) fees.eip1559_fee_source: title: EIP-1559 fee source (served estimates/s) description: 'Where each served estimate came from, at the serve boundary: provider = alternative provider (Infura/1inch) cache hit; onchain_fallback = provider configured but its cache was stale/unready, so eth_feeHistory was used; onchain = no provider configured. Healthy state is provider dominating on provider-backed coins; a burst of onchain_fallback at startup is cold-cache warmup, not an incident (the cache-age panel disambiguates). Sustained onchain_fallback means the provider is wedged.' diff --git a/configs/grafana/template.json b/configs/grafana/template.json index ce71452308..028cdb28c5 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5563,6 +5563,98 @@ "type": "timeseries", "x-panel-key": "fees.eip1559_fee_source" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*cutoff.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "id": 345, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "age" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "cutoff" + } + ], + "type": "timeseries", + "x-panel-key": "fees.alt_fee_provider_cache_age" + }, { "collapsed": false, "id": 333, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 4de41f8785..d4bac85127 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -373,6 +373,11 @@ metrics: type: counter_vec help: Requests to external fee providers, labeled by provider and outcome status labels: [provider, status] + alternative_fee_provider_last_sync_timestamp_seconds: + name: blockbook_alternative_fee_provider_last_sync_timestamp_seconds + type: gauge_vec + help: Unix time of the last successful refresh of the cached EVM alternative-provider fees, labeled by provider (infura or 1inch). Plot time() - this for cache age; the pull path falls back to on-chain once the cache is older than the stale window (30x the poll period, e.g. 30m at a 60s poll). Exported as a timestamp rather than a computed *_age_seconds gauge on purpose - it is only written on a successful refresh, so age keeps rising when a provider wedges and it is restart-safe + labels: [provider] eth_eip1559_fee: name: blockbook_eth_eip1559_fee type: gauge_vec From 45011b2f883ce4f7ba2878f13898ad57f8bbe93b Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:32:56 +0200 Subject: [PATCH 021/303] feat(server): export latest-block gas-used ratio (push path) Add blockbook_eth_block_gas_used_ratio, gasUsed/gasLimit of the most recently connected EVM block (0..1) - the congestion signal that drives the next base fee (>0.5 it rises up to +12.5%/block, <0.5 it falls). This is the push path's own view of why fees move, independent of any provider. Set synchronously in OnNewBlock before the async broadcast: OnNewBlock is invoked in monotonic height order by the single writeBlockWorker, whereas the per-block broadcast goroutines can reorder and let an older block clobber a newer gauge value. Last-value semantics, nil-guarded for non-EVM/pre-London. Dashboard: a "Latest block gas-used ratio" panel (percentunit) with a dashed 50% balance-point reference. Co-Authored-By: Claude Opus 4.8 --- common/metrics.go | 1 + configs/grafana/panels.yaml | 10 ++++ configs/grafana/template.json | 92 +++++++++++++++++++++++++++++++++++ configs/metrics.yaml | 4 ++ server/websocket.go | 23 +++++++++ server/ws_newblock_test.go | 43 ++++++++++++++++ 6 files changed, 173 insertions(+) diff --git a/common/metrics.go b/common/metrics.go index 4837641e36..fdfdaac274 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -100,6 +100,7 @@ type Metrics struct { EthEip1559Fee *prometheus.GaugeVec `metric:"eth_eip1559_fee"` EthEip1559BaseFee prometheus.Gauge `metric:"eth_eip1559_base_fee"` EthEip1559FeeSource *prometheus.CounterVec `metric:"eth_eip1559_fee_source_total"` + EthBlockGasUsedRatio prometheus.Gauge `metric:"eth_block_gas_used_ratio"` EthSyncRpcErrors *prometheus.CounterVec `metric:"eth_sync_rpc_errors"` } diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 0a725b112a..4244d9deff 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -687,6 +687,16 @@ panels: threshold: promql: vector(2) legend: target buffer (2x base fee) + fees.block_gas_used_ratio: + title: Latest block gas-used ratio (% of gas limit) + description: 'gasUsed / gasLimit of the most recently connected block, pushed on subscribeNewBlock. It is what moves the base fee: above the 50% balance point (dashed) the next block''s base fee rises up to +12.5%, below it falls. Read together with the base-fee and maxFeePerGas panels, this is the push-path explanation for "fees went up" - a sustained >50% ratio is real congestion, not an estimator issue.' + queries: + ratio: + promql: '{{name:eth_block_gas_used_ratio}}{job="blockbook", coin="$coin"}' + legend: '{{coin}} - {{instance}}' + balance: + promql: vector(0.5) + legend: balance point (50%) fees.alt_fee_provider_cache_age: title: Alternative fee provider cache age description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the stale window - 30x the poll period - after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). The dashed line marks the default 30-minute cutoff (30x a 60s poll); adjust if a coin polls at a different period. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' diff --git a/configs/grafana/template.json b/configs/grafana/template.json index 028cdb28c5..c6b9667fb3 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5655,6 +5655,98 @@ "type": "timeseries", "x-panel-key": "fees.alt_fee_provider_cache_age" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*balance point.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "id": 346, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "ratio" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "balance" + } + ], + "type": "timeseries", + "x-panel-key": "fees.block_gas_used_ratio" + }, { "collapsed": false, "id": 333, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index d4bac85127..1b6bdd66bb 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -392,6 +392,10 @@ metrics: type: counter_vec help: Served EIP-1559 pull-path estimates by source observed at the serve boundary - provider (alternative provider cache hit), onchain_fallback (provider configured but cache stale/unready, served from eth_feeHistory) or onchain (no provider configured). Complements alternative_fee_provider_requests (background fetch) and alternative_fee_provider_last_sync_timestamp_seconds (cache age), which together explain why a fallback happened labels: [source] + eth_block_gas_used_ratio: + name: blockbook_eth_block_gas_used_ratio + type: gauge + help: gasUsed / gasLimit of the most recently connected EVM block (0..1), the congestion signal that drives the next base fee - above 0.5 the next block's base fee rises (up to +12.5% per block), below 0.5 it falls. This is the push path's view of why fees move; updated once per connected block, so it sweeps catch-up blocks and settles on the tip eth_sync_rpc_errors: name: blockbook_eth_sync_rpc_errors type: counter_vec diff --git a/server/websocket.go b/server/websocket.go index 8ab15a3e9d..314fb0d7bc 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1429,6 +1429,26 @@ func newBlockNotification(block *bchain.Block) *WsNewBlock { return data } +// observeNewBlockGas records push-path block-gas metrics from the most recently connected block. +// It is called synchronously from OnNewBlock (which the single writeBlockWorker invokes in height +// order) before the async broadcast, so the gauges advance monotonically without a mutex; the +// per-block broadcast goroutines could otherwise reorder and let an older block clobber a newer +// value. Last-value semantics: it sweeps catch-up blocks and settles on the tip. Non-EVM and +// pre-London blocks (no EthereumBlockSpecificData with gas set) are skipped. +func (s *WebsocketServer) observeNewBlockGas(block *bchain.Block) { + if s.metrics == nil { + return + } + bsd, ok := block.CoinSpecificData.(*bchain.EthereumBlockSpecificData) + if !ok || bsd == nil { + return + } + if s.metrics.EthBlockGasUsedRatio != nil && bsd.GasUsed != nil && bsd.GasLimit != nil && bsd.GasLimit.Sign() > 0 { + ratio, _ := new(big.Float).Quo(new(big.Float).SetInt(bsd.GasUsed), new(big.Float).SetInt(bsd.GasLimit)).Float64() + s.metrics.EthBlockGasUsedRatio.Set(ratio) + } +} + func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { s.newBlockSubscriptionsLock.Lock() defer s.newBlockSubscriptionsLock.Unlock() @@ -1594,6 +1614,9 @@ func (s *WebsocketServer) publishNewBlockTxsByAddr(block *bchain.Block) { // OnNewBlock is a callback that broadcasts info about new block to subscribed clients func (s *WebsocketServer) OnNewBlock(block *bchain.Block) { + // Synchronous and before the async dispatch: OnNewBlock is called in monotonic height order, so + // the push-path gas gauges never get an older block's value written after a newer one's. + s.observeNewBlockGas(block) s.addressSubscriptionsLock.Lock() defer s.addressSubscriptionsLock.Unlock() go s.onNewBlockAsync(block) diff --git a/server/ws_newblock_test.go b/server/ws_newblock_test.go index 09cf5b566f..11c747f0ad 100644 --- a/server/ws_newblock_test.go +++ b/server/ws_newblock_test.go @@ -7,10 +7,53 @@ import ( "math/big" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/trezor/blockbook/api" "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/common" ) +// gaugeFloat reads a single gauge's value via a throwaway registry, avoiding the +// prometheus/testutil dependency (and its transitive modules). +func gaugeFloat(t *testing.T, g prometheus.Gauge) float64 { + t.Helper() + reg := prometheus.NewRegistry() + if err := reg.Register(g); err != nil { + t.Fatalf("register gauge: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather gauge: %v", err) + } + for _, mf := range families { + for _, m := range mf.GetMetric() { + return m.GetGauge().GetValue() + } + } + return 0 +} + +func TestObserveNewBlockGas(t *testing.T) { + m := &common.Metrics{ + EthBlockGasUsedRatio: prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_eth_block_gas_used_ratio"}), + } + s := &WebsocketServer{metrics: m} + + // half-full post-London block -> ratio 0.5 + s.observeNewBlockGas(&bchain.Block{CoinSpecificData: &bchain.EthereumBlockSpecificData{ + BaseFeePerGas: big.NewInt(7), GasUsed: big.NewInt(15000000), GasLimit: big.NewInt(30000000), + }}) + if got := gaugeFloat(t, m.EthBlockGasUsedRatio); got != 0.5 { + t.Errorf("gas-used ratio = %v, want 0.5", got) + } + + // non-EVM block and nil metrics must be safe no-ops + s.observeNewBlockGas(&bchain.Block{}) + (&WebsocketServer{}).observeNewBlockGas(&bchain.Block{CoinSpecificData: &bchain.EthereumBlockSpecificData{ + GasUsed: big.NewInt(1), GasLimit: big.NewInt(2), + }}) +} + // TestNewBlockNotification covers the block -> subscribeNewBlock payload mapping: only EVM // post-London blocks (EthereumBlockSpecificData with BaseFeePerGas set) carry evmData; non-EVM, // pre-London, and missing coin-specific data leave it nil. From 01e3dfdbe96ab4a5e25885902a69874e8afda74f Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:34:17 +0200 Subject: [PATCH 022/303] feat(server): export latest-block base fee (push path) + projection overlay Add blockbook_eth_block_base_fee, the realized base fee of the most recently connected block (push path), in raw wei. Set in the same OnNewBlock helper as the gas-used ratio, nil-guarded. Paired with eth_eip1559_base_fee (the pull path's next-block projection) on one overlay panel - solid mined vs dashed projected, both in Gwei. The two should track within a block's +/-12.5% step; a persistent gap means the feeHistory projection (or a provider's estimatedBaseFee) is drifting from the chain - the base-fee half of a "fees too high" investigation. Co-Authored-By: Claude Opus 4.8 --- common/metrics.go | 1 + configs/grafana/panels.yaml | 10 +++++ configs/grafana/template.json | 85 +++++++++++++++++++++++++++++++++++ configs/metrics.yaml | 4 ++ server/websocket.go | 4 ++ server/ws_newblock_test.go | 6 ++- 6 files changed, 109 insertions(+), 1 deletion(-) diff --git a/common/metrics.go b/common/metrics.go index fdfdaac274..d83bd33746 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -101,6 +101,7 @@ type Metrics struct { EthEip1559BaseFee prometheus.Gauge `metric:"eth_eip1559_base_fee"` EthEip1559FeeSource *prometheus.CounterVec `metric:"eth_eip1559_fee_source_total"` EthBlockGasUsedRatio prometheus.Gauge `metric:"eth_block_gas_used_ratio"` + EthBlockBaseFee prometheus.Gauge `metric:"eth_block_base_fee"` EthSyncRpcErrors *prometheus.CounterVec `metric:"eth_sync_rpc_errors"` } diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 4244d9deff..ada38b2b28 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -687,6 +687,16 @@ panels: threshold: promql: vector(2) legend: target buffer (2x base fee) + fees.base_fee_push_vs_pull: + title: Base fee - mined block (push) vs next-block projection (pull) (Gwei) + description: 'Solid = base fee of the latest connected block (push path, eth_block_base_fee); dashed = the next-block base fee the pull-path estimate is built on (eth_eip1559_base_fee). They should track within one block''s +/-12.5% step. A persistent gap means the eth_feeHistory projection is drifting from the chain (or, on a provider-backed coin, the provider''s estimatedBaseFee is off) - the base-fee half of a "fees too high" investigation. Note the projection''s meaning differs by source (provider estimate vs feeHistory).' + queries: + mined: + promql: ({{name:eth_block_base_fee}}{job="blockbook", coin="$coin"} / 1e9) + legend: '{{coin}} - {{instance}} - mined (push)' + projected: + promql: ({{name:eth_eip1559_base_fee}}{job="blockbook", coin="$coin"} / 1e9) + legend: '{{coin}} - {{instance}} - projected (pull)' fees.block_gas_used_ratio: title: Latest block gas-used ratio (% of gas limit) description: 'gasUsed / gasLimit of the most recently connected block, pushed on subscribeNewBlock. It is what moves the base fee: above the 50% balance point (dashed) the next block''s base fee rises up to +12.5%, below it falls. Read together with the base-fee and maxFeePerGas panels, this is the push-path explanation for "fees went up" - a sustained >50% ratio is real congestion, not an estimator issue.' diff --git a/configs/grafana/template.json b/configs/grafana/template.json index c6b9667fb3..aae664eed9 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5747,6 +5747,91 @@ "type": "timeseries", "x-panel-key": "fees.block_gas_used_ratio" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*projected.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "id": 347, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "mined" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "projected" + } + ], + "type": "timeseries", + "x-panel-key": "fees.base_fee_push_vs_pull" + }, { "collapsed": false, "id": 333, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 1b6bdd66bb..ed49e8f6a8 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -396,6 +396,10 @@ metrics: name: blockbook_eth_block_gas_used_ratio type: gauge help: gasUsed / gasLimit of the most recently connected EVM block (0..1), the congestion signal that drives the next base fee - above 0.5 the next block's base fee rises (up to +12.5% per block), below 0.5 it falls. This is the push path's view of why fees move; updated once per connected block, so it sweeps catch-up blocks and settles on the tip + eth_block_base_fee: + name: blockbook_eth_block_base_fee + type: gauge + help: Base fee per gas of the most recently connected EVM block, in wei - divide by 1e9 for Gwei. The realized base fee from the block header (push path); compare with eth_eip1559_base_fee (the pull path's next-block projection) to see whether the projection tracks the chain. Updated once per connected block eth_sync_rpc_errors: name: blockbook_eth_sync_rpc_errors type: counter_vec diff --git a/server/websocket.go b/server/websocket.go index 314fb0d7bc..ea45cd33fc 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1447,6 +1447,10 @@ func (s *WebsocketServer) observeNewBlockGas(block *bchain.Block) { ratio, _ := new(big.Float).Quo(new(big.Float).SetInt(bsd.GasUsed), new(big.Float).SetInt(bsd.GasLimit)).Float64() s.metrics.EthBlockGasUsedRatio.Set(ratio) } + if s.metrics.EthBlockBaseFee != nil && bsd.BaseFeePerGas != nil { + baseFee, _ := new(big.Float).SetInt(bsd.BaseFeePerGas).Float64() + s.metrics.EthBlockBaseFee.Set(baseFee) + } } func (s *WebsocketServer) onNewBlockAsync(block *bchain.Block) { diff --git a/server/ws_newblock_test.go b/server/ws_newblock_test.go index 11c747f0ad..4022f4645d 100644 --- a/server/ws_newblock_test.go +++ b/server/ws_newblock_test.go @@ -36,16 +36,20 @@ func gaugeFloat(t *testing.T, g prometheus.Gauge) float64 { func TestObserveNewBlockGas(t *testing.T) { m := &common.Metrics{ EthBlockGasUsedRatio: prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_eth_block_gas_used_ratio"}), + EthBlockBaseFee: prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_eth_block_base_fee"}), } s := &WebsocketServer{metrics: m} - // half-full post-London block -> ratio 0.5 + // half-full post-London block -> ratio 0.5, base fee in raw wei s.observeNewBlockGas(&bchain.Block{CoinSpecificData: &bchain.EthereumBlockSpecificData{ BaseFeePerGas: big.NewInt(7), GasUsed: big.NewInt(15000000), GasLimit: big.NewInt(30000000), }}) if got := gaugeFloat(t, m.EthBlockGasUsedRatio); got != 0.5 { t.Errorf("gas-used ratio = %v, want 0.5", got) } + if got := gaugeFloat(t, m.EthBlockBaseFee); got != 7 { + t.Errorf("base fee = %v, want 7 (raw wei)", got) + } // non-EVM block and nil metrics must be safe no-ops s.observeNewBlockGas(&bchain.Block{}) From 2f6de1b9e0107a786057d8f9bc3e940b8f2a9aea Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 24 Jun 2026 13:44:37 +0200 Subject: [PATCH 023/303] docs(grafana,eth): sharpen Fees panel/metric wording per review Documentation-only follow-up from an adversarial review of the new Fees section (no functional change, no Go change): - order the section pull-estimate -> provider-health -> push -> legacy, and label the relocated UTXO "Estimated fee rate" so an EVM $coin selection reads as expected-empty rather than an outage - buffer-ratio: note Infura's high tier sits ~2.5x by design, so the dashed 2x line is the on-chain estimator's target only - base-fee overlay & cache-age: scope the "tracks within +/-12.5%" and "30x poll period" expectations to the path/provider they actually hold for (provider coins serve a cached projection up to the stale window old) - eth_eip1559_base_fee help: drop the unconditional "next-block projection" claim (true only on Erigon-like backends) and Infura's estimatedBaseFee field name (1inch feeds the same gauge from baseFee) - priority-tip: "separate panel", not "own axis" Co-Authored-By: Claude Opus 4.8 --- configs/grafana/panels.yaml | 14 +-- configs/grafana/template.json | 182 +++++++++++++++++----------------- configs/metrics.yaml | 4 +- 3 files changed, 100 insertions(+), 100 deletions(-) diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index ada38b2b28..45be0a05c5 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -641,8 +641,8 @@ panels: row.fees: title: Fees fees.estimated_fee: - title: Estimated fee rate - description: Fee estimates for 1/3/5/8 block targets; units are chain-specific. + title: Estimated fee rate (UTXO chains) + description: 'Fee estimates for 1/3/5/8 block targets; units are chain-specific. UTXO chains only - estimated_fee is not populated for EVM coins, which expose their fees in the EIP-1559 panels above, so this panel is empty when $coin is an EVM chain (expected, not an outage).' queries: blocks_1: promql: ({{name:estimated_fee}}{job="blockbook", coin="$coin", blocks="1"}/1000) @@ -665,21 +665,21 @@ panels: legend: '{{instance}} - {{provider}} - {{status}}' fees.eip1559_max_fee: title: EIP-1559 maxFeePerGas by tier (Gwei) - description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On Infura-backed coins these are the provider's padded suggestions; on on-chain coins they are 2x baseFee + tip. Compare with the base fee and the buffer-ratio panel to tell a busy chain (high but ~2x base) from an estimator regression. + description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On Infura-backed coins these are the provider's padded suggestions; on on-chain coins they are 2x baseFee + tip. The instant tier exists only on the on-chain path (Infura has no instant), so an instant line on a provider coin is a leftover from an onchain_fallback. Compare with the base fee and the buffer-ratio panel to tell a busy chain from an estimator regression. queries: by_tier: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / 1e9) legend: '{{coin}} - {{instance}} - {{tier}}' fees.eip1559_priority_fee: title: EIP-1559 priority tip by tier (Gwei) - description: The maxPriorityFeePerGas (miner tip) served per tier, in Gwei (raw wei / 1e9), on its own axis because tips are an order of magnitude smaller than maxFeePerGas. On the on-chain path tiers map to eth_feeHistory reward percentiles 20/70/90/99; a near-zero flat tip is normal on quiet or L2 chains. Read exact values from the legend table. + description: The maxPriorityFeePerGas (miner tip) served per tier, in Gwei (raw wei / 1e9), shown in a separate panel from maxFeePerGas because tips are an order of magnitude smaller. On the on-chain path tiers map to eth_feeHistory reward percentiles 20/70/90/99; a near-zero flat tip is normal on quiet or L2 chains. Read exact values from the legend table. queries: by_tier: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="priority_fee"} / 1e9) legend: '{{coin}} - {{instance}} - {{tier}}' fees.eip1559_buffer_ratio: title: EIP-1559 buffer ratio (maxFeePerGas / baseFee) - description: 'Per-tier maxFeePerGas divided by the next-block base fee - the headroom over the base fee. The on-chain estimator targets 2x by design (eip1559BaseFeeMultiplier), shown as the dashed target line. How to read it: ratio flat near its target while absolute fees are high means the chain is congested (nothing to fix); ratio drifting far above target means the estimator (or the provider) is over-buffering - the "fees too high" signature.' + description: 'Per-tier maxFeePerGas divided by the next-block base fee - the headroom over the base fee. The on-chain estimator targets 2x by design (eip1559BaseFeeMultiplier), shown as the dashed line; on Infura-backed coins the high tier sits ~2.5x by design, so a steady ratio above 2 there is expected, not an anomaly - the 2x line is the on-chain target only. How to read it: a ratio flat near its source''s normal level while absolute fees are high means the chain is congested (nothing to fix); a ratio climbing well above that level means over-buffering - the "fees too high" signature.' queries: ratio: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / on(job, coin, instance) group_left {{name:eth_eip1559_base_fee}}{job="blockbook", coin="$coin"}) @@ -689,7 +689,7 @@ panels: legend: target buffer (2x base fee) fees.base_fee_push_vs_pull: title: Base fee - mined block (push) vs next-block projection (pull) (Gwei) - description: 'Solid = base fee of the latest connected block (push path, eth_block_base_fee); dashed = the next-block base fee the pull-path estimate is built on (eth_eip1559_base_fee). They should track within one block''s +/-12.5% step. A persistent gap means the eth_feeHistory projection is drifting from the chain (or, on a provider-backed coin, the provider''s estimatedBaseFee is off) - the base-fee half of a "fees too high" investigation. Note the projection''s meaning differs by source (provider estimate vs feeHistory).' + description: 'Solid = base fee of the latest connected block (push path, eth_block_base_fee); dashed = the next-block base fee the pull-path estimate is built on (eth_eip1559_base_fee). On the on-chain path they should track within ~one block''s +/-12.5% step, so a persistent gap there means the eth_feeHistory projection is drifting from the chain. On provider-backed coins the projection is a cached value (up to the stale window old), so a larger, slowly-updating gap is expected cache lag - cross-check the cache-age and fee-source panels before calling it drift.' queries: mined: promql: ({{name:eth_block_base_fee}}{job="blockbook", coin="$coin"} / 1e9) @@ -709,7 +709,7 @@ panels: legend: balance point (50%) fees.alt_fee_provider_cache_age: title: Alternative fee provider cache age - description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the stale window - 30x the poll period - after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). The dashed line marks the default 30-minute cutoff (30x a 60s poll); adjust if a coin polls at a different period. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' + description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the provider''s stale window, after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). For Infura (the EVM default) the window is 30x the poll period; the dashed line marks the default 30-minute cutoff (30x a 60s poll), so adjust it if a coin polls differently. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' queries: age: promql: (time() - {{name:alternative_fee_provider_last_sync_timestamp_seconds}}{job="blockbook", coin="$coin"}) diff --git a/configs/grafana/template.json b/configs/grafana/template.json index aae664eed9..131a04f5ce 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5184,7 +5184,7 @@ }, "overrides": [] }, - "id": 45, + "id": 341, "options": { "legend": { "calcs": [ @@ -5207,29 +5207,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "blocks_1" - }, - { - "editorMode": "code", - "range": true, - "refId": "B", - "x-query-key": "blocks_3" - }, - { - "editorMode": "code", - "range": true, - "refId": "C", - "x-query-key": "blocks_5" - }, - { - "editorMode": "code", - "range": true, - "refId": "D", - "x-query-key": "blocks_8" + "x-query-key": "by_tier" } ], "type": "timeseries", - "x-panel-key": "fees.estimated_fee" + "x-panel-key": "fees.eip1559_max_fee" }, { "fieldConfig": { @@ -5239,7 +5221,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5262,7 +5244,7 @@ }, "overrides": [] }, - "id": 57, + "id": 342, "options": { "legend": { "calcs": [ @@ -5275,7 +5257,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -5285,11 +5267,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "requests" + "x-query-key": "by_tier" } ], "type": "timeseries", - "x-panel-key": "fees.alt_fee_provider_requests" + "x-panel-key": "fees.eip1559_priority_fee" }, { "fieldConfig": { @@ -5299,7 +5281,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5320,9 +5302,35 @@ }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*target.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, - "id": 341, + "id": 343, "options": { "legend": { "calcs": [ @@ -5345,11 +5353,17 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "by_tier" + "x-query-key": "ratio" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "threshold" } ], "type": "timeseries", - "x-panel-key": "fees.eip1559_max_fee" + "x-panel-key": "fees.eip1559_buffer_ratio" }, { "fieldConfig": { @@ -5359,7 +5373,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5382,7 +5396,7 @@ }, "overrides": [] }, - "id": 342, + "id": 344, "options": { "legend": { "calcs": [ @@ -5405,11 +5419,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "by_tier" + "x-query-key": "by_source" } ], "type": "timeseries", - "x-panel-key": "fees.eip1559_priority_fee" + "x-panel-key": "fees.eip1559_fee_source" }, { "fieldConfig": { @@ -5438,13 +5452,13 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*target.*" + "options": ".*cutoff.*" }, "properties": [ { @@ -5468,7 +5482,7 @@ } ] }, - "id": 343, + "id": 345, "options": { "legend": { "calcs": [ @@ -5491,17 +5505,17 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "ratio" + "x-query-key": "age" }, { "editorMode": "code", "range": true, "refId": "B", - "x-query-key": "threshold" + "x-query-key": "cutoff" } ], "type": "timeseries", - "x-panel-key": "fees.eip1559_buffer_ratio" + "x-panel-key": "fees.alt_fee_provider_cache_age" }, { "fieldConfig": { @@ -5534,7 +5548,7 @@ }, "overrides": [] }, - "id": 344, + "id": 57, "options": { "legend": { "calcs": [ @@ -5547,7 +5561,7 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -5557,11 +5571,11 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "by_source" + "x-query-key": "requests" } ], "type": "timeseries", - "x-panel-key": "fees.eip1559_fee_source" + "x-panel-key": "fees.alt_fee_provider_requests" }, { "fieldConfig": { @@ -5571,7 +5585,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5590,13 +5604,13 @@ } ] }, - "unit": "s" + "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*cutoff.*" + "options": ".*balance point.*" }, "properties": [ { @@ -5620,7 +5634,7 @@ } ] }, - "id": 345, + "id": 346, "options": { "legend": { "calcs": [ @@ -5643,17 +5657,17 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "age" + "x-query-key": "ratio" }, { "editorMode": "code", "range": true, "refId": "B", - "x-query-key": "cutoff" + "x-query-key": "balance" } ], "type": "timeseries", - "x-panel-key": "fees.alt_fee_provider_cache_age" + "x-panel-key": "fees.block_gas_used_ratio" }, { "fieldConfig": { @@ -5663,7 +5677,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5682,13 +5696,13 @@ } ] }, - "unit": "percentunit" + "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*balance point.*" + "options": ".*projected.*" }, "properties": [ { @@ -5700,19 +5714,12 @@ ], "fill": "dash" } - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } } ] } ] }, - "id": 346, + "id": 347, "options": { "legend": { "calcs": [ @@ -5735,17 +5742,17 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "ratio" + "x-query-key": "mined" }, { "editorMode": "code", "range": true, "refId": "B", - "x-query-key": "balance" + "x-query-key": "projected" } ], "type": "timeseries", - "x-panel-key": "fees.block_gas_used_ratio" + "x-panel-key": "fees.base_fee_push_vs_pull" }, { "fieldConfig": { @@ -5755,7 +5762,7 @@ }, "custom": { "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, @@ -5776,28 +5783,9 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*projected.*" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - } - ] - } - ] + "overrides": [] }, - "id": 347, + "id": 45, "options": { "legend": { "calcs": [ @@ -5820,17 +5808,29 @@ "editorMode": "code", "range": true, "refId": "A", - "x-query-key": "mined" + "x-query-key": "blocks_1" }, { "editorMode": "code", "range": true, "refId": "B", - "x-query-key": "projected" + "x-query-key": "blocks_3" + }, + { + "editorMode": "code", + "range": true, + "refId": "C", + "x-query-key": "blocks_5" + }, + { + "editorMode": "code", + "range": true, + "refId": "D", + "x-query-key": "blocks_8" } ], "type": "timeseries", - "x-panel-key": "fees.base_fee_push_vs_pull" + "x-panel-key": "fees.estimated_fee" }, { "collapsed": false, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index ed49e8f6a8..5018502f5e 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -376,7 +376,7 @@ metrics: alternative_fee_provider_last_sync_timestamp_seconds: name: blockbook_alternative_fee_provider_last_sync_timestamp_seconds type: gauge_vec - help: Unix time of the last successful refresh of the cached EVM alternative-provider fees, labeled by provider (infura or 1inch). Plot time() - this for cache age; the pull path falls back to on-chain once the cache is older than the stale window (30x the poll period, e.g. 30m at a 60s poll). Exported as a timestamp rather than a computed *_age_seconds gauge on purpose - it is only written on a successful refresh, so age keeps rising when a provider wedges and it is restart-safe + help: Unix time of the last successful refresh of the cached EVM alternative-provider fees, labeled by provider (infura or 1inch). Plot time() - this for cache age; the pull path falls back to on-chain once the cache is older than the provider's stale window (for infura, 30x the poll period, e.g. 30m at a 60s poll). Exported as a timestamp rather than a computed *_age_seconds gauge on purpose - it is only written on a successful refresh, so age keeps rising when a provider wedges and it is restart-safe labels: [provider] eth_eip1559_fee: name: blockbook_eth_eip1559_fee @@ -386,7 +386,7 @@ metrics: eth_eip1559_base_fee: name: blockbook_eth_eip1559_base_fee type: gauge - help: Next-block base fee underlying the most recent pull-path EIP-1559 estimate, in wei - divide by 1e9 for Gwei. The alternative provider's estimatedBaseFee when served from the provider, otherwise eth_feeHistory's projected next-block base fee; pair with eth_block_base_fee to see projection vs realized + help: Base fee underlying the most recent pull-path EIP-1559 estimate, in wei - divide by 1e9 for Gwei. The alternative provider's reported base fee when served from a provider, otherwise eth_feeHistory's last requested-window base fee (baseFeePerGas[blocks-1]); on backends that drop the extra projected element (e.g. Erigon) that is already the next-block projection, on others it is the most recent included block. Pair with eth_block_base_fee to see projection vs realized eth_eip1559_fee_source_total: name: blockbook_eth_eip1559_fee_source_total type: counter_vec From 06f001a905eafb63d6396df1f207e1cd9c7ce5d5 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 26 Jun 2026 04:44:46 +0000 Subject: [PATCH 024/303] fix(eth): guard short eth_feeHistory reward rows in on-chain tiers Bounds-check the per-tier column index so a non-compliant backend returning a reward row shorter than the requested percentile count is skipped instead of panicking EthereumTypeGetEip1559Fees. Divide the tip average by the rows that actually contributed, so a skipped short row does not deflate the tier. Document the accepted zero-tip case on idle chains, and add a regression test. Co-Authored-By: Claude Opus 4.8 --- bchain/coins/eth/eip1559_blockgas_test.go | 45 +++++++++++++++++++++++ bchain/coins/eth/ethrpc.go | 16 +++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/eip1559_blockgas_test.go b/bchain/coins/eth/eip1559_blockgas_test.go index bc21815ba9..7a7bc916f9 100644 --- a/bchain/coins/eth/eip1559_blockgas_test.go +++ b/bchain/coins/eth/eip1559_blockgas_test.go @@ -87,6 +87,51 @@ func TestEthereumTypeGetEip1559FeesOnChain(t *testing.T) { } } +// TestEthereumTypeGetEip1559FeesOnChainShortRewardRow asserts the on-chain tier loop tolerates a +// non-compliant eth_feeHistory whose reward rows are shorter than the requested percentile count: it +// must not panic on h.Reward[j][i], and the per-tier average must divide only by the rows that +// actually contributed a value (so a skipped short row does not deflate the tip). +func TestEthereumTypeGetEip1559FeesOnChainShortRewardRow(t *testing.T) { + // Row 0 has all 4 percentiles; row 1 has only 2. For tiers high(i=2) and instant(i=3) row 1 is + // short and must be skipped, leaving the average over row 0 alone. + raw := `{"oldestBlock":"0x1",` + + `"reward":[["0x1","0x2","0x3","0x4"],["0x3","0x4"]],` + + `"baseFeePerGas":["0x10","0x20","0x30","0x64"],` + + `"gasUsedRatio":[0.5,0.5,0.5]}` + b := &EthereumRPC{ + RPC: &feeHistoryRPCStub{raw: raw}, + Timeout: time.Second, + ChainConfig: &Configuration{Eip1559Fees: true}, + } + fees, err := b.EthereumTypeGetEip1559Fees() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fees == nil { + t.Fatal("expected fees") + } + cases := []struct { + name string + fee *bchain.Eip1559Fee + wantTip int64 + }{ + {"low", fees.Low, 2}, // avg(1,3) over both rows + {"medium", fees.Medium, 3}, // avg(2,4) over both rows + {"high", fees.High, 3}, // row 1 short -> avg(3) over row 0 only + {"instant", fees.Instant, 4}, // row 1 short -> avg(4) over row 0 only + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if c.fee == nil { + t.Fatal("nil tier") + } + if c.fee.MaxPriorityFeePerGas.Int64() != c.wantTip { + t.Errorf("MaxPriorityFeePerGas = %v, want %d", c.fee.MaxPriorityFeePerGas, c.wantTip) + } + }) + } +} + // gaugeVecSeriesCount reports how many label series a GaugeVec currently holds, using a throwaway // registry (same approach as gaugeValue, no prometheus/testutil dependency). func gaugeVecSeriesCount(t *testing.T, gv *prometheus.GaugeVec) int { diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 726a3b3b60..a2cc4081e7 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1995,14 +1995,26 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) for i := 0; i < 4; i++ { var f bchain.Eip1559Fee // Per-tier tip: average of the requested reward percentile (low=20th .. instant=99th) over the window. + // A compliant eth_feeHistory row has one reward per requested percentile, but guard the column index + // so a non-conforming backend returning a short row skips that row instead of panicking; the divisor + // counts only the rows actually summed so skipped rows don't deflate the average. priorityFee := int64(0) + rows := int64(0) for j := 0; j < len(h.Reward); j++ { + if len(h.Reward[j]) <= i { + continue + } p, _ := hexutil.DecodeUint64(h.Reward[j][i]) priorityFee += int64(p) + rows++ } - if len(h.Reward) > 0 { - priorityFee /= int64(len(h.Reward)) + if rows > 0 { + priorityFee /= rows } + // A zero tip is a deliberate, accepted outcome on idle chains: when eth_feeHistory reports empty or + // all-zero reward percentiles (quiet testnets such as Sepolia/Holesky, or a backend that omits + // rewards) there is no priority competition to price, so maxPriorityFeePerGas is 0. maxFeePerGas + // still covers eip1559BaseFeeMultiplier*baseFee below, so the tx stays mineable. tip := big.NewInt(priorityFee) f.MaxPriorityFeePerGas = tip // maxFeePerGas must cover the next-block base fee plus the tip, with headroom for base-fee From b524e5778124c3b9914a7f06e9f645f98ec8cb55 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Mon, 29 Jun 2026 15:11:02 +0000 Subject: [PATCH 025/303] fix(eth): set staleSyncDuration on 1inch fee provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1inch alternative fee provider never set staleSyncDuration, so it defaulted to the zero value (0s), making the cache immediately stale after every write. GetEip1559Fees always fell back to on-chain estimation via eth_feeHistory, defeating the purpose of having a provider cache. Mirror the same pattern used by the Infura provider: introduce oneInchFeeStalePeriods=30 and oneInchFeeStaleDuration, and set p.staleSyncDuration in the constructor so cached fees are reused within the 30×pollPeriod window (e.g. 30min for a 60s poll). --- bchain/coins/eth/oneinchfees.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bchain/coins/eth/oneinchfees.go b/bchain/coins/eth/oneinchfees.go index cb3c7485c2..6bb2656e6b 100644 --- a/bchain/coins/eth/oneinchfees.go +++ b/bchain/coins/eth/oneinchfees.go @@ -60,6 +60,12 @@ type oneInchFeeProvider struct { apiKey string } +const oneInchFeeStalePeriods = 30 + +func oneInchFeeStaleDuration(periodSeconds int) time.Duration { + return time.Duration(periodSeconds*oneInchFeeStalePeriods) * time.Second +} + // NewOneInchFeesProvider initializes https://api.1inch.dev provider func NewOneInchFeesProvider(chain bchain.BlockChain, params string, metrics *common.Metrics) (alternativeFeeProviderInterface, error) { p := &oneInchFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{metrics: metrics, name: "1inch"}} @@ -75,6 +81,7 @@ func NewOneInchFeesProvider(chain bchain.BlockChain, params string, metrics *com return nil, errors.New("NewOneInchFeesProvider: missing ONE_INCH_API_KEY env variable.") } p.chain = chain + p.staleSyncDuration = oneInchFeeStaleDuration(p.params.PeriodSeconds) go p.FeeDownloader() return p, nil } From 7d76bfffea50eb8a77a5227d81a2a91c4e78e8e8 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Mon, 29 Jun 2026 15:20:13 +0000 Subject: [PATCH 026/303] chore(eth): switch ethereum_archive fee provider from Infura to 1inch Replace the Infura gas API with 1inch Gas Price API for Ethereum mainnet archive. The 1inch API returns raw wei integers and uses Bearer token auth (ONE_INCH_API_KEY). The staleSyncDuration bug that made 1inch cache always immediately stale was fixed in the preceding commit. --- configs/coins/ethereum_archive.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/coins/ethereum_archive.json b/configs/coins/ethereum_archive.json index bb609155a6..754aab5c90 100644 --- a/configs/coins/ethereum_archive.json +++ b/configs/coins/ethereum_archive.json @@ -69,8 +69,8 @@ "consensusNodeVersion": "http://localhost:7516/eth/v1/node/version", "address_aliases": true, "eip1559Fees": true, - "alternative_estimate_fee": "infura", - "alternative_estimate_fee_params": "{\"url\": \"https://gas.api.infura.io/v3/${api_key}/networks/1/suggestedGasFees\", \"periodSeconds\": 60}", + "alternative_estimate_fee": "1inch", + "alternative_estimate_fee_params": "{\"url\": \"https://api.1inch.dev/gas-price/v1.5/1\", \"periodSeconds\": 60}", "mempoolTxTimeoutHours": 48, "processInternalTransactions": true, "queryBackendOnMempoolResync": false, From 0aa12d883184ead3645fdbafc6e2eb7e02d8f508 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Mon, 29 Jun 2026 15:26:40 +0000 Subject: [PATCH 027/303] fix(grafana,docs): make fee panels and docs provider-agnostic for 1inch Grafana: - alt_fee_provider_requests tooltip mode: single -> multi (inconsistent with other fee panels) - eip1559_max_fee description: generalize Infura-only language to mention both providers and clarify instant tier behavior differs per provider - eip1559_buffer_ratio description: generalize Infura-only padding reference - alt_fee_provider_cache_age description: generalize "Infura (the EVM default)" to mention both providers use the same 30x stale window docs/fees.md: - Replace Infura-only references with provider-agnostic language throughout - Keep accurate historical context (Infura as original source of fees-too-high reports) - Update mermaid flowcharts from "Infura fees" to "provider fees" - Document both Infura and 1inch behavior in the provider section --- configs/grafana/panels.yaml | 6 +++--- configs/grafana/template.json | 2 +- docs/fees.md | 23 ++++++++++++----------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 45be0a05c5..a3aae3f2b4 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -665,7 +665,7 @@ panels: legend: '{{instance}} - {{provider}} - {{status}}' fees.eip1559_max_fee: title: EIP-1559 maxFeePerGas by tier (Gwei) - description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On Infura-backed coins these are the provider's padded suggestions; on on-chain coins they are 2x baseFee + tip. The instant tier exists only on the on-chain path (Infura has no instant), so an instant line on a provider coin is a leftover from an onchain_fallback. Compare with the base fee and the buffer-ratio panel to tell a busy chain from an estimator regression. + description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On provider-backed coins (e.g. Infura, 1inch) these are the provider's suggestions; on on-chain coins they are 2x baseFee + tip. The instant tier exists only on the on-chain path and on providers that return four tiers (1inch), so an instant line appearing on a provider that does not return it means the provider cache was stale and the on-chain fallback served the estimate. Compare with the base fee and the buffer-ratio panel to tell a busy chain from an estimator regression. queries: by_tier: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / 1e9) @@ -679,7 +679,7 @@ panels: legend: '{{coin}} - {{instance}} - {{tier}}' fees.eip1559_buffer_ratio: title: EIP-1559 buffer ratio (maxFeePerGas / baseFee) - description: 'Per-tier maxFeePerGas divided by the next-block base fee - the headroom over the base fee. The on-chain estimator targets 2x by design (eip1559BaseFeeMultiplier), shown as the dashed line; on Infura-backed coins the high tier sits ~2.5x by design, so a steady ratio above 2 there is expected, not an anomaly - the 2x line is the on-chain target only. How to read it: a ratio flat near its source''s normal level while absolute fees are high means the chain is congested (nothing to fix); a ratio climbing well above that level means over-buffering - the "fees too high" signature.' + description: 'Per-tier maxFeePerGas divided by the next-block base fee - the headroom over the base fee. The on-chain estimator targets 2x by design (eip1559BaseFeeMultiplier), shown as the dashed line; provider-backed coins return their own padded suggestions which may differ (e.g. Infura''s high tier sits ~2.5x), so a steady ratio above 2 there is expected, not an anomaly - the 2x line is the on-chain target only. How to read it: a ratio flat near its source''s normal level while absolute fees are high means the chain is congested (nothing to fix); a ratio climbing well above that level means over-buffering - the "fees too high" signature.' queries: ratio: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / on(job, coin, instance) group_left {{name:eth_eip1559_base_fee}}{job="blockbook", coin="$coin"}) @@ -709,7 +709,7 @@ panels: legend: balance point (50%) fees.alt_fee_provider_cache_age: title: Alternative fee provider cache age - description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the provider''s stale window, after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). For Infura (the EVM default) the window is 30x the poll period; the dashed line marks the default 30-minute cutoff (30x a 60s poll), so adjust it if a coin polls differently. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' + description: 'Age of the cached provider fees (time() - last successful refresh), per provider. The pull path keeps serving the cache until it crosses the provider''s stale window, after which estimateFee falls back to on-chain (visible as onchain_fallback in the fee-source panel). Both Infura and 1inch use a 30x stale window (e.g. 30 minutes for a 60s poll); the dashed line marks this default cutoff, so adjust it if a coin polls with a different period. A sawtooth that resets each poll is healthy; a line climbing past the cutoff is a wedged provider.' queries: age: promql: (time() - {{name:alternative_fee_provider_last_sync_timestamp_seconds}}{job="blockbook", coin="$coin"}) diff --git a/configs/grafana/template.json b/configs/grafana/template.json index 131a04f5ce..8c9f907c6d 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -5561,7 +5561,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, diff --git a/docs/fees.md b/docs/fees.md index 9d6eabb230..3105b19a14 100644 --- a/docs/fees.md +++ b/docs/fees.md @@ -11,7 +11,7 @@ The split is deliberate: Blockbook supplies facts, the wallet decides policy. - base fee — the next-block projection, taken from the chain - priority-fee tiers (tips) -- congestion and trend signals (via Infura, when configured) +- congestion and trend signals (via the configured alternative provider, when supported) - per-block gas — `baseFee`, `gasUsed`, `gasLimit` **Trezor Suite — owns the fee policy:** @@ -23,21 +23,21 @@ The split is deliberate: Blockbook supplies facts, the wallet decides policy. - displays the fee options and signs the transaction This is why the wallet computes `maxFeePerGas` itself instead of trusting a provider's pre-padded -value (Infura's `suggestedMaxFeePerGas`, which is the source of the "fees too high" reports). +value (e.g. Infura's `suggestedMaxFeePerGas`, which was the source of the "fees too high" reports). --- ## Blockbook — pull path (`estimateFee` / `EthereumTypeGetEip1559Fees`) On-demand estimate. Returns `Eip1559Fees`: a `baseFeePerGas` plus `low/medium/high` (and `instant` -on the on-chain path) tiers, and — on the Infura path — congestion/trend ranges. +on the on-chain path) tiers, and — on supported provider paths — congestion/trend ranges. ```mermaid flowchart TD A["estimateFee"] --> B{"EIP-1559 supported?"} B -->|no| Z["legacy feePerUnit only"] - B -->|yes| C{"fresh Infura fees available?"} - C -->|yes| D["use cached Infura fees"] + B -->|yes| C{"fresh provider fees available?"} + C -->|yes| D["use cached provider fees"] C -->|no| E["estimate on-chain from base fee + tips"] D --> R["return Eip1559Fees
baseFee + fee tiers"] E --> R @@ -50,9 +50,11 @@ How each source is built: (array index `blocks-1`) and per-tier reward percentiles (20/70/90/99) used as tips. Blockbook builds `maxFeePerGas = eip1559BaseFeeMultiplier(2) × baseFee + tip`. (Previously this field held the tip alone — below the base fee, so not mineable; that was fixed.) -- **Infura** (archive coins, served from an in-memory cache, no node RPC): blockbook returns Infura's - `suggestedMaxFeePerGas` **unchanged**, which is padded well above the base fee (~2.5× for the high - tier). Blockbook does not rewrite it — the wallet overrides it (see the Suite section). +- **Alternative provider** (archive coins, served from an in-memory cache, no node RPC): blockbook + returns the provider's `maxFeePerGas` **unchanged**. For Infura, this is `suggestedMaxFeePerGas`, + padded well above the base fee (~2.5× for the high tier); for 1inch, it is the provider's own + computed `maxFeePerGas`. Blockbook does not rewrite it — the wallet overrides it (see the Suite + section). --- @@ -103,10 +105,9 @@ Details and current coverage: - The block-based estimate derives the next-block base fee from the latest block (EIP-1559 formula on `gasUsed`/`gasLimit`), takes tips from the block's transaction `maxPriorityFeePerGas` percentiles, and sets `maxFeePerGas = 2 × nextBaseFee + tip`. The fallback uses the backend's `eip1559` tiers. -- **evm-rpc-backed networks**: `getBlock('latest')` returns block gas → block-based fees apply, so the - Infura padding is gone. +- **evm-rpc-backed networks**: `getBlock('latest')` returns block gas → block-based fees apply. - **blockbook-backed networks**: `getBlock('latest')` is unsupported → block-based estimate is empty → - Suite falls back to the backend (Infura) tiers. Wiring the `subscribeNewBlock` `evmData` push as the + Suite falls back to the backend (provider) tiers. Wiring the `subscribeNewBlock` `evmData` push as the block-data source for these networks is the remaining step. --- From a2ec3b3d0f33cafd0c5ef7cdcad70d6bd93965df Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 30 Jun 2026 10:46:26 +0000 Subject: [PATCH 028/303] fix(eth): remap 1inch fee tiers for stable fees across providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1inch returns 4 tiers (low/medium/high/instant) with tighter per-tier values than Infura (3 tiers). The previous 1:1 mapping (suite low/medium/high ← 1inch low/medium/high) produced fees that were too conservative compared to Infura, causing ~2x fee differences between providers. Remap so the suite's low/medium/high tiers are fed from comparable aggressiveness: 1inch medium → suite low 1inch high → suite medium 1inch instant → suite high 1inch's low tier is discarded (too conservative to be useful). The fee formula (maxFeePerGas = 2×baseFee + tip) is already correct on the suite side (fix/evm-fee branch) — this change ensures the input tier values are comparable regardless of which provider serves the response. --- bchain/coins/eth/oneinchfees.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bchain/coins/eth/oneinchfees.go b/bchain/coins/eth/oneinchfees.go index 6bb2656e6b..9fbb919e6c 100644 --- a/bchain/coins/eth/oneinchfees.go +++ b/bchain/coins/eth/oneinchfees.go @@ -118,10 +118,15 @@ func oneInchFeesFromResult(result *oneInchFeeFeeResult) *bchain.Eip1559Fee { func (p *oneInchFeeProvider) processData(data *oneInchFeeFeesResult) bool { fees := bchain.Eip1559Fees{} fees.BaseFeePerGas = bigIntFromString(data.BaseFee) - fees.Instant = oneInchFeesFromResult(&data.Instant) - fees.High = oneInchFeesFromResult(&data.High) - fees.Medium = oneInchFeesFromResult(&data.Medium) - fees.Low = oneInchFeesFromResult(&data.Low) + // 1inch's tiers are tighter than Infura's. Remap so the suite's low/medium/high map + // to comparable aggressiveness regardless of provider: + // 1inch medium → suite low + // 1inch high → suite medium + // 1inch instant → suite high + // 1inch's low tier is discarded (too conservative). + fees.Low = oneInchFeesFromResult(&data.Medium) + fees.Medium = oneInchFeesFromResult(&data.High) + fees.High = oneInchFeesFromResult(&data.Instant) p.mux.Lock() defer p.mux.Unlock() p.observeSync(time.Now()) From 1446247f7d5b2d7d0b049c2b494d474ebe7accd2 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 10:56:16 +0200 Subject: [PATCH 029/303] fix(L2-config): disable mempool sync on OP Stack and Arbitrum chains op-geth and arbitrum-nitro do not implement the newPendingTransactions subscription filter. Without disableMempoolSync: true, Blockbook logs: initializeMempool EthSubscribe newPendingTransactions: invalid subscription type for subscribe and wastes resources on repeated resubscribe attempts. This was already fixed for bsc_archive, polygon_archive, and base_archive. Apply the same fix to the remaining L2 coin configs: - optimism, optimism_archive - base - arbitrum, arbitrum_archive - arbitrum_nova, arbitrum_nova_archive --- configs/coins/arbitrum.json | 1 + configs/coins/arbitrum_archive.json | 1 + configs/coins/arbitrum_nova.json | 1 + configs/coins/arbitrum_nova_archive.json | 1 + configs/coins/base.json | 1 + configs/coins/optimism.json | 1 + configs/coins/optimism_archive.json | 1 + 7 files changed, 7 insertions(+) diff --git a/configs/coins/arbitrum.json b/configs/coins/arbitrum.json index ec8b5eb6ff..ff67f198bd 100644 --- a/configs/coins/arbitrum.json +++ b/configs/coins/arbitrum.json @@ -55,6 +55,7 @@ "averageBlockTimeMs": 250, "mempoolTxTimeoutHours": 12, "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"arbitrum-one\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}" diff --git a/configs/coins/arbitrum_archive.json b/configs/coins/arbitrum_archive.json index 4611e59218..7597005f57 100644 --- a/configs/coins/arbitrum_archive.json +++ b/configs/coins/arbitrum_archive.json @@ -67,6 +67,7 @@ "processInternalTransactions": true, "trace_timeout": "10s", "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"arbitrum-one\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}", diff --git a/configs/coins/arbitrum_nova.json b/configs/coins/arbitrum_nova.json index 513d159262..ac0819b226 100644 --- a/configs/coins/arbitrum_nova.json +++ b/configs/coins/arbitrum_nova.json @@ -54,6 +54,7 @@ "averageBlockTimeMs": 250, "mempoolTxTimeoutHours": 12, "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"ethereum\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}" diff --git a/configs/coins/arbitrum_nova_archive.json b/configs/coins/arbitrum_nova_archive.json index 41272e4bef..e3af7fe068 100644 --- a/configs/coins/arbitrum_nova_archive.json +++ b/configs/coins/arbitrum_nova_archive.json @@ -57,6 +57,7 @@ "processInternalTransactions": true, "trace_timeout": "10s", "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"ethereum\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}", diff --git a/configs/coins/base.json b/configs/coins/base.json index 218346c459..fdad2bf36a 100644 --- a/configs/coins/base.json +++ b/configs/coins/base.json @@ -56,6 +56,7 @@ "averageBlockTimeMs": 2000, "mempoolTxTimeoutHours": 12, "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"base\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}" diff --git a/configs/coins/optimism.json b/configs/coins/optimism.json index 8cd1a4cd0c..21f0a35e7d 100644 --- a/configs/coins/optimism.json +++ b/configs/coins/optimism.json @@ -56,6 +56,7 @@ "averageBlockTimeMs": 2000, "mempoolTxTimeoutHours": 12, "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"optimistic-ethereum\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}" diff --git a/configs/coins/optimism_archive.json b/configs/coins/optimism_archive.json index ec273cbe7c..fbf6d934a2 100644 --- a/configs/coins/optimism_archive.json +++ b/configs/coins/optimism_archive.json @@ -69,6 +69,7 @@ "processInternalTransactions": true, "trace_timeout": "10s", "queryBackendOnMempoolResync": false, + "disableMempoolSync": true, "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"ethereum\",\"platformIdentifier\": \"optimistic-ethereum\",\"platformVsCurrency\": \"eth\",\"periodSeconds\": 900}", From 3af322ef3a99ee6337c0ec689042a320cf073e2b Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 20:41:17 +0000 Subject: [PATCH 030/303] feat: default WS_GETACCOUNTINFO_LIMIT to 42, matching Suite client concurrency Trezor Suite's Blockbook websocket client caps concurrent in-flight requests at 42 (blockchain-link/src/workers/blockbook/index.ts:481). The old default of 0 disabled the server-side limit entirely, making per-connection getAccountInfo abuse bounded only by the 2500 msgs/10 min rate limiter. Change the default to 42 so the server-side descriptor-count limit activates automatically. The env var override is preserved for operators who want a different value. --- blockbook.go | 7 ++++++- docs/env.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/blockbook.go b/blockbook.go index 6888fdaaf5..4990aa02e9 100644 --- a/blockbook.go +++ b/blockbook.go @@ -571,7 +571,12 @@ func newInternalState(config *common.Config, d *db.RocksDB, enableSubNewTx bool) is.Host = name } - is.WsGetAccountInfoLimit, _ = strconv.Atoi(os.Getenv(strings.ToUpper(is.GetNetwork()) + "_WS_GETACCOUNTINFO_LIMIT")) + limitStr := os.Getenv(strings.ToUpper(is.GetNetwork()) + "_WS_GETACCOUNTINFO_LIMIT") + if limitStr == "" { + is.WsGetAccountInfoLimit = 42 + } else { + is.WsGetAccountInfoLimit, _ = strconv.Atoi(limitStr) + } if is.WsGetAccountInfoLimit > 0 { glog.Info("WsGetAccountInfoLimit enabled with limit ", is.WsGetAccountInfoLimit) is.WsLimitExceedingIPs = make(map[string]int) diff --git a/docs/env.md b/docs/env.md index f0b01df7c9..75217268bb 100644 --- a/docs/env.md +++ b/docs/env.md @@ -6,7 +6,7 @@ Blockbook reads these from its process environment. When installed from the Debi - `BB_ADMIN_USER` / `BB_ADMIN_PASSWORD` - **Global (not coin-prefixed).** HTTP Basic-auth credentials required to reach the internal server's `/admin` endpoints (the administrative pages and the state-mutating POST handlers such as internal-data refetch and contract-info updates). Basic auth is used so the admin pages and forms work directly in a browser via its native login prompt (and `curl -u user:pass` for scripts). The admin surface is **fail-closed**: unless **both** variables are set, every `/admin` route returns `503` and the endpoints are unusable; `/metrics`, the status page (`/`) and static assets are unaffected. A request with missing or wrong credentials gets `401`. Leading/trailing whitespace in either value is ignored, so a stray space or newline in `blockbook.env` will not lock you out. The internal server binds all interfaces by default (`internal_binding_template` is `:`), so set these on every host. Note that the packaged service serves the internal port over HTTPS using the **bundled self-signed certificate** (`cert/blockbook.{crt,key}`, symlinked to the repo's `testcert`), whose private key is public — so that TLS protects the credentials against passive sniffing but not against an active man-in-the-middle. This is acceptable on a trusted, firewalled internal segment (the intended deployment); from a shell you can reach it as e.g. `curl -k -u "$BB_ADMIN_USER:$BB_ADMIN_PASSWORD" https://host:/admin/...`. If the internal network is not trusted, terminate real TLS at a reverse proxy and/or restrict the internal port to trusted peers. Do not expose it directly to the internet. -- `_WS_GETACCOUNTINFO_LIMIT` - Limits the number of `getAccountInfo` requests per websocket connection to reduce server abuse. Accepts number as input. +- `_WS_GETACCOUNTINFO_LIMIT` - Limits the number of `getAccountInfo` requests per websocket connection to reduce server abuse. Accepts number as input. Defaults to `42` (matching the Trezor Suite client concurrency limit). - `_WS_BALANCE_HISTORY_MAX_TXS` / `_REST_BALANCE_HISTORY_MAX_TXS` - Maximum number of transactions a single balance-history request (for an address or an xpub) may aggregate, set independently for the WebSocket `getBalanceHistory` method and the REST `/api/v2/balancehistory/...` endpoint. Each aggregated transaction costs a database read, so an unbounded request over an address or xpub with a very large history (e.g. an exchange address) is a cheap-to-send, expensive-to-serve request. Past the cap the request is rejected with `400` and a message asking to narrow the `from`/`to` range, rather than returning a truncated (and therefore wrong) history. Accepts a non-negative integer; `0` disables the cap. From 1f19251ae00f7928dac3aa4ea48308a5ad9dde0f Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 12:46:21 +0200 Subject: [PATCH 031/303] perf(e2e): parallelize cross-coin execution, preload samples, pool WS connections, reduce timeouts - Use Promise.all for cross-coin execution: coins run concurrently (they connect to independent backends). Each coin buffers its own output and flushes contiguously after completion. - Add preloadSamples() to TestContext: eagerly resolves block, tx, address, and fiat samples once at coin startup so downstream tests hit the cache instead of triggering redundant probe chains. - Replace per-call WebSocket connections with a wsPool: maintains 3 persistent connections, eliminating the 5s handshake overhead for every WS test. Pool acquires/releases per request with automatic dead-connection replacement. - Reduce timeouts: HTTP 30s->15s, WS dial 5s->3s, WS message 15s->10s. --- tests/openapi/src/client.ts | 2 +- tests/openapi/src/constants.ts | 4 +- tests/openapi/src/context.ts | 265 +++++++++++++++++++++++++++------ tests/openapi/src/runner.ts | 54 ++++--- 4 files changed, 260 insertions(+), 65 deletions(-) diff --git a/tests/openapi/src/client.ts b/tests/openapi/src/client.ts index 28a405bd97..07c10c0e2a 100644 --- a/tests/openapi/src/client.ts +++ b/tests/openapi/src/client.ts @@ -61,7 +61,7 @@ export class OpenApiFetchClient { try { const response = await fetch(url, { method: "GET", - signal: AbortSignal.timeout(30_000), + signal: AbortSignal.timeout(15_000), }); const body = await response.text(); if (attempt < 2 && isRetryableHTTPStatus(response.status)) { diff --git a/tests/openapi/src/constants.ts b/tests/openapi/src/constants.ts index 0fc1fe54d7..3bc5ee9175 100644 --- a/tests/openapi/src/constants.ts +++ b/tests/openapi/src/constants.ts @@ -1,5 +1,5 @@ -export const wsDialTimeoutMs = 5_000; -export const wsMessageTimeoutMs = 15_000; +export const wsDialTimeoutMs = 3_000; +export const wsMessageTimeoutMs = 10_000; export const txSearchWindow = 12; export const blockPageSize = 1; export const sampleBlockPageSize = 3; diff --git a/tests/openapi/src/context.ts b/tests/openapi/src/context.ts index 9165bc03f8..15569422b1 100644 --- a/tests/openapi/src/context.ts +++ b/tests/openapi/src/context.ts @@ -31,6 +31,181 @@ import { import type { Capability, AddressResponse, BlockHashResponse, BlockResponse, BlockSummary, FiatTickerResponse, StatusResponse, TxResponse, UtxoResponse, WsEnvelope, WsInfoResponse, WsMethod, WsResponse } from "./types.js"; +// --------------------------------------------------------------------------- +// WebSocket connection pool +// --------------------------------------------------------------------------- + +// wsConnection wraps a single persistent WebSocket and multiplexes requests +// over it by matching responses to the originating request ID. +class wsConnection { + private ws: WebSocket | null = null; + private pendings = new Map< + string, + { resolve: (value: unknown) => void; reject: (reason: unknown) => void; timer: NodeJS.Timeout } + >(); + private closed = false; + + constructor( + private readonly url: string, + private readonly contract: OpenApiContract, + ) {} + + get isClosed(): boolean { + return this.closed; + } + + connect(): Promise { + return new Promise((resolve, reject) => { + this.ws = new WebSocket(this.url, { + handshakeTimeout: wsDialTimeoutMs, + rejectUnauthorized: process.env.OPENAPI_INSECURE_TLS === "0", + agent: wsProxyAgent(), + }); + this.ws.on("open", () => resolve()); + this.ws.on("error", (err) => { + this.closed = true; + reject(err); + }); + this.ws.on("close", () => { + this.closed = true; + for (const [, p] of this.pendings) { + clearTimeout(p.timer); + p.reject(new Error("websocket connection closed")); + } + this.pendings.clear(); + }); + this.ws.on("message", (data) => { + let response: WsResponse; + try { + response = JSON.parse(data.toString()) as WsResponse; + } catch { + return; + } + const pending = this.pendings.get(response.id); + if (!pending) { + return; + } + clearTimeout(pending.timer); + this.pendings.delete(response.id); + if (isWsError(response.data)) { + pending.reject( + new Error(`websocket ${response.id} returned error: ${response.data.error.message}`), + ); + return; + } + pending.resolve(response.data); + }); + }); + } + + async request(request: WsEnvelope, dataSchemaRef?: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pendings.delete(request.id); + reject(new Error(`websocket ${request.method} timed out for ${this.url}`)); + }, wsMessageTimeoutMs); + this.pendings.set(request.id, { + resolve: (value) => { + if (dataSchemaRef) { + this.contract.validateSchemaRef(dataSchemaRef, `WS ${request.method} response data`, value); + } + resolve(value as T); + }, + reject, + timer, + }); + if (this.ws) { + this.ws.send(JSON.stringify(request)); + } else { + clearTimeout(timer); + this.pendings.delete(request.id); + reject(new Error("websocket not connected")); + } + }); + } + + close(): void { + this.closed = true; + this.ws?.close(); + this.ws = null; + } +} + +// wsPool maintains a small set of persistent WebSocket connections so that +// WS tests don't pay a handshake timeout per call. acquire/release follow +// a simple lease pattern; callers must release after the request completes. +class wsPool { + private conns: wsConnection[] = []; + private busy = new Set(); + + constructor( + private readonly url: string, + private readonly contract: OpenApiContract, + private readonly size: number, + ) {} + + get aliveCount(): number { + return this.conns.length; + } + + async init(): Promise { + const errors: Error[] = []; + for (let i = 0; i < this.size; i++) { + try { + const c = new wsConnection(this.url, this.contract); + await c.connect(); + this.conns.push(c); + } catch (error) { + errors.push(error instanceof Error ? error : new Error(String(error))); + } + } + // If none connected, propagate the first error so the caller can attempt + // a protocol upgrade (ws→wss). A partial pool is still usable. + if (this.conns.length === 0 && errors.length > 0) { + throw errors[0]; + } + } + + async acquire(): Promise { + // eslint-disable-next-line no-constant-condition + while (true) { + for (const c of this.conns) { + if (!this.busy.has(c) && !c.isClosed) { + this.busy.add(c); + return c; + } + } + // All connections are busy or dead. If any died, replace them lazily. + for (let i = 0; i < this.conns.length; i++) { + if (this.conns[i].isClosed) { + try { + const c = new wsConnection(this.url, this.contract); + await c.connect(); + this.conns[i] = c; + this.busy.add(c); + return c; + } catch { + // Replacement failed, try the next slot. + } + } + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + } + + release(conn: wsConnection): void { + this.busy.delete(conn); + } + + closeAll(): void { + for (const c of this.conns) { + c.close(); + } + this.conns = []; + this.busy.clear(); + } +} + // proxyFromEnv returns the configured egress proxy URL (sandboxed/corporate networks that only allow // outbound traffic via HTTP(S)_PROXY), or "" when none is set. Shared by the fetch dispatcher setup // in runner.ts and the ws client below. @@ -51,6 +226,7 @@ function wsProxyAgent(): HttpsProxyAgent | undefined { export class TestContext { readonly client: OpenApiFetchClient; + private wsPool_: wsPool | undefined; private status?: NonNullable; private nextWSReq = 0; @@ -139,6 +315,26 @@ export class TestContext { return found; } + // preloadSamples eagerly resolves every sample type so that all downstream + // tests hit the cache rather than triggering redundant probe chains. Safe to + // call multiple times; subsequent calls are no-ops because every resolver is + // guarded by a resolved flag. A SkipTest from a fiat ticker that is + // temporarily unavailable is swallowed — tests that need fiat data check the + // availability flag and skip themselves. + async preloadSamples() { + await this.getSampleIndexedBlock(); + await this.getSampleTxID(); + await this.getSampleAddress(); + await this.getSampleAddressTx(); + try { + await this.sampleFiatTickerOrSkip(); + } catch (error) { + if (!(error instanceof SkipTest)) { + throw error; + } + } + } + async getSampleIndexedHeight() { if (this.sampleIndexResolved) { return this.sampleIndexHash ? { height: this.sampleIndexHeight, hash: this.sampleIndexHash } : undefined; @@ -498,17 +694,32 @@ export class TestContext { } async wsCallWithID(id: string, method: WsMethod, params: unknown, dataSchemaRef?: string) { + // Lazily initialise the WS connection pool. The first call tries the + // configured ws:// URL; if that fails the pool tries wss:// and updates + // this.wsURL so subsequent calls skip the fallback. + if (!this.wsPool_) { + this.wsPool_ = new wsPool(this.wsURL, this.contract, 3); + try { + await this.wsPool_.init(); + } catch (firstError) { + const upgraded = upgradeWSBaseToWSS(this.wsURL); + if (!upgraded) { + this.wsPool_ = undefined; + throw firstError; + } + const pool2 = new wsPool(upgraded, this.contract, 3); + await pool2.init(); + this.wsPool_ = pool2; + this.wsURL = upgraded; + } + } + const request: WsEnvelope = { id, method, params }; + const conn = await this.wsPool_.acquire(); try { - return await this.wsCallOnce(this.wsURL, request, dataSchemaRef); - } catch (error) { - const upgraded = upgradeWSBaseToWSS(this.wsURL); - if (!upgraded) { - throw error; - } - const result = await this.wsCallOnce(upgraded, request, dataSchemaRef); - this.wsURL = upgraded; - return result; + return await conn.request(request, dataSchemaRef); + } finally { + this.wsPool_.release(conn); } } @@ -660,41 +871,5 @@ export class TestContext { return summary; } - private wsCallOnce(wsURL: string, request: WsEnvelope, dataSchemaRef?: string) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsURL, { - handshakeTimeout: wsDialTimeoutMs, - rejectUnauthorized: process.env.OPENAPI_INSECURE_TLS === "0", - agent: wsProxyAgent(), - }); - const timeout = setTimeout(() => { - ws.terminate(); - reject(new Error(`websocket ${request.method} timed out for ${wsURL}`)); - }, wsMessageTimeoutMs); - ws.on("open", () => { - ws.send(JSON.stringify(request)); - }); - ws.on("message", (data) => { - const response = JSON.parse(data.toString()) as WsResponse; - if (response.id !== request.id) { - return; - } - clearTimeout(timeout); - ws.close(); - if (isWsError(response.data)) { - reject(new Error(`websocket ${request.method} returned error: ${response.data.error.message}`)); - return; - } - if (dataSchemaRef) { - this.contract.validateSchemaRef(dataSchemaRef, `WS ${request.method} response data`, response.data); - } - resolve(response.data as T); - }); - ws.on("error", (error) => { - clearTimeout(timeout); - reject(error); - }); - }); - } } diff --git a/tests/openapi/src/runner.ts b/tests/openapi/src/runner.ts index b774ee69dd..1021f1086a 100644 --- a/tests/openapi/src/runner.ts +++ b/tests/openapi/src/runner.ts @@ -34,17 +34,29 @@ export async function runOpenApiE2E() { return; } - const summaries: CoinSummary[] = []; - for (const coin of selectedCoins) { - summaries.push(await runCoin(coin, contract)); + // Run coins in parallel — they connect to independent Blockbook instances, + // so there is no shared state to contend over. Each coin buffers its own + // console output and flushes it after completion so per-coin lines are + // contiguous rather than interleaved. + type CoinResult = { summary: CoinSummary; output: string[] }; + const results: CoinResult[] = await Promise.all( + selectedCoins.map(async (coin) => runCoin(coin, contract)), + ); + + // Flush buffered per-coin output in the original order. + for (const { output } of results) { + for (const line of output) { + console.log(line); + } } // Per-coin and aggregate summary so a run that is green-but-empty (everything skipped) is // visible at a glance, not hidden behind a zero-failure exit. console.log("\nOpenAPI e2e summary:"); - for (const summary of summaries) { + for (const { summary } of results) { console.log(` ${summary.coin}: ${summary.ok} ok, ${summary.skip} skip, ${summary.fail} fail`); } + const summaries = results.map((r) => r.summary); const totals = summaries.reduce( (acc, s) => ({ ok: acc.ok + s.ok, skip: acc.skip + s.skip, fail: acc.fail + s.fail }), { ok: 0, skip: 0, fail: 0 }, @@ -78,17 +90,21 @@ export async function runOpenApiE2E() { console.log(`\nOpenAPI e2e passed for ${summaries.length} coin(s): ${selectedCoins.join(", ")}`); } -async function runCoin(coin: string, contract: OpenApiContract): Promise { +async function runCoin(coin: string, contract: OpenApiContract): Promise<{ summary: CoinSummary; output: string[] }> { const testsConfig = loadTestsConfig(); const apiTests = testsConfig[coin]?.api ?? []; const results: TestResult[] = []; + const output: string[] = []; + + const emit = (msg: string) => output.push(msg); + if (apiTests.length === 0) { - console.log(`OpenAPI e2e ${coin}: no api tests configured, skipping.`); - return summarize(coin, results); + emit(`OpenAPI e2e ${coin}: no api tests configured, skipping.`); + return { summary: summarize(coin, results), output }; } const ctx = await TestContext.create(coin, contract); - console.log(`\nOpenAPI e2e ${coin}: ${apiTests.length} tests`); + emit(`\nOpenAPI e2e ${coin}: ${apiTests.length} tests`); // Status preflight: if the node is unreachable or not in sync, every sample-derived test would // fail or skip downstream for the same root cause. Surface it once as a single coin-level failure @@ -97,16 +113,20 @@ async function runCoin(coin: string, contract: OpenApiContract): Promise 0) { const message = `coin ${coin} had 0 passing tests (${summary.skip} skipped) — likely not in sync or the recent-block window yielded no usable sample data`; - console.error(` fail (coin health): ${message}`); + emit(` fail (coin health): ${message}`); results.push({ coin, name: "CoinHealth", group: "health", status: "fail", durationMs: 0, message }); - return summarize(coin, results); + return { summary: summarize(coin, results), output }; } - return summary; + return { summary, output }; } From ca081c7bc8f0d9951a9e8fc687620378bcdaae5b Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 13:04:45 +0200 Subject: [PATCH 032/303] fix(e2e): add connect timeout to WS pool, skip heavy preload, flush output per-coin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a mandatory timeout to wsConnection.connect() using a 'settled' guard that rejects after wsDialTimeoutMs even when DNS or TCP stalls (the ws library's handshakeTimeout only applies after TCP connect). - Make wsPool.init() parallel (Promise.allSettled) so all connections are attempted concurrently; a partial pool is still usable. - Remove getSampleAddressTx from preloadSamples() — it probes up to 40 transactions with paginated lookups and is only needed by a few address-listing tests; keep it lazy. - Flush each coin's output immediately upon completion instead of waiting for Promise.all, so results from finished coins are visible even if another coin hangs. - Use Promise.allSettled instead of Promise.all so a rejection in one coin does not discard results from other coins. --- tests/openapi/src/context.ts | 63 ++++++++++++++++++++++++------------ tests/openapi/src/runner.ts | 26 +++++++-------- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/tests/openapi/src/context.ts b/tests/openapi/src/context.ts index 15569422b1..63ebcc2123 100644 --- a/tests/openapi/src/context.ts +++ b/tests/openapi/src/context.ts @@ -55,18 +55,40 @@ class wsConnection { } connect(): Promise { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { + // Guard against hangs below the ws handshake layer (DNS / TCP). + let settled = false; + const guard = setTimeout(() => { + if (settled) { return; } + settled = true; + this.ws?.terminate(); + reject(new Error(`websocket connect timed out for ${this.url}`)); + }, wsDialTimeoutMs); + this.ws = new WebSocket(this.url, { handshakeTimeout: wsDialTimeoutMs, rejectUnauthorized: process.env.OPENAPI_INSECURE_TLS === "0", agent: wsProxyAgent(), }); - this.ws.on("open", () => resolve()); + this.ws.on("open", () => { + if (settled) { return; } + settled = true; + clearTimeout(guard); + resolve(); + }); this.ws.on("error", (err) => { + if (settled) { return; } + settled = true; + clearTimeout(guard); this.closed = true; reject(err); }); this.ws.on("close", () => { + if (!settled) { + settled = true; + clearTimeout(guard); + reject(new Error("websocket closed before handshake completed")); + } this.closed = true; for (const [, p] of this.pendings) { clearTimeout(p.timer); @@ -149,20 +171,23 @@ class wsPool { } async init(): Promise { - const errors: Error[] = []; - for (let i = 0; i < this.size; i++) { - try { + const results = await Promise.allSettled( + Array.from({ length: this.size }, () => { const c = new wsConnection(this.url, this.contract); - await c.connect(); - this.conns.push(c); - } catch (error) { - errors.push(error instanceof Error ? error : new Error(String(error))); + return c.connect().then(() => c); + }), + ); + for (const r of results) { + if (r.status === "fulfilled") { + this.conns.push(r.value); } } - // If none connected, propagate the first error so the caller can attempt - // a protocol upgrade (ws→wss). A partial pool is still usable. - if (this.conns.length === 0 && errors.length > 0) { - throw errors[0]; + // A partial pool (fewer than this.size connections) is still usable — + // acquire() replaces dead slots lazily. If nothing connected, propagate + // the first rejection so the caller can attempt a protocol upgrade. + if (this.conns.length === 0) { + const first = results.find((r) => r.status === "rejected"); + throw first ? (first as PromiseRejectedResult).reason : new Error("wsPool init: no connections and no errors"); } } @@ -315,17 +340,15 @@ export class TestContext { return found; } - // preloadSamples eagerly resolves every sample type so that all downstream - // tests hit the cache rather than triggering redundant probe chains. Safe to - // call multiple times; subsequent calls are no-ops because every resolver is - // guarded by a resolved flag. A SkipTest from a fiat ticker that is - // temporarily unavailable is swallowed — tests that need fiat data check the - // availability flag and skip themselves. + // preloadSamples eagerly resolves the most commonly-needed sample types so + // that downstream tests hit the cache rather than triggering redundant probe + // chains. getSampleAddressTx is deliberately excluded here — it can probe + // 40+ transactions with paginated address lookups and is only needed by a + // handful of tests; it stays lazy. async preloadSamples() { await this.getSampleIndexedBlock(); await this.getSampleTxID(); await this.getSampleAddress(); - await this.getSampleAddressTx(); try { await this.sampleFiatTickerOrSkip(); } catch (error) { diff --git a/tests/openapi/src/runner.ts b/tests/openapi/src/runner.ts index 1021f1086a..675312fa99 100644 --- a/tests/openapi/src/runner.ts +++ b/tests/openapi/src/runner.ts @@ -36,27 +36,25 @@ export async function runOpenApiE2E() { // Run coins in parallel — they connect to independent Blockbook instances, // so there is no shared state to contend over. Each coin buffers its own - // console output and flushes it after completion so per-coin lines are - // contiguous rather than interleaved. - type CoinResult = { summary: CoinSummary; output: string[] }; - const results: CoinResult[] = await Promise.all( - selectedCoins.map(async (coin) => runCoin(coin, contract)), + // console output and flushes it immediately upon completion so that even + // if another coin hangs the completed results are still visible. + const summaries: CoinSummary[] = []; + await Promise.allSettled( + selectedCoins.map(async (coin) => { + const { summary, output } = await runCoin(coin, contract); + for (const line of output) { + console.log(line); + } + summaries.push(summary); + }), ); - // Flush buffered per-coin output in the original order. - for (const { output } of results) { - for (const line of output) { - console.log(line); - } - } - // Per-coin and aggregate summary so a run that is green-but-empty (everything skipped) is // visible at a glance, not hidden behind a zero-failure exit. console.log("\nOpenAPI e2e summary:"); - for (const { summary } of results) { + for (const summary of summaries) { console.log(` ${summary.coin}: ${summary.ok} ok, ${summary.skip} skip, ${summary.fail} fail`); } - const summaries = results.map((r) => r.summary); const totals = summaries.reduce( (acc, s) => ({ ok: acc.ok + s.ok, skip: acc.skip + s.skip, fail: acc.fail + s.fail }), { ok: 0, skip: 0, fail: 0 }, From b869ab56d439ebe4d577dc9a03355d6b9b780ac1 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 19:02:36 +0000 Subject: [PATCH 033/303] fix(e2e): reject pending WS request when ws.send() throws, guard pool init race E2e-only remainder of the original review-findings commit; its tests/integration.go part (configurable init retry) was dropped along with the Go-side test parallelization. --- tests/openapi/src/context.ts | 54 ++++++++++++++++++++++++------------ tests/openapi/src/runner.ts | 8 ++++-- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/openapi/src/context.ts b/tests/openapi/src/context.ts index 63ebcc2123..01e93bc360 100644 --- a/tests/openapi/src/context.ts +++ b/tests/openapi/src/context.ts @@ -137,7 +137,13 @@ class wsConnection { timer, }); if (this.ws) { - this.ws.send(JSON.stringify(request)); + try { + this.ws.send(JSON.stringify(request)); + } catch (err) { + clearTimeout(timer); + this.pendings.delete(request.id); + reject(err); + } } else { clearTimeout(timer); this.pendings.delete(request.id); @@ -252,6 +258,7 @@ function wsProxyAgent(): HttpsProxyAgent | undefined { export class TestContext { readonly client: OpenApiFetchClient; private wsPool_: wsPool | undefined; + private wsPoolInit_: Promise | undefined; private status?: NonNullable; private nextWSReq = 0; @@ -720,29 +727,36 @@ export class TestContext { // Lazily initialise the WS connection pool. The first call tries the // configured ws:// URL; if that fails the pool tries wss:// and updates // this.wsURL so subsequent calls skip the fallback. - if (!this.wsPool_) { - this.wsPool_ = new wsPool(this.wsURL, this.contract, 3); - try { - await this.wsPool_.init(); - } catch (firstError) { - const upgraded = upgradeWSBaseToWSS(this.wsURL); - if (!upgraded) { - this.wsPool_ = undefined; - throw firstError; + // wsPoolInit_ guards against concurrent calls creating two pools. + if (!this.wsPoolInit_) { + this.wsPoolInit_ = (async () => { + const pool = new wsPool(this.wsURL, this.contract, 3); + try { + await pool.init(); + } catch (firstError) { + const upgraded = upgradeWSBaseToWSS(this.wsURL); + if (!upgraded) { + throw firstError; + } + const pool2 = new wsPool(upgraded, this.contract, 3); + await pool2.init(); + this.wsURL = upgraded; + this.wsPool_ = pool2; + return; } - const pool2 = new wsPool(upgraded, this.contract, 3); - await pool2.init(); - this.wsPool_ = pool2; - this.wsURL = upgraded; - } + this.wsPool_ = pool; + })(); } + await this.wsPoolInit_; + // wsPoolInit_ resolved, so wsPool_ is guaranteed to be set. + const pool = this.wsPool_!; const request: WsEnvelope = { id, method, params }; - const conn = await this.wsPool_.acquire(); + const conn = await pool.acquire(); try { return await conn.request(request, dataSchemaRef); } finally { - this.wsPool_.release(conn); + pool.release(conn); } } @@ -755,6 +769,12 @@ export class TestContext { return isEVMAddressValue(address) || (this.coin === "tron" && isTronAddress(address)); } + close() { + this.wsPool_?.closeAll(); + this.wsPool_ = undefined; + this.wsPoolInit_ = undefined; + } + private async probeCapabilities() { return { utxo: await this.probeUTXOSupport(), diff --git a/tests/openapi/src/runner.ts b/tests/openapi/src/runner.ts index 675312fa99..6b96f2f046 100644 --- a/tests/openapi/src/runner.ts +++ b/tests/openapi/src/runner.ts @@ -102,7 +102,8 @@ async function runCoin(coin: string, contract: OpenApiContract): Promise<{ summa } const ctx = await TestContext.create(coin, contract); - emit(`\nOpenAPI e2e ${coin}: ${apiTests.length} tests`); + try { + emit(`\nOpenAPI e2e ${coin}: ${apiTests.length} tests`); // Status preflight: if the node is unreachable or not in sync, every sample-derived test would // fail or skip downstream for the same root cause. Surface it once as a single coin-level failure @@ -162,5 +163,8 @@ async function runCoin(coin: string, contract: OpenApiContract): Promise<{ summa return { summary: summarize(coin, results), output }; } - return { summary, output }; + return { summary, output }; + } finally { + ctx.close(); + } } From f4c518b3d64e540aafe203757c4e2b7a491337db Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 14:39:14 +0200 Subject: [PATCH 034/303] feat: add rpc_sync_latency histogram for sync-internal RPC calls --- bchain/coins/eth/ethrpc.go | 27 ++++++++++++-- common/metrics.go | 1 + configs/grafana/panels.yaml | 10 ++++++ configs/grafana/template.json | 66 +++++++++++++++++++++++++++++++++++ configs/metrics.yaml | 6 ++++ 5 files changed, 108 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index a2cc4081e7..12922ad9da 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -395,6 +395,17 @@ func (b *EthereumRPC) observeEthSyncRpcError(method string, err error) { b.metrics.EthSyncRpcErrors.With(common.Labels{"method": method, "status": ethSyncRpcErrStatus(err)}).Inc() } +func (b *EthereumRPC) observeSyncRPCLatency(method string, start time.Time, err error) { + if b.metrics == nil { + return + } + errorLabel := "" + if err != nil { + errorLabel = "failure" + } + b.metrics.RPCSyncLatency.With(common.Labels{"method": method, "error": errorLabel}).Observe(float64(time.Since(start)) / 1e6) +} + // EnsureSameRPCHost validates both RPC URLs and logs a warning if hosts differ. func EnsureSameRPCHost(httpURL, wsURL string) error { if httpURL == "" || wsURL == "" { @@ -960,12 +971,17 @@ func (b *EthereumRPC) GetChainInfo() (*bchain.ChainInfo, error) { } ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) defer cancel() + netStart := time.Now() id, err := b.Client.NetworkID(ctx) + b.observeSyncRPCLatency("net_version", netStart, err) if err != nil { return nil, err } var ver string - if err := b.RPC.CallContext(ctx, &ver, "web3_clientVersion"); err != nil { + web3Start := time.Now() + err = b.RPC.CallContext(ctx, &ver, "web3_clientVersion") + b.observeSyncRPCLatency("web3_clientVersion", web3Start, err) + if err != nil { return nil, err } rv := &bchain.ChainInfo{ @@ -996,7 +1012,9 @@ func (b *EthereumRPC) getBestHeader() (bchain.EVMHeader, error) { var err error ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) defer cancel() + headerStart := time.Now() b.bestHeader, err = b.Client.HeaderByNumber(ctx, nil) + b.observeSyncRPCLatency("eth_getBlockByNumber", headerStart, err) if err != nil { b.bestHeader = nil return nil, err @@ -1363,6 +1381,7 @@ func (b *EthereumRPC) getBlockRaw(hash string, height uint32, fullTxs bool) (jso var raw json.RawMessage var err error var method string + defer func(s time.Time) { b.observeSyncRPCLatency(method, s, err) }(time.Now()) if hash != "" { if hash == "pending" { method = "eth_getBlockByNumber" @@ -1395,7 +1414,9 @@ func (b *EthereumRPC) processEventsForBlock(blockNumber string) (map[string][]*b var logs []rpcLogWithTxHash var ensRecords []bchain.AddressAliasRecord var method = "eth_getLogs" - err := b.RPC.CallContext(ctx, &logs, method, map[string]interface{}{ + var err error + defer func(s time.Time) { b.observeSyncRPCLatency(method, s, err) }(time.Now()) + err = b.RPC.CallContext(ctx, &logs, method, map[string]interface{}{ "fromBlock": blockNumber, "toBlock": blockNumber, }) @@ -1499,7 +1520,9 @@ func (b *EthereumRPC) getInternalDataForBlock(ctx context.Context, blockHash str if b.ChainConfig.TraceTimeout != "" { traceConfig["timeout"] = b.ChainConfig.TraceTimeout } + traceStart := time.Now() err := b.RPC.CallContext(ctx, &trace, "debug_traceBlockByHash", blockHash, traceConfig) // Use caller-provided ctx for timeout/cancel. + b.observeSyncRPCLatency("debug_traceBlockByHash", traceStart, err) b.observeEthSyncRpcError("debug_traceBlockByHash", err) if err != nil { glog.Error("debug_traceBlockByHash block ", blockHash, ", error ", err) diff --git a/common/metrics.go b/common/metrics.go index d83bd33746..0e11783038 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -102,6 +102,7 @@ type Metrics struct { EthEip1559FeeSource *prometheus.CounterVec `metric:"eth_eip1559_fee_source_total"` EthBlockGasUsedRatio prometheus.Gauge `metric:"eth_block_gas_used_ratio"` EthBlockBaseFee prometheus.Gauge `metric:"eth_block_base_fee"` + RPCSyncLatency *prometheus.HistogramVec `metric:"rpc_sync_latency"` EthSyncRpcErrors *prometheus.CounterVec `metric:"eth_sync_rpc_errors"` } diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index a3aae3f2b4..de3b680d88 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -286,6 +286,16 @@ panels: events: promql: sum by (subscription, event) (rate({{name:backend_subscription_events}}{job="blockbook", coin="$coin"}[5m])) legend: '{{subscription}} - {{event}}' + sync.rpc_sync_latency: + title: Sync RPC latency (p50/p95) + description: Round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.) that are not tracked by the general RPC latency panel. These directly affect block indexing speed. + queries: + p95: + promql: histogram_quantile(0.95, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin"}[1m])) by (coin, method, le)) + legend: '{{coin}} - {{method}}' + p50: + promql: histogram_quantile(0.50, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin"}[1m])) by (coin, method, le)) + legend: '{{coin}} - {{method}}' row.websocket: title: Websocket websocket.clients: diff --git a/configs/grafana/template.json b/configs/grafana/template.json index 8c9f907c6d..bdc937d509 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -2436,6 +2436,72 @@ "type": "timeseries", "x-panel-key": "sync.tip_feed_events" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "id": 348, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "p95" + }, + { + "editorMode": "code", + "range": true, + "refId": "B", + "x-query-key": "p50" + } + ], + "type": "timeseries", + "x-panel-key": "sync.rpc_sync_latency" + }, { "collapsed": false, "id": 315, diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 5018502f5e..ec6311bcb4 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -400,6 +400,12 @@ metrics: name: blockbook_eth_block_base_fee type: gauge help: Base fee per gas of the most recently connected EVM block, in wei - divide by 1e9 for Gwei. The realized base fee from the block header (push path); compare with eth_eip1559_base_fee (the pull path's next-block projection) to see whether the projection tracks the chain. Updated once per connected block + rpc_sync_latency: + name: blockbook_rpc_sync_latency + type: histogram_vec + help: Blockbook-to-backend round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.), in milliseconds, labeled by method and whether the call failed + labels: [method, error] + buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 20000] eth_sync_rpc_errors: name: blockbook_eth_sync_rpc_errors type: counter_vec From 13801b01812f5f7fe4b1b06f039deb571d1500b3 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 18:37:43 +0000 Subject: [PATCH 035/303] fix(e2e): record coin aborts as failures instead of swallowing rejections Promise.allSettled discarded runCoin rejections: a coin whose context creation or preload threw vanished from the summaries with its buffered output unflushed, and the run printed 'passed for N coin(s)' with exit code 0. Catch the rejection per coin and push a failed CoinRun summary so the aggregate failure path fires. Co-Authored-By: Claude Fable 5 --- tests/openapi/src/runner.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/openapi/src/runner.ts b/tests/openapi/src/runner.ts index 6b96f2f046..142376ac94 100644 --- a/tests/openapi/src/runner.ts +++ b/tests/openapi/src/runner.ts @@ -39,13 +39,24 @@ export async function runOpenApiE2E() { // console output and flushes it immediately upon completion so that even // if another coin hangs the completed results are still visible. const summaries: CoinSummary[] = []; - await Promise.allSettled( + await Promise.all( selectedCoins.map(async (coin) => { - const { summary, output } = await runCoin(coin, contract); - for (const line of output) { - console.log(line); + try { + const { summary, output } = await runCoin(coin, contract); + for (const line of output) { + console.log(line); + } + summaries.push(summary); + } catch (error) { + // A rejection here means the coin aborted outside runCoin's own error + // handling. Record it as a coin-level failure so the run cannot go + // green while a selected coin was never tested. + const message = errorMessage(error); + console.error(`\nOpenAPI e2e ${coin}: aborted: ${message}`); + summaries.push( + summarize(coin, [{ coin, name: "CoinRun", group: "run", status: "fail", durationMs: 0, message }]), + ); } - summaries.push(summary); }), ); From 798847476ef93ca2ed4ff033f9705abf079bc3f3 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 18:38:02 +0000 Subject: [PATCH 036/303] fix(e2e): fail the coin visibly when sample preload throws preloadSamples() ran bare inside runCoin, outside the per-test try/catch, so one transient HTTP or schema-validation error during the probe chain rejected the whole coin. Handle it like the status preflight: emit a single SamplePreload failure with the buffered output preserved and stop the coin there. Co-Authored-By: Claude Fable 5 --- tests/openapi/src/runner.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/openapi/src/runner.ts b/tests/openapi/src/runner.ts index 142376ac94..c3cb663843 100644 --- a/tests/openapi/src/runner.ts +++ b/tests/openapi/src/runner.ts @@ -129,8 +129,17 @@ async function runCoin(coin: string, contract: OpenApiContract): Promise<{ summa } // Eagerly resolve all samples once so downstream tests hit the cache - // instead of each triggering its own probe chain. - await ctx.preloadSamples(); + // instead of each triggering its own probe chain. A probe error here is a + // coin-level failure like the status preflight — every sample-derived test + // would fail downstream for the same root cause. + try { + await ctx.preloadSamples(); + } catch (error) { + const message = errorMessage(error); + emit(` fail (sample preload): ${message}`); + results.push({ coin, name: "SamplePreload", group: "preflight", status: "fail", durationMs: 0, message }); + return { summary: summarize(coin, results), output }; + } for (const testName of apiTests) { const def = testRegistry[testName]; From 81c4c8714a0c66f8f0517239182ba3c5d114c75c Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 19:04:52 +0000 Subject: [PATCH 037/303] refactor(e2e): replace WS pool with a single persistent connection wsConnection already multiplexes concurrent requests by ID and tests within a coin run sequentially, so the 3-connection lease pool bought nothing over one persistent socket while adding an unbounded acquire retry loop and a permanently-cached failed init. One shared connection with redial-on-close keeps the handshake-per-coin win; every wait is bounded by the dial and message timeouts, and a failed dial is not cached so the next call retries. Co-Authored-By: Claude Fable 5 --- tests/openapi/src/context.ts | 152 ++++++++++------------------------- 1 file changed, 43 insertions(+), 109 deletions(-) diff --git a/tests/openapi/src/context.ts b/tests/openapi/src/context.ts index 01e93bc360..996df20a89 100644 --- a/tests/openapi/src/context.ts +++ b/tests/openapi/src/context.ts @@ -32,7 +32,7 @@ import { import type { Capability, AddressResponse, BlockHashResponse, BlockResponse, BlockSummary, FiatTickerResponse, StatusResponse, TxResponse, UtxoResponse, WsEnvelope, WsInfoResponse, WsMethod, WsResponse } from "./types.js"; // --------------------------------------------------------------------------- -// WebSocket connection pool +// WebSocket connection // --------------------------------------------------------------------------- // wsConnection wraps a single persistent WebSocket and multiplexes requests @@ -159,84 +159,6 @@ class wsConnection { } } -// wsPool maintains a small set of persistent WebSocket connections so that -// WS tests don't pay a handshake timeout per call. acquire/release follow -// a simple lease pattern; callers must release after the request completes. -class wsPool { - private conns: wsConnection[] = []; - private busy = new Set(); - - constructor( - private readonly url: string, - private readonly contract: OpenApiContract, - private readonly size: number, - ) {} - - get aliveCount(): number { - return this.conns.length; - } - - async init(): Promise { - const results = await Promise.allSettled( - Array.from({ length: this.size }, () => { - const c = new wsConnection(this.url, this.contract); - return c.connect().then(() => c); - }), - ); - for (const r of results) { - if (r.status === "fulfilled") { - this.conns.push(r.value); - } - } - // A partial pool (fewer than this.size connections) is still usable — - // acquire() replaces dead slots lazily. If nothing connected, propagate - // the first rejection so the caller can attempt a protocol upgrade. - if (this.conns.length === 0) { - const first = results.find((r) => r.status === "rejected"); - throw first ? (first as PromiseRejectedResult).reason : new Error("wsPool init: no connections and no errors"); - } - } - - async acquire(): Promise { - // eslint-disable-next-line no-constant-condition - while (true) { - for (const c of this.conns) { - if (!this.busy.has(c) && !c.isClosed) { - this.busy.add(c); - return c; - } - } - // All connections are busy or dead. If any died, replace them lazily. - for (let i = 0; i < this.conns.length; i++) { - if (this.conns[i].isClosed) { - try { - const c = new wsConnection(this.url, this.contract); - await c.connect(); - this.conns[i] = c; - this.busy.add(c); - return c; - } catch { - // Replacement failed, try the next slot. - } - } - } - await new Promise((resolve) => setTimeout(resolve, 20)); - } - } - - release(conn: wsConnection): void { - this.busy.delete(conn); - } - - closeAll(): void { - for (const c of this.conns) { - c.close(); - } - this.conns = []; - this.busy.clear(); - } -} - // proxyFromEnv returns the configured egress proxy URL (sandboxed/corporate networks that only allow // outbound traffic via HTTP(S)_PROXY), or "" when none is set. Shared by the fetch dispatcher setup // in runner.ts and the ws client below. @@ -257,8 +179,8 @@ function wsProxyAgent(): HttpsProxyAgent | undefined { export class TestContext { readonly client: OpenApiFetchClient; - private wsPool_: wsPool | undefined; - private wsPoolInit_: Promise | undefined; + private wsConn_: wsConnection | undefined; + private wsConnInit_: Promise | undefined; private status?: NonNullable; private nextWSReq = 0; @@ -724,40 +646,52 @@ export class TestContext { } async wsCallWithID(id: string, method: WsMethod, params: unknown, dataSchemaRef?: string) { - // Lazily initialise the WS connection pool. The first call tries the - // configured ws:// URL; if that fails the pool tries wss:// and updates - // this.wsURL so subsequent calls skip the fallback. - // wsPoolInit_ guards against concurrent calls creating two pools. - if (!this.wsPoolInit_) { - this.wsPoolInit_ = (async () => { - const pool = new wsPool(this.wsURL, this.contract, 3); + const conn = await this.ensureWSConnection(); + const request: WsEnvelope = { id, method, params }; + return conn.request(request, dataSchemaRef); + } + + // ensureWSConnection lazily dials the shared WebSocket connection (so WS + // tests pay a single handshake per coin, not one per call) and redials when + // the previous connection died. The connection multiplexes concurrent + // requests by ID, so callers share it freely. The first dial tries the + // configured ws:// URL; if that fails it tries wss:// and updates this.wsURL + // so later dials skip the fallback. wsConnInit_ guards against concurrent + // calls dialing twice; a failed dial is not cached, so a transient outage + // does not poison every later WS test of the coin. + private ensureWSConnection(): Promise { + if (this.wsConn_ && !this.wsConn_.isClosed) { + return Promise.resolve(this.wsConn_); + } + if (!this.wsConnInit_) { + this.wsConnInit_ = (async () => { + const conn = new wsConnection(this.wsURL, this.contract); try { - await pool.init(); + await conn.connect(); + return conn; } catch (firstError) { const upgraded = upgradeWSBaseToWSS(this.wsURL); if (!upgraded) { throw firstError; } - const pool2 = new wsPool(upgraded, this.contract, 3); - await pool2.init(); + const conn2 = new wsConnection(upgraded, this.contract); + await conn2.connect(); this.wsURL = upgraded; - this.wsPool_ = pool2; - return; + return conn2; } - this.wsPool_ = pool; - })(); - } - await this.wsPoolInit_; - // wsPoolInit_ resolved, so wsPool_ is guaranteed to be set. - const pool = this.wsPool_!; - - const request: WsEnvelope = { id, method, params }; - const conn = await pool.acquire(); - try { - return await conn.request(request, dataSchemaRef); - } finally { - pool.release(conn); + })().then( + (conn) => { + this.wsConn_ = conn; + this.wsConnInit_ = undefined; + return conn; + }, + (error) => { + this.wsConnInit_ = undefined; + throw error; + }, + ); } + return this.wsConnInit_; } isEVMTxID(txid: string) { @@ -770,9 +704,9 @@ export class TestContext { } close() { - this.wsPool_?.closeAll(); - this.wsPool_ = undefined; - this.wsPoolInit_ = undefined; + this.wsConn_?.close(); + this.wsConn_ = undefined; + this.wsConnInit_ = undefined; } private async probeCapabilities() { From 2ce18fad5becb2a4c1a9ee7646b43c7f8dbdd7b2 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 19:16:38 +0000 Subject: [PATCH 038/303] fix(metrics): reflect real sync RPC latency distribution in help text and panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit debug_traceBlockByHash is typically sub-100ms, reaches seconds only when the node is overloaded, and ~10-15s observations are trace_timeout / rpc_timeout expiries — not routine latency. Say so in the histogram help text, and exclude failed calls from the Grafana p50/p95 queries so timeout expiries do not skew the quantiles (failures are already charted in the sync RPC errors panel). Co-Authored-By: Claude Fable 5 --- configs/grafana/panels.yaml | 6 +++--- configs/metrics.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index de3b680d88..c694cb8e99 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -288,13 +288,13 @@ panels: legend: '{{subscription}} - {{event}}' sync.rpc_sync_latency: title: Sync RPC latency (p50/p95) - description: Round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.) that are not tracked by the general RPC latency panel. These directly affect block indexing speed. + description: Round-trip time of successful sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.) that are not tracked by the general RPC latency panel; these directly affect block indexing speed. Typically sub-100ms; a sustained multi-second p95 means the backend is overloaded. Failed calls are excluded so timeout expiries (~10-15s) do not skew the quantiles; they show up in the Ethereum sync RPC errors panel instead. queries: p95: - promql: histogram_quantile(0.95, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin"}[1m])) by (coin, method, le)) + promql: histogram_quantile(0.95, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin", error=""}[1m])) by (coin, method, le)) legend: '{{coin}} - {{method}}' p50: - promql: histogram_quantile(0.50, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin"}[1m])) by (coin, method, le)) + promql: histogram_quantile(0.50, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin", error=""}[1m])) by (coin, method, le)) legend: '{{coin}} - {{method}}' row.websocket: title: Websocket diff --git a/configs/metrics.yaml b/configs/metrics.yaml index ec6311bcb4..55d767fd83 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -403,7 +403,7 @@ metrics: rpc_sync_latency: name: blockbook_rpc_sync_latency type: histogram_vec - help: Blockbook-to-backend round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.), in milliseconds, labeled by method and whether the call failed + help: Blockbook-to-backend round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.), in milliseconds, labeled by method and whether the call failed. Typically sub-100ms; multi-second values mean an overloaded backend, and observations near 10-15s are trace_timeout/rpc_timeout expiries rather than real round trips labels: [method, error] buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 20000] eth_sync_rpc_errors: From 0deb2510615453231c64db1649153e3b7e16e2ba Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 20:43:25 +0000 Subject: [PATCH 039/303] fix(build): quote paths in root logrotate cleanup to prevent arbitrary deletion The generated Debian package installs a daily cron wrapper that runs bin/logrotate.sh as root, while postinst chowns the per-coin log directory to the unprivileged Blockbook service user. The cleanup script expanded the discovered log path unquoted in both `fuser` and `rm -f $log`, so a file whose name contains shell word-splitting tokens (e.g. "blockbook-bitcoin.log -rf /opt/coins/data/bitcoin/backend") would be split into multiple operands. On GNU rm, `-rf` is honored even after an earlier operand, letting the service user cause the root cron job to recursively delete an attacker-chosen path outside the log tree (backend/RocksDB data, config, other coins' data). Quote "$LOGS" and "$log" and pass `--` before operands so an attacker-controlled path is always treated as a single non-option argument. Co-Authored-By: Claude Fable 5 --- build/templates/blockbook/logrotate.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/templates/blockbook/logrotate.sh b/build/templates/blockbook/logrotate.sh index 267e1bdc78..56fe166d8d 100755 --- a/build/templates/blockbook/logrotate.sh +++ b/build/templates/blockbook/logrotate.sh @@ -4,10 +4,10 @@ set -e LOGS={{.Env.BlockbookInstallPath}}/{{.Coin.Alias}}/logs -find $LOGS -mtime +7 -type f -print0 | while read -r -d $'\0' log; do +find "$LOGS" -mtime +7 -type f -print0 | while read -r -d $'\0' log; do # remove log if isn't opened by any process - if ! fuser -s $log; then - rm -f $log + if ! fuser -s -- "$log"; then + rm -f -- "$log" fi done {{end}} From 572e93118ba89641f3216e31b0bdfdc3759f6ff1 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:34:37 +0000 Subject: [PATCH 040/303] fix(deploy): run post-deploy verification on partial matrix failure deploy is a fail-fast:false matrix, but wait-for-sync and e2e-tests were gated on needs.deploy.result == 'success'. A single failed coin therefore skipped sync verification and e2e coverage for every coin that DID deploy, leaving freshly deployed instances unverified. Run both jobs on partial failure too; only mode=build (skipped) and cancellation keep them off. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c1e2177174..382c3447d2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -180,7 +180,10 @@ jobs: wait-for-sync: name: Wait For Sync needs: [prepare_deploy, deploy] - if: ${{ needs.deploy.result == 'success' }} + # deploy is a fail-fast:false matrix: one failed coin must not skip sync + # verification for the coins that did deploy, so run on partial failure + # too. Only 'skipped' (mode=build) and cancellation keep this job off. + if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') }} runs-on: [self-hosted, bb-dev-selfhosted] timeout-minutes: 31 env: @@ -206,7 +209,9 @@ jobs: e2e-tests: name: E2E Tests (post-deploy) needs: [prepare_deploy, deploy, wait-for-sync] - if: ${{ needs.deploy.result == 'success' && needs.wait-for-sync.result == 'success' }} + # Mirror wait-for-sync: partially failed deploys still get e2e coverage + # for the coins that deployed, as long as they reached sync. + if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') && needs.wait-for-sync.result == 'success' }} runs-on: [self-hosted, bb-dev-selfhosted] env: RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} From 071fe4f72eced83c0069bbdfbb333442d7a29694 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:35:16 +0000 Subject: [PATCH 041/303] ci: run .github/scripts unit tests in the lint job Seven *_test.py files (~1000 lines covering runner.py, build/deploy plans, backend policy and wait_for_sync) existed but were executed by no workflow or Makefile target, so regressions in the pipeline scripts shipped with green CI. Run them via unittest discover in the lint job; they are stdlib-only, need no secrets, and are safe for fork PRs on the GitHub-hosted runner. Co-Authored-By: Claude Fable 5 --- .github/workflows/testing.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c86ab1a3da..289d5a9cfc 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -105,3 +105,8 @@ jobs: - name: Validate backend artifact sources run: python3 ./.github/scripts/validate_backend_artifacts.py + + - name: Run CI pipeline script tests + # Stdlib-only unit tests for the build/deploy plan scripts; without + # this step a regression in .github/scripts ships with green CI. + run: python3 -m unittest discover -s .github/scripts -p '*_test.py' From 21e1a2cdc0bff16630d8b5ced720214e0700bfff Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:35:36 +0000 Subject: [PATCH 042/303] ci: reduce wait-for-sync timeout from 30 to 5 minutes Lower SYNC_TIMEOUT_SECONDS to 300 and the job timeout-minutes to 6, keeping the one-minute buffer over the script deadline. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 382c3447d2..4a1372423f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -185,11 +185,11 @@ jobs: # too. Only 'skipped' (mode=build) and cancellation keep this job off. if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') }} runs-on: [self-hosted, bb-dev-selfhosted] - timeout-minutes: 31 + timeout-minutes: 6 env: RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} COINS_INPUT: ${{ needs.prepare_deploy.outputs.test_coins_csv }} - SYNC_TIMEOUT_SECONDS: "1800" + SYNC_TIMEOUT_SECONDS: "300" steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 From 8f3d60c7d071d8373af705243173ccef12cade04 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:41:54 +0000 Subject: [PATCH 043/303] ci: bound all long-running jobs with timeout-minutes Only wait-for-sync had a timeout; every other job fell back to GitHub's 360-minute default, so a hung apt lock, unresponsive backend RPC or stalled install could occupy a self-hosted runner for six hours and block everything queued behind it. Ceilings are sized well above normal runtimes so they only trip on genuine hangs. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 5 +++++ .github/workflows/testing.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4a1372423f..10b2aa8922 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -107,6 +107,9 @@ jobs: name: Build (${{ matrix.runner }}) needs: prepare_build if: ${{ inputs.mode == 'build' }} + # Generous ceilings so a hung step can't hold a self-hosted runner for + # the 360-minute default; sized well above normal runtimes. + timeout-minutes: 240 env: RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} strategy: @@ -139,6 +142,7 @@ jobs: name: Deploy (${{ matrix.coin }}) needs: prepare_deploy if: ${{ inputs.mode == 'deploy' }} + timeout-minutes: 45 env: RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} strategy: @@ -213,6 +217,7 @@ jobs: # for the coins that deployed, as long as they reached sync. if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') && needs.wait-for-sync.result == 'success' }} runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 60 env: RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} CONNECTIVITY_REGEX: ${{ needs.prepare_deploy.outputs.connectivity_regex }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 289d5a9cfc..1a1829ac36 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -13,6 +13,9 @@ jobs: unit-tests: name: Unit Tests runs-on: [self-hosted, bb-dev-selfhosted] + # Generous ceilings so a hung step can't hold a self-hosted runner for + # the 360-minute default; sized well above normal runtimes. + timeout-minutes: 60 # Defense-in-depth: never run untrusted fork-PR code on the self-hosted # runner. connectivity/integration already carry this guard; without it # here, unit-tests relied solely on the repo "require approval for fork @@ -33,6 +36,7 @@ jobs: connectivity-tests: name: Connectivity Tests runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 30 needs: unit-tests if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -58,6 +62,7 @@ jobs: integration-tests: name: Integration Tests (RPC + Sync) runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 180 needs: connectivity-tests if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} From c39a3d6806e342e9835686c616cde1a9636d2ed5 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:42:29 +0000 Subject: [PATCH 044/303] fix(deploy): harden shell interpolation and checkout credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass matrix.coin to deploy-bb-and-backend.sh via an env var instead of expanding ${{ }} directly into the run: script, matching how every other dynamic value in this workflow is handled. Set persist-credentials: false on all checkouts so the GITHUB_TOKEN is not left in .git/config on the persistent self-hosted runners when post-step cleanup is skipped (cancelled job, crashed runner agent) — testing.yml already does this on the same runner pool. Nothing in the build/deploy path performs authenticated git operations after checkout. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 10b2aa8922..025a836dc0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -52,6 +52,8 @@ jobs: steps: - name: Checkout workflow code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false - name: Validate branch or tag env: @@ -62,6 +64,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Build build plan id: plan @@ -85,6 +88,8 @@ jobs: steps: - name: Checkout workflow code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false - name: Validate branch or tag env: @@ -95,6 +100,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Build deploy/e2e plan id: plan @@ -122,6 +128,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -155,6 +162,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -179,7 +187,8 @@ jobs: env: BRANCH_OR_TAG: ${{ env.RESOLVED_BRANCH_OR_TAG }} BB_BUILD_ENV: dev - run: ./contrib/scripts/deploy-bb-and-backend.sh "${{ matrix.coin }}" --force-confnew + DEPLOY_COIN: ${{ matrix.coin }} + run: ./contrib/scripts/deploy-bb-and-backend.sh "$DEPLOY_COIN" --force-confnew wait-for-sync: name: Wait For Sync @@ -199,6 +208,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -226,6 +236,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars From cef523fd59332c50fd4e2d240e05ccecdacca98d Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:42:47 +0000 Subject: [PATCH 045/303] ci: cancel superseded PR test runs with a concurrency group Without a concurrency group, every push to a PR queued a full unit->connectivity->integration chain on the limited self-hosted pool even when a newer commit had already superseded it. Cancel-in-progress applies only to pull_request events; push runs on master/develop always run to completion. Co-Authored-By: Claude Fable 5 --- .github/workflows/testing.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1a1829ac36..55d1c8dacb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,6 +9,13 @@ on: permissions: contents: read +# Rapid pushes queue full unit->connectivity->integration chains for +# already-superseded commits on the shared self-hosted pool. Cancel only +# superseded PR runs; pushes to master/develop are never cancelled. +concurrency: + group: testing-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: unit-tests: name: Unit Tests From f8918ccf932f37b27b3db83f6ec5bbdf68cc655a Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:43:35 +0000 Subject: [PATCH 046/303] fix(ci/cd): require exact ref match in branch/tag validation git ls-remote arguments are tail-matching glob patterns, so 'mas*' passed validation, and 'master' validated even if only 'feature/master' existed -- after which actions/checkout used the literal (nonexistent or different) ref. Check the returned ref name for an exact match and pass '--' so a ref can never be parsed as a git option. Git forbids glob characters in ref names, so no legitimate ref is rejected. Co-Authored-By: Claude Fable 5 --- .github/scripts/validate_branch_or_tag.py | 14 ++++++-- .../scripts/validate_branch_or_tag_test.py | 32 ++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.github/scripts/validate_branch_or_tag.py b/.github/scripts/validate_branch_or_tag.py index 5ae9fe3320..2aa95fef9b 100755 --- a/.github/scripts/validate_branch_or_tag.py +++ b/.github/scripts/validate_branch_or_tag.py @@ -23,8 +23,8 @@ def fail(message: str) -> None: def ref_exists(repo: str, ref: str, kind: str) -> bool: remote = f"https://github.com/{repo}.git" try: - subprocess.run( - ["git", "ls-remote", "--exit-code", f"--{kind}", remote, ref], + result = subprocess.run( + ["git", "ls-remote", "--exit-code", f"--{kind}", remote, "--", ref], check=True, capture_output=True, text=True, @@ -34,7 +34,15 @@ def ref_exists(repo: str, ref: str, kind: str) -> bool: raise AssertionError("unreachable") from exc except subprocess.CalledProcessError: return False - return True + # ls-remote patterns are tail-matching globs, so e.g. 'mas*' or 'master' + # can match a different ref than the one actions/checkout will later use; + # accept only an exact ref-name match. + expected = f"refs/{kind}/{ref}" + for line in result.stdout.splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[1] == expected: + return True + return False def validate_branch_or_tag(repo: str, ref: str) -> str: diff --git a/.github/scripts/validate_branch_or_tag_test.py b/.github/scripts/validate_branch_or_tag_test.py index 630115730e..0592566625 100644 --- a/.github/scripts/validate_branch_or_tag_test.py +++ b/.github/scripts/validate_branch_or_tag_test.py @@ -1,7 +1,8 @@ +import subprocess import unittest from unittest.mock import patch -from validate_branch_or_tag import validate_branch_or_tag +from validate_branch_or_tag import ref_exists, validate_branch_or_tag class ValidateBranchOrTagTest(unittest.TestCase): @@ -21,5 +22,34 @@ def test_rejects_missing_ref(self) -> None: validate_branch_or_tag("trezor/blockbook", "missing-ref") +class RefExistsTest(unittest.TestCase): + @staticmethod + def _ls_remote(stdout: str) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(args=[], returncode=0, stdout=stdout, stderr="") + + def test_accepts_exact_ref_name(self) -> None: + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/master\n"), + ): + self.assertTrue(ref_exists("trezor/blockbook", "master", "heads")) + + def test_rejects_glob_pattern_matching_another_ref(self) -> None: + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/master\n"), + ): + self.assertFalse(ref_exists("trezor/blockbook", "mas*", "heads")) + + def test_rejects_tail_match_of_longer_ref(self) -> None: + # ls-remote patterns tail-match, so 'master' would match + # 'feature/master' even when no branch named 'master' exists. + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/feature/master\n"), + ): + self.assertFalse(ref_exists("trezor/blockbook", "master", "heads")) + + if __name__ == "__main__": unittest.main() From 3828705d92e3dd2da080b468a250eb11523dd527 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:44:08 +0000 Subject: [PATCH 047/303] fix(ci/cd): keep wait_for_sync retrying through flaky error responses exc.read() in the HTTPError handlers was unprotected: a connection reset while reading an error body (e.g. proxy 502) raised out of the handler and killed a wait that still had minutes of budget, failing the deploy on one flaky poll. Degrade an unreadable error body to empty instead. Also stop truncating the remaining budget to whole seconds, which declared a timeout with up to a second (and one final poll) left. Co-Authored-By: Claude Fable 5 --- .github/scripts/wait_for_sync.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/scripts/wait_for_sync.py b/.github/scripts/wait_for_sync.py index 41c4770396..c4760efff4 100644 --- a/.github/scripts/wait_for_sync.py +++ b/.github/scripts/wait_for_sync.py @@ -21,6 +21,15 @@ def log(message: str) -> None: print(f"{LOG_PREFIX} {message}", flush=True) +def read_error_body(exc: urllib.error.HTTPError) -> bytes: + # The connection can be reset while reading the error body (e.g. a proxy + # 502); that must degrade to an empty body, not kill the retry loop. + try: + return exc.read() + except Exception: + return b"" + + def parse_requested_coins(raw: str) -> list[str]: text = raw.strip() if not text: @@ -192,7 +201,7 @@ def main() -> None: status, body = fetch_status(base_url, request_timeout, ssl_context) except urllib.error.HTTPError as exc: status = exc.code - body = exc.read() + body = read_error_body(exc) except Exception as exc: last_seen[coin] = f"{base_url}/api/status request failed: {exc}" continue @@ -204,7 +213,7 @@ def main() -> None: status, body = fetch_status(base_url, request_timeout, ssl_context) except urllib.error.HTTPError as exc: status = exc.code - body = exc.read() + body = read_error_body(exc) except Exception as exc: last_seen[coin] = f"{base_url}/api/status request failed: {exc}" continue @@ -224,14 +233,16 @@ def main() -> None: if not pending: break - remaining_seconds = int(max(0, deadline - time.monotonic())) - if remaining_seconds == 0: + # Keep sub-second remainders: int() truncation used to declare a + # timeout with up to a second of budget (and one last poll) unused. + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: break details = "; ".join( f"{coin}: {last_seen[coin]}" for coin in sorted(pending) ) - log(f"Still waiting for Blockbook sync ({remaining_seconds}s left): {details}") + log(f"Still waiting for Blockbook sync ({remaining_seconds:.0f}s left): {details}") time.sleep(min(poll_seconds, remaining_seconds)) if pending: From 1c62322d29ca9abedff2cd8862d9ecbdfefeb368 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 21:45:08 +0000 Subject: [PATCH 048/303] refactor(deploy): deduplicate ref resolution and drop unused outputs The RESOLVED_BRANCH_OR_TAG ternary was copy-pasted as job-level env into all six jobs; a one-sided edit would make jobs resolve different refs. Hoist it to workflow-level env (inputs/github contexts are available there) so it exists once. Also drop the prepare_* coins_csv job outputs, which no downstream job consumes. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 025a836dc0..e4e9032704 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,16 +39,16 @@ on: permissions: contents: read +env: + RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} + jobs: prepare_build: name: Prepare Build Plan runs-on: ubuntu-latest if: ${{ inputs.mode == 'build' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} outputs: runner_matrix: ${{ steps.plan.outputs.runner_matrix }} - coins_csv: ${{ steps.plan.outputs.coins_csv }} steps: - name: Checkout workflow code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -78,12 +78,9 @@ jobs: name: Prepare Deploy Plan runs-on: ubuntu-latest if: ${{ inputs.mode == 'deploy' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} outputs: runner_matrix: ${{ steps.plan.outputs.runner_matrix }} connectivity_regex: ${{ steps.plan.outputs.connectivity_regex }} - coins_csv: ${{ steps.plan.outputs.coins_csv }} test_coins_csv: ${{ steps.plan.outputs.test_coins_csv }} steps: - name: Checkout workflow code @@ -116,8 +113,6 @@ jobs: # Generous ceilings so a hung step can't hold a self-hosted runner for # the 360-minute default; sized well above normal runtimes. timeout-minutes: 240 - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} strategy: fail-fast: false matrix: @@ -150,8 +145,6 @@ jobs: needs: prepare_deploy if: ${{ inputs.mode == 'deploy' }} timeout-minutes: 45 - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} strategy: fail-fast: false matrix: @@ -200,7 +193,6 @@ jobs: runs-on: [self-hosted, bb-dev-selfhosted] timeout-minutes: 6 env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} COINS_INPUT: ${{ needs.prepare_deploy.outputs.test_coins_csv }} SYNC_TIMEOUT_SECONDS: "300" steps: @@ -229,7 +221,6 @@ jobs: runs-on: [self-hosted, bb-dev-selfhosted] timeout-minutes: 60 env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} CONNECTIVITY_REGEX: ${{ needs.prepare_deploy.outputs.connectivity_regex }} steps: - name: Checkout code From 05b241f5334b514a4861d675e91c2ef947be5255 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 3 Jul 2026 03:41:24 +0000 Subject: [PATCH 049/303] feat(ci/cd): show built/deployed coins in the Actions run UI A dispatched run only showed "Prepare Build Plan -> Build (production_builder)", so the run gave no hint which coins it actually built or deployed. Surface it in three places: a run-name with mode/coins/env/ref so the runs list says it, the coin list in the Build job name, and a runner->coins markdown table written by build_plan.py / deploy_plan.py to GITHUB_STEP_SUMMARY so it renders on the run's summary page. Co-Authored-By: Claude Fable 5 --- .github/scripts/build_plan.py | 17 +++++++++++++++++ .github/scripts/deploy_plan.py | 15 +++++++++++++++ .github/scripts/runner.py | 9 +++++++++ .github/workflows/deploy.yml | 7 ++++++- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/scripts/build_plan.py b/.github/scripts/build_plan.py index 22ceb38e6b..603aa68b00 100644 --- a/.github/scripts/build_plan.py +++ b/.github/scripts/build_plan.py @@ -13,6 +13,7 @@ log, parse_json_object, resolve_build_selection, + write_step_summary, ) @@ -67,6 +68,22 @@ def main() -> None: + ", ".join(item["coins"]) ) + summary_lines = [ + f"### Build plan ({build_env})", + "", + "| Runner | Coins |", + "| --- | --- |", + ] + for item in runner_matrix: + summary_lines.append(f"| {item['runner']} | {', '.join(item['coins'])} |") + if selection.skipped_prod_only and selection.requested_all: + summary_lines += [ + "", + "Skipped prod-only coins for env=dev: " + + ", ".join(selection.skipped_prod_only), + ] + write_step_summary("\n".join(summary_lines)) + if __name__ == "__main__": main() diff --git a/.github/scripts/deploy_plan.py b/.github/scripts/deploy_plan.py index 8539cf7101..903d05590d 100644 --- a/.github/scripts/deploy_plan.py +++ b/.github/scripts/deploy_plan.py @@ -14,6 +14,7 @@ parse_json_object, require_coin_config, resolve_deploy_selection, + write_step_summary, ) @@ -90,6 +91,20 @@ def main() -> None: log("Selected coins: " + ", ".join(requested)) log("Connectivity regex: " + connectivity_regex) + summary_lines = [ + "### Deploy plan", + "", + "| Coin | Runner |", + "| --- | --- |", + ] + for item in runner_matrix: + summary_lines.append(f"| {item['coin']} | {item['runner']} |") + summary_lines += [ + "", + "E2E test coins: " + ", ".join(unique_test_coins), + ] + write_step_summary("\n".join(summary_lines)) + if __name__ == "__main__": main() diff --git a/.github/scripts/runner.py b/.github/scripts/runner.py index 06de28d61f..21fa350821 100644 --- a/.github/scripts/runner.py +++ b/.github/scripts/runner.py @@ -41,6 +41,15 @@ def log(message: str) -> None: print(f"{LOG_PREFIX} {message}", flush=True) +def write_step_summary(markdown: str) -> None: + # Renders on the workflow run's summary page; a no-op outside Actions. + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + with open(summary_path, "a", encoding="utf-8") as summary: + summary.write(markdown.rstrip("\n") + "\n") + + def parse_json_object(raw: str, description: str) -> dict: try: payload = json.loads(raw) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e4e9032704..97a133685e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,5 +1,10 @@ name: Build / Deploy +# Surface what the run actually does in the run list; without this every +# dispatch shows only the generic workflow name. env context is not +# available in run-name, so the ref fallback is inlined. +run-name: "${{ inputs.mode }} ${{ inputs.coins }} (${{ inputs.mode == 'build' && inputs.env || 'dev' }}) @ ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }}" + on: workflow_dispatch: inputs: @@ -107,7 +112,7 @@ jobs: run: python3 ./.github/scripts/deploy_plan.py build: - name: Build (${{ matrix.runner }}) + name: "Build (${{ matrix.runner }}: ${{ matrix.coins_csv }})" needs: prepare_build if: ${{ inputs.mode == 'build' }} # Generous ceilings so a hung step can't hold a self-hosted runner for From b311de19636060fc9ca1e23b2289c0863e61277f Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Sat, 4 Jul 2026 18:49:48 +0200 Subject: [PATCH 050/303] fix(eth): avoid int64 overflow in on-chain base fee conversion The pull-path EIP-1559 estimate decoded baseFeePerGas as uint64 and then cast it to int64 for big.NewInt. A value above math.MaxInt64 (~9.22e18 wei) would wrap to a negative big.Int. Unreachable on mainnet today but possible on high-fee L2s. Use new(big.Int).SetUint64() instead, matching the header path in attachBlockGas. Co-Authored-By: Claude Opus 4.8 (1M context) --- bchain/coins/eth/ethrpc.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 12922ad9da..8b09a0c6fa 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2004,7 +2004,10 @@ func (b *EthereumRPC) EthereumTypeGetEip1559Fees() (*bchain.Eip1559Fees, error) hs, _ := json.Marshal(h) baseFee, _ := hexutil.DecodeUint64(h.BaseFeePerGas[blocks-1]) - fees.BaseFeePerGas = big.NewInt(int64(baseFee)) + // SetUint64, not big.NewInt(int64(...)): base fee is wei and a value above math.MaxInt64 + // (~9.22e18) would wrap negative on the int64 cast. Unreachable on mainnet today but possible + // on high-fee L2s. Matches the header path in attachBlockGas. + fees.BaseFeePerGas = new(big.Int).SetUint64(baseFee) // We expose only baseFeePerGas here and deliberately do NOT add a separate "next block" base-fee // field. eth_feeHistory returns one extra projected element beyond the requested range, but its // meaning is backend-dependent: nodes with no distinct pending block (e.g. Erigon, which ethereum From b93a530d4e9085bab9296328cf77bb6ab510241c Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 1 Jul 2026 12:39:21 +0000 Subject: [PATCH 051/303] fix(grafana): correct stale instant-tier claim in EIP-1559 max-fee panel a2ec3b3d remapped 1inch's four fee tiers onto low/medium/high (discarding the low tier and folding instant into high), so 1inch no longer surfaces a separate instant tier. The panel description still claimed 1inch was a "provider that return[s] four tiers", which is no longer true and would send an on-call engineer chasing a phantom instant-tier discrepancy. Co-Authored-By: Claude Sonnet 5 --- configs/grafana/panels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index c694cb8e99..7496287079 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -675,7 +675,7 @@ panels: legend: '{{instance}} - {{provider}} - {{status}}' fees.eip1559_max_fee: title: EIP-1559 maxFeePerGas by tier (Gwei) - description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On provider-backed coins (e.g. Infura, 1inch) these are the provider's suggestions; on on-chain coins they are 2x baseFee + tip. The instant tier exists only on the on-chain path and on providers that return four tiers (1inch), so an instant line appearing on a provider that does not return it means the provider cache was stale and the on-chain fallback served the estimate. Compare with the base fee and the buffer-ratio panel to tell a busy chain from an estimator regression. + description: The maxFeePerGas the pull path (estimateFee) serves per tier, in Gwei (raw wei / 1e9). This is what wallets such as Trezor Suite spend as the fee cap, so it is the headline panel for "fees too high". On provider-backed coins (e.g. Infura, 1inch) these are the provider's suggestions, remapped per-provider onto low/medium/high for comparable aggressiveness across providers; on on-chain coins they are 2x baseFee + tip. The instant tier exists only on the on-chain path, so an instant line on a provider-backed coin means the provider cache was stale and the on-chain fallback served the estimate. Compare with the base fee and the buffer-ratio panel to tell a busy chain from an estimator regression. queries: by_tier: promql: ({{name:eth_eip1559_fee}}{job="blockbook", coin="$coin", kind="max_fee"} / 1e9) From 965a104b4d00b8dea3062951f5b226d28a0c30b4 Mon Sep 17 00:00:00 2001 From: Matus Balascak Date: Fri, 3 Jul 2026 14:03:47 +0200 Subject: [PATCH 052/303] feat(tron): expose latestWithdrawTime in account staking info --- bchain/coins/tron/tronhttp_endpoints.go | 17 ++++++++++++----- bchain/coins/tron/tronrpc_test.go | 4 +++- bchain/types_tron_chainextradata.go | 1 + blockbook-api.ts | 1 + docs/api-tron.md | 1 + server/public_tron_test.go | 7 ++++--- static/templates/address_chainextra_tron.html | 6 ++++++ tests/dbtestdata/fakechain_tron.go | 1 + 8 files changed, 29 insertions(+), 9 deletions(-) diff --git a/bchain/coins/tron/tronhttp_endpoints.go b/bchain/coins/tron/tronhttp_endpoints.go index 0e117033fe..465ce3f941 100644 --- a/bchain/coins/tron/tronhttp_endpoints.go +++ b/bchain/coins/tron/tronhttp_endpoints.go @@ -49,11 +49,12 @@ type tronUnfrozenV2Entry struct { } type tronGetAccountResponse struct { - Address string `json:"address,omitempty"` - FrozenV2 []tronFrozenV2Entry `json:"frozenV2,omitempty"` - UnfrozenV2 []tronUnfrozenV2Entry `json:"unfrozenV2,omitempty"` - Votes []tronTxVote `json:"votes,omitempty"` - AccountResource struct { + Address string `json:"address,omitempty"` + FrozenV2 []tronFrozenV2Entry `json:"frozenV2,omitempty"` + UnfrozenV2 []tronUnfrozenV2Entry `json:"unfrozenV2,omitempty"` + Votes []tronTxVote `json:"votes,omitempty"` + LatestWithdrawTime *int64 `json:"latest_withdraw_time,omitempty"` + AccountResource struct { DelegatedFrozenV2BalanceForEnergy int64 `json:"delegated_frozenV2_balance_for_energy"` } `json:"account_resource,omitempty"` DelegatedFrozenV2BalanceForBandwidth int64 `json:"delegated_frozenV2_balance_for_bandwidth"` @@ -257,6 +258,11 @@ func tronBuildStakingInfo(accountResp *tronGetAccountResponse, resourceResp *tro delegatedEnergy := max(accountResp.AccountResource.DelegatedFrozenV2BalanceForEnergy, 0) delegatedBandwidth := max(accountResp.DelegatedFrozenV2BalanceForBandwidth, 0) + latestWithdrawTime := int64(0) + if accountResp.LatestWithdrawTime != nil && *accountResp.LatestWithdrawTime > 0 { + latestWithdrawTime = *accountResp.LatestWithdrawTime / 1000 + } + return &bchain.TronStakingInfo{ StakedBalance: stakedBalance.String(), StakedBalanceEnergy: stakedEnergy.String(), @@ -268,6 +274,7 @@ func tronBuildStakingInfo(accountResp *tronGetAccountResponse, resourceResp *tro UnclaimedReward: strconv.FormatInt(unclaimedReward, 10), DelegatedBalanceEnergy: strconv.FormatInt(delegatedEnergy, 10), DelegatedBalanceBandwidth: strconv.FormatInt(delegatedBandwidth, 10), + LatestWithdrawTime: latestWithdrawTime, } } diff --git a/bchain/coins/tron/tronrpc_test.go b/bchain/coins/tron/tronrpc_test.go index 29e995fde0..528bdf1a10 100644 --- a/bchain/coins/tron/tronrpc_test.go +++ b/bchain/coins/tron/tronrpc_test.go @@ -577,6 +577,7 @@ func TestTronRPC_GetAddressChainExtraData(t *testing.T) { "delegated_frozenV2_balance_for_energy": int64(3210000), }, "delegated_frozenV2_balance_for_bandwidth": int64(654000), + "latest_withdraw_time": int64(1777000000000), }, "/wallet/getReward": map[string]any{ "reward": int64(42767), @@ -618,7 +619,8 @@ func TestTronRPC_GetAddressChainExtraData(t *testing.T) { "votes":[{"address":"TJvaAeFb8Lykt9RQcVyyTFN2iDvGMuyD4M","voteCount":"20"}], "unclaimedReward":"42767", "delegatedBalanceEnergy":"3210000", - "delegatedBalanceBandwidth":"654000" + "delegatedBalanceBandwidth":"654000", + "latestWithdrawTime":1777000000 } }`, string(payload)) paths, bodies := mockHTTP.SnapshotRequests() diff --git a/bchain/types_tron_chainextradata.go b/bchain/types_tron_chainextradata.go index f79348e8ac..01281ddaf6 100644 --- a/bchain/types_tron_chainextradata.go +++ b/bchain/types_tron_chainextradata.go @@ -53,6 +53,7 @@ type TronStakingInfo struct { UnclaimedReward string `json:"unclaimedReward"` DelegatedBalanceEnergy string `json:"delegatedBalanceEnergy"` DelegatedBalanceBandwidth string `json:"delegatedBalanceBandwidth"` + LatestWithdrawTime int64 `json:"latestWithdrawTime"` } // TronAccountExtraData contains normalized Tron-specific account resource metadata. diff --git a/blockbook-api.ts b/blockbook-api.ts index 4bc7f72fc7..2ae68c05b3 100644 --- a/blockbook-api.ts +++ b/blockbook-api.ts @@ -49,6 +49,7 @@ export interface TronStakingInfo { unclaimedReward: string; delegatedBalanceEnergy: string; delegatedBalanceBandwidth: string; + latestWithdrawTime: number; } export interface TronAccountExtraData { availableStakedBandwidth: number; diff --git a/docs/api-tron.md b/docs/api-tron.md index e206615eb7..9a8cfa9a30 100644 --- a/docs/api-tron.md +++ b/docs/api-tron.md @@ -101,3 +101,4 @@ Resource fields: - `unclaimedReward` (`string`): unclaimed voting reward (sun) - `delegatedBalanceEnergy` (`string`): delegated staked energy in sun - `delegatedBalanceBandwidth` (`string`): delegated staked bandwidth in sun +- `latestWithdrawTime` (`number`): Unix timestamp in **seconds** of the account's last voting-reward withdrawal, or `0` if it has never withdrawn. diff --git a/server/public_tron_test.go b/server/public_tron_test.go index caae9b6709..1018350bf1 100644 --- a/server/public_tron_test.go +++ b/server/public_tron_test.go @@ -30,6 +30,7 @@ func httpTestsTron(t *testing.T, ts *httptest.Server) { `Voting Power7 / 10`, `href="/address/TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD"`, `Unclaimed Reward`, + `Last Reward Claim`, }, }, { @@ -88,7 +89,7 @@ func httpTestsTron(t *testing.T, ts *httptest.Server) { status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ - `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"TZEZWXYQS44388xBoMhQdpL1HrBZFLfDpt","balance":"123450255","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":1,"nonTokenTxs":1,"internalTxs":1,"txids":["a431984fef1d014620504d02f821f872221cf44c250a81a31e81fa4855b2b302"],"nonce":"255","confirmedNonce":"255","tokens":[{"type":"TRC20","standard":"TRC20","name":"TronTestContract236","contract":"TXYZopYRdj2D9XRtbG411XZZ3kM5VkAeBf","transfers":1,"symbol":"TRC236","decimals":6,"balance":"1000255236"}],"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":255,"totalStakedBandwidth":1255,"availableFreeBandwidth":755,"totalFreeBandwidth":1755,"availableEnergy":25500,"totalEnergy":35500,"totalEnergyLimit":45500,"totalEnergyWeight":2255,"totalBandwidthLimit":2755,"totalBandwidthWeight":3255,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000"}}}}`, + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"TZEZWXYQS44388xBoMhQdpL1HrBZFLfDpt","balance":"123450255","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":1,"nonTokenTxs":1,"internalTxs":1,"txids":["a431984fef1d014620504d02f821f872221cf44c250a81a31e81fa4855b2b302"],"nonce":"255","confirmedNonce":"255","tokens":[{"type":"TRC20","standard":"TRC20","name":"TronTestContract236","contract":"TXYZopYRdj2D9XRtbG411XZZ3kM5VkAeBf","transfers":1,"symbol":"TRC236","decimals":6,"balance":"1000255236"}],"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":255,"totalStakedBandwidth":1255,"availableFreeBandwidth":755,"totalFreeBandwidth":1755,"availableEnergy":25500,"totalEnergy":35500,"totalEnergyLimit":45500,"totalEnergyWeight":2255,"totalBandwidthLimit":2755,"totalBandwidthWeight":3255,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000","latestWithdrawTime":1777018452}}}}`, }, }, { @@ -110,7 +111,7 @@ func httpTestsTron(t *testing.T, ts *httptest.Server) { contentType: "application/json; charset=utf-8", body: []string{ `"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD"`, - `"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":36,"totalStakedBandwidth":1036,"availableFreeBandwidth":536,"totalFreeBandwidth":1536,"availableEnergy":3600,"totalEnergy":13600,"totalEnergyLimit":23600,"totalEnergyWeight":2036,"totalBandwidthLimit":2536,"totalBandwidthWeight":3036,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000"}}`, + `"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":36,"totalStakedBandwidth":1036,"availableFreeBandwidth":536,"totalFreeBandwidth":1536,"availableEnergy":3600,"totalEnergy":13600,"totalEnergyLimit":23600,"totalEnergyWeight":2036,"totalBandwidthLimit":2536,"totalBandwidthWeight":3036,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000","latestWithdrawTime":1777018452}}`, `"transactions":[{"txid":"a431984fef1d014620504d02f821f872221cf44c250a81a31e81fa4855b2b302"`, `"chainExtraData":{"contractType":"TriggerSmartContract","operation":"contractCall","assetIssueID":"1002001","totalFee":"3076500","energyUsage":"14650","energyUsageTotal":"14650","bandwidthUsage":"345","bandwidthFee":"0","result":"SUCCESS"}`, `"nonce":"36"`, @@ -181,7 +182,7 @@ var websocketTestsTron = []websocketTest{ "confirmedNonce": true, }, }, - want: `{"id":"2","data":{"page":1,"totalPages":1,"itemsOnPage":25,"address":"TZEZWXYQS44388xBoMhQdpL1HrBZFLfDpt","balance":"123450255","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":1,"nonTokenTxs":1,"internalTxs":1,"txids":["a431984fef1d014620504d02f821f872221cf44c250a81a31e81fa4855b2b302"],"nonce":"255","confirmedNonce":"255","tokens":[{"type":"TRC20","standard":"TRC20","name":"TronTestContract236","contract":"TXYZopYRdj2D9XRtbG411XZZ3kM5VkAeBf","transfers":1,"symbol":"TRC236","decimals":6,"balance":"1000255236"}],"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":255,"totalStakedBandwidth":1255,"availableFreeBandwidth":755,"totalFreeBandwidth":1755,"availableEnergy":25500,"totalEnergy":35500,"totalEnergyLimit":45500,"totalEnergyWeight":2255,"totalBandwidthLimit":2755,"totalBandwidthWeight":3255,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000"}}}}}`, + want: `{"id":"2","data":{"page":1,"totalPages":1,"itemsOnPage":25,"address":"TZEZWXYQS44388xBoMhQdpL1HrBZFLfDpt","balance":"123450255","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":1,"nonTokenTxs":1,"internalTxs":1,"txids":["a431984fef1d014620504d02f821f872221cf44c250a81a31e81fa4855b2b302"],"nonce":"255","confirmedNonce":"255","tokens":[{"type":"TRC20","standard":"TRC20","name":"TronTestContract236","contract":"TXYZopYRdj2D9XRtbG411XZZ3kM5VkAeBf","transfers":1,"symbol":"TRC236","decimals":6,"balance":"1000255236"}],"chainExtraData":{"payloadType":"tron","payload":{"availableStakedBandwidth":255,"totalStakedBandwidth":1255,"availableFreeBandwidth":755,"totalFreeBandwidth":1755,"availableEnergy":25500,"totalEnergy":35500,"totalEnergyLimit":45500,"totalEnergyWeight":2255,"totalBandwidthLimit":2755,"totalBandwidthWeight":3255,"stakingInfo":{"stakedBalance":"7000000","stakedBalanceEnergy":"5000000","stakedBalanceBandwidth":"2000000","unstakingBatches":[{"amount":"1112757","expireTime":1777018452}],"totalVotingPower":"10","availableVotingPower":"7","votes":[{"address":"TDGSR64oU4QDpViKfdwawSiqwyqpUB6JUD","voteCount":"20"}],"unclaimedReward":"42767","delegatedBalanceEnergy":"3210000","delegatedBalanceBandwidth":"654000","latestWithdrawTime":1777018452}}}}}`, }, } diff --git a/static/templates/address_chainextra_tron.html b/static/templates/address_chainextra_tron.html index dc081d7d1b..64c9727df6 100644 --- a/static/templates/address_chainextra_tron.html +++ b/static/templates/address_chainextra_tron.html @@ -95,6 +95,12 @@ {{$staking.UnclaimedReward}} sun {{end}} +{{if $staking.LatestWithdrawTime}} + + Last Reward Claim + {{unixTimeSpan $staking.LatestWithdrawTime}} + +{{end}} {{if $staking.DelegatedBalanceEnergyValue}} Delegated for Energy diff --git a/tests/dbtestdata/fakechain_tron.go b/tests/dbtestdata/fakechain_tron.go index 2b3a7ba705..5e7c6aa585 100644 --- a/tests/dbtestdata/fakechain_tron.go +++ b/tests/dbtestdata/fakechain_tron.go @@ -188,6 +188,7 @@ func (c *fakeBlockChainTronType) GetAddressChainExtraData(addrDesc bchain.Addres UnclaimedReward: "42767", DelegatedBalanceEnergy: "3210000", DelegatedBalanceBandwidth: "654000", + LatestWithdrawTime: 1777018452, } payload, err := json.Marshal(extra) From 05415342ee34df9035ffa7734b3830bc710f6fbb Mon Sep 17 00:00:00 2001 From: Jakub Jerabek <116381722+cranycrane@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:18:41 +0200 Subject: [PATCH 053/303] =?UTF-8?q?fix(tron):=20the=20receipt=20for=20unso?= =?UTF-8?q?lidified=20block=20should=20not=20be=20set=20to=20ni=E2=80=A6?= =?UTF-8?q?=20(#1583)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tron): the receipt for unsolidified block should not be set to nil so all token transfers are parsed and indexed correctly * feat(tron): increase number of connections in the HTTP client to speed up syncing * feat(tron): remove zeromq nil check to reduce number of requests per block to speed up syncing * feat(tron): use HTTP endpoint for getting block has by number to fasten sequential sync path * fix(tron): drain HTTP response body before close to enable connection reuse net/http only returns a connection to the idle pool when the response body is read to EOF and then closed. Request() closed the body without draining: the >=300 early return reads nothing, and json.Decoder can leave trailing bytes unconsumed on larger responses, so connections were torn down instead of pooled — defeating the MaxIdleConns/MaxIdleConnsPerHost tuning this branch added to speed up syncing. Drain the body in the deferred close. Co-Authored-By: Claude Opus 4.8 * refactor(tron): drop dead isSolidified param from requestBlockHashByNum requestBlockHashByNum is only called from GetBlockHash's fast path, which is guarded by isBlockSolidified() && solidityNodeHTTP \!= nil, so the sole caller always passed true and the /wallet/getblock (full-node) branch was unreachable. Hardcode the solidity-node client and /walletsolidity/getblock path so the function's actual single-purpose role is explicit and no untested branch lingers. Co-Authored-By: Claude Opus 4.8 * docs(tron): add initial sync tuning --------- Co-authored-by: pragmaxim Co-authored-by: Claude Opus 4.8 --- bchain/coins/tron/tronhttp.go | 22 ++++++- bchain/coins/tron/tronhttp_endpoints.go | 27 +++++++++ bchain/coins/tron/tronrpc.go | 49 +++++----------- bchain/coins/tron/txextra_test.go | 57 ++++++++++++++++++ docs/tron-config.md | 5 ++ docs/tron-sync-tuning.md | 78 +++++++++++++++++++++++++ 6 files changed, 202 insertions(+), 36 deletions(-) create mode 100644 docs/tron-sync-tuning.md diff --git a/bchain/coins/tron/tronhttp.go b/bchain/coins/tron/tronhttp.go index 5172e37ce5..91d7258f82 100644 --- a/bchain/coins/tron/tronhttp.go +++ b/bchain/coins/tron/tronhttp.go @@ -5,10 +5,16 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "time" ) +const ( + tronMaxIdleConnsPerHost = 64 + tronMaxIdleConns = 100 +) + type TronHTTP interface { Request(ctx context.Context, path string, reqBody interface{}, respBody interface{}) error } @@ -19,10 +25,15 @@ type TronHTTPClient struct { } func NewTronHTTPClient(baseURL string, timeout time.Duration) *TronHTTPClient { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = tronMaxIdleConns + transport.MaxIdleConnsPerHost = tronMaxIdleConnsPerHost + return &TronHTTPClient{ baseURL: baseURL, httpClient: &http.Client{ - Timeout: timeout, + Timeout: timeout, + Transport: transport, }, } } @@ -43,7 +54,14 @@ func (c *TronHTTPClient) Request(ctx context.Context, path string, reqBody inter if err != nil { return fmt.Errorf("HTTP error calling Tron API %s: %w", path, err) } - defer resp.Body.Close() + // Drain the body before closing so net/http can return the connection to the + // idle pool for keep-alive reuse. Without this, the >=300 early return (which + // reads nothing) and any bytes the JSON decoder leaves unconsumed would force + // the connection closed, defeating the MaxIdleConns* pooling configured above. + defer func() { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() if resp.StatusCode >= 300 { return fmt.Errorf("Tron API returned status %d at path: %s %s", resp.StatusCode, c.baseURL, path) diff --git a/bchain/coins/tron/tronhttp_endpoints.go b/bchain/coins/tron/tronhttp_endpoints.go index 465ce3f941..5d2ea1ca6b 100644 --- a/bchain/coins/tron/tronhttp_endpoints.go +++ b/bchain/coins/tron/tronhttp_endpoints.go @@ -69,6 +69,7 @@ type tronGetBlockResponse struct { } type tronGetBlockHeaderResponse struct { + BlockID string `json:"blockID"` BlockHeader struct { RawData struct { Number *uint64 `json:"number"` @@ -474,6 +475,32 @@ func (b *TronRPC) requestBlockByID(ctx context.Context, blockHash string, isSoli return &resp, nil } +// requestBlockHashByNum resolves a block hash by height via the solidity node's +// /walletsolidity/getblock endpoint. It is only called for already-solidified +// heights (see GetBlockHash), so the lookup always targets the solidity node. +func (b *TronRPC) requestBlockHashByNum(ctx context.Context, blockNum uint32) (string, error) { + req := map[string]any{ + "id_or_num": strconv.FormatUint(uint64(blockNum), 10), + "detail": false, + } + http := b.getLookupHTTPClient(true) + var resp tronGetBlockHeaderResponse + if err := http.Request(ctx, "/walletsolidity/getblock", req, &resp); err != nil { + return "", err + } + if resp.BlockHeader.RawData.Number == nil { + return "", errors.Errorf("Tron getblock returned missing block_header.raw_data.number for height %d", blockNum) + } + if *resp.BlockHeader.RawData.Number != uint64(blockNum) { + return "", errors.Errorf("Tron getblock returned height %d, want %d", *resp.BlockHeader.RawData.Number, blockNum) + } + hash := strip0xPrefix(resp.BlockID) + if hash == "" { + return "", errors.Errorf("Tron getblock returned empty blockID for height %d", blockNum) + } + return hash, nil +} + func (b *TronRPC) requestLatestSolidifiedBlockHeight(ctx context.Context) (uint64, error) { http := b.solidityNodeHTTP if http == nil { diff --git a/bchain/coins/tron/tronrpc.go b/bchain/coins/tron/tronrpc.go index 0fc653ffdf..cd9299a208 100644 --- a/bchain/coins/tron/tronrpc.go +++ b/bchain/coins/tron/tronrpc.go @@ -287,6 +287,16 @@ func (b *TronRPC) GetBestBlockHash() (string, error) { // GetBlockHash returns block hash in Tron API format (without 0x prefix). func (b *TronRPC) GetBlockHash(height uint32) (string, error) { + if b.isBlockSolidified(uint64(height)) && b.solidityNodeHTTP != nil { + ctx, cancel := context.WithTimeout(b.requestContext(), b.Timeout) + defer cancel() + hash, err := b.requestBlockHashByNum(ctx, height) + if err == nil { + return hash, nil + } + glog.V(1).Infof("Tron native getblock hash lookup failed for height %d, falling back to JSON-RPC: %v", height, err) + } + hash, err := b.EthereumRPC.GetBlockHash(height) if err != nil { return "", err @@ -347,21 +357,6 @@ func (b *TronRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { } func (b *TronRPC) getBestHeader() (bchain.EVMHeader, error) { - // During initial sync (before ZeroMQ is initialized) there is no push-based - // tip refresh, so always read the latest header from the backend. - if b.mq == nil { - _, err := b.refreshBestHeaderFromChain() - if err != nil { - return nil, err - } - b.bestHeaderLock.Lock() - defer b.bestHeaderLock.Unlock() - if b.bestHeader == nil || b.bestHeader.Number() == nil { - return nil, errors.New("best header is nil") - } - return b.bestHeader, nil - } - b.bestHeaderLock.Lock() cachedHeader := b.bestHeader cachedAt := b.bestHeaderTime @@ -384,22 +379,6 @@ func (b *TronRPC) getBestHeader() (bchain.EVMHeader, error) { return b.bestHeader, nil } -// setBestHeader stores h as the cached tip and reports whether it changed. -// -// Unlike EthereumRPC.setBestHeader this is intentionally NON-monotonic: a lower -// height is accepted. Tron's tip is never taken from the feed header (the ZeroMQ -// notification carries none) — it is always an HTTP re-query (refreshBestHeaderFromChain), -// refreshed on every notification and on a tronBestHeaderMaxAge timer, so the cache -// is meant to track whatever the backend currently reports. Accepting a lower height -// is what lets a genuine rollback surface immediately to resyncIndex, so Tron is not -// subject to the frozen-tip masking that the EVM monotonic guard introduces (and which -// EVM has to undo with a watchdog regress). -// -// Tradeoff: with a load-balanced Tron RPC, a single lagging node answering a re-query -// could regress the tip and trip a spurious fork in resyncIndex (the case the EVM -// monotonic guard exists to prevent). That is acceptable for the common single-node -// java-tron backend; if Tron is ever fronted by a load balancer, port the EVM pattern -// here (monotonic hot path + on-advance liveness + allowRegress watchdog poll). func (b *TronRPC) setBestHeader(h bchain.EVMHeader) bool { if h == nil || h.Number() == nil { return false @@ -697,11 +676,11 @@ func (b *TronRPC) computeBlockConfirmations(blockNumber uint64) (uint32, error) return uint32(bestHeight - blockNumber + 1), nil } -func (b *TronRPC) buildTxFromHTTPData(txByID *tronGetTransactionByIDResponse, txInfo *tronGetTransactionInfoByIDResponse, blockTime int64, confirmations uint32, internalData *bchain.EthereumInternalData, isSolidified bool) (*bchain.Tx, error) { +func (b *TronRPC) buildTxFromHTTPData(txByID *tronGetTransactionByIDResponse, txInfo *tronGetTransactionInfoByIDResponse, blockTime int64, confirmations uint32, internalData *bchain.EthereumInternalData, keepReceipt bool) (*bchain.Tx, error) { csd := tronBuildEthereumSpecificData(txByID, txInfo) csd.InternalData = internalData - if !isSolidified { + if !keepReceipt { csd.Receipt = nil // set to nil so it can be considered as pending } @@ -948,7 +927,9 @@ func (b *TronRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { txInternalData = &internalData[i] } - rebuiltTx, err := b.buildTxFromHTTPData(txByID, txInfo, bbh.Time, confirmations, txInternalData, isSolidified) + // Keep receipts for block indexing even before solidity. If logs are dropped + // here, token-transfer participants never reach the address/contract indexes. + rebuiltTx, err := b.buildTxFromHTTPData(txByID, txInfo, bbh.Time, confirmations, txInternalData, true) if err != nil { return nil, err } diff --git a/bchain/coins/tron/txextra_test.go b/bchain/coins/tron/txextra_test.go index f7ed4de45a..64eca6721f 100644 --- a/bchain/coins/tron/txextra_test.go +++ b/bchain/coins/tron/txextra_test.go @@ -512,3 +512,60 @@ func TestTronBuildTxFromHTTPData_WithSynthesizedGenesisData(t *testing.T) { require.NotNil(t, receipt) require.Equal(t, "0x1", receipt.Status) } + +func TestTronBuildTxFromHTTPData_KeepReceiptControlsTokenLogs(t *testing.T) { + txByID := &tronGetTransactionByIDResponse{ + TxID: "b7a97862b0c719b714f0cf7e250ebc2dcdf1e5f05c54ddb21ff3a748c1aa45e4", + } + txByID.RawData.Contract = []tronTxContract{{ + Type: "TriggerSmartContract", + }} + txByID.RawData.Contract[0].Parameter.Value.OwnerAddress = "410746a05c314538e3e21faae3d702cc7939efc07a" + txByID.RawData.Contract[0].Parameter.Value.ContractAddress = "4139dd12a54e2bab7c82aa14a1e158b34263d2d510" + txByID.RawData.Contract[0].Parameter.Value.Data = "6f21b898" + + txInfo := &tronGetTransactionInfoByIDResponse{ + ID: txByID.TxID, + Log: []*bchain.RpcLog{ + { + Address: "a614f803b6fd780986a42c78ec9c7f77e6ded13c", + Data: "000000000000000000000000000000000000000000000000000000002d4cae00", + Topics: []string{ + "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0000000000000000000000000746a05c314538e3e21faae3d702cc7939efc07a", + "000000000000000000000000f14cca91e8d5fff29cbcd42b26a19a7395b1aa84", + }, + }, + }, + } + txInfo.Receipt.Result = "SUCCESS" + + parser := NewTronParser(1, false) + tronRPC := &TronRPC{ + Parser: parser, + } + + pendingTx, err := tronRPC.buildTxFromHTTPData(txByID, txInfo, 0, 0, nil, false) + require.NoError(t, err) + pendingSpecific, ok := pendingTx.CoinSpecificData.(bchain.EthereumSpecificData) + require.True(t, ok) + require.Nil(t, pendingSpecific.Receipt) + pendingTransfers, err := parser.EthereumTypeGetTokenTransfersFromTx(pendingTx) + require.NoError(t, err) + require.Empty(t, pendingTransfers) + + indexedTx, err := tronRPC.buildTxFromHTTPData(txByID, txInfo, 0, 1, nil, true) + require.NoError(t, err) + indexedSpecific, ok := indexedTx.CoinSpecificData.(bchain.EthereumSpecificData) + require.True(t, ok) + require.NotNil(t, indexedSpecific.Receipt) + require.Len(t, indexedSpecific.Receipt.Logs, 1) + + indexedTransfers, err := parser.EthereumTypeGetTokenTransfersFromTx(indexedTx) + require.NoError(t, err) + require.Len(t, indexedTransfers, 1) + require.Equal(t, "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t", indexedTransfers[0].Contract) + require.Equal(t, "TAdgLcNFZjeF1AspMobkz2bbxs5yDXpwYx", indexedTransfers[0].From) + require.Equal(t, "TXy5qqpAJNykdqsMU2dZM7B7mZbVF2izAN", indexedTransfers[0].To) + require.Equal(t, "760000000", indexedTransfers[0].Value.String()) +} diff --git a/docs/tron-config.md b/docs/tron-config.md index 8cfd24546c..107abf2634 100644 --- a/docs/tron-config.md +++ b/docs/tron-config.md @@ -61,7 +61,12 @@ Set explicit `tron_fullnode_http_url_template` and `tron_solidity_http_url_templ * the scheme differs from the JSON-RPC endpoint * the deployment uses non-standard ports +## Sync Performance Tuning + +For initial-sync tuning notes, see [Tron Sync Tuning](/docs/tron-sync-tuning.md). + ## Related Docs * [Config](/docs/config.md) +* [Tron Sync Tuning](/docs/tron-sync-tuning.md) * [Testing](/docs/testing.md) diff --git a/docs/tron-sync-tuning.md b/docs/tron-sync-tuning.md new file mode 100644 index 0000000000..33d3104f49 --- /dev/null +++ b/docs/tron-sync-tuning.md @@ -0,0 +1,78 @@ +# Tron Sync Tuning + +Fast Tron initial sync needs tuning on both sides of the RPC boundary. In the +observed setup, the bottleneck was backend block-hash/block lookup latency, not +Blockbook Go GC or RocksDB writes. + +## java-tron + +Keep the backend on LevelDB, keep transaction history enabled, enable the LevelDB read tuning and raise the +HTTP connection limit enough for Blockbook's worker count: + +```hocon +storage { + db.engine = "LEVELDB" + db.sync = false + transHistory.switch = "on" + + # LevelDB read tuning from Tron TIP-343. Raises fd usage. + default = { + maxOpenFiles = 100 + } + defaultM = { + maxOpenFiles = 500 + } + defaultL = { + maxOpenFiles = 1000 + } +} + +node { + # With Blockbook -workers=32, start around 200. + maxHttpConnectNumber = 200 +} +``` + +Run java-tron with enough heap, but leave RAM for the OS page cache. On large +hosts, `-Xms32g -Xmx32g` with G1GC is a good starting point. Validate with GC +logs and disk latency; a larger heap is not automatically faster. + +Keep event subscription minimal for Blockbook sync: + +```hocon +event.subscribe { + topics = [ + { triggerName = "block", enable = true, topic = "block", solidified = false }, + { triggerName = "solidity", enable = true, topic = "solidity" } + ] +} +``` + +Do not enable `transaction`, `contractevent`, or `contractlog` topics just for +Blockbook initial sync. + +## Blockbook + +Run initial sync with enough workers to keep java-tron busy: + +```ini +ExecStart=.../blockbook ... -sync -workers=32 ... +``` + +Pair this with a sufficiently large Tron HTTP connection pool in Blockbook and +`node.maxHttpConnectNumber` in java-tron. Raising workers without raising the +backend connection limit just moves the queue. + +## Verify + +Useful indicators while syncing: + +* Blockbook log `elapsed` per 1000 blocks +* `blockbook_rpc_latency{method="GetBlockHash"}` and `method="GetBlock"`, increased latency means the bottleneck is on the Java-Tron side +* java-tron GC pauses and heap after GC +* disk utilization and await, for example `iostat -xz 1` +* open file descriptors for both services +* `blockbook_index_block_not_found_retries` + +With the tuning above and native hash lookup, observed throughput reached about +7.5-9.5 seconds per 1000 historical blocks on the tested setup. From fb2efb7c75e8c5329bc8bba1462030e9ec5eaea7 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Sun, 5 Jul 2026 14:48:04 +0000 Subject: [PATCH 054/303] fix(pivx): compute correct txid for Sapling (v>=3) transactions PIVX Sapling transactions (version >= 3) include shielded fields (valueBalance, vShieldedSpend, vShieldedOutput, bindingSig) after the transparent part. wire.MsgTx.TxHash() only hashes the transparent fields, producing a phantom txid that is stored in the address index. Fix by computing the txid as sha256d of the full raw transaction bytes for Sapling transactions, matching PIVX's consensus definition. Also fix a pre-existing encoding bug in ParseTx: it used WitnessEncoding via Deserialize(), which can misinterpret a Sapling tx with 0 inputs as a segwit transaction (0x00 0x01 marker). Now reads the version first and uses BaseEncoding for version >= 3, matching PivxDecodeTransactions. Fixes https://github.com/trezor/blockbook/issues/1522 --- bchain/coins/pivx/pivxparser.go | 53 ++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/bchain/coins/pivx/pivxparser.go b/bchain/coins/pivx/pivxparser.go index 57acff4463..4377c1a9c0 100644 --- a/bchain/coins/pivx/pivxparser.go +++ b/bchain/coins/pivx/pivxparser.go @@ -11,6 +11,7 @@ import ( "github.com/juju/errors" "github.com/martinboehm/btcd/blockchain" + "github.com/martinboehm/btcd/chaincfg/chainhash" "github.com/martinboehm/btcd/wire" "github.com/martinboehm/btcutil/chaincfg" "github.com/trezor/blockbook/bchain" @@ -106,7 +107,7 @@ func (p *PivXParser) ParseBlock(b []byte) (*bchain.Block, error) { r.Seek(32, io.SeekCurrent) } - err = p.PivxDecodeTransactions(r, 0, &w) + rawTxs, err := p.PivxDecodeTransactions(r, 0, &w, b) if err != nil { return nil, errors.Annotatef(err, "DecodeTransactions") } @@ -114,6 +115,9 @@ func (p *PivXParser) ParseBlock(b []byte) (*bchain.Block, error) { txs := make([]bchain.Tx, len(w.Transactions)) for ti, t := range w.Transactions { txs[ti] = p.TxFromMsgTx(t, false) + if int(t.Version)&0xffff >= 3 { + txs[ti].Txid = chainhash.DoubleHashH(rawTxs[ti]).String() + } } return &bchain.Block{ @@ -140,11 +144,29 @@ func (p *PivXParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { func (p *PivXParser) ParseTx(b []byte) (*bchain.Tx, error) { t := wire.MsgTx{} r := bytes.NewReader(b) - if err := t.Deserialize(r); err != nil { + + // read version and choose encoding (same logic as PivxDecodeTransactions) + var version uint32 + if err := binary.Read(r, binary.LittleEndian, &version); err != nil { + return nil, err + } + if _, err := r.Seek(-4, io.SeekCurrent); err != nil { + return nil, err + } + + enc := wire.WitnessEncoding + if int32(version)&0xffff >= 3 { + enc = wire.BaseEncoding + } + + if err := t.BtcDecode(r, 0, enc); err != nil { return nil, err } tx := p.TxFromMsgTx(&t, true) tx.Hex = hex.EncodeToString(b) + if int(t.Version)&0xffff >= 3 { + tx.Txid = chainhash.DoubleHashH(b).String() + } return &tx, nil } @@ -262,12 +284,12 @@ func (p *PivXParser) GetAddrDescForUnknownInput(tx *bchain.Tx, input int) bchain return s } -func (p *PivXParser) PivxDecodeTransactions(r *bytes.Reader, pver uint32, blk *wire.MsgBlock) error { +func (p *PivXParser) PivxDecodeTransactions(r *bytes.Reader, pver uint32, blk *wire.MsgBlock, blockBytes []byte) ([][]byte, error) { maxTxPerBlock := uint64((wire.MaxBlockPayload / 10) + 1) txCount, err := wire.ReadVarInt(r, pver) if err != nil { - return err + return nil, err } // Prevent more transactions than could possibly fit into a block. @@ -276,20 +298,26 @@ func (p *PivXParser) PivxDecodeTransactions(r *bytes.Reader, pver uint32, blk *w if txCount > maxTxPerBlock { str := fmt.Sprintf("too many transactions to fit into a block "+ "[count %d, max %d]", txCount, maxTxPerBlock) - return &wire.MessageError{Func: "utils.decodeTransactions", Description: str} + return nil, &wire.MessageError{Func: "utils.decodeTransactions", Description: str} } blk.Transactions = make([]*wire.MsgTx, 0, txCount) + rawTxs := make([][]byte, 0, txCount) for i := uint64(0); i < txCount; i++ { tx := wire.MsgTx{} + txStart, err := r.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + // read version & seek back to original state var version uint32 = 0 if err = binary.Read(r, binary.LittleEndian, &version); err != nil { - return err + return nil, err } if _, err = r.Seek(-4, io.SeekCurrent); err != nil { - return err + return nil, err } txVersion := version & 0xffff @@ -300,14 +328,19 @@ func (p *PivXParser) PivxDecodeTransactions(r *bytes.Reader, pver uint32, blk *w enc = wire.BaseEncoding } - err := p.PivxDecode(&tx, r, pver, enc) + err = p.PivxDecode(&tx, r, pver, enc) if err != nil { - return err + return nil, err + } + txEnd, err := r.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err } blk.Transactions = append(blk.Transactions, &tx) + rawTxs = append(rawTxs, blockBytes[txStart:txEnd]) } - return nil + return rawTxs, nil } func (p *PivXParser) PivxDecode(MsgTx *wire.MsgTx, r *bytes.Reader, pver uint32, enc wire.MessageEncoding) error { From 3ae71efcb1fada6ae7e546de106570489d569012 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 8 Jul 2026 09:03:39 +0000 Subject: [PATCH 055/303] fix(eth): recover GetTransaction on archive backends with pruned tx index EthereumRPC.GetTransaction relied solely on eth_getTransactionByHash, which returns null for mined transactions older than the backend's transaction-by-hash retention window on some archive nodes (observed on QuikNode Base, now the Base backend). Such transactions are still fully retained and reachable via eth_getBlockByHash and eth_getTransactionReceipt, but blockbook reported them as ErrTxNotFound, failing both production lookups and the base=main/rpc/GetTransaction integration test. Fall back to reconstructing the transaction from its receipt (for the block hash) plus the full block body when eth_getTransactionByHash returns null. Genuinely unknown transactions still return ErrTxNotFound, and the primary path is unchanged for backends that retain the full index. Fixes #1609 Co-Authored-By: Claude Opus 4.8 (1M context) --- bchain/coins/eth/ethrpc.go | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 8b09a0c6fa..4ff5372e35 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1727,6 +1727,49 @@ func (b *EthereumRPC) removeTransactionFromMempool(txid string) { } } +// recoverMinedTransaction reconstructs a mined transaction that eth_getTransactionByHash +// failed to return. Some archive backends prune the transaction-by-hash index beyond a +// recent window while still serving full block bodies and receipts (observed on QuikNode +// Base), so an older-but-retained transaction reads as missing. The receipt still carries +// the block hash, from which the full block body yields the transaction. Returns nil when +// the backend genuinely has no record of the transaction (so the caller falls back to +// ErrTxNotFound) or when the recovery lookups fail; failures are logged, not propagated, +// to preserve the not-found semantics of the primary lookup. +func (b *EthereumRPC) recoverMinedTransaction(txid string) *bchain.RpcTransaction { + ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) + defer cancel() + // eth_getTransactionReceipt keeps working on such backends; the block hash is enough + // to fetch the block body, which contains the full transaction. + var receipt struct { + BlockHash string `json:"blockHash"` + } + if err := b.RPC.CallContext(ctx, &receipt, "eth_getTransactionReceipt", ethcommon.HexToHash(txid)); err != nil { + glog.Warningf("recoverMinedTransaction %s: eth_getTransactionReceipt failed: %v", txid, err) + return nil + } + if receipt.BlockHash == "" { + // No receipt: the transaction is genuinely unknown to the backend. + return nil + } + raw, err := b.getBlockRaw(receipt.BlockHash, 0, true) + if err != nil { + glog.Warningf("recoverMinedTransaction %s: getBlockRaw %s failed: %v", txid, receipt.BlockHash, err) + return nil + } + var body rpcBlockTransactions + if err := json.Unmarshal(raw, &body); err != nil { + glog.Warningf("recoverMinedTransaction %s: decode block %s failed: %v", txid, receipt.BlockHash, err) + return nil + } + for i := range body.Transactions { + if strings.EqualFold(body.Transactions[i].Hash, txid) { + return &body.Transactions[i] + } + } + glog.Warningf("recoverMinedTransaction %s: not present in block %s body", txid, receipt.BlockHash) + return nil +} + // GetTransaction returns a transaction by the transaction ID. func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) @@ -1746,8 +1789,18 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { } } if *tx == (bchain.RpcTransaction{}) { - b.removeTransactionFromMempool(txid) - return nil, bchain.ErrTxNotFound + // eth_getTransactionByHash returned null. Some archive backends (observed on + // QuikNode Base) prune the transaction-by-hash index beyond a recent window + // while still serving block bodies and receipts, so a mined transaction older + // than that window is invisible to this call even though it is fully retained. + // Recover it from its receipt (which still carries the block hash) plus the + // block body before treating it as not found. + if recovered := b.recoverMinedTransaction(txid); recovered != nil { + tx = recovered + } else { + b.removeTransactionFromMempool(txid) + return nil, bchain.ErrTxNotFound + } } var btx *bchain.Tx if tx.BlockNumber == "" { From 15126e835ba1780d5148ae180c70fc8ab7cd46a5 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 9 Jul 2026 08:13:13 +0000 Subject: [PATCH 056/303] perf(eth): recover pruned tx via getTransactionByBlockHashAndIndex recoverMinedTransaction reconstructed a pruned-index transaction by fetching the entire block body (eth_getBlockByHash, fullTxs=true) and scanning it for the hash. On endpoints that call GetTransaction per tx (address/xpub/block/balancehistory), a cold-cache page of transactions from the pruned window fetched and re-parsed the full block body once per tx - the same ~592 KB block up to 1000x on a block page. Fetch the transaction directly by its receipt's (blockHash, transactionIndex) via eth_getTransactionByBlockHashAndIndex - an O(1), ~900x smaller lookup - and reuse the already-fetched receipt for EthTxToTx instead of fetching it a second time. Verified against the QuikNode Base archive backend: the by-index result is byte-identical to the block-body scan across transactions 64-926 days old. Fall back to the block-body scan when the positional lookup is unavailable, so the change is strictly non-regressive for any backend, and give each recovery RPC its own timeout budget. Co-Authored-By: Claude Opus 4.8 (1M context) --- bchain/coins/eth/ethrpc.go | 100 ++++++++++----- bchain/coins/eth/ethrpc_test.go | 212 ++++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 32 deletions(-) create mode 100644 bchain/coins/eth/ethrpc_test.go diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 4ff5372e35..32c46a8b63 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -1727,38 +1727,27 @@ func (b *EthereumRPC) removeTransactionFromMempool(txid string) { } } -// recoverMinedTransaction reconstructs a mined transaction that eth_getTransactionByHash -// failed to return. Some archive backends prune the transaction-by-hash index beyond a -// recent window while still serving full block bodies and receipts (observed on QuikNode -// Base), so an older-but-retained transaction reads as missing. The receipt still carries -// the block hash, from which the full block body yields the transaction. Returns nil when -// the backend genuinely has no record of the transaction (so the caller falls back to -// ErrTxNotFound) or when the recovery lookups fail; failures are logged, not propagated, -// to preserve the not-found semantics of the primary lookup. -func (b *EthereumRPC) recoverMinedTransaction(txid string) *bchain.RpcTransaction { +// callContextWithTimeout issues a single JSON-RPC call under its own fresh b.Timeout +// deadline, so sequential calls in a recovery sequence do not share (and progressively +// shrink) one deadline budget. +func (b *EthereumRPC) callContextWithTimeout(result interface{}, method string, args ...interface{}) error { ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) defer cancel() - // eth_getTransactionReceipt keeps working on such backends; the block hash is enough - // to fetch the block body, which contains the full transaction. - var receipt struct { - BlockHash string `json:"blockHash"` - } - if err := b.RPC.CallContext(ctx, &receipt, "eth_getTransactionReceipt", ethcommon.HexToHash(txid)); err != nil { - glog.Warningf("recoverMinedTransaction %s: eth_getTransactionReceipt failed: %v", txid, err) - return nil - } - if receipt.BlockHash == "" { - // No receipt: the transaction is genuinely unknown to the backend. - return nil - } - raw, err := b.getBlockRaw(receipt.BlockHash, 0, true) + return b.RPC.CallContext(ctx, result, method, args...) +} + +// txFromBlockBody fetches the full block body and returns the transaction matching txid, or +// nil if the fetch fails or the transaction is not present. This is the recovery fallback +// used when the O(1) positional lookup is unavailable. +func (b *EthereumRPC) txFromBlockBody(txid, blockHash string) *bchain.RpcTransaction { + raw, err := b.getBlockRaw(blockHash, 0, true) if err != nil { - glog.Warningf("recoverMinedTransaction %s: getBlockRaw %s failed: %v", txid, receipt.BlockHash, err) + glog.Warningf("recoverMinedTransaction %s: getBlockRaw %s failed: %v", txid, blockHash, err) return nil } var body rpcBlockTransactions if err := json.Unmarshal(raw, &body); err != nil { - glog.Warningf("recoverMinedTransaction %s: decode block %s failed: %v", txid, receipt.BlockHash, err) + glog.Warningf("recoverMinedTransaction %s: decode block %s failed: %v", txid, blockHash, err) return nil } for i := range body.Transactions { @@ -1766,10 +1755,49 @@ func (b *EthereumRPC) recoverMinedTransaction(txid string) *bchain.RpcTransactio return &body.Transactions[i] } } - glog.Warningf("recoverMinedTransaction %s: not present in block %s body", txid, receipt.BlockHash) return nil } +// recoverMinedTransaction reconstructs a mined transaction that eth_getTransactionByHash +// returned null because the backend pruned its tx-by-hash index (observed on QuikNode Base). +// It looks the tx up by the receipt's (blockHash, transactionIndex) and returns it with the +// receipt for reuse. Returns (nil, nil) when the tx is genuinely unknown or recovery fails, +// so the caller yields ErrTxNotFound; recovery-lookup failures are logged, not propagated. +func (b *EthereumRPC) recoverMinedTransaction(txid string) (*bchain.RpcTransaction, *bchain.RpcReceipt) { + // The receipt still works on such backends and carries the block hash and tx index; + // decode it in one pass (embedding RpcReceipt) so it can be reused for EthTxToTx. + var receipt struct { + bchain.RpcReceipt + BlockHash string `json:"blockHash"` + TransactionIndex string `json:"transactionIndex"` + } + if err := b.callContextWithTimeout(&receipt, "eth_getTransactionReceipt", ethcommon.HexToHash(txid)); err != nil { + glog.Warningf("recoverMinedTransaction %s: eth_getTransactionReceipt failed: %v", txid, err) + return nil, nil + } + if receipt.BlockHash == "" { + // No receipt: the transaction is genuinely unknown to the backend. + return nil, nil + } + // Fast path: fetch the single tx by (blockHash, index) - an O(1), ~900x smaller lookup + // than scanning the whole block body. transactionIndex is already a hex quantity. + tx := &bchain.RpcTransaction{} + err := b.callContextWithTimeout(tx, "eth_getTransactionByBlockHashAndIndex", ethcommon.HexToHash(receipt.BlockHash), receipt.TransactionIndex) + if err == nil && strings.EqualFold(tx.Hash, txid) { + return tx, &receipt.RpcReceipt + } + if err != nil { + glog.Warningf("recoverMinedTransaction %s: eth_getTransactionByBlockHashAndIndex %s/%s failed, falling back to block body: %v", txid, receipt.BlockHash, receipt.TransactionIndex, err) + } + // Fallback for backends that prune tx-by-hash but do not serve the positional lookup (or + // returned an empty/mismatched result): scan the block body, as before the optimization. + if scanned := b.txFromBlockBody(txid, receipt.BlockHash); scanned != nil { + return scanned, &receipt.RpcReceipt + } + glog.Warningf("recoverMinedTransaction %s: not recoverable from block %s (index %s)", txid, receipt.BlockHash, receipt.TransactionIndex) + return nil, nil +} + // GetTransaction returns a transaction by the transaction ID. func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) @@ -1788,15 +1816,19 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { return nil, err } } + // recoveredReceipt is set only when the transaction was reconstructed via the pruned-index + // fallback below; the mined branch reuses it instead of fetching the receipt again. + var recoveredReceipt *bchain.RpcReceipt if *tx == (bchain.RpcTransaction{}) { // eth_getTransactionByHash returned null. Some archive backends (observed on // QuikNode Base) prune the transaction-by-hash index beyond a recent window // while still serving block bodies and receipts, so a mined transaction older // than that window is invisible to this call even though it is fully retained. - // Recover it from its receipt (which still carries the block hash) plus the - // block body before treating it as not found. - if recovered := b.recoverMinedTransaction(txid); recovered != nil { + // Recover it from its receipt (which carries the block hash and index) before + // treating it as not found. + if recovered, receipt := b.recoverMinedTransaction(txid); recovered != nil { tx = recovered + recoveredReceipt = receipt } else { b.removeTransactionFromMempool(txid) return nil, bchain.ErrTxNotFound @@ -1827,9 +1859,13 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { return nil, errors.Annotatef(err, "txid %v", txid) } tx.BaseFeePerGas = ht.BaseFeePerGas - receipt, err := b.EthereumTypeGetTransactionReceipt(txid) - if err != nil { - return nil, errors.Annotatef(err, "txid %v", txid) + // Reuse the receipt already fetched during pruned-index recovery; otherwise fetch it. + receipt := recoveredReceipt + if receipt == nil { + receipt, err = b.EthereumTypeGetTransactionReceipt(txid) + if err != nil { + return nil, errors.Annotatef(err, "txid %v", txid) + } } n, err := ethNumber(tx.BlockNumber) if err != nil { diff --git a/bchain/coins/eth/ethrpc_test.go b/bchain/coins/eth/ethrpc_test.go new file mode 100644 index 0000000000..0ef5b49a7c --- /dev/null +++ b/bchain/coins/eth/ethrpc_test.go @@ -0,0 +1,212 @@ +//go:build unittest + +package eth + +import ( + "context" + "encoding/json" + stdErrors "errors" + "testing" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/trezor/blockbook/bchain" +) + +// recoveryStub is a method-aware bchain.EVMRPCClient fake for exercising the pruned-index +// recovery path in recoverMinedTransaction / GetTransaction. It serves canned JSON per +// JSON-RPC method, can force the positional lookup to error (to drive the block-body +// fallback), and records call counts plus arguments so tests can assert exactly which RPCs +// were made (single receipt fetch, block body only when needed). +type recoveryStub struct { + byHashJSON string // eth_getTransactionByHash result; "" leaves the target zero (null) + receiptJSON string // eth_getTransactionReceipt result ("null" => unknown tx) + byIndexJSON string // eth_getTransactionByBlockHashAndIndex result + byIndexErr error // when set, the positional lookup errors (drives the fallback) + blockJSON string // eth_getBlockByHash raw result (header fields and/or transactions) + calls map[string]int + byIndexArgs []interface{} + blockFullTxs []bool // fullTxs arg recorded per eth_getBlockByHash call +} + +func newRecoveryStub() *recoveryStub { return &recoveryStub{calls: map[string]int{}} } + +func (s *recoveryStub) EthSubscribe(context.Context, interface{}, ...interface{}) (bchain.EVMClientSubscription, error) { + return nil, stdErrors.New("not implemented") +} + +func (s *recoveryStub) Close() {} + +func (s *recoveryStub) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + s.calls[method]++ + switch method { + case "eth_getTransactionByHash": + if s.byHashJSON == "" { + return nil // null result: leaves the caller's zero-value tx untouched + } + return json.Unmarshal([]byte(s.byHashJSON), result) + case "eth_getTransactionReceipt": + return json.Unmarshal([]byte(s.receiptJSON), result) + case "eth_getTransactionByBlockHashAndIndex": + s.byIndexArgs = args + if s.byIndexErr != nil { + return s.byIndexErr + } + return json.Unmarshal([]byte(s.byIndexJSON), result) + case "eth_getBlockByHash": + if len(args) >= 2 { + if full, ok := args[1].(bool); ok { + s.blockFullTxs = append(s.blockFullTxs, full) + } + } + if p, ok := result.(*json.RawMessage); ok { + *p = json.RawMessage(s.blockJSON) + } + return nil + default: + return stdErrors.New("unexpected method: " + method) + } +} + +const ( + recTxid = "0xd20ea523eee594f82d481b4da6a8c2f1ce1e7fee34cd6369ee0ba6093c1d19bb" + recBlockHash = "0x4437c2e020a9c940532e2431babb3550cfdd16f498d2c7b5bb6c0f728567d69d" + recTxIndex = "0x21" + recReceipt = `{"blockHash":"` + recBlockHash + `","transactionIndex":"` + recTxIndex + `","status":"0x1","gasUsed":"0x5208","logs":[]}` + recTxObject = `{"blockHash":"` + recBlockHash + `","blockNumber":"0x2b83e54","from":"0xc68eff0a07180ce4b6e490ddb080b6b8f3867024","gas":"0x5208","gasPrice":"0xf4240","hash":"` + recTxid + `","input":"0x","nonce":"0x5","to":"0x22b51ee43ccab63ec03c50794a841c9189d94ed2","transactionIndex":"` + recTxIndex + `","value":"0x2386f26fc10000"}` + // block header fields (timestamp, baseFeePerGas) plus the tx body, so one JSON serves + // both the mined-branch header read (fullTxs=false) and the block-body fallback scan. + recBlockJSON = `{"timestamp":"0x6819a2e0","baseFeePerGas":"0x5f5e100","transactions":[` + recTxObject + `]}` +) + +// Fast path: a pruned-but-retained tx is recovered via receipt + +// eth_getTransactionByBlockHashAndIndex, the block body is never fetched, the receipt is +// fetched exactly once and returned for reuse, and the index lookup is addressed by the +// receipt's block hash and index. +func TestRecoverMinedTransaction_UsesByBlockHashAndIndex(t *testing.T) { + stub := newRecoveryStub() + stub.receiptJSON, stub.byIndexJSON = recReceipt, recTxObject + b := &EthereumRPC{RPC: stub, Timeout: time.Second} + + tx, receipt := b.recoverMinedTransaction(recTxid) + + if tx == nil || tx.Hash != recTxid { + t.Fatalf("recovered tx = %+v, want hash %s", tx, recTxid) + } + if tx.BlockNumber == "" { + t.Error("recovered tx has empty BlockNumber; would be misrouted as a mempool tx") + } + if receipt == nil || receipt.Status != "0x1" || receipt.GasUsed != "0x5208" { + t.Fatalf("reused receipt = %+v, want Status 0x1 / GasUsed 0x5208", receipt) + } + if got := stub.calls["eth_getBlockByHash"]; got != 0 { + t.Errorf("eth_getBlockByHash called %d times, want 0 (no block-body fetch on fast path)", got) + } + if got := stub.calls["eth_getTransactionReceipt"]; got != 1 { + t.Errorf("eth_getTransactionReceipt called %d times, want exactly 1", got) + } + if got := stub.calls["eth_getTransactionByBlockHashAndIndex"]; got != 1 { + t.Errorf("eth_getTransactionByBlockHashAndIndex called %d times, want 1", got) + } + if len(stub.byIndexArgs) != 2 { + t.Fatalf("eth_getTransactionByBlockHashAndIndex got %d args, want 2", len(stub.byIndexArgs)) + } + if bh, ok := stub.byIndexArgs[0].(ethcommon.Hash); !ok || bh != ethcommon.HexToHash(recBlockHash) { + t.Errorf("byIndex arg[0] = %v, want block hash %s", stub.byIndexArgs[0], recBlockHash) + } + if idx, ok := stub.byIndexArgs[1].(string); !ok || idx != recTxIndex { + t.Errorf("byIndex arg[1] = %v, want index %s", stub.byIndexArgs[1], recTxIndex) + } +} + +// A genuinely unknown tx has a null receipt: recovery returns (nil, nil) so the caller +// yields ErrTxNotFound, and neither the index lookup nor the block body is fetched. +func TestRecoverMinedTransaction_UnknownReturnsNil(t *testing.T) { + stub := newRecoveryStub() + stub.receiptJSON = "null" + b := &EthereumRPC{RPC: stub, Timeout: time.Second} + + tx, receipt := b.recoverMinedTransaction(recTxid) + + if tx != nil || receipt != nil { + t.Errorf("recoverMinedTransaction(unknown) = (%v, %v), want (nil, nil)", tx, receipt) + } + if got := stub.calls["eth_getTransactionByBlockHashAndIndex"] + stub.calls["eth_getBlockByHash"]; got != 0 { + t.Errorf("made %d lookup calls for unknown tx, want 0", got) + } +} + +// Fallback: when eth_getTransactionByBlockHashAndIndex is unavailable (errors), recovery +// falls back to scanning the block body, still recovering the tx and returning the receipt. +func TestRecoverMinedTransaction_FallsBackToBlockBody(t *testing.T) { + stub := newRecoveryStub() + stub.receiptJSON = recReceipt + stub.byIndexErr = stdErrors.New("the method eth_getTransactionByBlockHashAndIndex does not exist") + stub.blockJSON = recBlockJSON + b := &EthereumRPC{RPC: stub, Timeout: time.Second} + + tx, receipt := b.recoverMinedTransaction(recTxid) + + if tx == nil || tx.Hash != recTxid { + t.Fatalf("fallback recovered tx = %+v, want hash %s", tx, recTxid) + } + if receipt == nil || receipt.Status != "0x1" { + t.Fatalf("fallback receipt = %+v, want Status 0x1", receipt) + } + if len(stub.blockFullTxs) != 1 || !stub.blockFullTxs[0] { + t.Errorf("block-body fallback should fetch eth_getBlockByHash once with fullTxs=true, got %v", stub.blockFullTxs) + } +} + +// If both the positional lookup fails and the block body lacks the tx, recovery returns +// (nil, nil). +func TestRecoverMinedTransaction_FallbackMissReturnsNil(t *testing.T) { + stub := newRecoveryStub() + stub.receiptJSON = recReceipt + stub.byIndexErr = stdErrors.New("unsupported") + stub.blockJSON = `{"timestamp":"0x1","transactions":[]}` + b := &EthereumRPC{RPC: stub, Timeout: time.Second} + + tx, receipt := b.recoverMinedTransaction(recTxid) + + if tx != nil || receipt != nil { + t.Errorf("recoverMinedTransaction(fallback miss) = (%v, %v), want (nil, nil)", tx, receipt) + } +} + +// End-to-end GetTransaction on a pruned tx: eth_getTransactionByHash returns null, the tx is +// recovered, and the recovered receipt is reused for EthTxToTx instead of being fetched a +// second time - so eth_getTransactionReceipt is called exactly once and no block body is read. +func TestGetTransaction_RecoveryReusesReceipt(t *testing.T) { + stub := newRecoveryStub() + stub.receiptJSON, stub.byIndexJSON, stub.blockJSON = recReceipt, recTxObject, recBlockJSON + b := &EthereumRPC{ + RPC: stub, + Timeout: time.Second, + Parser: NewEthereumParser(1, false), + bestHeader: stubHeader{n: 45629030}, // > tx block 0x2b83e54, for computeConfirmations + } + + got, err := b.GetTransaction(recTxid) + if err != nil { + t.Fatalf("GetTransaction returned error: %v", err) + } + if got == nil || got.Txid != recTxid { + t.Fatalf("GetTransaction = %+v, want Txid %s", got, recTxid) + } + if c := stub.calls["eth_getTransactionReceipt"]; c != 1 { + t.Errorf("eth_getTransactionReceipt called %d times, want exactly 1 (receipt must be reused, not re-fetched)", c) + } + if c := stub.calls["eth_getTransactionByHash"]; c != 1 { + t.Errorf("eth_getTransactionByHash called %d times, want 1", c) + } + // Only the mined-branch header read (fullTxs=false); no block-body fetch. + for _, full := range stub.blockFullTxs { + if full { + t.Errorf("GetTransaction fetched a full block body during recovery, want header-only reads: %v", stub.blockFullTxs) + } + } + if got.Confirmations <= 0 { + t.Errorf("Confirmations = %d, want > 0", got.Confirmations) + } +} From b72e0727b373e1dc101737d2252062005015812d Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 05:34:31 +0000 Subject: [PATCH 057/303] feat(eth): allow rpcCall by whitelisted EVM method selectors Add _ALLOWED_EVM_CALL_METHODS, a comma-separated list of 4-byte method selectors that permits websocket rpcCall requests to any address when the calldata starts with an allowed selector (e.g. 0xdd62ed3e for ERC-20 allowance). Complements ALLOWED_RPC_CALL_TO: a call passes when either its target address or its selector is allowed; with neither set rpcCall stays unrestricted. Malformed calldata never matches (full hex decode, fail closed) and malformed or empty selector config fails startup. Co-Authored-By: Claude Fable 5 --- docs/env.md | 4 +- server/public_ethereumtype_test.go | 69 ++++++++++++ server/websocket.go | 90 +++++++++++++++- server/websocket_test.go | 168 +++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+), 6 deletions(-) diff --git a/docs/env.md b/docs/env.md index 75217268bb..5fd821e24e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -81,7 +81,9 @@ Blockbook reads these from its process environment. When installed from the Debi 3. `COINGECKO_API_KEY` Example: for Optimism, `network=OP` and `coin shortcut=ETH`, so `OP_COINGECKO_API_KEY` is preferred over `ETH_COINGECKO_API_KEY`. -- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. If omitted, `rpcCall` is enabled for all addresses. +- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. If omitted (and `ALLOWED_EVM_CALL_METHODS` is not set either), `rpcCall` is enabled for all addresses. + +- `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all, is a configuration error and Blockbook fails on startup. - `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. diff --git a/server/public_ethereumtype_test.go b/server/public_ethereumtype_test.go index 3fac995e55..ca6bd77442 100644 --- a/server/public_ethereumtype_test.go +++ b/server/public_ethereumtype_test.go @@ -361,6 +361,75 @@ func Test_PublicServer_EthereumType(t *testing.T) { httpTestsEthereumType(t, ts) runWebsocketTests(t, ts, websocketTestsEthereumType) } + +// allowance(owner, spender) calldata with the 0xdd62ed3e selector +const rpcCallAllowanceData = "0xdd62ed3e0000000000000000000000009ea3721b5bf3b64b4418c38b603154d2d597fae3000000000000000000000000e4db1c5a1b709ce4d2ada6985d9d506e58f73829" + +var websocketTestsEthereumTypeRpcCallAllowlist = []websocketTest{ + { + name: "rpcCall to allowed address", + req: websocketReq{ + Method: "rpcCall", + Params: WsRpcCallReq{ + To: "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9", + Data: "0x4567", + }, + }, + want: `{"id":"0","data":{"data":"0x4567abcd"}}`, + }, + { + name: "rpcCall with allowed method selector to any address", + req: websocketReq{ + Method: "rpcCall", + Params: WsRpcCallReq{ + To: "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + Data: rpcCallAllowanceData, + }, + }, + want: `{"id":"1","data":{"data":"` + rpcCallAllowanceData + `abcd"}}`, + }, + { + name: "rpcCall with not allowed address and method selector", + req: websocketReq{ + Method: "rpcCall", + Params: WsRpcCallReq{ + To: "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + Data: "0x70a08231000000000000000000000000e4db1c5a1b709ce4d2ada6985d9d506e58f73829", + }, + }, + want: `{"id":"2","data":{"error":{"message":"Not supported"}}}`, + }, + { + name: "rpcCall with allowed selector but malformed calldata", + req: websocketReq{ + Method: "rpcCall", + Params: WsRpcCallReq{ + To: "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + Data: rpcCallAllowanceData[2:], + }, + }, + want: `{"id":"3","data":{"error":{"message":"Not supported"}}}`, + }, +} + +func Test_PublicServer_EthereumType_RpcCallAllowlist(t *testing.T) { + timeNow = fixedTimeNow + t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9") + t.Setenv("FAKE_ALLOWED_EVM_CALL_METHODS", "0xdd62ed3e") + parser := eth.NewEthereumParser(1, true) + chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) + if err != nil { + glog.Fatal("fakechain: ", err) + } + + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) + defer closeAndDestroyPublicServer(t, s, dbpath) + s.ConnectFullPublicInterface() + ts := httptest.NewServer(s.https.Handler) + defer ts.Close() + + runWebsocketTests(t, ts, websocketTestsEthereumTypeRpcCallAllowlist) +} func TestENSResolution(t *testing.T) { parser := eth.NewEthereumParser(1, true) chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) diff --git a/server/websocket.go b/server/websocket.go index ea45cd33fc..621ebb9dba 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -2,6 +2,7 @@ package server import ( "context" + "encoding/hex" "encoding/json" "math/big" "net/http" @@ -106,7 +107,13 @@ type WebsocketServer struct { fiatRatesSubscriptionsLock sync.Mutex allowedOrigins map[string]struct{} allowedRpcCallTo map[string]struct{} - trustedProxyPrefixes []netip.Prefix + // allowedRpcCallMethods holds 4-byte EVM call selectors (lowercase hex, + // no 0x prefix) that rpcCall may invoke on any address; nil when + // unconfigured. Together with allowedRpcCallTo it forms the rpcCall + // allowlist: a call passes when either the target address or the calldata + // selector is allowed; if both are nil, rpcCall is unrestricted. + allowedRpcCallMethods map[string]struct{} + trustedProxyPrefixes []netip.Prefix // cloudflarePrefixes gates trust of the CF-Connecting-* headers: when // non-empty, those headers are honored only when the TCP peer is inside one // of these ranges (or a loopback/private proxy). Empty disables verification @@ -183,6 +190,15 @@ func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, mempool bchain. } glog.Info("Support of rpcCall for these contracts: ", envRpcCall) } + methodsEnvName := strings.ToUpper(is.GetNetwork()) + "_ALLOWED_EVM_CALL_METHODS" + envRpcCallMethods := os.Getenv(methodsEnvName) + s.allowedRpcCallMethods, err = parseAllowedEvmCallMethods(methodsEnvName, envRpcCallMethods) + if err != nil { + return nil, err + } + if s.allowedRpcCallMethods != nil { + glog.Info("Support of rpcCall for these method selectors: ", envRpcCallMethods) + } clientIPCfg, err := readClientIPConfig(is.GetNetwork()) if err != nil { return nil, err @@ -240,6 +256,35 @@ func parseAllowedOrigins(originEnvName, envAllowedOrigins string) map[string]str return allowedOrigins } +// parseAllowedEvmCallMethods parses a comma-separated list of 4-byte EVM call +// selectors (optional 0x prefix, case-insensitive) into a set keyed by +// lowercase hex without the prefix. Returns nil when the variable is unset. +// A malformed selector or a set variable that parses to no selectors at all +// is a configuration error that aborts startup so a typo cannot silently +// disable the intended allowlist. +func parseAllowedEvmCallMethods(envName, envValue string) (map[string]struct{}, error) { + if envValue == "" { + return nil, nil + } + methods := make(map[string]struct{}) + for _, m := range strings.Split(envValue, ",") { + m = strings.TrimSpace(m) + if m == "" { + continue + } + selector := strings.TrimPrefix(strings.ToLower(m), "0x") + b, err := hex.DecodeString(selector) + if err != nil || len(b) != 4 { + return nil, errors.Errorf("Invalid EVM call method selector %q in %s, expecting 4 bytes in hex", m, envName) + } + methods[selector] = struct{}{} + } + if len(methods) == 0 { + return nil, errors.Errorf("%s is set but contains no method selectors", envName) + } + return methods, nil +} + func (s *WebsocketServer) checkOrigin(r *http.Request) bool { origin := r.Header.Get("Origin") if origin == "" { @@ -1201,13 +1246,48 @@ func (s *WebsocketServer) getBlockFiltersBatch(r *WsBlockFiltersBatchReq) (res i }, nil } -func (s *WebsocketServer) rpcCall(r *WsRpcCallReq) (*WsRpcCallRes, error) { +// evmCallSelector extracts the 4-byte function selector from hex-encoded EVM +// calldata as lowercase hex without the 0x prefix. It decodes the whole data +// string so malformed calldata (missing 0x prefix, odd length, non-hex +// characters, fewer than 4 bytes) never yields a selector and thus fails +// closed in the rpcCall allowlist check. +func evmCallSelector(data string) (string, bool) { + if len(data) < 2 || data[0] != '0' || (data[1] != 'x' && data[1] != 'X') { + return "", false + } + b, err := hex.DecodeString(data[2:]) + if err != nil || len(b) < 4 { + return "", false + } + return hex.EncodeToString(b[:4]), true +} + +// rpcCallAllowed reports whether a rpcCall request passes the allowlists. With +// no allowlist configured rpcCall is unrestricted; otherwise the call must +// target an allowed address or invoke an allowed method selector. +func (s *WebsocketServer) rpcCallAllowed(r *WsRpcCallReq) bool { + if s.allowedRpcCallTo == nil && s.allowedRpcCallMethods == nil { + return true + } if s.allowedRpcCallTo != nil { - _, ok := s.allowedRpcCallTo[strings.ToLower(r.To)] - if !ok { - return nil, errors.New("Not supported") + if _, ok := s.allowedRpcCallTo[strings.ToLower(r.To)]; ok { + return true } } + if s.allowedRpcCallMethods != nil { + if selector, ok := evmCallSelector(r.Data); ok { + if _, ok := s.allowedRpcCallMethods[selector]; ok { + return true + } + } + } + return false +} + +func (s *WebsocketServer) rpcCall(r *WsRpcCallReq) (*WsRpcCallRes, error) { + if !s.rpcCallAllowed(r) { + return nil, errors.New("Not supported") + } data, err := s.chain.EthereumTypeRpcCall(r.Data, r.To, r.From) if err != nil { return nil, err diff --git a/server/websocket_test.go b/server/websocket_test.go index fa98bbcef9..a9ccf9d7fc 100644 --- a/server/websocket_test.go +++ b/server/websocket_test.go @@ -167,6 +167,174 @@ func TestParseAllowedOrigins(t *testing.T) { } } +func TestParseAllowedEvmCallMethods(t *testing.T) { + tests := []struct { + name string + env string + want []string + wantErr bool + }{ + {name: "empty", env: "", want: nil}, + {name: "empty entries only", env: " , ,", wantErr: true}, + {name: "single selector", env: "0xdd62ed3e", want: []string{"dd62ed3e"}}, + {name: "without prefix", env: "dd62ed3e", want: []string{"dd62ed3e"}}, + {name: "uppercase", env: "0XDD62ED3E", want: []string{"dd62ed3e"}}, + {name: "multiple with spaces", env: " 0xdd62ed3e , 0x70a08231 ", want: []string{"dd62ed3e", "70a08231"}}, + {name: "empty entries skipped", env: "0xdd62ed3e,,0x70a08231", want: []string{"dd62ed3e", "70a08231"}}, + {name: "too short", env: "0xdd62ed", wantErr: true}, + {name: "too long", env: "0xdd62ed3e00", wantErr: true}, + {name: "odd length", env: "0xdd62ed3", wantErr: true}, + {name: "non hex", env: "0xdd62ed3g", wantErr: true}, + {name: "bare prefix", env: "0x", wantErr: true}, + {name: "one invalid among valid", env: "0xdd62ed3e,invalid", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseAllowedEvmCallMethods("FAKE_ALLOWED_EVM_CALL_METHODS", tt.env) + if tt.wantErr { + if err == nil { + t.Fatalf("parseAllowedEvmCallMethods(%q) = nil err, want error", tt.env) + } + return + } + if err != nil { + t.Fatalf("parseAllowedEvmCallMethods(%q) unexpected error: %v", tt.env, err) + } + if tt.want == nil && got != nil { + t.Fatalf("parseAllowedEvmCallMethods(%q) = %v, want nil", tt.env, got) + } + if len(got) != len(tt.want) { + t.Fatalf("parseAllowedEvmCallMethods(%q) len = %d, want %d", tt.env, len(got), len(tt.want)) + } + for _, selector := range tt.want { + if _, ok := got[selector]; !ok { + t.Fatalf("parseAllowedEvmCallMethods(%q) missing %q", tt.env, selector) + } + } + }) + } +} + +func TestRpcCallAllowed(t *testing.T) { + // allowance(owner, spender) calldata with the 0xdd62ed3e selector + const allowanceData = "0xdd62ed3e0000000000000000000000009ea3721b5bf3b64b4418c38b603154d2d597fae3000000000000000000000000e4db1c5a1b709ce4d2ada6985d9d506e58f73829" + const allowedTo = "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9" + const otherTo = "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" + allowedToSet := map[string]struct{}{strings.ToLower(allowedTo): {}} + allowanceSelectorSet := map[string]struct{}{"dd62ed3e": {}} + + tests := []struct { + name string + to map[string]struct{} + methods map[string]struct{} + req WsRpcCallReq + want bool + }{ + { + name: "no allowlists configured allows all", + req: WsRpcCallReq{To: otherTo, Data: "0x12345678"}, + want: true, + }, + { + name: "allowed address passes", + to: allowedToSet, + req: WsRpcCallReq{To: allowedTo, Data: "0x12345678"}, + want: true, + }, + { + name: "allowed address is case-insensitive", + to: allowedToSet, + req: WsRpcCallReq{To: strings.ToUpper(allowedTo), Data: "0x12345678"}, + want: true, + }, + { + name: "address not allowed without methods list", + to: allowedToSet, + req: WsRpcCallReq{To: otherTo, Data: allowanceData}, + want: false, + }, + { + name: "allowed address passes regardless of selector", + to: allowedToSet, + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: allowedTo, Data: "0x12345678"}, + want: true, + }, + { + name: "allowed selector passes to any address", + to: allowedToSet, + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: allowanceData}, + want: true, + }, + { + name: "selector alone is enough for exact 4 byte calldata", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0xdd62ed3e"}, + want: true, + }, + { + name: "uppercase hex calldata matches", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0XDD62ED3E"}, + want: true, + }, + { + name: "only methods list set rejects other selectors", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0x70a08231000000000000000000000000e4db1c5a1b709ce4d2ada6985d9d506e58f73829"}, + want: false, + }, + { + name: "not allowed address nor selector", + to: allowedToSet, + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0x12345678"}, + want: false, + }, + { + name: "missing 0x prefix fails closed", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: allowanceData[2:]}, + want: false, + }, + { + name: "odd length calldata fails closed", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: allowanceData + "0"}, + want: false, + }, + { + name: "non hex tail after valid selector fails closed", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0xdd62ed3exx"}, + want: false, + }, + { + name: "calldata shorter than selector fails closed", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: "0xdd62ed"}, + want: false, + }, + { + name: "empty calldata fails closed", + methods: allowanceSelectorSet, + req: WsRpcCallReq{To: otherTo, Data: ""}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &WebsocketServer{allowedRpcCallTo: tt.to, allowedRpcCallMethods: tt.methods} + if got := s.rpcCallAllowed(&tt.req); got != tt.want { + t.Fatalf("rpcCallAllowed(to=%q, data=%q) = %v, want %v", tt.req.To, tt.req.Data, got, tt.want) + } + }) + } +} + func TestParseTrustedProxies(t *testing.T) { tests := []struct { name string From a28a85e0c5e2e959508e8f61ad0d1f037ec40245 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 06:11:53 +0000 Subject: [PATCH 058/303] feat(server): make rpcCall allowlists runtime-configurable via admin API Turn ALLOWED_RPC_CALL_TO and ALLOWED_EVM_CALL_METHODS into runtime settings: the environment variables remain the startup defaults, while an override written through the authenticated internal API GET/POST/DELETE /admin/runtime-settings/ is persisted in RocksDB (own cfDefault rows, independent of the periodically stored internalState blob and invisible to older versions), survives restarts, takes precedence over the environment and takes effect on websocket rpcCall immediately, without a restart. The live allowlists are an immutable snapshot behind an atomic pointer on InternalState, so the rpcCall hot path stays lock-free and both servers (public websocket, internal admin) share one view regardless of which of them a deployment runs. Writes are strictly ordered: validate (400 on invalid input) -> store to DB (500 and untouched live state on failure) -> publish snapshot -> log the change with old/new value, source and client address. POST of an exactly empty value explicitly unconfigures a dimension; whitespace- or separator-only values are rejected so a botched input cannot silently un-restrict rpcCall; DELETE reverts to the environment default and validates it first so a malformed environment cannot brick the next restart. The TO-list parser is unified with the selector parser: entries are trimmed, empty entries skipped and a set-but-empty value fails startup (previously "," allowlisted the empty string, which matched rpcCall requests with an empty to field). A minimal /admin/runtime-settings page lists the current values and their sources with curl hints. Co-Authored-By: Claude Fable 5 --- common/internalstate.go | 50 +++ db/rocksdb_runtime_settings.go | 56 +++ db/rocksdb_runtime_settings_test.go | 97 +++++ docs/env.md | 16 +- server/internal.go | 34 +- server/internal_test.go | 2 +- server/public_ethereumtype_test.go | 114 ++++++ server/runtime_settings.go | 373 ++++++++++++++++++ server/runtime_settings_test.go | 292 ++++++++++++++ server/websocket.go | 70 +--- server/websocket_test.go | 12 +- static/internal_templates/index.html | 3 + .../internal_templates/runtime_settings.html | 38 ++ 13 files changed, 1086 insertions(+), 71 deletions(-) create mode 100644 db/rocksdb_runtime_settings.go create mode 100644 db/rocksdb_runtime_settings_test.go create mode 100644 server/runtime_settings.go create mode 100644 server/runtime_settings_test.go create mode 100644 static/internal_templates/runtime_settings.html diff --git a/common/internalstate.go b/common/internalstate.go index 1964d287c4..0d8a22617b 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -119,6 +119,56 @@ type InternalState struct { // shared /64. Guarded by its own mutex (consulted on every connection attempt). wsBlockMux sync.Mutex wsBlockedIPs map[string]*WsBlockedIP + + // rpcCallAllowlists is the effective websocket rpcCall allowlist snapshot, + // resolved from the DB runtime-setting overrides and the environment + // defaults (see server initRpcCallAllowlists). Unexported so it is never + // serialized by Pack; replaced wholesale via the accessors below, giving + // the rpcCall hot path a lock-free, consistent view. + rpcCallAllowlists atomic.Pointer[RpcCallAllowlists] +} + +// Sources of a runtime setting value, reported by the /admin runtime-settings +// interface. +const ( + RuntimeSettingSourceUnset = "unset" + RuntimeSettingSourceEnv = "env" + RuntimeSettingSourceDB = "db" +) + +// RpcCallAllowlists is an immutable snapshot of the websocket rpcCall +// allowlists. Readers must not mutate the maps; writers build a new snapshot +// and replace it via SetRpcCallAllowlists. +type RpcCallAllowlists struct { + // To and Methods are the parsed allowlists; a nil map means that dimension + // is unconfigured. With both nil, rpcCall is unrestricted. + To map[string]struct{} + Methods map[string]struct{} + // Raw comma-separated values and their sources (unset/env/db), kept for + // the admin interface and logging. + ToValue string + MethodsValue string + ToSource string + MethodsSource string +} + +// GetRpcCallAllowlists returns the current rpcCall allowlist snapshot, nil +// when not yet initialized. +func (is *InternalState) GetRpcCallAllowlists() *RpcCallAllowlists { + return is.rpcCallAllowlists.Load() +} + +// SetRpcCallAllowlists atomically replaces the rpcCall allowlist snapshot. +func (is *InternalState) SetRpcCallAllowlists(a *RpcCallAllowlists) { + is.rpcCallAllowlists.Store(a) +} + +// InitRpcCallAllowlists publishes the initial snapshot only when none exists +// yet and reports whether it did. The compare-and-swap keeps a snapshot that +// was already published — possibly updated by the admin interface in the +// meantime — intact when several server constructors initialize in any order. +func (is *InternalState) InitRpcCallAllowlists(a *RpcCallAllowlists) bool { + return is.rpcCallAllowlists.CompareAndSwap(nil, a) } // WsBlockedIP records a websocket client key (IPv4 address or full IPv6 /128) diff --git a/db/rocksdb_runtime_settings.go b/db/rocksdb_runtime_settings.go new file mode 100644 index 0000000000..50a7535c0a --- /dev/null +++ b/db/rocksdb_runtime_settings.go @@ -0,0 +1,56 @@ +package db + +import ( + "encoding/json" + "time" + + "github.com/juju/errors" +) + +// runtimeSettingKeyPrefix prefixes cfDefault keys holding runtime setting +// overrides written through the internal /admin interface. Each setting is +// stored under its own key, so a write or delete touches nothing else in the +// database (in particular it is independent of the periodically stored +// internalState blob) and the rows are ignored by older blockbook versions. +const runtimeSettingKeyPrefix = "runtimeSetting:" + +// runtimeSettingEnvelope wraps a stored runtime setting value. The envelope +// keeps a stored empty value distinguishable from a missing row and records +// when the value was last changed. +type runtimeSettingEnvelope struct { + Value string `json:"value"` + Updated time.Time `json:"updated"` +} + +// GetRuntimeSetting returns the stored override of the given runtime setting +// and whether an override exists. +func (d *RocksDB) GetRuntimeSetting(name string) (string, bool, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfDefault], []byte(runtimeSettingKeyPrefix+name)) + if err != nil { + return "", false, err + } + defer val.Free() + data := val.Data() + if len(data) == 0 { + return "", false, nil + } + var e runtimeSettingEnvelope + if err := json.Unmarshal(data, &e); err != nil { + return "", false, errors.Annotatef(err, "cannot unpack runtime setting %s", name) + } + return e.Value, true, nil +} + +// StoreRuntimeSetting persists an override of the given runtime setting. +func (d *RocksDB) StoreRuntimeSetting(name, value string) error { + buf, err := json.Marshal(&runtimeSettingEnvelope{Value: value, Updated: time.Now()}) + if err != nil { + return err + } + return d.db.PutCF(d.wo, d.cfh[cfDefault], []byte(runtimeSettingKeyPrefix+name), buf) +} + +// DeleteRuntimeSetting removes the override of the given runtime setting. +func (d *RocksDB) DeleteRuntimeSetting(name string) error { + return d.db.DeleteCF(d.wo, d.cfh[cfDefault], []byte(runtimeSettingKeyPrefix+name)) +} diff --git a/db/rocksdb_runtime_settings_test.go b/db/rocksdb_runtime_settings_test.go new file mode 100644 index 0000000000..ce22e5eb3f --- /dev/null +++ b/db/rocksdb_runtime_settings_test.go @@ -0,0 +1,97 @@ +//go:build unittest + +package db + +import ( + "os" + "testing" + + "github.com/trezor/blockbook/common" +) + +func TestRocksDB_RuntimeSettings(t *testing.T) { + parser := &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + } + d := setupRocksDB(t, parser) + path := d.path + dClosed := false + defer func() { + if !dClosed { + if err := d.Close(); err != nil { + t.Error(err) + } + } + os.RemoveAll(path) + }() + + getSetting := func(db *RocksDB, name string) (string, bool) { + t.Helper() + value, found, err := db.GetRuntimeSetting(name) + if err != nil { + t.Fatalf("GetRuntimeSetting(%q) unexpected error: %v", name, err) + } + return value, found + } + + if value, found := getSetting(d, "MISSING"); found || value != "" { + t.Fatalf("GetRuntimeSetting(MISSING) = %q, %v, want empty, false", value, found) + } + + if err := d.StoreRuntimeSetting("ALLOWED_RPC_CALL_TO", "0xabcd,0xef01"); err != nil { + t.Fatal(err) + } + if value, found := getSetting(d, "ALLOWED_RPC_CALL_TO"); !found || value != "0xabcd,0xef01" { + t.Fatalf("GetRuntimeSetting after store = %q, %v, want 0xabcd,0xef01, true", value, found) + } + + // overwrite + if err := d.StoreRuntimeSetting("ALLOWED_RPC_CALL_TO", "0x1234"); err != nil { + t.Fatal(err) + } + if value, found := getSetting(d, "ALLOWED_RPC_CALL_TO"); !found || value != "0x1234" { + t.Fatalf("GetRuntimeSetting after overwrite = %q, %v, want 0x1234, true", value, found) + } + + // an explicitly stored empty value must stay distinguishable from a missing row + if err := d.StoreRuntimeSetting("ALLOWED_EVM_CALL_METHODS", ""); err != nil { + t.Fatal(err) + } + if value, found := getSetting(d, "ALLOWED_EVM_CALL_METHODS"); !found || value != "" { + t.Fatalf("GetRuntimeSetting of stored empty value = %q, %v, want empty, true", value, found) + } + + if err := d.DeleteRuntimeSetting("ALLOWED_EVM_CALL_METHODS"); err != nil { + t.Fatal(err) + } + if value, found := getSetting(d, "ALLOWED_EVM_CALL_METHODS"); found || value != "" { + t.Fatalf("GetRuntimeSetting after delete = %q, %v, want empty, false", value, found) + } + // deleting a missing row is not an error + if err := d.DeleteRuntimeSetting("ALLOWED_EVM_CALL_METHODS"); err != nil { + t.Fatal(err) + } + + // settings survive a database reopen + dClosed = true + if err := d.Close(); err != nil { + t.Fatal(err) + } + d2, err := NewRocksDB(path, 100000, -1, parser, nil, false) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d2.Close(); err != nil { + t.Error(err) + } + }() + is, err := d2.LoadInternalState(&common.Config{CoinName: "coin-unittest"}) + if err != nil { + t.Fatal(err) + } + d2.SetInternalState(is) + if value, found := getSetting(d2, "ALLOWED_RPC_CALL_TO"); !found || value != "0x1234" { + t.Fatalf("GetRuntimeSetting after reopen = %q, %v, want 0x1234, true", value, found) + } +} diff --git a/docs/env.md b/docs/env.md index 5fd821e24e..1fb80a3ba4 100644 --- a/docs/env.md +++ b/docs/env.md @@ -81,9 +81,9 @@ Blockbook reads these from its process environment. When installed from the Debi 3. `COINGECKO_API_KEY` Example: for Optimism, `network=OP` and `coin shortcut=ETH`, so `OP_COINGECKO_API_KEY` is preferred over `ETH_COINGECKO_API_KEY`. -- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. If omitted (and `ALLOWED_EVM_CALL_METHODS` is not set either), `rpcCall` is enabled for all addresses. +- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. Entries are trimmed and empty entries skipped; a set value that contains no addresses at all is a configuration error and Blockbook fails on startup. If omitted (and `ALLOWED_EVM_CALL_METHODS` is not set either), `rpcCall` is enabled for all addresses. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all, is a configuration error and Blockbook fails on startup. +- `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all, is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. - `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. @@ -93,6 +93,18 @@ Blockbook reads these from its process environment. When installed from the Debi WebSocket `sendTransaction` can bypass the alternative provider for a single request by setting `disableAlternativeRPC` to `true`. +## Runtime settings + +The `rpcCall` allowlists can be changed at runtime, without a restart, through the internal server's authenticated admin API (see `BB_ADMIN_USER`/`BB_ADMIN_PASSWORD` above). An override is persisted in the Blockbook database, survives restarts and takes precedence over the corresponding environment variable, which serves only as the startup default. Every change is logged. The `/admin/runtime-settings` page shows the current values and their sources. + +`GET/POST/DELETE /admin/runtime-settings/` where `` is `ALLOWED_RPC_CALL_TO` or `ALLOWED_EVM_CALL_METHODS`: + +- `GET` returns the effective value and its source (`db` = stored override, `env` = environment default, `unset` = neither): `{"key":"ALLOWED_EVM_CALL_METHODS","value":"0xdd62ed3e","source":"env"}`. +- `POST` (or `PUT`) with body `{"value":"0xdd62ed3e,0x70a08231"}` validates the value (invalid values are rejected with `400` and change nothing), stores it in the database and only then applies it to the live allowlists — a database failure returns `500` and leaves the live state unchanged. A `"value"` of exactly `""` is a valid override meaning "explicitly unconfigured" (that allowlist dimension is disabled, as if its environment variable was not set) — it is the only way to un-restrict at runtime while the environment variable has a value. A value containing only whitespace or separators is rejected with `400`, so a botched automation input cannot silently disable the allowlist. +- `DELETE` removes the stored override and reverts to the environment default. If the environment value is malformed the request is rejected with `400` and the override is kept, so a later restart cannot fail on it. + +Old Blockbook versions ignore the stored overrides (the environment applies again after a version rollback) and keep them intact, so rolling forward resumes the override. + ## Build-time variables - `BB_BUILD_ENV` - Selects the active RPC URL override family during package/config generation. Defaults to `dev`. diff --git a/server/internal.go b/server/internal.go index 30584f05a8..72c17565c7 100644 --- a/server/internal.go +++ b/server/internal.go @@ -14,6 +14,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/golang/glog" @@ -45,6 +46,12 @@ type InternalServer struct { adminAuthEnabled bool adminUserHash [32]byte adminPassHash [32]byte + // runtimeSettingsMux serializes runtime-setting writes (validate → store + // to DB → publish snapshot) so concurrent admin requests cannot publish a + // snapshot whose value lost the database write. runtimeSettings is the + // persistence backend of those writes (the RocksDB in production). + runtimeSettingsMux sync.Mutex + runtimeSettings runtimeSettingStore } // NewInternalServer creates new internal http interface to blockbook and returns its handle @@ -64,15 +71,16 @@ func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.B htmlTemplates: htmlTemplates[InternalTemplateData]{ debug: true, }, - https: https, - certFiles: certFiles, - db: db, - txCache: txCache, - chain: chain, - chainParser: chain.GetChainParser(), - mempool: mempool, - is: is, - api: api, + https: https, + certFiles: certFiles, + db: db, + txCache: txCache, + chain: chain, + chainParser: chain.GetChainParser(), + mempool: mempool, + is: is, + api: api, + runtimeSettings: db, } s.htmlTemplates.newTemplateData = s.newTemplateData s.htmlTemplates.newTemplateDataWithError = s.newTemplateDataWithError @@ -104,9 +112,14 @@ func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.B serveMux.HandleFunc(adminPath+"/", s.requireAdminAuth(s.adminSubtreeHandler(adminPath))) serveMux.HandleFunc(adminPath+"/ws-limit-exceeding-ips", s.requireAdminAuth(s.htmlTemplateHandler(s.wsLimitExceedingIPs))) if s.chainParser.GetChainType() == bchain.ChainEthereumType { + if err := initRpcCallAllowlists(db, is); err != nil { + return nil, err + } serveMux.HandleFunc(adminPath+"/internal-data-errors", s.requireAdminAuth(s.htmlTemplateHandler(s.internalDataErrors))) serveMux.HandleFunc(adminPath+"/contract-info", s.requireAdminAuth(s.htmlTemplateHandler(s.contractInfoPage))) serveMux.HandleFunc(adminPath+"/contract-info/", s.requireAdminAuth(s.jsonHandler(s.apiContractInfo, 0))) + serveMux.HandleFunc(adminPath+"/runtime-settings", s.requireAdminAuth(s.htmlTemplateHandler(s.runtimeSettingsPage))) + serveMux.HandleFunc(adminPath+"/runtime-settings/", s.requireAdminAuth(s.jsonHandler(s.apiRuntimeSetting, 0))) } return s, nil } @@ -219,6 +232,7 @@ const ( adminInternalErrorsTpl adminLimitExceedingIPSTpl adminContractInfoTpl + adminRuntimeSettingsTpl internalTplCount ) @@ -252,6 +266,7 @@ type InternalTemplateData struct { WsGetAccountInfoLimit int WsLimitExceedingIPs []WsLimitExceedingIP WsBlockedIPs []WsBlockedIPView + RuntimeSettings []RuntimeSettingView } func (s *InternalServer) newTemplateData(r *http.Request) *InternalTemplateData { @@ -287,6 +302,7 @@ func (s *InternalServer) parseTemplates() []*template.Template { t[adminInternalErrorsTpl] = createTemplate("./static/internal_templates/block_internal_data_errors.html", "./static/internal_templates/base.html") t[adminLimitExceedingIPSTpl] = createTemplate("./static/internal_templates/ws_limit_exceeding_ips.html", "./static/internal_templates/base.html") t[adminContractInfoTpl] = createTemplate("./static/internal_templates/contract_info.html", "./static/internal_templates/base.html") + t[adminRuntimeSettingsTpl] = createTemplate("./static/internal_templates/runtime_settings.html", "./static/internal_templates/base.html") return t } diff --git a/server/internal_test.go b/server/internal_test.go index a0ab85fb5c..6c753293d9 100644 --- a/server/internal_test.go +++ b/server/internal_test.go @@ -81,7 +81,7 @@ func TestAdminSubtreeIsGated(t *testing.T) { // Without credentials, every /admin path is gated (401) and never reaches the // unauthenticated index handler. - for _, p := range []string{"/admin", "/admin/", "/admin/unknown", "/admin/contract-info"} { + for _, p := range []string{"/admin", "/admin/", "/admin/unknown", "/admin/contract-info", "/admin/runtime-settings/ALLOWED_RPC_CALL_TO"} { t.Run("no-auth "+p, func(t *testing.T) { r := httptest.NewRequest(http.MethodGet, p, nil) w := httptest.NewRecorder() diff --git a/server/public_ethereumtype_test.go b/server/public_ethereumtype_test.go index ca6bd77442..725e92f8b1 100644 --- a/server/public_ethereumtype_test.go +++ b/server/public_ethereumtype_test.go @@ -4,9 +4,11 @@ package server import ( + "io" "net/http" "net/http/httptest" "reflect" + "strings" "testing" "time" @@ -412,6 +414,118 @@ var websocketTestsEthereumTypeRpcCallAllowlist = []websocketTest{ }, } +// Test_PublicServer_EthereumType_RuntimeSettingsLiveUpdate proves the full +// runtime-settings flow: a websocket rpcCall rejected under the env-seeded +// allowlist starts passing right after an authenticated admin POST (no +// restart), the override is resolved from the DB on a simulated restart with +// precedence over the environment, and DELETE reverts to the env default. +func Test_PublicServer_EthereumType_RuntimeSettingsLiveUpdate(t *testing.T) { + timeNow = fixedTimeNow + t.Setenv("FAKE_ALLOWED_EVM_CALL_METHODS", "0xdd62ed3e") + t.Setenv("BB_ADMIN_USER", "admin") + t.Setenv("BB_ADMIN_PASSWORD", "password") + parser := eth.NewEthereumParser(1, true) + chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) + if err != nil { + glog.Fatal("fakechain: ", err) + } + + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) + defer closeAndDestroyPublicServer(t, s, dbpath) + s.ConnectFullPublicInterface() + ts := httptest.NewServer(s.https.Handler) + defer ts.Close() + + internal, err := NewInternalServer("localhost:12346", "", s.db, s.chain, s.mempool, s.txCache, metrics, s.is, s.fiatRates) + if err != nil { + t.Fatal(err) + } + tsAdmin := httptest.NewServer(internal.https.Handler) + defer tsAdmin.Close() + + const balanceOfData = "0x70a08231000000000000000000000000e4db1c5a1b709ce4d2ada6985d9d506e58f73829" + balanceOfCall := func(name, want string) []websocketTest { + return []websocketTest{{ + name: name, + req: websocketReq{ + Method: "rpcCall", + Params: WsRpcCallReq{ + To: "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + Data: balanceOfData, + }, + }, + want: want, + }} + } + rejected := `{"id":"0","data":{"error":{"message":"Not supported"}}}` + allowed := `{"id":"0","data":{"data":"` + balanceOfData + `abcd"}}` + + adminURL := tsAdmin.URL + "/admin/runtime-settings/ALLOWED_EVM_CALL_METHODS" + doAdmin := func(method, body string) (int, string) { + t.Helper() + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + req, err := http.NewRequest(method, adminURL, rdr) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth("admin", "password") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, strings.TrimSpace(string(b)) + } + + // the balanceOf selector is not in the env-seeded allowlist + runWebsocketTests(t, ts, balanceOfCall("rpcCall balanceOf rejected by env allowlist", rejected)) + + // the admin endpoint requires authentication + resp, err := http.Post(adminURL, "application/json", strings.NewReader(`{"value":"0x70a08231"}`)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated POST status = %d, want 401", resp.StatusCode) + } + + // authenticated POST adds the selector and takes effect without a restart + code, body := doAdmin(http.MethodPost, `{"value":"0xdd62ed3e,0x70a08231"}`) + if code != http.StatusOK || body != `{"key":"ALLOWED_EVM_CALL_METHODS","value":"0xdd62ed3e,0x70a08231","source":"db"}` { + t.Fatalf("admin POST = %d %s", code, body) + } + runWebsocketTests(t, ts, balanceOfCall("rpcCall balanceOf allowed after admin update", allowed)) + + // simulated restart: a fresh InternalState resolves the override from the + // DB with precedence over the environment variable + is2 := &common.InternalState{CoinShortcut: "FAKE"} + if err := initRpcCallAllowlists(s.db, is2); err != nil { + t.Fatal(err) + } + a := is2.GetRpcCallAllowlists() + if a.MethodsSource != common.RuntimeSettingSourceDB || a.MethodsValue != "0xdd62ed3e,0x70a08231" { + t.Fatalf("after simulated restart got source %q value %q, want db override", a.MethodsSource, a.MethodsValue) + } + if _, ok := a.Methods["70a08231"]; !ok { + t.Fatal("after simulated restart the stored selector is missing") + } + + // DELETE reverts to the env default and the call is rejected again + code, body = doAdmin(http.MethodDelete, "") + if code != http.StatusOK || body != `{"key":"ALLOWED_EVM_CALL_METHODS","value":"0xdd62ed3e","source":"env"}` { + t.Fatalf("admin DELETE = %d %s", code, body) + } + runWebsocketTests(t, ts, balanceOfCall("rpcCall balanceOf rejected after revert", rejected)) +} + func Test_PublicServer_EthereumType_RpcCallAllowlist(t *testing.T) { timeNow = fixedTimeNow t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9") diff --git a/server/runtime_settings.go b/server/runtime_settings.go new file mode 100644 index 0000000000..396ab76897 --- /dev/null +++ b/server/runtime_settings.go @@ -0,0 +1,373 @@ +package server + +import ( + "encoding/hex" + "encoding/json" + "io" + "net/http" + "os" + "strings" + + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/trezor/blockbook/api" + "github.com/trezor/blockbook/common" + "github.com/trezor/blockbook/db" +) + +// Runtime settings are configuration values that can be changed at runtime +// through the internal /admin/runtime-settings interface. Each setting is +// named after the environment variable that provides its startup default +// (prefixed with the network name, e.g. ETH_ALLOWED_RPC_CALL_TO); an override +// written through the admin interface is persisted in the database, survives +// restarts and takes precedence over the environment variable. +const ( + runtimeSettingAllowedRpcCallTo = "ALLOWED_RPC_CALL_TO" + runtimeSettingAllowedEvmCallMethods = "ALLOWED_EVM_CALL_METHODS" +) + +// runtimeSettingDef describes one runtime setting: how to validate its raw +// value and how to read/write it in the RpcCallAllowlists snapshot. Future +// runtime settings plug in here. +type runtimeSettingDef struct { + key string + // parse validates the raw comma-separated value and returns the parsed + // set; name is used in error messages. + parse func(name, value string) (map[string]struct{}, error) + // apply writes the parsed value into a snapshot under construction. + apply func(a *common.RpcCallAllowlists, parsed map[string]struct{}, value, source string) + // get reads the raw value and its source from a snapshot. + get func(a *common.RpcCallAllowlists) (value, source string) +} + +var runtimeSettingDefs = []*runtimeSettingDef{ + { + key: runtimeSettingAllowedRpcCallTo, + parse: parseAllowedRpcCallTo, + apply: func(a *common.RpcCallAllowlists, parsed map[string]struct{}, value, source string) { + a.To, a.ToValue, a.ToSource = parsed, value, source + }, + get: func(a *common.RpcCallAllowlists) (string, string) { + return a.ToValue, a.ToSource + }, + }, + { + key: runtimeSettingAllowedEvmCallMethods, + parse: parseAllowedEvmCallMethods, + apply: func(a *common.RpcCallAllowlists, parsed map[string]struct{}, value, source string) { + a.Methods, a.MethodsValue, a.MethodsSource = parsed, value, source + }, + get: func(a *common.RpcCallAllowlists) (string, string) { + return a.MethodsValue, a.MethodsSource + }, + }, +} + +func runtimeSettingDefByKey(key string) *runtimeSettingDef { + for _, def := range runtimeSettingDefs { + if def.key == key { + return def + } + } + return nil +} + +func runtimeSettingKeys() string { + keys := make([]string, len(runtimeSettingDefs)) + for i, def := range runtimeSettingDefs { + keys[i] = def.key + } + return strings.Join(keys, ", ") +} + +// parseAllowedRpcCallTo parses a comma-separated list of contract addresses +// into a set keyed by the lowercase address string. Entries are trimmed and +// empty entries skipped; a set value that parses to no addresses at all is a +// configuration error — silently ignoring it would leave rpcCall unrestricted, +// and keeping an empty-string entry would allowlist rpcCall requests with an +// empty to field. +func parseAllowedRpcCallTo(name, value string) (map[string]struct{}, error) { + if value == "" { + return nil, nil + } + addresses := make(map[string]struct{}) + for _, a := range strings.Split(value, ",") { + a = strings.TrimSpace(a) + if a == "" { + continue + } + addresses[strings.ToLower(a)] = struct{}{} + } + if len(addresses) == 0 { + return nil, errors.Errorf("%s is set but contains no addresses", name) + } + return addresses, nil +} + +// parseAllowedEvmCallMethods parses a comma-separated list of 4-byte EVM call +// selectors (optional 0x prefix, case-insensitive) into a set keyed by +// lowercase hex without the prefix. Returns nil when the value is unset. +// A malformed selector or a set value that parses to no selectors at all is a +// configuration error so a typo cannot silently disable the intended +// allowlist. +func parseAllowedEvmCallMethods(name, value string) (map[string]struct{}, error) { + if value == "" { + return nil, nil + } + methods := make(map[string]struct{}) + for _, m := range strings.Split(value, ",") { + m = strings.TrimSpace(m) + if m == "" { + continue + } + selector := strings.TrimPrefix(strings.ToLower(m), "0x") + b, err := hex.DecodeString(selector) + if err != nil || len(b) != 4 { + return nil, errors.Errorf("Invalid EVM call method selector %q in %s, expecting 4 bytes in hex", m, name) + } + methods[selector] = struct{}{} + } + if len(methods) == 0 { + return nil, errors.Errorf("%s is set but contains no method selectors", name) + } + return methods, nil +} + +// runtimeSettingEnvName returns the environment variable that provides the +// startup default of the given runtime setting. +func runtimeSettingEnvName(network, key string) string { + return strings.ToUpper(network) + "_" + key +} + +// resolveRuntimeSetting returns the effective raw value of a runtime setting +// and its source: a stored override wins over the environment variable. +func resolveRuntimeSetting(d *db.RocksDB, network, key string) (string, string, error) { + if d != nil { + value, found, err := d.GetRuntimeSetting(key) + if err != nil { + return "", "", err + } + if found { + return value, common.RuntimeSettingSourceDB, nil + } + } + if value := os.Getenv(runtimeSettingEnvName(network, key)); value != "" { + return value, common.RuntimeSettingSourceEnv, nil + } + return "", common.RuntimeSettingSourceUnset, nil +} + +// buildRpcCallAllowlists resolves and parses all runtime settings into a new +// snapshot. An unparsable value — a malformed environment variable or a +// corrupted stored override (writes are validated, so it cannot get there +// through the admin interface) — is an error; falling back could silently +// widen access. +func buildRpcCallAllowlists(d *db.RocksDB, is *common.InternalState) (*common.RpcCallAllowlists, error) { + network := is.GetNetwork() + a := &common.RpcCallAllowlists{} + for _, def := range runtimeSettingDefs { + value, source, err := resolveRuntimeSetting(d, network, def.key) + if err != nil { + return nil, err + } + var parsed map[string]struct{} + if value != "" { + name := def.key + if source == common.RuntimeSettingSourceEnv { + name = runtimeSettingEnvName(network, def.key) + } + parsed, err = def.parse(name, value) + if err != nil { + return nil, err + } + } + def.apply(a, parsed, value, source) + } + return a, nil +} + +// initRpcCallAllowlists resolves and publishes the rpcCall allowlist snapshot +// if none exists yet. It is called from both NewWebsocketServer and +// NewInternalServer so that each server gets correct allowlists regardless of +// which of them (or both, in any order) a deployment runs; the CAS in +// InitRpcCallAllowlists keeps an already published — and possibly already +// admin-updated — snapshot intact. +func initRpcCallAllowlists(d *db.RocksDB, is *common.InternalState) error { + if is.GetRpcCallAllowlists() != nil { + return nil + } + a, err := buildRpcCallAllowlists(d, is) + if err != nil { + return err + } + if !is.InitRpcCallAllowlists(a) { + return nil + } + if a.To != nil { + glog.Info("Support of rpcCall for these contracts (source ", a.ToSource, "): ", a.ToValue) + } + if a.Methods != nil { + glog.Info("Support of rpcCall for these method selectors (source ", a.MethodsSource, "): ", a.MethodsValue) + } + return nil +} + +// runtimeSettingStore persists runtime setting overrides; *db.RocksDB in +// production, replaceable in tests to exercise storage failures. +type runtimeSettingStore interface { + StoreRuntimeSetting(name, value string) error + DeleteRuntimeSetting(name string) error +} + +// runtimeSettingResponse is the JSON shape returned by the +// /admin/runtime-settings/ endpoint. +type runtimeSettingResponse struct { + Key string `json:"key"` + Value string `json:"value"` + Source string `json:"source"` +} + +// apiRuntimeSetting handles GET/POST/PUT/DELETE of a single runtime setting at +// /admin/runtime-settings/. +func (s *InternalServer) apiRuntimeSetting(r *http.Request, apiVersion int) (interface{}, error) { + var key string + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + key = strings.ToUpper(strings.TrimSpace(r.URL.Path[i+1:])) + } + def := runtimeSettingDefByKey(key) + if def == nil { + return nil, api.NewAPIError("Unknown runtime setting, supported: "+runtimeSettingKeys(), true) + } + switch r.Method { + case http.MethodGet: + return s.getRuntimeSetting(def) + case http.MethodPost, http.MethodPut: + return s.updateRuntimeSetting(def, r) + case http.MethodDelete: + return s.deleteRuntimeSetting(def, r) + } + return nil, api.NewAPIError("Unsupported method "+r.Method, true) +} + +// currentRpcCallAllowlists returns the live snapshot; it is published by +// NewInternalServer before the routes are registered, so a nil snapshot means +// a broken test setup rather than a runtime condition. +func (s *InternalServer) currentRpcCallAllowlists() (*common.RpcCallAllowlists, error) { + a := s.is.GetRpcCallAllowlists() + if a == nil { + return nil, errors.New("runtime settings not initialized") + } + return a, nil +} + +func (s *InternalServer) getRuntimeSetting(def *runtimeSettingDef) (interface{}, error) { + a, err := s.currentRpcCallAllowlists() + if err != nil { + return nil, err + } + value, source := def.get(a) + return &runtimeSettingResponse{Key: def.key, Value: value, Source: source}, nil +} + +// updateRuntimeSetting validates the new value, persists it to the database +// and only then publishes it to the live snapshot, so an admin never gets a +// success response for a change that would not survive a restart. A failed +// database write leaves the live allowlists untouched and returns HTTP 500; +// an invalid value returns HTTP 400. +func (s *InternalServer) updateRuntimeSetting(def *runtimeSettingDef, r *http.Request) (interface{}, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, api.NewAPIError("Cannot get request body", true) + } + var req struct { + Value *string `json:"value"` + } + if err := json.Unmarshal(body, &req); err != nil || req.Value == nil { + return nil, api.NewAPIError(`Body must be a JSON object {"value":"..."}`, true) + } + // A value of exactly "" is a valid override: it explicitly unconfigures + // the dimension (as if the environment variable was not set), which is + // the only way to un-restrict at runtime when the environment has a + // value. Anything else — including whitespace- or separator-only values — + // must parse to at least one entry, so a botched automation input cannot + // silently disable the allowlist. + value := strings.TrimSpace(*req.Value) + var parsed map[string]struct{} + if *req.Value != "" { + parsed, err = def.parse(def.key, *req.Value) + if err != nil { + return nil, api.NewAPIError(err.Error(), true) + } + } + s.runtimeSettingsMux.Lock() + defer s.runtimeSettingsMux.Unlock() + old, err := s.currentRpcCallAllowlists() + if err != nil { + return nil, err + } + if err := s.runtimeSettings.StoreRuntimeSetting(def.key, value); err != nil { + glog.Error("admin: storing runtime setting ", def.key, "=", value, " failed: ", err) + return nil, api.NewAPIError("Cannot store runtime setting "+def.key+": "+err.Error(), false) + } + oldValue, oldSource := def.get(old) + a := *old + def.apply(&a, parsed, value, common.RuntimeSettingSourceDB) + s.is.SetRpcCallAllowlists(&a) + glog.Infof("admin: runtime setting %s changed from %q (%s) to %q (%s), client %s", + def.key, oldValue, oldSource, value, common.RuntimeSettingSourceDB, r.RemoteAddr) + return &runtimeSettingResponse{Key: def.key, Value: value, Source: common.RuntimeSettingSourceDB}, nil +} + +// deleteRuntimeSetting removes the stored override and reverts the setting to +// its environment default. The environment fallback is validated before the +// row is deleted — deleting with a malformed environment value would leave +// the database and the live state divergent and fail the next restart. +func (s *InternalServer) deleteRuntimeSetting(def *runtimeSettingDef, r *http.Request) (interface{}, error) { + envName := runtimeSettingEnvName(s.is.GetNetwork(), def.key) + envValue := os.Getenv(envName) + var parsed map[string]struct{} + source := common.RuntimeSettingSourceUnset + if envValue != "" { + var err error + parsed, err = def.parse(envName, envValue) + if err != nil { + return nil, api.NewAPIError("Cannot remove override, fallback environment value is invalid: "+err.Error()+"; fix the environment or set a value instead", true) + } + source = common.RuntimeSettingSourceEnv + } + s.runtimeSettingsMux.Lock() + defer s.runtimeSettingsMux.Unlock() + old, err := s.currentRpcCallAllowlists() + if err != nil { + return nil, err + } + if err := s.runtimeSettings.DeleteRuntimeSetting(def.key); err != nil { + glog.Error("admin: deleting runtime setting ", def.key, " failed: ", err) + return nil, api.NewAPIError("Cannot delete runtime setting "+def.key+": "+err.Error(), false) + } + oldValue, oldSource := def.get(old) + a := *old + def.apply(&a, parsed, envValue, source) + s.is.SetRpcCallAllowlists(&a) + glog.Infof("admin: runtime setting %s override removed, reverted from %q (%s) to %q (%s), client %s", + def.key, oldValue, oldSource, envValue, source, r.RemoteAddr) + return &runtimeSettingResponse{Key: def.key, Value: envValue, Source: source}, nil +} + +// RuntimeSettingView is a single row of the admin runtime-settings page. +type RuntimeSettingView struct { + Key string + Value string + Source string +} + +func (s *InternalServer) runtimeSettingsPage(w http.ResponseWriter, r *http.Request) (tpl, *InternalTemplateData, error) { + data := s.newTemplateData(r) + if a := s.is.GetRpcCallAllowlists(); a != nil { + for _, def := range runtimeSettingDefs { + value, source := def.get(a) + data.RuntimeSettings = append(data.RuntimeSettings, RuntimeSettingView{Key: def.key, Value: value, Source: source}) + } + } + return adminRuntimeSettingsTpl, data, nil +} diff --git a/server/runtime_settings_test.go b/server/runtime_settings_test.go new file mode 100644 index 0000000000..288721089c --- /dev/null +++ b/server/runtime_settings_test.go @@ -0,0 +1,292 @@ +//go:build unittest +// +build unittest + +package server + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/trezor/blockbook/bchain/coins/eth" + "github.com/trezor/blockbook/common" + "github.com/trezor/blockbook/tests/dbtestdata" +) + +func TestParseAllowedRpcCallTo(t *testing.T) { + tests := []struct { + name string + value string + want []string + wantErr bool + }{ + {name: "empty", value: "", want: nil}, + {name: "single address", value: "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9", want: []string{"0xcda9fc258358ecaa88845f19af595e908bb7efe9"}}, + {name: "multiple with spaces", value: " 0xABCD , 0xEF01 ", want: []string{"0xabcd", "0xef01"}}, + {name: "empty entries skipped", value: "0xabcd,,0xef01", want: []string{"0xabcd", "0xef01"}}, + {name: "separators only", value: ",", wantErr: true}, + {name: "whitespace entries only", value: " , ,", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseAllowedRpcCallTo("FAKE_ALLOWED_RPC_CALL_TO", tt.value) + if tt.wantErr { + if err == nil { + t.Fatalf("parseAllowedRpcCallTo(%q) = nil err, want error", tt.value) + } + return + } + if err != nil { + t.Fatalf("parseAllowedRpcCallTo(%q) unexpected error: %v", tt.value, err) + } + if tt.want == nil && got != nil { + t.Fatalf("parseAllowedRpcCallTo(%q) = %v, want nil", tt.value, got) + } + if len(got) != len(tt.want) { + t.Fatalf("parseAllowedRpcCallTo(%q) len = %d, want %d", tt.value, len(got), len(tt.want)) + } + for _, a := range tt.want { + if _, ok := got[a]; !ok { + t.Fatalf("parseAllowedRpcCallTo(%q) missing %q", tt.value, a) + } + } + }) + } +} + +func doRuntimeSettingRequest(t *testing.T, handler http.HandlerFunc, method, key, body string) (int, map[string]string) { + t.Helper() + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + r := httptest.NewRequest(method, "/admin/runtime-settings/"+key, rdr) + w := httptest.NewRecorder() + handler(w, r) + var resp map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("%s %s: cannot unmarshal response %q: %v", method, key, w.Body.String(), err) + } + return w.Code, resp +} + +// failingRuntimeSettingStore fails every persistence call; it stands in for a +// broken database to exercise the store-before-publish error paths. +type failingRuntimeSettingStore struct{} + +func (failingRuntimeSettingStore) StoreRuntimeSetting(name, value string) error { + return errors.New("store failed") +} + +func (failingRuntimeSettingStore) DeleteRuntimeSetting(name string) error { + return errors.New("delete failed") +} + +// The subtests of TestRuntimeSettingsAPI intentionally share the DB and +// snapshot state and depend on running in order; run the whole test, not +// individual subtests. +func TestRuntimeSettingsAPI(t *testing.T) { + const envTo = "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9" + t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", envTo) + parser := eth.NewEthereumParser(1, true) + chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) + if err != nil { + t.Fatal(err) + } + d, is, path := setupRocksDB(parser, chain, t, false, &common.Config{CoinName: "Fakecoin", CoinShortcut: "FAKE"}) + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + os.RemoveAll(path) + }() + if err := initRpcCallAllowlists(d, is); err != nil { + t.Fatal(err) + } + s := &InternalServer{ + htmlTemplates: htmlTemplates[InternalTemplateData]{debug: true}, + db: d, + is: is, + chainParser: parser, + runtimeSettings: d, + } + handler := s.jsonHandler(s.apiRuntimeSetting, 0) + + t.Run("GET unknown key", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodGet, "NO_SUCH_SETTING", "") + if code != http.StatusBadRequest || !strings.Contains(resp["error"], "Unknown runtime setting") { + t.Fatalf("got %d %v, want 400 unknown runtime setting", code, resp) + } + }) + + t.Run("GET env-sourced value", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodGet, "ALLOWED_RPC_CALL_TO", "") + if code != http.StatusOK || resp["value"] != envTo || resp["source"] != common.RuntimeSettingSourceEnv { + t.Fatalf("got %d %v, want 200 value %q source env", code, resp, envTo) + } + }) + + t.Run("GET unset value", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodGet, "ALLOWED_EVM_CALL_METHODS", "") + if code != http.StatusOK || resp["value"] != "" || resp["source"] != common.RuntimeSettingSourceUnset { + t.Fatalf("got %d %v, want 200 empty value source unset", code, resp) + } + }) + + t.Run("key is case-insensitive", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodGet, "allowed_rpc_call_to", "") + if code != http.StatusOK || resp["key"] != runtimeSettingAllowedRpcCallTo { + t.Fatalf("got %d %v, want 200 key %s", code, resp, runtimeSettingAllowedRpcCallTo) + } + }) + + t.Run("POST invalid selector is rejected", func(t *testing.T) { + code, _ := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_EVM_CALL_METHODS", `{"value":"0x12"}`) + if code != http.StatusBadRequest { + t.Fatalf("got %d, want 400", code) + } + if _, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedEvmCallMethods); found { + t.Fatal("rejected POST must not create a DB row") + } + if a := is.GetRpcCallAllowlists(); a.Methods != nil || a.MethodsSource != common.RuntimeSettingSourceUnset { + t.Fatalf("rejected POST must not change the snapshot, got %+v", a) + } + }) + + t.Run("POST invalid to list is rejected", func(t *testing.T) { + code, _ := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_RPC_CALL_TO", `{"value":","}`) + if code != http.StatusBadRequest { + t.Fatalf("got %d, want 400", code) + } + }) + + t.Run("POST bad body is rejected", func(t *testing.T) { + for _, body := range []string{"not json", `{"other":"field"}`} { + code, _ := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_EVM_CALL_METHODS", body) + if code != http.StatusBadRequest { + t.Fatalf("body %q: got %d, want 400", body, code) + } + } + }) + + t.Run("POST whitespace-only value is rejected", func(t *testing.T) { + // only a value of exactly "" is an explicit unconfigure; a + // whitespace- or separator-only value is a typo that must not + // silently un-restrict rpcCall + for _, body := range []string{`{"value":" "}`, `{"value":"\n"}`} { + code, _ := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_EVM_CALL_METHODS", body) + if code != http.StatusBadRequest { + t.Fatalf("body %q: got %d, want 400", body, code) + } + } + if _, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedEvmCallMethods); found { + t.Fatal("rejected POST must not create a DB row") + } + }) + + t.Run("unsupported method is rejected", func(t *testing.T) { + code, _ := doRuntimeSettingRequest(t, handler, http.MethodPatch, "ALLOWED_EVM_CALL_METHODS", "") + if code != http.StatusBadRequest { + t.Fatalf("got %d, want 400", code) + } + }) + + t.Run("POST stores override and updates snapshot", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_EVM_CALL_METHODS", `{"value":"0xdd62ed3e,0x70a08231"}`) + if code != http.StatusOK || resp["value"] != "0xdd62ed3e,0x70a08231" || resp["source"] != common.RuntimeSettingSourceDB { + t.Fatalf("got %d %v, want 200 source db", code, resp) + } + value, found, err := d.GetRuntimeSetting(runtimeSettingAllowedEvmCallMethods) + if err != nil || !found || value != "0xdd62ed3e,0x70a08231" { + t.Fatalf("DB row = %q, %v, %v, want stored value", value, found, err) + } + a := is.GetRpcCallAllowlists() + if _, ok := a.Methods["70a08231"]; !ok || a.MethodsSource != common.RuntimeSettingSourceDB { + t.Fatalf("snapshot not updated: %+v", a) + } + }) + + t.Run("POST empty value explicitly unconfigures", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_RPC_CALL_TO", `{"value":""}`) + if code != http.StatusOK || resp["value"] != "" || resp["source"] != common.RuntimeSettingSourceDB { + t.Fatalf("got %d %v, want 200 empty value source db", code, resp) + } + a := is.GetRpcCallAllowlists() + if a.To != nil || a.ToSource != common.RuntimeSettingSourceDB { + t.Fatalf("snapshot not cleared: %+v", a) + } + if value, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedRpcCallTo); !found || value != "" { + t.Fatalf("DB row = %q, %v, want stored empty value", value, found) + } + }) + + t.Run("stored empty override beats env after restart", func(t *testing.T) { + // simulated restart: the stored empty override must keep the + // dimension unconfigured even though the env var has a value + restarted := &common.InternalState{CoinShortcut: "FAKE"} + if err := initRpcCallAllowlists(d, restarted); err != nil { + t.Fatal(err) + } + a := restarted.GetRpcCallAllowlists() + if a.To != nil || a.ToValue != "" || a.ToSource != common.RuntimeSettingSourceDB { + t.Fatalf("after restart got To %v value %q source %q, want unconfigured db override", a.To, a.ToValue, a.ToSource) + } + }) + + t.Run("DELETE reverts to env", func(t *testing.T) { + code, resp := doRuntimeSettingRequest(t, handler, http.MethodDelete, "ALLOWED_RPC_CALL_TO", "") + if code != http.StatusOK || resp["value"] != envTo || resp["source"] != common.RuntimeSettingSourceEnv { + t.Fatalf("got %d %v, want 200 value %q source env", code, resp, envTo) + } + if _, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedRpcCallTo); found { + t.Fatal("DELETE must remove the DB row") + } + a := is.GetRpcCallAllowlists() + if _, ok := a.To[strings.ToLower(envTo)]; !ok || a.ToSource != common.RuntimeSettingSourceEnv { + t.Fatalf("snapshot not reverted to env: %+v", a) + } + }) + + t.Run("DELETE with malformed env fallback is rejected", func(t *testing.T) { + t.Setenv("FAKE_ALLOWED_EVM_CALL_METHODS", "not-a-selector") + code, resp := doRuntimeSettingRequest(t, handler, http.MethodDelete, "ALLOWED_EVM_CALL_METHODS", "") + if code != http.StatusBadRequest { + t.Fatalf("got %d %v, want 400", code, resp) + } + if _, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedEvmCallMethods); !found { + t.Fatal("rejected DELETE must keep the DB row") + } + if a := is.GetRpcCallAllowlists(); a.MethodsSource != common.RuntimeSettingSourceDB { + t.Fatalf("rejected DELETE must not change the snapshot: %+v", a) + } + }) + + // proves the DB write happens before the snapshot swap: with a failing + // store the requests take the error branch (glog.Error + 500) and the + // live allowlists stay untouched + t.Run("failed store returns 500 and keeps snapshot", func(t *testing.T) { + s.runtimeSettings = failingRuntimeSettingStore{} + defer func() { s.runtimeSettings = d }() + before := is.GetRpcCallAllowlists() + code, resp := doRuntimeSettingRequest(t, handler, http.MethodPost, "ALLOWED_EVM_CALL_METHODS", `{"value":"0xa9059cbb"}`) + if code != http.StatusInternalServerError || !strings.Contains(resp["error"], "Cannot store runtime setting") { + t.Fatalf("got %d %v, want 500 store error", code, resp) + } + if is.GetRpcCallAllowlists() != before { + t.Fatal("failed DB write must not publish a new snapshot") + } + code, resp = doRuntimeSettingRequest(t, handler, http.MethodDelete, "ALLOWED_EVM_CALL_METHODS", "") + if code != http.StatusInternalServerError || !strings.Contains(resp["error"], "Cannot delete runtime setting") { + t.Fatalf("got %d %v, want 500 delete error", code, resp) + } + if is.GetRpcCallAllowlists() != before { + t.Fatal("failed DB delete must not publish a new snapshot") + } + }) +} diff --git a/server/websocket.go b/server/websocket.go index 621ebb9dba..3c7acc256a 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -106,14 +106,7 @@ type WebsocketServer struct { fiatRatesTokenSubscriptions map[*websocketChannel][]string fiatRatesSubscriptionsLock sync.Mutex allowedOrigins map[string]struct{} - allowedRpcCallTo map[string]struct{} - // allowedRpcCallMethods holds 4-byte EVM call selectors (lowercase hex, - // no 0x prefix) that rpcCall may invoke on any address; nil when - // unconfigured. Together with allowedRpcCallTo it forms the rpcCall - // allowlist: a call passes when either the target address or the calldata - // selector is allowed; if both are nil, rpcCall is unrestricted. - allowedRpcCallMethods map[string]struct{} - trustedProxyPrefixes []netip.Prefix + trustedProxyPrefixes []netip.Prefix // cloudflarePrefixes gates trust of the CF-Connecting-* headers: when // non-empty, those headers are honored only when the TCP peer is inside one // of these ranges (or a loopback/private proxy). Empty disables verification @@ -182,23 +175,9 @@ func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, mempool bchain. } originEnvName := strings.ToUpper(is.GetNetwork()) + "_WS_ALLOWED_ORIGINS" s.allowedOrigins = parseAllowedOrigins(originEnvName, os.Getenv(originEnvName)) - envRpcCall := os.Getenv(strings.ToUpper(is.GetNetwork()) + "_ALLOWED_RPC_CALL_TO") - if envRpcCall != "" { - s.allowedRpcCallTo = make(map[string]struct{}) - for _, c := range strings.Split(envRpcCall, ",") { - s.allowedRpcCallTo[strings.ToLower(c)] = struct{}{} - } - glog.Info("Support of rpcCall for these contracts: ", envRpcCall) - } - methodsEnvName := strings.ToUpper(is.GetNetwork()) + "_ALLOWED_EVM_CALL_METHODS" - envRpcCallMethods := os.Getenv(methodsEnvName) - s.allowedRpcCallMethods, err = parseAllowedEvmCallMethods(methodsEnvName, envRpcCallMethods) - if err != nil { + if err := initRpcCallAllowlists(db, is); err != nil { return nil, err } - if s.allowedRpcCallMethods != nil { - glog.Info("Support of rpcCall for these method selectors: ", envRpcCallMethods) - } clientIPCfg, err := readClientIPConfig(is.GetNetwork()) if err != nil { return nil, err @@ -256,35 +235,6 @@ func parseAllowedOrigins(originEnvName, envAllowedOrigins string) map[string]str return allowedOrigins } -// parseAllowedEvmCallMethods parses a comma-separated list of 4-byte EVM call -// selectors (optional 0x prefix, case-insensitive) into a set keyed by -// lowercase hex without the prefix. Returns nil when the variable is unset. -// A malformed selector or a set variable that parses to no selectors at all -// is a configuration error that aborts startup so a typo cannot silently -// disable the intended allowlist. -func parseAllowedEvmCallMethods(envName, envValue string) (map[string]struct{}, error) { - if envValue == "" { - return nil, nil - } - methods := make(map[string]struct{}) - for _, m := range strings.Split(envValue, ",") { - m = strings.TrimSpace(m) - if m == "" { - continue - } - selector := strings.TrimPrefix(strings.ToLower(m), "0x") - b, err := hex.DecodeString(selector) - if err != nil || len(b) != 4 { - return nil, errors.Errorf("Invalid EVM call method selector %q in %s, expecting 4 bytes in hex", m, envName) - } - methods[selector] = struct{}{} - } - if len(methods) == 0 { - return nil, errors.Errorf("%s is set but contains no method selectors", envName) - } - return methods, nil -} - func (s *WebsocketServer) checkOrigin(r *http.Request) bool { origin := r.Header.Get("Origin") if origin == "" { @@ -1264,19 +1214,23 @@ func evmCallSelector(data string) (string, bool) { // rpcCallAllowed reports whether a rpcCall request passes the allowlists. With // no allowlist configured rpcCall is unrestricted; otherwise the call must -// target an allowed address or invoke an allowed method selector. +// target an allowed address or invoke an allowed method selector. The snapshot +// is nil only when initRpcCallAllowlists has not run (bare test servers); +// NewWebsocketServer and NewInternalServer fail construction when they cannot +// publish one. func (s *WebsocketServer) rpcCallAllowed(r *WsRpcCallReq) bool { - if s.allowedRpcCallTo == nil && s.allowedRpcCallMethods == nil { + a := s.is.GetRpcCallAllowlists() + if a == nil || (a.To == nil && a.Methods == nil) { return true } - if s.allowedRpcCallTo != nil { - if _, ok := s.allowedRpcCallTo[strings.ToLower(r.To)]; ok { + if a.To != nil { + if _, ok := a.To[strings.ToLower(r.To)]; ok { return true } } - if s.allowedRpcCallMethods != nil { + if a.Methods != nil { if selector, ok := evmCallSelector(r.Data); ok { - if _, ok := s.allowedRpcCallMethods[selector]; ok { + if _, ok := a.Methods[selector]; ok { return true } } diff --git a/server/websocket_test.go b/server/websocket_test.go index a9ccf9d7fc..f29ab9a390 100644 --- a/server/websocket_test.go +++ b/server/websocket_test.go @@ -18,6 +18,7 @@ import ( "github.com/trezor/blockbook/api" "github.com/trezor/blockbook/bchain" "github.com/trezor/blockbook/bchain/coins/eth" + "github.com/trezor/blockbook/common" "github.com/trezor/blockbook/tests/dbtestdata" ) @@ -327,12 +328,21 @@ func TestRpcCallAllowed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := &WebsocketServer{allowedRpcCallTo: tt.to, allowedRpcCallMethods: tt.methods} + is := &common.InternalState{} + is.SetRpcCallAllowlists(&common.RpcCallAllowlists{To: tt.to, Methods: tt.methods}) + s := &WebsocketServer{is: is} if got := s.rpcCallAllowed(&tt.req); got != tt.want { t.Fatalf("rpcCallAllowed(to=%q, data=%q) = %v, want %v", tt.req.To, tt.req.Data, got, tt.want) } }) } + + t.Run("nil snapshot allows all", func(t *testing.T) { + s := &WebsocketServer{is: &common.InternalState{}} + if !s.rpcCallAllowed(&WsRpcCallReq{To: otherTo, Data: "0x12345678"}) { + t.Fatal("rpcCallAllowed with uninitialized snapshot = false, want true") + } + }) } func TestParseTrustedProxies(t *testing.T) { diff --git a/static/internal_templates/index.html b/static/internal_templates/index.html index 7a94bce8f0..196b0027e5 100644 --- a/static/internal_templates/index.html +++ b/static/internal_templates/index.html @@ -11,4 +11,7 @@ + {{end}}{{end}} diff --git a/static/internal_templates/runtime_settings.html b/static/internal_templates/runtime_settings.html new file mode 100644 index 0000000000..f3a73aa4f1 --- /dev/null +++ b/static/internal_templates/runtime_settings.html @@ -0,0 +1,38 @@ +{{define "specific"}} {{if eq .ChainType 1}} + + + + + + + + + + {{range .RuntimeSettings}} + + + + + + {{end}} + +
SettingValueSource
{{.Key}}{{.Value}}{{.Source}}
+
+ Runtime settings override their environment variable defaults, are stored in the database and survive restarts. + Use the /admin/runtime-settings/<KEY> endpoint to manage them. Examples: +
+
+            # read the current value and its source (db, env or unset)
+            curl -k -u user:password 'https://<internaladdress>/admin/runtime-settings/ALLOWED_EVM_CALL_METHODS'
+
+            # set an override (an empty value explicitly unconfigures the setting)
+            curl -k -u user:password -X POST 'https://<internaladdress>/admin/runtime-settings/ALLOWED_EVM_CALL_METHODS' \
+            -H 'Content-Type: application/json' \
+            --data '{"value":"0xdd62ed3e,0x70a08231"}'
+
+            # remove the override and revert to the environment default
+            curl -k -u user:password -X DELETE 'https://<internaladdress>/admin/runtime-settings/ALLOWED_EVM_CALL_METHODS'
+        
+
+
+{{else}} Not supported {{end}}{{end}} From 522354829336306709dd8172d48c7261153ee8cc Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 09:36:33 +0000 Subject: [PATCH 059/303] perf(eth): decode only first 4 bytes in evmCallSelector Avoids allocating a decoded copy of the full calldata (up to 4 MiB) on every rpcCall allowlist check. The full hex encoding is still validated byte-by-byte to preserve the existing fail-closed behavior for malformed calldata. --- server/websocket.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/server/websocket.go b/server/websocket.go index 3c7acc256a..5f0b628743 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -1197,19 +1197,32 @@ func (s *WebsocketServer) getBlockFiltersBatch(r *WsBlockFiltersBatchReq) (res i } // evmCallSelector extracts the 4-byte function selector from hex-encoded EVM -// calldata as lowercase hex without the 0x prefix. It decodes the whole data -// string so malformed calldata (missing 0x prefix, odd length, non-hex -// characters, fewer than 4 bytes) never yields a selector and thus fails -// closed in the rpcCall allowlist check. +// calldata as lowercase hex without the 0x prefix. It validates the full +// calldata hex (odd length or non-hex characters fail closed) but decodes +// only the first 4 bytes (8 hex chars) so that arbitrarily long calldata +// does not cause a large allocation that is then discarded. func evmCallSelector(data string) (string, bool) { - if len(data) < 2 || data[0] != '0' || (data[1] != 'x' && data[1] != 'X') { + if len(data) < 10 || data[0] != '0' || (data[1] != 'x' && data[1] != 'X') { return "", false } - b, err := hex.DecodeString(data[2:]) - if err != nil || len(b) < 4 { + s := data[2:] + if len(s)&1 == 1 { return "", false } - return hex.EncodeToString(b[:4]), true + for i := 0; i < len(s); i++ { + switch { + case s[i] >= '0' && s[i] <= '9': + case s[i] >= 'a' && s[i] <= 'f': + case s[i] >= 'A' && s[i] <= 'F': + default: + return "", false + } + } + b, err := hex.DecodeString(s[:8]) + if err != nil { + return "", false + } + return hex.EncodeToString(b), true } // rpcCallAllowed reports whether a rpcCall request passes the allowlists. With From fb44b13b72a7a0e73d114c946bb533cf0dff5a81 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 10:20:35 +0000 Subject: [PATCH 060/303] refactor(server): unify admin contract-info and runtime-settings APIs Addresses the API unification review on PR #1597: - POST /admin/contract-info/ returns a typed {"updated":N} object instead of a pre-serialized string that jsonHandler double-encoded on the wire; a malformed body is now a 400 (public APIError) rather than a 500 - apiContractInfo dispatches on method like apiRuntimeSetting: POST/PUT write the collection path only (an address segment is rejected instead of silently ignored), unknown methods are a 400 instead of falling into the GET path - new DELETE /admin/contract-info/
purges cached contract metadata (DB row + LRU entry) so it is re-fetched from the backend on the next read; idempotent, missing row reports deleted:false - shared urlPathSegment helper unifies path-segment extraction; contract addresses get no case normalization (the chain parser is the authority), runtime-setting keys keep their uppercasing at the call site - runtime-settings routes and initRpcCallAllowlists move out of the ChainEthereumType guard (the mechanism is chain-generic and the init already runs unconditionally in NewWebsocketServer); contract-info and internal-data-errors stay EVM-gated, nav links follow the same split Co-Authored-By: Claude Fable 5 --- db/rocksdb_contracts.go | 27 +++ db/rocksdb_contracts_test.go | 58 +++++ docs/env.md | 10 +- server/internal.go | 91 ++++++-- server/internal_contract_info_test.go | 212 +++++++++++++++++++ server/internal_test.go | 22 ++ server/runtime_settings.go | 5 +- static/internal_templates/contract_info.html | 16 +- static/internal_templates/index.html | 6 +- 9 files changed, 417 insertions(+), 30 deletions(-) create mode 100644 server/internal_contract_info_test.go diff --git a/db/rocksdb_contracts.go b/db/rocksdb_contracts.go index 62e62eadc6..5709be6750 100644 --- a/db/rocksdb_contracts.go +++ b/db/rocksdb_contracts.go @@ -2,6 +2,7 @@ package db import ( vlq "github.com/bsm/go-vlq" + "github.com/juju/errors" "github.com/linxGnu/grocksdb" "github.com/trezor/blockbook/bchain" ) @@ -186,3 +187,29 @@ func (d *RocksDB) storeContractInfo(wb *grocksdb.WriteBatch, contractInfo *bchai } return nil } + +// DeleteContractInfoForAddress removes the stored contract metadata for the given +// address (and its in-memory cache entry) so the next read re-fetches it from the +// backend node. Returns whether a stored row existed. ERC-4626 protocol-detection +// rows in cfErcProtocols are kept: they are sync-owned, chain-derived state with +// their own lifecycle, merged on read — not cached backend metadata. +func (d *RocksDB) DeleteContractInfoForAddress(address string) (bool, error) { + contract, err := d.chainParser.GetAddrDescFromAddress(address) + if err != nil { + return false, err + } + if contract == nil { + return false, errors.Errorf("invalid address %s", address) + } + val, err := d.db.GetCF(d.ro, d.cfh[cfContracts], contract) + if err != nil { + return false, err + } + found := len(val.Data()) > 0 + val.Free() + if err := d.db.DeleteCF(d.wo, d.cfh[cfContracts], contract); err != nil { + return false, err + } + cachedContracts.delete(string(contract)) + return found, nil +} diff --git a/db/rocksdb_contracts_test.go b/db/rocksdb_contracts_test.go index 0df1b9bcc0..cd8aca68ce 100644 --- a/db/rocksdb_contracts_test.go +++ b/db/rocksdb_contracts_test.go @@ -7,8 +7,66 @@ import ( "testing" "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/tests/dbtestdata" ) +func TestRocksDB_DeleteContractInfoForAddress(t *testing.T) { + d := setupRocksDB(t, &testEthereumParser{ + EthereumParser: ethereumTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + address := "0x" + dbtestdata.EthAddr20 + ci := &bchain.ContractInfo{ + Standard: bchain.ERC20TokenStandard, + Type: bchain.ERC20TokenStandard, + Contract: address, + Name: "Test contract", + Symbol: "TCT", + Decimals: 18, + } + if err := d.StoreContractInfo(ci); err != nil { + t.Fatal(err) + } + // The get populates the in-memory cache, so a successful delete below also + // proves the cache entry is purged along with the DB row. + got, err := d.GetContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if got == nil || got.Name != ci.Name { + t.Fatalf("GetContractInfoForAddress() = %+v, want stored contract", got) + } + + found, err := d.DeleteContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if !found { + t.Error("DeleteContractInfoForAddress() = false, want true for a stored row") + } + got, err = d.GetContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if got != nil { + t.Errorf("GetContractInfoForAddress() after delete = %+v, want nil", got) + } + + // Idempotent: deleting a missing row is not an error. + found, err = d.DeleteContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if found { + t.Error("DeleteContractInfoForAddress() = true, want false for a missing row") + } + + if _, err = d.DeleteContractInfoForAddress("not-an-address"); err == nil { + t.Error("DeleteContractInfoForAddress() with invalid address: expected error") + } +} + // packContractInfo only carries the sync-owned core fields. ERC4626 detection // data lives in the cfErcProtocols column family and is exercised // separately in rocksdb_protocols_test.go. diff --git a/docs/env.md b/docs/env.md index 1fb80a3ba4..01f4317f0c 100644 --- a/docs/env.md +++ b/docs/env.md @@ -95,7 +95,7 @@ Blockbook reads these from its process environment. When installed from the Debi ## Runtime settings -The `rpcCall` allowlists can be changed at runtime, without a restart, through the internal server's authenticated admin API (see `BB_ADMIN_USER`/`BB_ADMIN_PASSWORD` above). An override is persisted in the Blockbook database, survives restarts and takes precedence over the corresponding environment variable, which serves only as the startup default. Every change is logged. The `/admin/runtime-settings` page shows the current values and their sources. +The `rpcCall` allowlists can be changed at runtime, without a restart, through the internal server's authenticated admin API (see `BB_ADMIN_USER`/`BB_ADMIN_PASSWORD` above). An override is persisted in the Blockbook database, survives restarts and takes precedence over the corresponding environment variable, which serves only as the startup default. Every change is logged. The `/admin/runtime-settings` page shows the current values and their sources. The endpoint and the page are registered on every chain type — the runtime-settings mechanism is chain-generic; the currently defined settings only affect EVM `rpcCall` and are simply unset elsewhere. `GET/POST/DELETE /admin/runtime-settings/` where `` is `ALLOWED_RPC_CALL_TO` or `ALLOWED_EVM_CALL_METHODS`: @@ -105,6 +105,14 @@ The `rpcCall` allowlists can be changed at runtime, without a restart, through t Old Blockbook versions ignore the stored overrides (the environment applies again after a version rollback) and keep them intact, so rolling forward resumes the override. +## Contract-info admin endpoint + +On EVM chains the internal server also exposes `/admin/contract-info/` (same Basic auth) to manage the contract metadata Blockbook caches from the backend node: + +- `GET /admin/contract-info/
` returns the contract's metadata as a `ContractInfo` JSON object (fetching it from the backend node and storing it if not cached yet). +- `POST` (or `PUT`) `/admin/contract-info/` with a JSON array body `[{ContractInfo},…]` updates the stored metadata of the listed contracts; the response is `{"updated":N}`. The write targets the collection path — a `POST` to an address path is rejected with `400`. +- `DELETE /admin/contract-info/
` purges the cached metadata of one contract so it is re-fetched from the backend node on the next read; the response is `{"contract":"
","deleted":true|false}` (`false` when nothing was stored — the delete is idempotent). + ## Build-time variables - `BB_BUILD_ENV` - Selects the active RPC URL override family during package/config generation. Defaults to `dev`. diff --git a/server/internal.go b/server/internal.go index 72c17565c7..e2e0038f88 100644 --- a/server/internal.go +++ b/server/internal.go @@ -12,13 +12,11 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "sync" "time" "github.com/golang/glog" - "github.com/juju/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/trezor/blockbook/api" "github.com/trezor/blockbook/bchain" @@ -111,15 +109,19 @@ func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.B serveMux.HandleFunc(adminPath, s.requireAdminAuth(s.htmlTemplateHandler(s.adminIndex))) serveMux.HandleFunc(adminPath+"/", s.requireAdminAuth(s.adminSubtreeHandler(adminPath))) serveMux.HandleFunc(adminPath+"/ws-limit-exceeding-ips", s.requireAdminAuth(s.htmlTemplateHandler(s.wsLimitExceedingIPs))) + // Runtime settings are chain-generic (initRpcCallAllowlists already runs + // unconditionally in NewWebsocketServer); the currently defined settings only + // affect EVM rpcCall and are simply unset on other chains. The init must stay + // before the route registration — currentRpcCallAllowlists relies on it. + if err := initRpcCallAllowlists(db, is); err != nil { + return nil, err + } + serveMux.HandleFunc(adminPath+"/runtime-settings", s.requireAdminAuth(s.htmlTemplateHandler(s.runtimeSettingsPage))) + serveMux.HandleFunc(adminPath+"/runtime-settings/", s.requireAdminAuth(s.jsonHandler(s.apiRuntimeSetting, 0))) if s.chainParser.GetChainType() == bchain.ChainEthereumType { - if err := initRpcCallAllowlists(db, is); err != nil { - return nil, err - } serveMux.HandleFunc(adminPath+"/internal-data-errors", s.requireAdminAuth(s.htmlTemplateHandler(s.internalDataErrors))) serveMux.HandleFunc(adminPath+"/contract-info", s.requireAdminAuth(s.htmlTemplateHandler(s.contractInfoPage))) serveMux.HandleFunc(adminPath+"/contract-info/", s.requireAdminAuth(s.jsonHandler(s.apiContractInfo, 0))) - serveMux.HandleFunc(adminPath+"/runtime-settings", s.requireAdminAuth(s.htmlTemplateHandler(s.runtimeSettingsPage))) - serveMux.HandleFunc(adminPath+"/runtime-settings/", s.requireAdminAuth(s.jsonHandler(s.apiRuntimeSetting, 0))) } return s, nil } @@ -151,6 +153,20 @@ func (s *InternalServer) adminSubtreeHandler(adminPath string) http.HandlerFunc } } +// urlPathSegment returns the last segment of the request path with surrounding +// whitespace trimmed, or "" when the path ends in "/" or has no sub-segment +// below a registered subtree (a root-level path like "/admin" yields ""). Callers +// apply their own normalization on top (runtime-setting keys are uppercased, +// contract addresses are left to the chain parser, the authority on case and +// checksum handling). +func urlPathSegment(r *http.Request) string { + var seg string + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + seg = r.URL.Path[i+1:] + } + return strings.TrimSpace(seg) +} + // requireAdminAuth wraps an internal-server handler so it is reachable only with // valid HTTP Basic credentials. The admin surface is fail-closed: when the // credentials are not configured the endpoints return 503 rather than serving @@ -372,20 +388,44 @@ func (s *InternalServer) contractInfoPage(w http.ResponseWriter, r *http.Request return adminContractInfoTpl, data, nil } +// contractInfoUpdateResponse is the JSON shape returned by POST /admin/contract-info/. +type contractInfoUpdateResponse struct { + Updated int `json:"updated"` +} + +// contractInfoDeleteResponse is the JSON shape returned by +// DELETE /admin/contract-info/
. +type contractInfoDeleteResponse struct { + Contract string `json:"contract"` + Deleted bool `json:"deleted"` +} + +// apiContractInfo handles GET/POST/PUT/DELETE of cached contract metadata at +// /admin/contract-info/
(POST/PUT write the collection path +// /admin/contract-info/ with a JSON array body). func (s *InternalServer) apiContractInfo(r *http.Request, apiVersion int) (interface{}, error) { - if r.Method == http.MethodPost { + address := urlPathSegment(r) + switch r.Method { + case http.MethodGet: + return s.getContractInfo(address) + case http.MethodPost, http.MethodPut: + // The bulk write addresses each contract in its body; reject a POST to + // an address path instead of silently ignoring the address segment. + if address != "" { + return nil, api.NewAPIError("POST updates the collection; use /admin/contract-info/ with a JSON array body", true) + } return s.updateContracts(r) + case http.MethodDelete: + return s.deleteContractInfo(address) } - var contractAddress string - i := strings.LastIndexByte(r.URL.Path, '/') - if i > 0 { - contractAddress = r.URL.Path[i+1:] - } - if len(contractAddress) == 0 { + return nil, api.NewAPIError("Unsupported method "+r.Method, true) +} + +func (s *InternalServer) getContractInfo(address string) (interface{}, error) { + if address == "" { return nil, api.NewAPIError("Missing contract address", true) } - - contractInfo, valid, err := s.api.GetContractInfo(contractAddress, bchain.UnknownTokenStandard) + contractInfo, valid, err := s.api.GetContractInfo(address, bchain.UnknownTokenStandard) if err != nil { return nil, api.NewAPIError(err.Error(), true) } @@ -403,7 +443,7 @@ func (s *InternalServer) updateContracts(r *http.Request) (interface{}, error) { var contractInfos []bchain.ContractInfo err = json.Unmarshal(data, &contractInfos) if err != nil { - return nil, errors.Annotatef(err, "Cannot unmarshal body to array of ContractInfo objects") + return nil, api.NewAPIError("Cannot unmarshal body to array of ContractInfo objects: "+err.Error(), true) } for i := range contractInfos { c := &contractInfos[i] @@ -413,5 +453,20 @@ func (s *InternalServer) updateContracts(r *http.Request) (interface{}, error) { } } - return "{\"success\":\"Updated " + strconv.Itoa(len(contractInfos)) + " contracts\"}", nil + return &contractInfoUpdateResponse{Updated: len(contractInfos)}, nil +} + +// deleteContractInfo purges the cached metadata of one contract so the next +// read re-fetches it from the backend node. Deleting is idempotent: a missing +// row reports deleted=false rather than an error, matching the runtime-settings +// DELETE semantics. +func (s *InternalServer) deleteContractInfo(address string) (interface{}, error) { + if address == "" { + return nil, api.NewAPIError("Missing contract address", true) + } + found, err := s.db.DeleteContractInfoForAddress(address) + if err != nil { + return nil, api.NewAPIError(err.Error(), true) + } + return &contractInfoDeleteResponse{Contract: address, Deleted: found}, nil } diff --git a/server/internal_contract_info_test.go b/server/internal_contract_info_test.go new file mode 100644 index 0000000000..7842e92945 --- /dev/null +++ b/server/internal_contract_info_test.go @@ -0,0 +1,212 @@ +//go:build unittest + +package server + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/golang/glog" + "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/bchain/coins/eth" + "github.com/trezor/blockbook/tests/dbtestdata" +) + +// newInternalTestServer builds a full internal server (including the api.Worker +// the contract-info GET path needs) on top of the public-server test harness. +// The caller must have set BB_ADMIN_USER/BB_ADMIN_PASSWORD beforehand — the +// credentials are read in NewInternalServer. +func newInternalTestServer(t *testing.T, s *PublicServer) *httptest.Server { + t.Helper() + internal, err := NewInternalServer("localhost:12346", "", s.db, s.chain, s.mempool, s.txCache, metrics, s.is, s.fiatRates) + if err != nil { + t.Fatal(err) + } + ts := httptest.NewServer(internal.https.Handler) + t.Cleanup(ts.Close) + return ts +} + +func adminRequest(t *testing.T, ts *httptest.Server, method, path, body string) (int, string) { + t.Helper() + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + req, err := http.NewRequest(method, ts.URL+path, rdr) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth("admin", "password") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, strings.TrimSpace(string(b)) +} + +// TestContractInfoAdminAPI exercises the /admin/contract-info/ JSON API +// end-to-end: GET of one contract, bulk POST on the collection path (the +// response must decode as a JSON object — a regression check for the formerly +// double-encoded string body), DELETE with idempotent semantics, and the +// request-validation failure paths. +func TestContractInfoAdminAPI(t *testing.T) { + t.Setenv("BB_ADMIN_USER", "admin") + t.Setenv("BB_ADMIN_PASSWORD", "password") + parser := eth.NewEthereumParser(1, true) + chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) + if err != nil { + glog.Fatal("fakechain: ", err) + } + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) + defer closeAndDestroyPublicServer(t, s, dbpath) + ts := newInternalTestServer(t, s) + + address := "0x" + dbtestdata.EthAddr20 + + t.Run("GET missing address", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodGet, "/admin/contract-info/", "") + if code != http.StatusBadRequest || !strings.Contains(body, "Missing contract address") { + t.Fatalf("GET = %d %s, want 400 Missing contract address", code, body) + } + }) + + t.Run("GET address", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodGet, "/admin/contract-info/"+address, "") + if code != http.StatusOK { + t.Fatalf("GET = %d %s, want 200", code, body) + } + var ci bchain.ContractInfo + if err := json.Unmarshal([]byte(body), &ci); err != nil { + t.Fatalf("GET body %q does not decode to ContractInfo: %v", body, err) + } + if ci.Contract == "" || ci.Name == "" || ci.Standard != bchain.ERC20TokenStandard { + t.Fatalf("GET returned unexpected contract info: %+v", ci) + } + }) + + t.Run("POST collection", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodPost, "/admin/contract-info/", + `[{"contract":"`+address+`","name":"Renamed","symbol":"RNM","standard":"ERC20","decimals":18}]`) + if code != http.StatusOK { + t.Fatalf("POST = %d %s, want 200", code, body) + } + // The old handler returned a pre-serialized string that jsonHandler + // encoded again; decoding into a typed object fails on that shape. + var resp contractInfoUpdateResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("POST body %q does not decode to an object: %v", body, err) + } + if resp.Updated != 1 { + t.Fatalf("POST updated = %d, want 1", resp.Updated) + } + stored, err := s.db.GetContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if stored == nil || stored.Name != "Renamed" { + t.Fatalf("stored contract = %+v, want name Renamed", stored) + } + }) + + t.Run("POST with address segment", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodPost, "/admin/contract-info/"+address, `[]`) + if code != http.StatusBadRequest || !strings.Contains(body, "collection") { + t.Fatalf("POST = %d %s, want 400 pointing to the collection path", code, body) + } + }) + + t.Run("POST non-array body", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodPost, "/admin/contract-info/", `{"contract":"x"}`) + if code != http.StatusBadRequest { + t.Fatalf("POST = %d %s, want 400", code, body) + } + }) + + t.Run("unsupported method", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodPatch, "/admin/contract-info/"+address, "") + if code != http.StatusBadRequest || !strings.Contains(body, "Unsupported method") { + t.Fatalf("PATCH = %d %s, want 400 Unsupported method", code, body) + } + }) + + t.Run("DELETE", func(t *testing.T) { + // stored by the POST subtest above + code, body := adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/"+address, "") + if code != http.StatusOK || body != `{"contract":"`+address+`","deleted":true}` { + t.Fatalf("DELETE = %d %s, want deleted:true", code, body) + } + // assert on the DB directly — the GET endpoint would re-fetch from the + // backend and re-store the row + stored, err := s.db.GetContractInfoForAddress(address) + if err != nil { + t.Fatal(err) + } + if stored != nil { + t.Fatalf("stored contract after DELETE = %+v, want nil", stored) + } + // idempotent: deleting a missing row reports deleted:false, not an error + code, body = adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/"+address, "") + if code != http.StatusOK || body != `{"contract":"`+address+`","deleted":false}` { + t.Fatalf("repeated DELETE = %d %s, want deleted:false", code, body) + } + }) + + t.Run("DELETE invalid address", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/not-an-address", "") + if code != http.StatusBadRequest { + t.Fatalf("DELETE = %d %s, want 400", code, body) + } + }) + + t.Run("DELETE missing address", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/", "") + if code != http.StatusBadRequest || !strings.Contains(body, "Missing contract address") { + t.Fatalf("DELETE = %d %s, want 400 Missing contract address", code, body) + } + }) +} + +// TestInternalServerRouteGatingNonEVM verifies that the chain-generic +// runtime-settings admin routes are registered on a non-EVM chain while the +// EVM-specific contract-info routes stay gated (404 via the authenticated +// catch-all). +func TestInternalServerRouteGatingNonEVM(t *testing.T) { + t.Setenv("BB_ADMIN_USER", "admin") + t.Setenv("BB_ADMIN_PASSWORD", "password") + parser, chain := setupChain(t) + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) + defer closeAndDestroyPublicServer(t, s, dbpath) + ts := newInternalTestServer(t, s) + + code, body := adminRequest(t, ts, http.MethodGet, "/admin/runtime-settings/ALLOWED_RPC_CALL_TO", "") + if code != http.StatusOK { + t.Fatalf("GET runtime setting = %d %s, want 200 on a non-EVM chain", code, body) + } + var resp runtimeSettingResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("runtime setting body %q does not decode: %v", body, err) + } + if resp.Key != "ALLOWED_RPC_CALL_TO" || resp.Source != "unset" { + t.Fatalf("runtime setting = %+v, want unset ALLOWED_RPC_CALL_TO", resp) + } + + code, _ = adminRequest(t, ts, http.MethodGet, "/admin/runtime-settings", "") + if code != http.StatusOK { + t.Fatalf("GET runtime-settings page = %d, want 200 on a non-EVM chain", code) + } + + code, _ = adminRequest(t, ts, http.MethodGet, "/admin/contract-info/0x20cd153de35d469ba46127a0c8f18626b59a256a", "") + if code != http.StatusNotFound { + t.Fatalf("GET contract-info = %d, want 404 on a non-EVM chain", code) + } +} diff --git a/server/internal_test.go b/server/internal_test.go index 6c753293d9..4a3776a4cb 100644 --- a/server/internal_test.go +++ b/server/internal_test.go @@ -65,6 +65,28 @@ func TestRequireAdminAuth(t *testing.T) { } } +func Test_urlPathSegment(t *testing.T) { + tests := []struct { + path string + want string + }{ + {"/admin/contract-info/0xAbC123", "0xAbC123"}, // case preserved + {"/admin/runtime-settings/allowed_rpc_call_to", "allowed_rpc_call_to"}, + {"/admin/contract-info/ 0xabc \t", "0xabc"}, // whitespace trimmed + {"/admin/contract-info/", ""}, // collection path + {"/admin", ""}, // root-level path has no sub-segment + {"/", ""}, + {"", ""}, + } + for _, tt := range tests { + r := httptest.NewRequest(http.MethodGet, "/x", nil) + r.URL.Path = tt.path // bypass NewRequest path validation to test raw shapes + if got := urlPathSegment(r); got != tt.want { + t.Errorf("urlPathSegment(%q) = %q, want %q", tt.path, got, tt.want) + } + } +} + // TestAdminSubtreeIsGated guards against the ServeMux fall-through where // trailing-slash or unknown /admin paths would otherwise reach the unauthenticated // index handler. It mirrors the route registration in NewInternalServer. diff --git a/server/runtime_settings.go b/server/runtime_settings.go index 396ab76897..513a61b0fa 100644 --- a/server/runtime_settings.go +++ b/server/runtime_settings.go @@ -230,10 +230,7 @@ type runtimeSettingResponse struct { // apiRuntimeSetting handles GET/POST/PUT/DELETE of a single runtime setting at // /admin/runtime-settings/. func (s *InternalServer) apiRuntimeSetting(r *http.Request, apiVersion int) (interface{}, error) { - var key string - if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - key = strings.ToUpper(strings.TrimSpace(r.URL.Path[i+1:])) - } + key := strings.ToUpper(urlPathSegment(r)) def := runtimeSettingDefByKey(key) if def == nil { return nil, api.NewAPIError("Unknown runtime setting, supported: "+runtimeSettingKeys(), true) diff --git a/static/internal_templates/contract_info.html b/static/internal_templates/contract_info.html index 57cbfece24..1f640e263d 100644 --- a/static/internal_templates/contract_info.html +++ b/static/internal_templates/contract_info.html @@ -26,13 +26,21 @@
- To update contract, use POST request to /admin/contract-info/ endpoint. Example: + Use the /admin/contract-info/ endpoint to manage cached contract metadata. Examples:
-            curl -k -v  \
-            'https://<internaladdress>/admin/contract-info/' \
+            # read the cached metadata of one contract
+            curl -k -u user:password 'https://<internaladdress>/admin/contract-info/<address>'
+
+            # update contracts; POST targets the collection path with a JSON array body,
+            # the response is {"updated":N}
+            curl -k -u user:password -X POST 'https://<internaladdress>/admin/contract-info/' \
             -H 'Content-Type: application/json' \
-            --data '[{ContractInfo},{ContractInfo},...]'        
+            --data '[{ContractInfo},{ContractInfo},...]'
+
+            # purge the cached metadata of one contract so it is re-fetched from the
+            # backend node on the next read; the response is {"contract":"...","deleted":true|false}
+            curl -k -u user:password -X DELETE 'https://<internaladdress>/admin/contract-info/<address>'
         
diff --git a/static/internal_templates/index.html b/static/internal_templates/index.html index 196b0027e5..166a68278d 100644 --- a/static/internal_templates/index.html +++ b/static/internal_templates/index.html @@ -4,6 +4,9 @@ IP addresses that exceeded websocket usage limit + {{if eq .ChainType 1}}
@@ -11,7 +14,4 @@ - {{end}}{{end}} From 06a411d4b6f2ac2eec54c69e9aeb6d23e5208c69 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 2 Jul 2026 10:49:12 +0000 Subject: [PATCH 061/303] fix(server,db): address review nits and harden the admin endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Smaller items from the PR #1597 review: - lowercase the "invalid EVM call method selector" parse error to match the sibling parser's message convention - trim environment values wherever they are resolved (startup and the DELETE fallback), so stray whitespace from an env file never surfaces in values; a set but whitespace-only env value is a configuration error rather than unset — treating it as unset would silently un-restrict rpcCall - document the differing To/Methods map key formats on RpcCallAllowlists so a future allowlist dimension picks a convention deliberately - runtimeSettingStore gains GetRuntimeSetting; resolveRuntimeSetting reads through the interface instead of requiring *db.RocksDB Admin runtime-settings page gets an inline editing UI (edit/save/delete per setting, replacing the curl-only instructions) and the page/nav/ routes are consistently available on all chain types. Hardening found by review of the new DELETE /admin/contract-info/: - bump protocolGen before the cache purge in DeleteContractInfoForAddress so a concurrent GetContractInfo cannot re-insert the deleted row into the LRU (same idiom as SetErcProtocol); without it the purge could be silently undone until eviction or restart - DELETE returns (and logs) the purged record: the backend re-fetch restores only name/symbol/decimals, not the sync-owned createdInBlock/destructedInBlock, so the response gives the operator a POST-back restore path and the docs now state exactly what is discarded - a failed save in the admin UI keeps the row in edit state instead of stranding an orphaned input, and error responses are unwrapped to their message Co-Authored-By: Claude Fable 5 --- common/internalstate.go | 11 ++ db/rocksdb_contracts.go | 39 +++-- db/rocksdb_contracts_test.go | 38 +++-- docs/env.md | 6 +- server/internal.go | 32 ++-- server/internal_contract_info_test.go | 24 ++- server/runtime_settings.go | 49 ++++-- server/runtime_settings_test.go | 35 +++- static/internal_templates/contract_info.html | 7 +- .../internal_templates/runtime_settings.html | 149 +++++++++++++++--- 10 files changed, 311 insertions(+), 79 deletions(-) diff --git a/common/internalstate.go b/common/internalstate.go index 0d8a22617b..ef6c537f7b 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -142,6 +142,17 @@ const ( type RpcCallAllowlists struct { // To and Methods are the parsed allowlists; a nil map means that dimension // is unconfigured. With both nil, rpcCall is unrestricted. + // + // Each dimension's key format must match what the rpcCall check looks up + // (server rpcCallAllowed), and the two deliberately differ: + // - To keys are the configured entries trimmed and lowercased verbatim — + // in practice 0x-prefixed hex addresses, because they are matched + // against the lowercased `to` field of the request as sent by clients. + // - Methods keys are 4-byte selectors as 8 lowercase hex characters + // without the 0x prefix, because they are matched against selectors + // hex-decoded from the request calldata (server evmCallSelector). + // A future dimension should document its key format here and keep the + // parser (server runtimeSettingDefs) and the rpcCall lookup in lockstep. To map[string]struct{} Methods map[string]struct{} // Raw comma-separated values and their sources (unset/env/db), kept for diff --git a/db/rocksdb_contracts.go b/db/rocksdb_contracts.go index 5709be6750..962c58153c 100644 --- a/db/rocksdb_contracts.go +++ b/db/rocksdb_contracts.go @@ -190,26 +190,45 @@ func (d *RocksDB) storeContractInfo(wb *grocksdb.WriteBatch, contractInfo *bchai // DeleteContractInfoForAddress removes the stored contract metadata for the given // address (and its in-memory cache entry) so the next read re-fetches it from the -// backend node. Returns whether a stored row existed. ERC-4626 protocol-detection -// rows in cfErcProtocols are kept: they are sync-owned, chain-derived state with -// their own lifecycle, merged on read — not cached backend metadata. -func (d *RocksDB) DeleteContractInfoForAddress(address string) (bool, error) { +// backend node. It returns the purged record (nil when no row was stored): the +// whole row is discarded, including the sync-owned CreatedInBlock and +// DestructedInBlock, which a backend re-fetch cannot restore — only a reindex or +// storing the returned record back can. ERC-4626 protocol-detection rows in +// cfErcProtocols are kept: they live in their own column family with their own +// lifecycle and are merged on read. +func (d *RocksDB) DeleteContractInfoForAddress(address string) (*bchain.ContractInfo, error) { contract, err := d.chainParser.GetAddrDescFromAddress(address) if err != nil { - return false, err + return nil, err } if contract == nil { - return false, errors.Errorf("invalid address %s", address) + return nil, errors.Errorf("invalid address %s", address) } val, err := d.db.GetCF(d.ro, d.cfh[cfContracts], contract) if err != nil { - return false, err + return nil, err + } + buf := val.Data() + var purged *bchain.ContractInfo + if len(buf) > 0 { + purged, _ = unpackContractInfo(buf) + addresses, _, _ := d.chainParser.GetAddressesFromAddrDesc(contract) + if len(addresses) > 0 { + purged.Contract = addresses[0] + } } - found := len(val.Data()) > 0 val.Free() + if purged == nil { + return nil, nil + } if err := d.db.DeleteCF(d.wo, d.cfh[cfContracts], contract); err != nil { - return false, err + return nil, err } + // Bump before the cache delete: a concurrent GetContractInfo that already + // sampled the old protocolGen and is about to re-add the just-deleted row + // will mismatch on the next read and miss, even if its add lands after our + // delete clears the slot (same idiom as SetErcProtocol). + d.protocolGen.Add(1) cachedContracts.delete(string(contract)) - return found, nil + return purged, nil } diff --git a/db/rocksdb_contracts_test.go b/db/rocksdb_contracts_test.go index cd8aca68ce..3132c13a53 100644 --- a/db/rocksdb_contracts_test.go +++ b/db/rocksdb_contracts_test.go @@ -18,12 +18,13 @@ func TestRocksDB_DeleteContractInfoForAddress(t *testing.T) { address := "0x" + dbtestdata.EthAddr20 ci := &bchain.ContractInfo{ - Standard: bchain.ERC20TokenStandard, - Type: bchain.ERC20TokenStandard, - Contract: address, - Name: "Test contract", - Symbol: "TCT", - Decimals: 18, + Standard: bchain.ERC20TokenStandard, + Type: bchain.ERC20TokenStandard, + Contract: address, + Name: "Test contract", + Symbol: "TCT", + Decimals: 18, + CreatedInBlock: 1234567, } if err := d.StoreContractInfo(ci); err != nil { t.Fatal(err) @@ -38,12 +39,18 @@ func TestRocksDB_DeleteContractInfoForAddress(t *testing.T) { t.Fatalf("GetContractInfoForAddress() = %+v, want stored contract", got) } - found, err := d.DeleteContractInfoForAddress(address) + genBefore := d.protocolGen.Load() + purged, err := d.DeleteContractInfoForAddress(address) if err != nil { t.Fatal(err) } - if !found { - t.Error("DeleteContractInfoForAddress() = false, want true for a stored row") + if purged == nil || purged.Name != ci.Name || purged.CreatedInBlock != ci.CreatedInBlock { + t.Errorf("DeleteContractInfoForAddress() = %+v, want the stored record", purged) + } + // The generation bump protects against a concurrent GetContractInfo + // re-inserting the deleted row into the cache (see SetErcProtocol). + if d.protocolGen.Load() != genBefore+1 { + t.Error("DeleteContractInfoForAddress() did not bump protocolGen") } got, err = d.GetContractInfoForAddress(address) if err != nil { @@ -53,13 +60,18 @@ func TestRocksDB_DeleteContractInfoForAddress(t *testing.T) { t.Errorf("GetContractInfoForAddress() after delete = %+v, want nil", got) } - // Idempotent: deleting a missing row is not an error. - found, err = d.DeleteContractInfoForAddress(address) + // Idempotent: deleting a missing row is not an error and does not bump + // the generation (nothing a concurrent reader could re-insert). + genBefore = d.protocolGen.Load() + purged, err = d.DeleteContractInfoForAddress(address) if err != nil { t.Fatal(err) } - if found { - t.Error("DeleteContractInfoForAddress() = true, want false for a missing row") + if purged != nil { + t.Errorf("DeleteContractInfoForAddress() = %+v, want nil for a missing row", purged) + } + if d.protocolGen.Load() != genBefore { + t.Error("DeleteContractInfoForAddress() of a missing row must not bump protocolGen") } if _, err = d.DeleteContractInfoForAddress("not-an-address"); err == nil { diff --git a/docs/env.md b/docs/env.md index 01f4317f0c..4468e2f2e7 100644 --- a/docs/env.md +++ b/docs/env.md @@ -81,9 +81,9 @@ Blockbook reads these from its process environment. When installed from the Debi 3. `COINGECKO_API_KEY` Example: for Optimism, `network=OP` and `coin shortcut=ETH`, so `OP_COINGECKO_API_KEY` is preferred over `ETH_COINGECKO_API_KEY`. -- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. Entries are trimmed and empty entries skipped; a set value that contains no addresses at all is a configuration error and Blockbook fails on startup. If omitted (and `ALLOWED_EVM_CALL_METHODS` is not set either), `rpcCall` is enabled for all addresses. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. +- `_ALLOWED_RPC_CALL_TO` - Addresses to which `rpcCall` websocket requests can be made, as a comma-separated list. The value and its entries are trimmed and empty entries skipped; a set value that contains no addresses at all (a whitespace-only value included) is a configuration error and Blockbook fails on startup. If omitted (and `ALLOWED_EVM_CALL_METHODS` is not set either), `rpcCall` is enabled for all addresses. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all, is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. +- `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all (a whitespace-only value included), is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. - `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. @@ -111,7 +111,7 @@ On EVM chains the internal server also exposes `/admin/contract-info/` (same Bas - `GET /admin/contract-info/
` returns the contract's metadata as a `ContractInfo` JSON object (fetching it from the backend node and storing it if not cached yet). - `POST` (or `PUT`) `/admin/contract-info/` with a JSON array body `[{ContractInfo},…]` updates the stored metadata of the listed contracts; the response is `{"updated":N}`. The write targets the collection path — a `POST` to an address path is rejected with `400`. -- `DELETE /admin/contract-info/
` purges the cached metadata of one contract so it is re-fetched from the backend node on the next read; the response is `{"contract":"
","deleted":true|false}` (`false` when nothing was stored — the delete is idempotent). +- `DELETE /admin/contract-info/
` purges the stored metadata of one contract so it is re-fetched from the backend node on the next read; the response is `{"contract":"
","deleted":true|false,"purged":{ContractInfo}}` (`deleted` is `false` and `purged` absent when nothing was stored — the delete is idempotent). Note that the whole record is discarded: the backend re-fetch restores only name/symbol/decimals, not the sync-owned `createdInBlock`/`destructedInBlock` fields, which are otherwise recoverable only by a reindex. The `purged` record in the response (also logged) can be `POST`ed back to restore them. ## Build-time variables diff --git a/server/internal.go b/server/internal.go index e2e0038f88..87a7c47f16 100644 --- a/server/internal.go +++ b/server/internal.go @@ -394,10 +394,14 @@ type contractInfoUpdateResponse struct { } // contractInfoDeleteResponse is the JSON shape returned by -// DELETE /admin/contract-info/
. +// DELETE /admin/contract-info/
. Purged carries the removed record so +// the operator can restore it verbatim with a POST — the row includes the +// sync-owned createdInBlock/destructedInBlock fields, which the backend +// re-fetch on the next read cannot recover. type contractInfoDeleteResponse struct { - Contract string `json:"contract"` - Deleted bool `json:"deleted"` + Contract string `json:"contract"` + Deleted bool `json:"deleted"` + Purged *bchain.ContractInfo `json:"purged,omitempty"` } // apiContractInfo handles GET/POST/PUT/DELETE of cached contract metadata at @@ -416,7 +420,7 @@ func (s *InternalServer) apiContractInfo(r *http.Request, apiVersion int) (inter } return s.updateContracts(r) case http.MethodDelete: - return s.deleteContractInfo(address) + return s.deleteContractInfo(address, r) } return nil, api.NewAPIError("Unsupported method "+r.Method, true) } @@ -456,17 +460,23 @@ func (s *InternalServer) updateContracts(r *http.Request) (interface{}, error) { return &contractInfoUpdateResponse{Updated: len(contractInfos)}, nil } -// deleteContractInfo purges the cached metadata of one contract so the next -// read re-fetches it from the backend node. Deleting is idempotent: a missing -// row reports deleted=false rather than an error, matching the runtime-settings -// DELETE semantics. -func (s *InternalServer) deleteContractInfo(address string) (interface{}, error) { +// deleteContractInfo purges the stored metadata of one contract so the next +// read re-fetches it from the backend node. The whole row is discarded — the +// backend re-fetch restores only name/symbol/decimals, not the sync-owned +// createdInBlock/destructedInBlock, so the purged record is logged and +// returned for a POST restore. Deleting is idempotent: a missing row reports +// deleted=false rather than an error, matching the runtime-settings DELETE +// semantics. +func (s *InternalServer) deleteContractInfo(address string, r *http.Request) (interface{}, error) { if address == "" { return nil, api.NewAPIError("Missing contract address", true) } - found, err := s.db.DeleteContractInfoForAddress(address) + purged, err := s.db.DeleteContractInfoForAddress(address) if err != nil { return nil, api.NewAPIError(err.Error(), true) } - return &contractInfoDeleteResponse{Contract: address, Deleted: found}, nil + if purged != nil { + glog.Infof("admin: contract info %s purged (%+v), client %s", address, *purged, r.RemoteAddr) + } + return &contractInfoDeleteResponse{Contract: address, Deleted: purged != nil, Purged: purged}, nil } diff --git a/server/internal_contract_info_test.go b/server/internal_contract_info_test.go index 7842e92945..4466467b87 100644 --- a/server/internal_contract_info_test.go +++ b/server/internal_contract_info_test.go @@ -142,8 +142,17 @@ func TestContractInfoAdminAPI(t *testing.T) { t.Run("DELETE", func(t *testing.T) { // stored by the POST subtest above code, body := adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/"+address, "") - if code != http.StatusOK || body != `{"contract":"`+address+`","deleted":true}` { - t.Fatalf("DELETE = %d %s, want deleted:true", code, body) + if code != http.StatusOK { + t.Fatalf("DELETE = %d %s, want 200", code, body) + } + var resp contractInfoDeleteResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("DELETE body %q does not decode: %v", body, err) + } + // the purged record is returned so the operator can restore it (incl. + // the sync-owned fields the backend re-fetch cannot recover) via POST + if !resp.Deleted || resp.Contract != address || resp.Purged == nil || resp.Purged.Name != "Renamed" { + t.Fatalf("DELETE = %+v, want deleted:true with the purged record", resp) } // assert on the DB directly — the GET endpoint would re-fetch from the // backend and re-store the row @@ -156,8 +165,15 @@ func TestContractInfoAdminAPI(t *testing.T) { } // idempotent: deleting a missing row reports deleted:false, not an error code, body = adminRequest(t, ts, http.MethodDelete, "/admin/contract-info/"+address, "") - if code != http.StatusOK || body != `{"contract":"`+address+`","deleted":false}` { - t.Fatalf("repeated DELETE = %d %s, want deleted:false", code, body) + if code != http.StatusOK { + t.Fatalf("repeated DELETE = %d %s, want 200", code, body) + } + resp = contractInfoDeleteResponse{} + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("repeated DELETE body %q does not decode: %v", body, err) + } + if resp.Deleted || resp.Purged != nil { + t.Fatalf("repeated DELETE = %+v, want deleted:false without a purged record", resp) } }) diff --git a/server/runtime_settings.go b/server/runtime_settings.go index 513a61b0fa..3c8c1742f4 100644 --- a/server/runtime_settings.go +++ b/server/runtime_settings.go @@ -123,7 +123,7 @@ func parseAllowedEvmCallMethods(name, value string) (map[string]struct{}, error) selector := strings.TrimPrefix(strings.ToLower(m), "0x") b, err := hex.DecodeString(selector) if err != nil || len(b) != 4 { - return nil, errors.Errorf("Invalid EVM call method selector %q in %s, expecting 4 bytes in hex", m, name) + return nil, errors.Errorf("invalid EVM call method selector %q in %s, expecting 4 bytes in hex", m, name) } methods[selector] = struct{}{} } @@ -139,11 +139,26 @@ func runtimeSettingEnvName(network, key string) string { return strings.ToUpper(network) + "_" + key } +// runtimeSettingEnvValue returns the trimmed value of the given environment +// variable, so a stray space or newline in an env file does not leak into the +// value (admin POSTs are trimmed the same way). A set but whitespace-only +// value is a configuration error rather than unset: treating it as unset +// would silently un-restrict rpcCall, unlike every other malformed value, +// which fails loudly. +func runtimeSettingEnvValue(envName string) (string, error) { + raw := os.Getenv(envName) + value := strings.TrimSpace(raw) + if raw != "" && value == "" { + return "", errors.Errorf("%s contains only whitespace", envName) + } + return value, nil +} + // resolveRuntimeSetting returns the effective raw value of a runtime setting // and its source: a stored override wins over the environment variable. -func resolveRuntimeSetting(d *db.RocksDB, network, key string) (string, string, error) { - if d != nil { - value, found, err := d.GetRuntimeSetting(key) +func resolveRuntimeSetting(store runtimeSettingStore, network, key string) (string, string, error) { + if store != nil { + value, found, err := store.GetRuntimeSetting(key) if err != nil { return "", "", err } @@ -151,7 +166,11 @@ func resolveRuntimeSetting(d *db.RocksDB, network, key string) (string, string, return value, common.RuntimeSettingSourceDB, nil } } - if value := os.Getenv(runtimeSettingEnvName(network, key)); value != "" { + value, err := runtimeSettingEnvValue(runtimeSettingEnvName(network, key)) + if err != nil { + return "", "", err + } + if value != "" { return value, common.RuntimeSettingSourceEnv, nil } return "", common.RuntimeSettingSourceUnset, nil @@ -163,10 +182,16 @@ func resolveRuntimeSetting(d *db.RocksDB, network, key string) (string, string, // through the admin interface) — is an error; falling back could silently // widen access. func buildRpcCallAllowlists(d *db.RocksDB, is *common.InternalState) (*common.RpcCallAllowlists, error) { + // the explicit nil check avoids a typed-nil interface on which the store + // methods would be called + var store runtimeSettingStore + if d != nil { + store = d + } network := is.GetNetwork() a := &common.RpcCallAllowlists{} for _, def := range runtimeSettingDefs { - value, source, err := resolveRuntimeSetting(d, network, def.key) + value, source, err := resolveRuntimeSetting(store, network, def.key) if err != nil { return nil, err } @@ -212,9 +237,11 @@ func initRpcCallAllowlists(d *db.RocksDB, is *common.InternalState) error { return nil } -// runtimeSettingStore persists runtime setting overrides; *db.RocksDB in -// production, replaceable in tests to exercise storage failures. +// runtimeSettingStore reads and persists runtime setting overrides; +// *db.RocksDB in production, replaceable in tests to exercise storage +// failures. type runtimeSettingStore interface { + GetRuntimeSetting(name string) (value string, found bool, err error) StoreRuntimeSetting(name, value string) error DeleteRuntimeSetting(name string) error } @@ -321,11 +348,13 @@ func (s *InternalServer) updateRuntimeSetting(def *runtimeSettingDef, r *http.Re // the database and the live state divergent and fail the next restart. func (s *InternalServer) deleteRuntimeSetting(def *runtimeSettingDef, r *http.Request) (interface{}, error) { envName := runtimeSettingEnvName(s.is.GetNetwork(), def.key) - envValue := os.Getenv(envName) + envValue, err := runtimeSettingEnvValue(envName) + if err != nil { + return nil, api.NewAPIError("Cannot remove override, fallback environment value is invalid: "+err.Error()+"; fix the environment or set a value instead", true) + } var parsed map[string]struct{} source := common.RuntimeSettingSourceUnset if envValue != "" { - var err error parsed, err = def.parse(envName, envValue) if err != nil { return nil, api.NewAPIError("Cannot remove override, fallback environment value is invalid: "+err.Error()+"; fix the environment or set a value instead", true) diff --git a/server/runtime_settings_test.go b/server/runtime_settings_test.go index 288721089c..9684198db6 100644 --- a/server/runtime_settings_test.go +++ b/server/runtime_settings_test.go @@ -60,6 +60,17 @@ func TestParseAllowedRpcCallTo(t *testing.T) { } } +// A whitespace-only environment value must fail startup resolution the same +// way a value with no parseable entries does — treating it as unset would +// silently leave rpcCall unrestricted. +func TestRuntimeSettingWhitespaceOnlyEnvFailsInit(t *testing.T) { + t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", " \n") + is := &common.InternalState{CoinShortcut: "FAKE"} + if err := initRpcCallAllowlists(nil, is); err == nil || !strings.Contains(err.Error(), "only whitespace") { + t.Fatalf("initRpcCallAllowlists() err = %v, want whitespace configuration error", err) + } +} + func doRuntimeSettingRequest(t *testing.T, handler http.HandlerFunc, method, key, body string) (int, map[string]string) { t.Helper() var rdr io.Reader @@ -76,10 +87,14 @@ func doRuntimeSettingRequest(t *testing.T, handler http.HandlerFunc, method, key return w.Code, resp } -// failingRuntimeSettingStore fails every persistence call; it stands in for a +// failingRuntimeSettingStore fails every store call; it stands in for a // broken database to exercise the store-before-publish error paths. type failingRuntimeSettingStore struct{} +func (failingRuntimeSettingStore) GetRuntimeSetting(name string) (string, bool, error) { + return "", false, errors.New("get failed") +} + func (failingRuntimeSettingStore) StoreRuntimeSetting(name, value string) error { return errors.New("store failed") } @@ -93,7 +108,9 @@ func (failingRuntimeSettingStore) DeleteRuntimeSetting(name string) error { // individual subtests. func TestRuntimeSettingsAPI(t *testing.T) { const envTo = "0xcdA9FC258358EcaA88845f19Af595e908bb7EfE9" - t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", envTo) + // the surrounding whitespace must never surface: the env value is trimmed + // wherever it is resolved (startup and DELETE fallback alike) + t.Setenv("FAKE_ALLOWED_RPC_CALL_TO", " "+envTo+"\n") parser := eth.NewEthereumParser(1, true) chain, err := dbtestdata.NewFakeBlockChainEthereumType(parser) if err != nil { @@ -267,6 +284,20 @@ func TestRuntimeSettingsAPI(t *testing.T) { } }) + t.Run("DELETE with whitespace-only env fallback is rejected", func(t *testing.T) { + // a whitespace-only env value is a configuration error, not unset — + // reverting to it would silently un-restrict rpcCall and the next + // restart would resolve the same error + t.Setenv("FAKE_ALLOWED_EVM_CALL_METHODS", " \n") + code, resp := doRuntimeSettingRequest(t, handler, http.MethodDelete, "ALLOWED_EVM_CALL_METHODS", "") + if code != http.StatusBadRequest || !strings.Contains(resp["error"], "only whitespace") { + t.Fatalf("got %d %v, want 400 whitespace error", code, resp) + } + if _, found, _ := d.GetRuntimeSetting(runtimeSettingAllowedEvmCallMethods); !found { + t.Fatal("rejected DELETE must keep the DB row") + } + }) + // proves the DB write happens before the snapshot swap: with a failing // store the requests take the error branch (glog.Error + 500) and the // live allowlists stay untouched diff --git a/static/internal_templates/contract_info.html b/static/internal_templates/contract_info.html index 1f640e263d..e9656cdc88 100644 --- a/static/internal_templates/contract_info.html +++ b/static/internal_templates/contract_info.html @@ -38,8 +38,11 @@ -H 'Content-Type: application/json' \ --data '[{ContractInfo},{ContractInfo},...]' - # purge the cached metadata of one contract so it is re-fetched from the - # backend node on the next read; the response is {"contract":"...","deleted":true|false} + # purge the stored metadata of one contract so it is re-fetched from the + # backend node on the next read; the response + # {"contract":"...","deleted":true|false,"purged":{ContractInfo}} + # returns the removed record — POST it back to restore the sync-owned + # createdInBlock/destructedInBlock fields the re-fetch cannot recover curl -k -u user:password -X DELETE 'https://<internaladdress>/admin/contract-info/<address>'
diff --git a/static/internal_templates/runtime_settings.html b/static/internal_templates/runtime_settings.html index f3a73aa4f1..43d1e21d97 100644 --- a/static/internal_templates/runtime_settings.html +++ b/static/internal_templates/runtime_settings.html @@ -1,38 +1,139 @@ -{{define "specific"}} {{if eq .ChainType 1}} - +{{define "specific"}} + +
+
+ - + {{range .RuntimeSettings}} - - - - + + + + + {{end}}
Setting Value SourceActions
{{.Key}}{{.Value}}{{.Source}}
{{.Key}}{{.Value}}{{.Source}} + + +
-
- Runtime settings override their environment variable defaults, are stored in the database and survive restarts. - Use the /admin/runtime-settings/<KEY> endpoint to manage them. Examples: -
-
-            # read the current value and its source (db, env or unset)
-            curl -k -u user:password 'https://<internaladdress>/admin/runtime-settings/ALLOWED_EVM_CALL_METHODS'
+
+{{end}}

From f7ba6a4f9ef6472f4e8b012b0a78ed251cdfe2fc Mon Sep 17 00:00:00 2001
From: pragmaxim 
Date: Fri, 3 Jul 2026 04:36:58 +0000
Subject: [PATCH 062/303] feat(server): list all runtime settings, warn on
 env/override drift
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Supports the bbctl counterpart (trezor/bbctl#10) of the runtime-settings
admin API in a two-replica deployment where each replica owns its DB:

- GET /admin/runtime-settings/ (bare collection path) returns every
  setting with its effective value and source as a JSON array, so a
  management tool reads the whole state per replica in one request;
  other methods on the bare path stay a 400
- initRpcCallAllowlists warns at startup when a stored override shadows
  a different (or malformed) environment value, making env/DB drift —
  a replica that missed an admin update, or an env change rolled out
  while an override exists — visible; the shadowed value is only
  compared, never parsed, so drift cannot fail a restart
- docs/env.md explains the deployment roles of the two sources: the env
  var is the deploy-managed baseline that survives a replica resync
  (the DB is wiped, and unset allowlists would silently un-restrict
  rpcCall), the stored override is the runtime layer that survives
  restarts until the next deploy ships an updated environment

Co-Authored-By: Claude Fable 5 
---
 docs/env.md                     |  4 ++-
 server/runtime_settings.go      | 47 ++++++++++++++++++++++++++++++-
 server/runtime_settings_test.go | 49 +++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 2 deletions(-)

diff --git a/docs/env.md b/docs/env.md
index 4468e2f2e7..4c348dbf88 100644
--- a/docs/env.md
+++ b/docs/env.md
@@ -99,12 +99,14 @@ The `rpcCall` allowlists can be changed at runtime, without a restart, through t
 
 `GET/POST/DELETE /admin/runtime-settings/` where `` is `ALLOWED_RPC_CALL_TO` or `ALLOWED_EVM_CALL_METHODS`:
 
--   `GET` returns the effective value and its source (`db` = stored override, `env` = environment default, `unset` = neither): `{"key":"ALLOWED_EVM_CALL_METHODS","value":"0xdd62ed3e","source":"env"}`.
+-   `GET` returns the effective value and its source (`db` = stored override, `env` = environment default, `unset` = neither): `{"key":"ALLOWED_EVM_CALL_METHODS","value":"0xdd62ed3e","source":"env"}`. A `GET` of the bare collection path `/admin/runtime-settings/` returns all settings as a JSON array.
 -   `POST` (or `PUT`) with body `{"value":"0xdd62ed3e,0x70a08231"}` validates the value (invalid values are rejected with `400` and change nothing), stores it in the database and only then applies it to the live allowlists — a database failure returns `500` and leaves the live state unchanged. A `"value"` of exactly `""` is a valid override meaning "explicitly unconfigured" (that allowlist dimension is disabled, as if its environment variable was not set) — it is the only way to un-restrict at runtime while the environment variable has a value. A value containing only whitespace or separators is rejected with `400`, so a botched automation input cannot silently disable the allowlist.
 -   `DELETE` removes the stored override and reverts to the environment default. If the environment value is malformed the request is rejected with `400` and the override is kept, so a later restart cannot fail on it.
 
 Old Blockbook versions ignore the stored overrides (the environment applies again after a version rollback) and keep them intact, so rolling forward resumes the override.
 
+The two sources play distinct roles in a replicated deployment. The environment variable is the deploy-managed baseline: it is shipped identically to every replica with the deployment's env file, applies from the first second of the process (before the admin port is even reachable) and — unlike the database, which is wiped when a replica is resynced — survives a database rebuild, so a freshly synced replica never starts with the allowlists silently unconfigured. The stored override is the runtime layer on top: it takes effect without a restart and persists across restarts until the deployment ships an updated environment and the override is removed. Because an override shadows the environment value, the two can drift (a replica that missed an admin update, or an env change rolled out while an override exists); the drift is visible in the `source` field and Blockbook logs a warning at startup when a stored override shadows a different environment value.
+
 ## Contract-info admin endpoint
 
 On EVM chains the internal server also exposes `/admin/contract-info/` (same Basic auth) to manage the contract metadata Blockbook caches from the backend node:
diff --git a/server/runtime_settings.go b/server/runtime_settings.go
index 3c8c1742f4..20f845ed0b 100644
--- a/server/runtime_settings.go
+++ b/server/runtime_settings.go
@@ -234,9 +234,35 @@ func initRpcCallAllowlists(d *db.RocksDB, is *common.InternalState) error {
 	if a.Methods != nil {
 		glog.Info("Support of rpcCall for these method selectors (source ", a.MethodsSource, "): ", a.MethodsValue)
 	}
+	warnShadowedRuntimeSettingEnv(a, is.GetNetwork())
 	return nil
 }
 
+// warnShadowedRuntimeSettingEnv logs a warning for every stored override that
+// shadows a different environment value, so drift between the deployed env
+// file and the database (for example a replica that missed an admin update,
+// or an env change rolled out while an override exists) is visible at
+// startup. It only compares — a shadowed environment value is never parsed or
+// validated, so it cannot fail the start.
+func warnShadowedRuntimeSettingEnv(a *common.RpcCallAllowlists, network string) {
+	for _, def := range runtimeSettingDefs {
+		value, source := def.get(a)
+		if source != common.RuntimeSettingSourceDB {
+			continue
+		}
+		envName := runtimeSettingEnvName(network, def.key)
+		envValue, err := runtimeSettingEnvValue(envName)
+		if err != nil {
+			glog.Warning("runtime setting ", def.key, ": stored override shadows a malformed environment value: ", err)
+			continue
+		}
+		if envValue != "" && envValue != value {
+			glog.Warningf("runtime setting %s: stored override %q shadows a different environment value %q (%s); the override wins until it is removed",
+				def.key, value, envValue, envName)
+		}
+	}
+}
+
 // runtimeSettingStore reads and persists runtime setting overrides;
 // *db.RocksDB in production, replaceable in tests to exercise storage
 // failures.
@@ -255,9 +281,13 @@ type runtimeSettingResponse struct {
 }
 
 // apiRuntimeSetting handles GET/POST/PUT/DELETE of a single runtime setting at
-// /admin/runtime-settings/.
+// /admin/runtime-settings/; a GET of the bare collection path returns all
+// settings, so a management tool can read the whole state in one request.
 func (s *InternalServer) apiRuntimeSetting(r *http.Request, apiVersion int) (interface{}, error) {
 	key := strings.ToUpper(urlPathSegment(r))
+	if key == "" && r.Method == http.MethodGet {
+		return s.listRuntimeSettings()
+	}
 	def := runtimeSettingDefByKey(key)
 	if def == nil {
 		return nil, api.NewAPIError("Unknown runtime setting, supported: "+runtimeSettingKeys(), true)
@@ -293,6 +323,21 @@ func (s *InternalServer) getRuntimeSetting(def *runtimeSettingDef) (interface{},
 	return &runtimeSettingResponse{Key: def.key, Value: value, Source: source}, nil
 }
 
+// listRuntimeSettings returns the effective value and source of every runtime
+// setting as a JSON array.
+func (s *InternalServer) listRuntimeSettings() (interface{}, error) {
+	a, err := s.currentRpcCallAllowlists()
+	if err != nil {
+		return nil, err
+	}
+	settings := make([]runtimeSettingResponse, len(runtimeSettingDefs))
+	for i, def := range runtimeSettingDefs {
+		value, source := def.get(a)
+		settings[i] = runtimeSettingResponse{Key: def.key, Value: value, Source: source}
+	}
+	return settings, nil
+}
+
 // updateRuntimeSetting validates the new value, persists it to the database
 // and only then publishes it to the live snapshot, so an admin never gets a
 // success response for a change that would not survive a restart. A failed
diff --git a/server/runtime_settings_test.go b/server/runtime_settings_test.go
index 9684198db6..23f935b74b 100644
--- a/server/runtime_settings_test.go
+++ b/server/runtime_settings_test.go
@@ -156,6 +156,40 @@ func TestRuntimeSettingsAPI(t *testing.T) {
 		}
 	})
 
+	t.Run("GET all settings", func(t *testing.T) {
+		r := httptest.NewRequest(http.MethodGet, "/admin/runtime-settings/", nil)
+		w := httptest.NewRecorder()
+		handler(w, r)
+		if w.Code != http.StatusOK {
+			t.Fatalf("got %d %s, want 200", w.Code, w.Body.String())
+		}
+		var settings []runtimeSettingResponse
+		if err := json.Unmarshal(w.Body.Bytes(), &settings); err != nil {
+			t.Fatalf("cannot unmarshal response %q: %v", w.Body.String(), err)
+		}
+		want := map[string]runtimeSettingResponse{
+			runtimeSettingAllowedRpcCallTo:      {Key: runtimeSettingAllowedRpcCallTo, Value: envTo, Source: common.RuntimeSettingSourceEnv},
+			runtimeSettingAllowedEvmCallMethods: {Key: runtimeSettingAllowedEvmCallMethods, Value: "", Source: common.RuntimeSettingSourceUnset},
+		}
+		if len(settings) != len(want) {
+			t.Fatalf("got %d settings %v, want %d", len(settings), settings, len(want))
+		}
+		for _, s := range settings {
+			if s != want[s.Key] {
+				t.Fatalf("setting %s = %+v, want %+v", s.Key, s, want[s.Key])
+			}
+		}
+	})
+
+	t.Run("empty key accepts only GET", func(t *testing.T) {
+		for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
+			code, resp := doRuntimeSettingRequest(t, handler, method, "", `{"value":""}`)
+			if code != http.StatusBadRequest || !strings.Contains(resp["error"], "Unknown runtime setting") {
+				t.Fatalf("%s: got %d %v, want 400 unknown runtime setting", method, code, resp)
+			}
+		}
+	})
+
 	t.Run("key is case-insensitive", func(t *testing.T) {
 		code, resp := doRuntimeSettingRequest(t, handler, http.MethodGet, "allowed_rpc_call_to", "")
 		if code != http.StatusOK || resp["key"] != runtimeSettingAllowedRpcCallTo {
@@ -256,6 +290,21 @@ func TestRuntimeSettingsAPI(t *testing.T) {
 		}
 	})
 
+	t.Run("stored override shadows malformed env without failing init", func(t *testing.T) {
+		// drift between a stored override and the environment must not fail a
+		// restart — the shadowed env value is never parsed, the override wins
+		// and the mismatch is only logged
+		t.Setenv("FAKE_ALLOWED_EVM_CALL_METHODS", "not-a-selector")
+		restarted := &common.InternalState{CoinShortcut: "FAKE"}
+		if err := initRpcCallAllowlists(d, restarted); err != nil {
+			t.Fatal(err)
+		}
+		a := restarted.GetRpcCallAllowlists()
+		if a.MethodsSource != common.RuntimeSettingSourceDB || a.MethodsValue != "0xdd62ed3e,0x70a08231" {
+			t.Fatalf("got source %q value %q, want the db override", a.MethodsSource, a.MethodsValue)
+		}
+	})
+
 	t.Run("DELETE reverts to env", func(t *testing.T) {
 		code, resp := doRuntimeSettingRequest(t, handler, http.MethodDelete, "ALLOWED_RPC_CALL_TO", "")
 		if code != http.StatusOK || resp["value"] != envTo || resp["source"] != common.RuntimeSettingSourceEnv {

From a9b3f5d67deb7955230783fc1d9262e11cacf594 Mon Sep 17 00:00:00 2001
From: pragmaxim 
Date: Fri, 3 Jul 2026 04:43:28 +0000
Subject: [PATCH 063/303] feat(server,db): paginated list endpoint for admin
 contract-info
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

GET /admin/contract-info/ (bare collection path) lists the stored
records, mirroring the runtime-settings list convention. Unlike runtime
settings the collection is unbounded — sync stores a record per contract
creation, millions on a busy chain — so the list is paginated:
?limit=<1..10000> (default 1000) with a from cursor, and the response
{"contracts":[...],"next":"
"} carries the from of the next page. ListContractInfos iterates cfContracts in address-descriptor order (the CF holds only plain addrDesc keys; protocol rows live in their own CF). The former 400 for GET of the bare path moves to the list; DELETE and POST semantics on the bare path are unchanged. Co-Authored-By: Claude Fable 5 --- db/rocksdb_contracts.go | 47 ++++++++++++++++ db/rocksdb_contracts_test.go | 56 ++++++++++++++++++++ docs/env.md | 1 + server/internal.go | 42 +++++++++++++-- server/internal_contract_info_test.go | 39 ++++++++++++-- static/internal_templates/contract_info.html | 5 ++ 6 files changed, 183 insertions(+), 7 deletions(-) diff --git a/db/rocksdb_contracts.go b/db/rocksdb_contracts.go index 962c58153c..483ff4bd00 100644 --- a/db/rocksdb_contracts.go +++ b/db/rocksdb_contracts.go @@ -188,6 +188,53 @@ func (d *RocksDB) storeContractInfo(wb *grocksdb.WriteBatch, contractInfo *bchai return nil } +// ListContractInfos returns up to limit stored contract records ordered by +// address descriptor, starting at the optional from address (inclusive), and +// the address to pass as from to fetch the next page ("" when the listing is +// complete). A page limit is mandatory: the rows are sync-populated (every +// contract creation when internal data processing is enabled), so the full +// set can run into millions on a busy chain. +func (d *RocksDB) ListContractInfos(from string, limit int) ([]bchain.ContractInfo, string, error) { + var start bchain.AddressDescriptor + if from != "" { + var err error + start, err = d.chainParser.GetAddrDescFromAddress(from) + if err != nil { + return nil, "", err + } + if start == nil { + return nil, "", errors.Errorf("invalid address %s", from) + } + } + it := d.db.NewIteratorCF(d.ro, d.cfh[cfContracts]) + defer it.Close() + if start != nil { + it.Seek(start) + } else { + it.SeekToFirst() + } + contracts := make([]bchain.ContractInfo, 0, limit) + for ; it.Valid(); it.Next() { + addresses, _, _ := d.chainParser.GetAddressesFromAddrDesc(it.Key().Data()) + if len(contracts) == limit { + // one more row exists — its address is the cursor of the next page + if len(addresses) > 0 { + return contracts, addresses[0], nil + } + return contracts, "", nil + } + contractInfo, err := unpackContractInfo(it.Value().Data()) + if err != nil { + return nil, "", err + } + if len(addresses) > 0 { + contractInfo.Contract = addresses[0] + } + contracts = append(contracts, *contractInfo) + } + return contracts, "", nil +} + // DeleteContractInfoForAddress removes the stored contract metadata for the given // address (and its in-memory cache entry) so the next read re-fetches it from the // backend node. It returns the purged record (nil when no row was stored): the diff --git a/db/rocksdb_contracts_test.go b/db/rocksdb_contracts_test.go index 3132c13a53..322714c6ff 100644 --- a/db/rocksdb_contracts_test.go +++ b/db/rocksdb_contracts_test.go @@ -4,12 +4,68 @@ package db import ( "reflect" + "strconv" + "strings" "testing" "github.com/trezor/blockbook/bchain" "github.com/trezor/blockbook/tests/dbtestdata" ) +func TestRocksDB_ListContractInfos(t *testing.T) { + d := setupRocksDB(t, &testEthereumParser{ + EthereumParser: ethereumTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + // ordered by address descriptor: 0x20… < 0x4b… < 0x55… + addresses := []string{"0x" + dbtestdata.EthAddr20, "0x" + dbtestdata.EthAddr4b, "0x" + dbtestdata.EthAddr55} + for i, a := range addresses { + if err := d.StoreContractInfo(&bchain.ContractInfo{ + Standard: bchain.ERC20TokenStandard, + Type: bchain.ERC20TokenStandard, + Contract: a, + Name: "Contract " + strconv.Itoa(i), + Decimals: 18, + }); err != nil { + t.Fatal(err) + } + } + + contracts, next, err := d.ListContractInfos("", 10) + if err != nil { + t.Fatal(err) + } + if len(contracts) != 3 || next != "" { + t.Fatalf("ListContractInfos() = %d rows, next %q, want 3 rows and no next", len(contracts), next) + } + for i, c := range contracts { + if !strings.EqualFold(c.Contract, addresses[i]) { + t.Errorf("row %d = %s, want %s", i, c.Contract, addresses[i]) + } + } + + // paging: a full first page and a next cursor pointing at the third row + contracts, next, err = d.ListContractInfos("", 2) + if err != nil { + t.Fatal(err) + } + if len(contracts) != 2 || !strings.EqualFold(next, addresses[2]) { + t.Fatalf("ListContractInfos(limit 2) = %d rows, next %q, want 2 rows and next %s", len(contracts), next, addresses[2]) + } + contracts, next, err = d.ListContractInfos(next, 2) + if err != nil { + t.Fatal(err) + } + if len(contracts) != 1 || next != "" || !strings.EqualFold(contracts[0].Contract, addresses[2]) { + t.Fatalf("ListContractInfos(from next) = %+v next %q, want only the third row", contracts, next) + } + + if _, _, err = d.ListContractInfos("not-an-address", 2); err == nil { + t.Error("ListContractInfos() with invalid from: expected error") + } +} + func TestRocksDB_DeleteContractInfoForAddress(t *testing.T) { d := setupRocksDB(t, &testEthereumParser{ EthereumParser: ethereumTestnetParser(), diff --git a/docs/env.md b/docs/env.md index 4c348dbf88..11e3c0df3a 100644 --- a/docs/env.md +++ b/docs/env.md @@ -112,6 +112,7 @@ The two sources play distinct roles in a replicated deployment. The environment On EVM chains the internal server also exposes `/admin/contract-info/` (same Basic auth) to manage the contract metadata Blockbook caches from the backend node: - `GET /admin/contract-info/
` returns the contract's metadata as a `ContractInfo` JSON object (fetching it from the backend node and storing it if not cached yet). +- `GET /admin/contract-info/` (bare collection path) lists the stored records page by page as `{"contracts":[{ContractInfo},…],"next":"
"}`. Unlike runtime settings, the collection is unbounded (sync stores a record per contract creation, millions on a busy chain), so the page size is limited: `?limit=<1..10000>` (default `1000`) and `?from=
` continue from a cursor; `next` (present only when more records exist) is the `from` of the next page. - `POST` (or `PUT`) `/admin/contract-info/` with a JSON array body `[{ContractInfo},…]` updates the stored metadata of the listed contracts; the response is `{"updated":N}`. The write targets the collection path — a `POST` to an address path is rejected with `400`. - `DELETE /admin/contract-info/
` purges the stored metadata of one contract so it is re-fetched from the backend node on the next read; the response is `{"contract":"
","deleted":true|false,"purged":{ContractInfo}}` (`deleted` is `false` and `purged` absent when nothing was stored — the delete is idempotent). Note that the whole record is discarded: the backend re-fetch restores only name/symbol/decimals, not the sync-owned `createdInBlock`/`destructedInBlock` fields, which are otherwise recoverable only by a reindex. The `purged` record in the response (also logged) can be `POST`ed back to restore them. diff --git a/server/internal.go b/server/internal.go index 87a7c47f16..726dcf0c96 100644 --- a/server/internal.go +++ b/server/internal.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -393,6 +394,18 @@ type contractInfoUpdateResponse struct { Updated int `json:"updated"` } +// contractInfoListResponse is the JSON shape returned by GET /admin/contract-info/. +// Next, when present, is the from parameter of the next page. +type contractInfoListResponse struct { + Contracts []bchain.ContractInfo `json:"contracts"` + Next string `json:"next,omitempty"` +} + +const ( + contractInfoListDefaultLimit = 1000 + contractInfoListMaxLimit = 10000 +) + // contractInfoDeleteResponse is the JSON shape returned by // DELETE /admin/contract-info/
. Purged carries the removed record so // the operator can restore it verbatim with a POST — the row includes the @@ -406,11 +419,15 @@ type contractInfoDeleteResponse struct { // apiContractInfo handles GET/POST/PUT/DELETE of cached contract metadata at // /admin/contract-info/
(POST/PUT write the collection path -// /admin/contract-info/ with a JSON array body). +// /admin/contract-info/ with a JSON array body; a GET of the collection path +// lists the stored records page by page). func (s *InternalServer) apiContractInfo(r *http.Request, apiVersion int) (interface{}, error) { address := urlPathSegment(r) switch r.Method { case http.MethodGet: + if address == "" { + return s.listContractInfos(r) + } return s.getContractInfo(address) case http.MethodPost, http.MethodPut: // The bulk write addresses each contract in its body; reject a POST to @@ -425,10 +442,27 @@ func (s *InternalServer) apiContractInfo(r *http.Request, apiVersion int) (inter return nil, api.NewAPIError("Unsupported method "+r.Method, true) } -func (s *InternalServer) getContractInfo(address string) (interface{}, error) { - if address == "" { - return nil, api.NewAPIError("Missing contract address", true) +// listContractInfos returns one page of the stored contract records. Unlike +// the runtime-settings list, the collection is unbounded (sync stores a row +// per contract creation), so the page size is limited and the response +// carries a next cursor. +func (s *InternalServer) listContractInfos(r *http.Request) (interface{}, error) { + limit := contractInfoListDefaultLimit + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > contractInfoListMaxLimit { + return nil, api.NewAPIError("Invalid limit, expecting a number in 1.."+strconv.Itoa(contractInfoListMaxLimit), true) + } + limit = n + } + contracts, next, err := s.db.ListContractInfos(r.URL.Query().Get("from"), limit) + if err != nil { + return nil, api.NewAPIError(err.Error(), true) } + return &contractInfoListResponse{Contracts: contracts, Next: next}, nil +} + +func (s *InternalServer) getContractInfo(address string) (interface{}, error) { contractInfo, valid, err := s.api.GetContractInfo(address, bchain.UnknownTokenStandard) if err != nil { return nil, api.NewAPIError(err.Error(), true) diff --git a/server/internal_contract_info_test.go b/server/internal_contract_info_test.go index 4466467b87..7f8d917f65 100644 --- a/server/internal_contract_info_test.go +++ b/server/internal_contract_info_test.go @@ -73,10 +73,23 @@ func TestContractInfoAdminAPI(t *testing.T) { address := "0x" + dbtestdata.EthAddr20 - t.Run("GET missing address", func(t *testing.T) { + t.Run("GET collection lists stored contracts", func(t *testing.T) { code, body := adminRequest(t, ts, http.MethodGet, "/admin/contract-info/", "") - if code != http.StatusBadRequest || !strings.Contains(body, "Missing contract address") { - t.Fatalf("GET = %d %s, want 400 Missing contract address", code, body) + if code != http.StatusOK { + t.Fatalf("GET = %d %s, want 200", code, body) + } + var resp contractInfoListResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("GET body %q does not decode: %v", body, err) + } + }) + + t.Run("GET collection with invalid limit", func(t *testing.T) { + for _, q := range []string{"limit=0", "limit=-1", "limit=abc", "limit=10001"} { + code, body := adminRequest(t, ts, http.MethodGet, "/admin/contract-info/?"+q, "") + if code != http.StatusBadRequest { + t.Fatalf("%s: GET = %d %s, want 400", q, code, body) + } } }) @@ -118,6 +131,26 @@ func TestContractInfoAdminAPI(t *testing.T) { } }) + t.Run("GET collection contains the updated contract", func(t *testing.T) { + code, body := adminRequest(t, ts, http.MethodGet, "/admin/contract-info/?limit=10", "") + if code != http.StatusOK { + t.Fatalf("GET = %d %s, want 200", code, body) + } + var resp contractInfoListResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("GET body %q does not decode: %v", body, err) + } + found := false + for _, c := range resp.Contracts { + if c.Name == "Renamed" { + found = true + } + } + if !found { + t.Fatalf("list %+v does not contain the contract stored by POST", resp.Contracts) + } + }) + t.Run("POST with address segment", func(t *testing.T) { code, body := adminRequest(t, ts, http.MethodPost, "/admin/contract-info/"+address, `[]`) if code != http.StatusBadRequest || !strings.Contains(body, "collection") { diff --git a/static/internal_templates/contract_info.html b/static/internal_templates/contract_info.html index e9656cdc88..f98813d932 100644 --- a/static/internal_templates/contract_info.html +++ b/static/internal_templates/contract_info.html @@ -32,6 +32,11 @@ # read the cached metadata of one contract curl -k -u user:password 'https://<internaladdress>/admin/contract-info/<address>' + # list the stored records page by page; the response + # {"contracts":[...],"next":"<address>"} carries the from cursor of the + # next page (the collection can hold millions of sync-stored records) + curl -k -u user:password 'https://<internaladdress>/admin/contract-info/?limit=1000&from=<address>' + # update contracts; POST targets the collection path with a JSON array body, # the response is {"updated":N} curl -k -u user:password -X POST 'https://<internaladdress>/admin/contract-info/' \ From 7f0c0664ceb6be94609b61313905361346539362 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 7 Jul 2026 08:42:49 +0000 Subject: [PATCH 064/303] ci: idempotent admin API e2e tests against the deployed dev instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-deploy step in the e2e-tests job exercising the admin API (/admin/runtime-settings/, /admin/contract-info/) of the deployed base_archive dev instance, leaving its database and live rpcCall policy exactly as found — at the end of a run and at every instant during it: - runtime settings use a zero-policy-impact roundtrip: the POSTed value is always the current effective value (or the explicit-empty override when unset), so the parsed allowlist never changes; the final DELETE restores the original source; db-sourced settings are only rewritten in place, since their env fallback is unknown from outside - the contract roundtrip uses a reserved never-real address, pre-cleans it (healing a crashed run) and deletes it at the end; the cleanup and restore flags are raised before the POSTs, so a response lost after a server-side commit still gets cleaned up - negative tests: 401 without credentials (503 = admin disabled fails loudly), unknown key, invalid values proven to change nothing, POST to an address path, unsupported method, list pagination limits - target URL derives from BB_DEV_API_URL_HTTP_ plus the coin config's ports.blockbook_internal (BB_ADMIN_E2E_URL overrides for local runs); the coin gate compares against coins_csv deploy aliases (test_coins_csv carries test names and would never match); skips are annotated (::warning:: for a missing BB_RUNTIME_ENV, ::notice:: for coin-not-deployed) so a dead signal cannot look like a pass - credentials are parsed from BB_ENV inside the script (surrounding quotes stripped like systemd EnvironmentFile) and never printed; transient network errors are retried once, then fail with a clean pipeline error; helpers reuse runner.py and wait_for_sync.py Verified live against blockbook-dev3: two consecutive runs green via the derived URL, instance state identical before and after. Co-Authored-By: Claude Fable 5 --- .github/scripts/admin_api_e2e.py | 380 ++++++++++++++++++++++++++ .github/scripts/admin_api_e2e_test.py | 87 ++++++ .github/workflows/deploy.yml | 16 ++ 3 files changed, 483 insertions(+) create mode 100644 .github/scripts/admin_api_e2e.py create mode 100644 .github/scripts/admin_api_e2e_test.py diff --git a/.github/scripts/admin_api_e2e.py b/.github/scripts/admin_api_e2e.py new file mode 100644 index 0000000000..a2221a4fcc --- /dev/null +++ b/.github/scripts/admin_api_e2e.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""E2E tests of the blockbook admin API (/admin/runtime-settings/, +/admin/contract-info/) against a deployed dev instance. + +The tests are idempotent and leave the instance's database and its live +rpcCall policy exactly as they were found: + +* runtime settings are exercised with a zero-policy-impact roundtrip — the + POSTed value is always the *current effective value* (or the explicit-empty + override when unset), so the parsed allowlist is identical at every instant, + and the final DELETE restores the original source; +* the contract-info roundtrip uses a reserved never-real address and deletes + its record at the end (and before starting, which heals a crashed run). + +Configuration (environment): + BB_ADMIN_E2E_COIN coin this test targets (default base_archive) + BB_ADMIN_E2E_URL explicit internal-server base URL; when unset, the host + is derived from the BB_DEV_API_URL_HTTP_ repo + variable and the port from the coin config's + ports.blockbook_internal, so a host move is a one-place + repo-variable change + DEPLOYED_COINS comma-separated deploy config names of this deploy run; + when set and the target coin is absent, the test skips + itself + BB_ENV content of blockbook.env; BB_ADMIN_USER and + BB_ADMIN_PASSWORD are extracted from it (never printed) + BB_ADMIN_USER/BB_ADMIN_PASSWORD direct credentials, override BB_ENV + SYNC_CA_FILE/SYNC_TLS_INSECURE TLS verification, as in wait_for_sync.py +""" + +import base64 +import json +import os +import ssl +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +from runner import LOG_PREFIX, fail, log +from wait_for_sync import build_ssl_context, resolve_http_base + +REPO_ROOT = Path(__file__).resolve().parents[2] + +SETTING_KEYS = ("ALLOWED_RPC_CALL_TO", "ALLOWED_EVM_CALL_METHODS") +# Invalid values per setting; each must be rejected with 400 and change nothing. +INVALID_SETTING_VALUES = { + "ALLOWED_RPC_CALL_TO": ",", + "ALLOWED_EVM_CALL_METHODS": "0x12", +} +# Reserved, never-real address the contract roundtrip may create and delete. +TEST_CONTRACT_ADDRESS = "0xE2Ee00000000000000000000000000000000e2eE" + + +def strip_matching_quotes(value: str) -> str: + """Remove one pair of surrounding quotes, the way systemd's + EnvironmentFile does before the value reaches the server process.""" + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + return value + + +def parse_env_credentials(blob: str) -> tuple[str, str]: + """Extract BB_ADMIN_USER/BB_ADMIN_PASSWORD from KEY=value lines.""" + values = {} + for line in blob.splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + if key in ("BB_ADMIN_USER", "BB_ADMIN_PASSWORD"): + values[key] = strip_matching_quotes(value.strip()) + return values.get("BB_ADMIN_USER", ""), values.get("BB_ADMIN_PASSWORD", "") + + +def coin_deployed(deployed_csv: str, coin: str) -> bool: + """Whether coin is in the comma-separated deploy list; an empty list means + the caller did not restrict the run (local use) and the test proceeds.""" + csv = deployed_csv.strip() + if not csv: + return True + coins = {part.strip().lower() for part in csv.split(",") if part.strip()} + return coin.strip().lower() in coins + + +def resolve_admin_url(coin: str) -> str: + """Resolve the internal-server base URL: an explicit BB_ADMIN_E2E_URL + wins; otherwise the host follows the same BB_DEV_API_URL_HTTP_ repo + variable every other post-deploy check uses, with the internal port from + the coin config. The internal server always serves HTTPS (the packaged + self-signed certificate).""" + explicit = os.environ.get("BB_ADMIN_E2E_URL", "").strip() + if explicit: + return explicit.rstrip("/") + host = urllib.parse.urlparse(resolve_http_base(coin)).hostname + config_path = REPO_ROOT / "configs" / "coins" / (coin + ".json") + try: + with open(config_path, encoding="utf-8") as f: + port = json.load(f)["ports"]["blockbook_internal"] + except (OSError, ValueError, KeyError) as exc: + fail(f"cannot read ports.blockbook_internal from {config_path}: {exc}") + return f"https://{host}:{port}" + + +def skip(message: str, annotation: str) -> None: + """Exit 0 with a visible workflow annotation, so a skip can never be + mistaken for a pass in the run summary.""" + print(f"::{annotation}::admin API e2e tests skipped: {message}") + log(f"{message}; skipping admin API e2e tests") + raise SystemExit(0) + + +def plan_settings_roundtrip(value: str, source: str) -> list[tuple[str, str | None, str, str]]: + """Return the (method, body value, expected value, expected source) steps + that exercise POST/GET/DELETE for a setting currently at (value, source) + while keeping the parsed policy identical at every instant and ending in + the starting state. A db-sourced setting is only rewritten in place: its + env fallback is unknown, so a DELETE could change policy.""" + if source == "db": + return [("POST", value, value, "db")] + if source == "env": + return [("POST", value, value, "db"), ("DELETE", None, value, "env")] + if source == "unset": + # the explicit-empty override behaves exactly like unset + return [("POST", "", "", "db"), ("DELETE", None, "", "unset")] + raise ValueError(f"unknown setting source {source!r}") + + +class AdminClient: + # one retry absorbs a transient connection error without masking a real + # outage; every request in this test is idempotent, so retrying is safe + RETRIES = 1 + RETRY_DELAY_SECONDS = 3 + + def __init__(self, base_url: str, user: str, password: str, timeout: int, context: ssl.SSLContext): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.context = context + token = base64.b64encode(f"{user}:{password}".encode()).decode() + self.auth_header = "Basic " + token + + def request(self, method: str, path: str, body: object = None, auth: bool = True) -> tuple[int, object]: + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(self.base_url + path, data=data, method=method, headers=headers) + if auth: + req.add_header("Authorization", self.auth_header) + for attempt in range(self.RETRIES + 1): + try: + with urllib.request.urlopen(req, timeout=self.timeout, context=self.context) as resp: + return resp.getcode(), self._parse(resp.read()) + except urllib.error.HTTPError as exc: + return exc.code, self._parse(exc.read()) + except OSError as exc: # URLError, TLS errors, timeouts + if attempt < self.RETRIES: + log(f"{method} {path} failed ({exc}), retrying in {self.RETRY_DELAY_SECONDS}s") + time.sleep(self.RETRY_DELAY_SECONDS) + continue + fail(f"{method} {self.base_url + path} failed after {attempt + 1} attempts: {exc}") + + @staticmethod + def _parse(raw: bytes) -> object: + try: + return json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + return {"raw": raw.decode("utf-8", "replace")} + + +def check(condition: bool, what: str, actual: object) -> None: + if not condition: + fail(f"{what}: got {actual!r}") + + +def get_setting(client: AdminClient, key: str) -> dict: + status, resp = client.request("GET", f"/admin/runtime-settings/{key}") + check(status == 200 and isinstance(resp, dict), f"GET {key} expected 200 object", (status, resp)) + return resp + + +def check_auth_gating(client: AdminClient) -> None: + status, resp = client.request("GET", "/admin/runtime-settings/", auth=False) + if status == 503: + fail("admin interface is disabled on the instance (503) — BB_ADMIN_USER/BB_ADMIN_PASSWORD not configured there") + check(status == 401, "unauthenticated GET expected 401", (status, resp)) + log("PASS auth: admin API rejects unauthenticated requests with 401") + + +def check_settings_list(client: AdminClient) -> dict[str, dict]: + status, resp = client.request("GET", "/admin/runtime-settings/") + check(status == 200 and isinstance(resp, list), "GET runtime-settings list expected 200 array", (status, resp)) + settings = {entry.get("key"): entry for entry in resp if isinstance(entry, dict)} + for key in SETTING_KEYS: + check(key in settings, f"runtime-settings list must contain {key}", sorted(settings)) + entry = settings[key] + check(entry.get("source") in ("db", "env", "unset"), f"{key} has a valid source", entry) + log("PASS settings: list returns all settings " + json.dumps([settings[k] for k in SETTING_KEYS])) + return settings + + +def check_settings_negative(client: AdminClient, before: dict[str, dict]) -> None: + status, resp = client.request("GET", "/admin/runtime-settings/NO_SUCH_SETTING") + check(status == 400, "GET unknown setting expected 400", (status, resp)) + for key, invalid in INVALID_SETTING_VALUES.items(): + status, resp = client.request("POST", f"/admin/runtime-settings/{key}", {"value": invalid}) + check(status == 400, f"POST invalid value to {key} expected 400", (status, resp)) + after = get_setting(client, key) + check(after == before[key], f"rejected POST must not change {key}", (before[key], after)) + log("PASS settings: unknown key and invalid values are rejected without effect") + + +def settings_roundtrip(client: AdminClient, key: str, current: dict) -> None: + value, source = current.get("value", ""), current.get("source", "") + steps = plan_settings_roundtrip(value, source) + # If the run dies between POST and DELETE, still try to restore the + # original source. The flag is raised before the POST is attempted: a + # spurious restore DELETE of a missing override is a harmless no-op that + # reverts to the env value, i.e. the state the failed POST left in place. + restore_delete = False + try: + for method, body_value, want_value, want_source in steps: + if method == "POST" and source != "db": + restore_delete = True + body = {"value": body_value} if method == "POST" else None + status, resp = client.request(method, f"/admin/runtime-settings/{key}", body) + want = {"key": key, "value": want_value, "source": want_source} + check(status == 200 and resp == want, f"{method} {key} expected {want}", (status, resp)) + after = get_setting(client, key) + check(after == want, f"GET {key} after {method} expected {want}", after) + if method == "DELETE": + restore_delete = False + finally: + if restore_delete: + try: + status, resp = client.request("DELETE", f"/admin/runtime-settings/{key}") + log(f"restore: DELETE {key} -> {status} {json.dumps(resp)}") + except Exception as exc: # do not mask the original failure + print(f"{LOG_PREFIX} restore DELETE {key} failed: {exc}", file=sys.stderr) + log(f"PASS settings: {key} roundtrip from source={source!r} left state unchanged") + + +def contract_roundtrip(client: AdminClient) -> None: + address = TEST_CONTRACT_ADDRESS + path = f"/admin/contract-info/{address}" + + # pre-clean: heals a previously crashed run; deleted:false is the norm + status, resp = client.request("DELETE", path) + check(status == 200 and isinstance(resp, dict), "pre-clean DELETE expected 200", (status, resp)) + if resp.get("deleted"): + log(f"pre-clean removed a leftover test record: {json.dumps(resp)}") + + # list pagination + status, resp = client.request("GET", "/admin/contract-info/?limit=1") + check( + status == 200 and isinstance(resp, dict) and isinstance(resp.get("contracts"), list), + "GET contract list expected 200 with contracts array", + (status, resp), + ) + check(len(resp["contracts"]) <= 1, "contract list must honor limit=1", len(resp["contracts"])) + if resp.get("next"): + status, page2 = client.request("GET", f"/admin/contract-info/?limit=1&from={resp['next']}") + check( + status == 200 and isinstance(page2, dict) and isinstance(page2.get("contracts"), list), + "GET contract list next page expected 200", + (status, page2), + ) + status, resp = client.request("GET", "/admin/contract-info/?limit=0") + check(status == 400, "GET contract list with limit=0 expected 400", (status, resp)) + + # negative writes + status, resp = client.request("POST", path, []) + check(status == 400, "POST to an address path expected 400", (status, resp)) + status, resp = client.request("PATCH", "/admin/contract-info/") + check(status == 400, "PATCH expected 400", (status, resp)) + + # create -> read -> list from cursor -> delete -> delete again + record = { + "contract": address, + "name": "TestToken-e2e", + "symbol": "TSTE2E", + "decimals": 18, + "standard": "ERC20", + "type": "ERC20", + } + # raised before the POST is attempted: the server may commit even when the + # response is lost, and a spurious cleanup DELETE of a missing row is a + # harmless no-op (same reasoning as the settings restore flag) + created = False + try: + created = True + status, resp = client.request("POST", "/admin/contract-info/", [record]) + check(status == 200 and resp == {"updated": 1}, "POST test contract expected {'updated': 1}", (status, resp)) + + status, resp = client.request("GET", path) + check(status == 200 and isinstance(resp, dict), "GET test contract expected 200", (status, resp)) + got = {k: resp.get(k) for k in ("name", "symbol", "decimals", "standard")} + want = {k: record[k] for k in ("name", "symbol", "decimals", "standard")} + check(got == want, f"GET test contract expected {want}", got) + check( + str(resp.get("contract", "")).lower() == address.lower(), + "GET test contract address echo", + resp.get("contract"), + ) + + status, resp = client.request("GET", f"/admin/contract-info/?limit=1&from={address}") + check( + status == 200 + and isinstance(resp, dict) + and len(resp.get("contracts", [])) == 1 + and str(resp["contracts"][0].get("contract", "")).lower() == address.lower(), + "list from= must start with the test record", + (status, resp), + ) + + status, resp = client.request("DELETE", path) + check( + status == 200 + and isinstance(resp, dict) + and resp.get("deleted") is True + and resp.get("purged", {}).get("name") == record["name"], + "DELETE must report deleted:true with the purged record", + (status, resp), + ) + created = False + status, resp = client.request("DELETE", path) + check( + status == 200 and isinstance(resp, dict) and resp.get("deleted") is False and "purged" not in resp, + "repeated DELETE must report deleted:false", + (status, resp), + ) + finally: + if created: # a failure after the POST must not leave the record behind + try: + status, resp = client.request("DELETE", path) + log(f"cleanup: DELETE test contract -> {status} {json.dumps(resp)}") + except Exception as exc: # do not mask the original failure + print(f"{LOG_PREFIX} cleanup DELETE {address} failed: {exc}", file=sys.stderr) + log("PASS contracts: list/POST/GET/DELETE roundtrip left no record behind") + + +def main() -> None: + coin = os.environ.get("BB_ADMIN_E2E_COIN", "base_archive") + if not coin_deployed(os.environ.get("DEPLOYED_COINS", ""), coin): + skip(f"{coin} is not part of this deploy run", "notice") + + user = os.environ.get("BB_ADMIN_USER", "").strip() + password = os.environ.get("BB_ADMIN_PASSWORD", "").strip() + if not (user and password): + blob = os.environ.get("BB_ENV", "") + if not blob.strip(): + # a warning, not a notice: in CI the secret is expected to exist, + # so this skip usually means a lost/renamed BB_RUNTIME_ENV + skip("BB_RUNTIME_ENV secret not set", "warning") + user, password = parse_env_credentials(blob) + if not (user and password): + fail("BB_ENV does not contain BB_ADMIN_USER and BB_ADMIN_PASSWORD") + + base_url = resolve_admin_url(coin) + ssl_context, tls_mode = build_ssl_context() + log(f"TLS certificate verification: {tls_mode}") + timeout = int(os.environ.get("ADMIN_E2E_REQUEST_TIMEOUT_SECONDS", "20")) + client = AdminClient(base_url, user, password, timeout, ssl_context) + log(f"Running admin API e2e tests against {base_url} ({coin})") + + check_auth_gating(client) + settings = check_settings_list(client) + check_settings_negative(client, settings) + for key in SETTING_KEYS: + settings_roundtrip(client, key, settings[key]) + contract_roundtrip(client) + log("All admin API e2e tests passed; instance state is unchanged.") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/admin_api_e2e_test.py b/.github/scripts/admin_api_e2e_test.py new file mode 100644 index 0000000000..605c896135 --- /dev/null +++ b/.github/scripts/admin_api_e2e_test.py @@ -0,0 +1,87 @@ +import unittest + +from admin_api_e2e import coin_deployed, parse_env_credentials, plan_settings_roundtrip + + +class ParseEnvCredentialsTest(unittest.TestCase): + def test_plain(self) -> None: + blob = "ETH_ALLOWED_RPC_CALL_TO=0xabc\nBB_ADMIN_USER=admin\nBB_ADMIN_PASSWORD=s3cr3t\n" + self.assertEqual(parse_env_credentials(blob), ("admin", "s3cr3t")) + + def test_whitespace_and_cr(self) -> None: + blob = " BB_ADMIN_USER = admin \r\nBB_ADMIN_PASSWORD=s3cr3t \r\n" + self.assertEqual(parse_env_credentials(blob), ("admin", "s3cr3t")) + + def test_value_containing_equals(self) -> None: + blob = "BB_ADMIN_USER=admin\nBB_ADMIN_PASSWORD=pa=ss=word\n" + self.assertEqual(parse_env_credentials(blob), ("admin", "pa=ss=word")) + + def test_surrounding_quotes_stripped_like_systemd(self) -> None: + # systemd's EnvironmentFile strips one pair of quotes before the value + # reaches the server, so the script must send the same unquoted value + blob = "BB_ADMIN_USER=\"admin\"\nBB_ADMIN_PASSWORD='s3c r3t'\n" + self.assertEqual(parse_env_credentials(blob), ("admin", "s3c r3t")) + + def test_inner_and_unmatched_quotes_kept(self) -> None: + blob = "BB_ADMIN_USER=ad\"min\nBB_ADMIN_PASSWORD=\"s3cr3t\n" + self.assertEqual(parse_env_credentials(blob), ('ad"min', '"s3cr3t')) + + def test_comments_and_missing_keys(self) -> None: + blob = "# BB_ADMIN_USER=commented\nOTHER=x\n" + self.assertEqual(parse_env_credentials(blob), ("", "")) + + +class CoinDeployedTest(unittest.TestCase): + def test_empty_list_means_unrestricted(self) -> None: + self.assertTrue(coin_deployed("", "base_archive")) + self.assertTrue(coin_deployed(" ", "base_archive")) + + def test_membership(self) -> None: + self.assertTrue(coin_deployed("bitcoin, base_archive ,zcash", "base_archive")) + self.assertTrue(coin_deployed("Base_Archive", "base_archive")) + self.assertFalse(coin_deployed("bitcoin,zcash", "base_archive")) + + +class PlanSettingsRoundtripTest(unittest.TestCase): + """The planner must keep the parsed policy identical at every instant and + end in the starting state (except db-sourced settings, which are only + rewritten in place because their env fallback is unknown).""" + + def test_env_sourced(self) -> None: + self.assertEqual( + plan_settings_roundtrip("0xabc,0xdef", "env"), + [ + ("POST", "0xabc,0xdef", "0xabc,0xdef", "db"), + ("DELETE", None, "0xabc,0xdef", "env"), + ], + ) + + def test_unset(self) -> None: + self.assertEqual( + plan_settings_roundtrip("", "unset"), + [("POST", "", "", "db"), ("DELETE", None, "", "unset")], + ) + + def test_db_sourced_is_rewritten_in_place_only(self) -> None: + self.assertEqual( + plan_settings_roundtrip("0xabc", "db"), + [("POST", "0xabc", "0xabc", "db")], + ) + + def test_every_step_preserves_the_effective_value(self) -> None: + for value, source in (("0xabc", "env"), ("", "unset"), ("0xdd62ed3e", "db")): + for _, _, expected_value, _ in plan_settings_roundtrip(value, source): + self.assertEqual(expected_value, value, (value, source)) + + def test_end_state_matches_start_state(self) -> None: + for value, source in (("0xabc", "env"), ("", "unset")): + _, _, final_value, final_source = plan_settings_roundtrip(value, source)[-1] + self.assertEqual((final_value, final_source), (value, source)) + + def test_unknown_source_raises(self) -> None: + with self.assertRaises(ValueError): + plan_settings_roundtrip("x", "bogus") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 97a133685e..c10dc011c0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -284,3 +284,19 @@ jobs: name: openapi-e2e-report path: tests/openapi/reports/ if-no-files-found: warn + + # Idempotent roundtrip of the internal-server admin API (runtime settings + # + contract info) against the deployed dev instance. The target URL is + # derived from the BB_DEV_API_URL_HTTP_ repo variable (exported + # above) plus the coin config's internal port. The script skips itself — + # with a visible annotation — when the target coin is not part of this + # deploy run (DEPLOYED_COINS carries deploy config names, hence + # coins_csv, not test_coins_csv) or the BB_RUNTIME_ENV secret is not + # set; credentials are parsed inside the script from BB_ENV, which is + # intentionally not exported to other steps. + - name: Run admin API e2e tests + env: + BB_ENV: ${{ secrets.BB_RUNTIME_ENV }} + BB_ADMIN_E2E_COIN: base_archive + DEPLOYED_COINS: ${{ needs.prepare_deploy.outputs.coins_csv }} + run: python3 ./.github/scripts/admin_api_e2e.py From 6d7c112eca188eae3954a934b4d19b9fb7690ca0 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 7 Jul 2026 09:06:24 +0000 Subject: [PATCH 065/303] fix(ci): resolve the admin e2e URL variable by test coin name The BB_DEV_API_URL_HTTP_ repo variables follow the tests.json test-name convention (BB_DEV_API_URL_HTTP_base for base_archive), so resolving them by the deploy alias failed with "missing BB_DEV_API_URL_HTTP_base_archive". Map the deploy alias through runner.load_test_coin_name (the same helper deploy_plan.py uses) before the lookup; the coin gate keeps comparing deploy aliases. Unit test pins the mapping against the real base_archive.json. Co-Authored-By: Claude Fable 5 --- .github/scripts/admin_api_e2e.py | 17 ++++++++++------ .github/scripts/admin_api_e2e_test.py | 28 +++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/scripts/admin_api_e2e.py b/.github/scripts/admin_api_e2e.py index a2221a4fcc..e851dbc475 100644 --- a/.github/scripts/admin_api_e2e.py +++ b/.github/scripts/admin_api_e2e.py @@ -39,7 +39,7 @@ import urllib.request from pathlib import Path -from runner import LOG_PREFIX, fail, log +from runner import LOG_PREFIX, ValidationError, fail, load_test_coin_name, log from wait_for_sync import build_ssl_context, resolve_http_base REPO_ROOT = Path(__file__).resolve().parents[2] @@ -88,15 +88,20 @@ def coin_deployed(deployed_csv: str, coin: str) -> bool: def resolve_admin_url(coin: str) -> str: """Resolve the internal-server base URL: an explicit BB_ADMIN_E2E_URL - wins; otherwise the host follows the same BB_DEV_API_URL_HTTP_ repo - variable every other post-deploy check uses, with the internal port from - the coin config. The internal server always serves HTTPS (the packaged - self-signed certificate).""" + wins; otherwise the host follows the same repo variable every other + post-deploy check uses — keyed by the coin's *test name* (tests.json + convention, e.g. BB_DEV_API_URL_HTTP_base for base_archive), not the + deploy alias — with the internal port from the coin config. The internal + server always serves HTTPS (the packaged self-signed certificate).""" explicit = os.environ.get("BB_ADMIN_E2E_URL", "").strip() if explicit: return explicit.rstrip("/") - host = urllib.parse.urlparse(resolve_http_base(coin)).hostname config_path = REPO_ROOT / "configs" / "coins" / (coin + ".json") + try: + test_coin = load_test_coin_name(config_path) + except ValidationError as exc: + fail(str(exc)) + host = urllib.parse.urlparse(resolve_http_base(test_coin)).hostname try: with open(config_path, encoding="utf-8") as f: port = json.load(f)["ports"]["blockbook_internal"] diff --git a/.github/scripts/admin_api_e2e_test.py b/.github/scripts/admin_api_e2e_test.py index 605c896135..3ecaf9b2ce 100644 --- a/.github/scripts/admin_api_e2e_test.py +++ b/.github/scripts/admin_api_e2e_test.py @@ -1,6 +1,30 @@ +import os import unittest - -from admin_api_e2e import coin_deployed, parse_env_credentials, plan_settings_roundtrip +from unittest import mock + +from admin_api_e2e import ( + coin_deployed, + parse_env_credentials, + plan_settings_roundtrip, + resolve_admin_url, +) + + +class ResolveAdminUrlTest(unittest.TestCase): + def test_explicit_url_wins(self) -> None: + with mock.patch.dict(os.environ, {"BB_ADMIN_E2E_URL": "https://example.test:9999/"}, clear=False): + self.assertEqual(resolve_admin_url("base_archive"), "https://example.test:9999") + + def test_derives_from_test_name_variable_and_config_port(self) -> None: + # the URL repo variables follow the tests.json test-name convention + # (BB_DEV_API_URL_HTTP_base for base_archive, not ..._base_archive); + # the port comes from the real coin config's ports.blockbook_internal + env = { + "BB_ADMIN_E2E_URL": "", + "BB_DEV_API_URL_HTTP_base": "https://blockbook-dev.test:9311", + } + with mock.patch.dict(os.environ, env, clear=False): + self.assertEqual(resolve_admin_url("base_archive"), "https://blockbook-dev.test:9211") class ParseEnvCredentialsTest(unittest.TestCase): From f9a86772b10513bb8f2299c75e06f48754eab4a7 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 8 Jul 2026 20:50:06 +0000 Subject: [PATCH 066/303] fix(db): fail loudly on undecodable key in ListContractInfos The paginated contract listing discarded the error from GetAddressesFromAddrDesc and fell through to return next="" for a boundary row whose key did not decode to an address. That silent truncation is indistinguishable from a completed listing and drops the remaining rows; in-page rows with no decoded address were likewise returned with an empty Contract. A corrupt key now returns an error, matching the unpackContractInfo error handling in the same loop. Unreachable for well-formed EVM/Tron contract descriptors, but a visible failure beats a confusing partial page. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/rocksdb_contracts.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/db/rocksdb_contracts.go b/db/rocksdb_contracts.go index 483ff4bd00..ce30ebadb8 100644 --- a/db/rocksdb_contracts.go +++ b/db/rocksdb_contracts.go @@ -215,21 +215,28 @@ func (d *RocksDB) ListContractInfos(from string, limit int) ([]bchain.ContractIn } contracts := make([]bchain.ContractInfo, 0, limit) for ; it.Valid(); it.Next() { - addresses, _, _ := d.chainParser.GetAddressesFromAddrDesc(it.Key().Data()) + // The key is the contract's address descriptor. A key that fails to + // decode to an address is a corrupt row: fail loudly (as the + // unpackContractInfo error below does) rather than fall through to + // return next="", which a caller cannot tell apart from a completed + // listing and which would silently drop the boundary row's cursor — + // or return an in-page record with an empty Contract. + addresses, _, err := d.chainParser.GetAddressesFromAddrDesc(it.Key().Data()) + if err != nil { + return nil, "", err + } + if len(addresses) == 0 { + return nil, "", errors.Errorf("no address for contract descriptor %x", it.Key().Data()) + } if len(contracts) == limit { // one more row exists — its address is the cursor of the next page - if len(addresses) > 0 { - return contracts, addresses[0], nil - } - return contracts, "", nil + return contracts, addresses[0], nil } contractInfo, err := unpackContractInfo(it.Value().Data()) if err != nil { return nil, "", err } - if len(addresses) > 0 { - contractInfo.Contract = addresses[0] - } + contractInfo.Contract = addresses[0] contracts = append(contracts, *contractInfo) } return contracts, "", nil From 3665ff7f281f20fde50fa26dd849b3373bd6540b Mon Sep 17 00:00:00 2001 From: Jakub Jerabek <116381722+cranycrane@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:03:05 +0200 Subject: [PATCH 067/303] Tron testnet backend update 4 8 2 (#1611) * chore(tron-testnet): upgrade backend to 4.8.2 * fix(tron-testnet): path to the .jar backend file * fix(tron-testnet): update backend version --- configs/coins/tron_testnet_nile.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/coins/tron_testnet_nile.json b/configs/coins/tron_testnet_nile.json index 35739d1a3e..25de3b16a9 100644 --- a/configs/coins/tron_testnet_nile.json +++ b/configs/coins/tron_testnet_nile.json @@ -22,13 +22,13 @@ "package_name": "backend-tron-testnet-nile", "package_revision": "latest", "system_user": "tron", - "version": "4.8.1-build4", - "binary_url": "https://github.com/tron-nile-testnet/nile-testnet/releases/download/GreatVoyage-Nile-v4.8.1-build4/FullNode-Nile-x64-4.8.1-build4.jar", + "version": "4.8.2-pq1-build1", + "binary_url": "https://github.com/tron-nile-testnet/nile-testnet/releases/download/GreatVoyage-Nile-v4.8.2-PQ1-build1/FullNode-Nile-x64-4.8.2-pq1-build1.jar", "verification_type": "sha256", - "verification_source": "fe71b6ea77d4506ff53cb34a8d8c0d8230bace2d4ab1c79bc32e4458c6531878", + "verification_source": "c013cbfdccc17bf57c300b637b6d2e24a725cab671760a6f63f7af7b7b16c0a9", "extract_command": "mv ${ARCHIVE} backend/ && wget -q https://raw.githubusercontent.com/tron-nile-testnet/nile-testnet/3f6f53ca3b1f113dabd7bcba346d66b1fb71b710/framework/src/main/resources/config-nile.conf -O test_net_config.conf && sha256sum test_net_config.conf | grep -q '^296455a150477bc0f80a4125a57809cafcd843c271f1d7973ee051576a827b84' && sed -i 's/^[ \\t]*#*[ \\t]*httpFullNodeEnable.*/httpFullNodeEnable = true/' test_net_config.conf && sed -i 's/^[ \\t]*#*[ \\t]*httpFullNodePort.*/httpFullNodePort = 8545/' test_net_config.conf && sed -i '/triggerName *= *\"block\"/{n;s/enable *= *.*/enable = true/;}' test_net_config.conf && sed -i 's/^[ \\t]*supportConstant[ \\t]*=[ \\t]*false/supportConstant = true/' test_net_config.conf && mv test_net_config.conf backend/ && echo ", "exclude_files": [], - "exec_command_template": "/usr/bin/java -Xms9G -Xmx16G -XX:ReservedCodeCacheSize=256m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/gc.log -XX:+UseConcMarkSweepGC -XX:NewRatio=2 -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -jar {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/FullNode-Nile-x64-4.8.1-build4.jar --es --output-directory {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/output-directory -c {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/test_net_config.conf", + "exec_command_template": "/usr/bin/java -Xms9G -Xmx16G -XX:ReservedCodeCacheSize=256m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/gc.log -XX:+UseConcMarkSweepGC -XX:NewRatio=2 -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -jar {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/FullNode-Nile-x64-4.8.2-pq1-build1.jar --es --output-directory {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/output-directory -c {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/test_net_config.conf", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", "postinst_script_template": "mkdir -p {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/logs && chown {{.Backend.SystemUser}}:{{.Backend.SystemUser}} {{.Env.BackendInstallPath}}/{{.Coin.Alias}}/logs", "service_type": "simple", From 302d08af9902422c8f6c45e006f5231edc3cb1fa Mon Sep 17 00:00:00 2001 From: Jakub Jerabek <116381722+cranycrane@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:33:25 +0200 Subject: [PATCH 068/303] 1581 add upper cap limits to chosen websocket endpoints to increase resilliency (#1582) * feat(fiat-api): add cap limits for fiat rate endpoints * feat(evm-contract): verify contract logs to avoid possible OOM * feat(websocket): add lower concurrency cap on getMempoolFilters * fix(fiat-api): apply currency count cap to raw selector list length The MaxFiatRatesCurrencies check fired only when a unique normalized code was appended, so an all-duplicate list bypassed the cap while still costing a full trim/lowercase/dedup pass per element. Reject over-long lists up front so the normalization work itself is bounded by the cap; this also makes the in-loop unique-count check unreachable, so drop it. Co-Authored-By: Claude Fable 5 * refactor(fiat-api): normalize currency selectors once at entry points setFiatRateToBalanceHistories re-normalized a slice that both of its callers (GetBalanceHistory, GetXpubBalanceHistory) had already passed through normalizeFiatCurrencies, and getFiatRatesResult re-lowercased selectors already lowercased by its callers. Drop the redundant passes and document the pre-normalized precondition instead. Normalization was the only error path of setFiatRateToBalanceHistories, so it no longer returns an error. Co-Authored-By: Claude Fable 5 * refactor(evm-contract): drop unreachable ERC-1155 count overflow guard The endValues <= len(data) check already bounds countValues to len(data)/evmWordHex, far below maxInt, so the truncation guard on the int conversion could never fire (the load-bearing overflow check lives in erc1155BatchArrayEnd). Co-Authored-By: Claude Fable 5 * docs(websocket): clarify scope of getMempoolFilters response cap The slot is deliberately held from before the handler computes the response until it is written to the websocket (or drained on close): the cap exists to bound peak memory in computed-but-unwritten filter responses, so it covers compute, queueing, and write together rather than compute concurrency alone. Co-Authored-By: Claude Fable 5 * fix(eth-parsing): skip unparseable log with a warning and continue with next logs * test(websocket): cover mempool filter slot release after successful write Adds TestWebsocketOutputLoopReleasesMempoolFilterSlotAfterWrite, which drives a release-bearing WsRes through outputLoop over a live websocket connection and asserts the mempool-filters slot is freed after WriteJSON. This pins the primary release path (outputLoop's c.finalize(m)) that the existing semaphore and drain-on-close tests do not exercise; without it, removing that call would leak a slot on every successful response and go undetected. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Fable 5 --- api/fiat_balance_history.go | 20 ++--- api/fiat_balance_history_test.go | 25 ++---- api/fiat_rates_api.go | 54 +++++++++--- api/fiat_rates_api_test.go | 122 +++++++++++++++++++++++++-- api/worker.go | 11 +-- api/xpub.go | 10 ++- bchain/coins/eth/contract.go | 134 +++++++++++++++++++++++------- bchain/coins/eth/contract_test.go | 79 ++++++++++++++++-- bchain/coins/eth/ethparser.go | 8 +- server/websocket.go | 99 +++++++++++++++++----- server/websocket_test.go | 99 ++++++++++++++++++++++ server/ws_types.go | 2 + 12 files changed, 535 insertions(+), 128 deletions(-) diff --git a/api/fiat_balance_history.go b/api/fiat_balance_history.go index 0c6f3698ba..8c5afdfb7d 100644 --- a/api/fiat_balance_history.go +++ b/api/fiat_balance_history.go @@ -1,7 +1,6 @@ package api import ( - "strings" "time" "github.com/golang/glog" @@ -15,14 +14,6 @@ func normalizeBalanceHistoryPathLabel(pathLabel string) string { return pathLabel } -func normalizeCurrenciesToLowercase(currencies []string) []string { - currenciesLowercase := make([]string, len(currencies)) - for i := range currencies { - currenciesLowercase[i] = strings.ToLower(currencies[i]) - } - return currenciesLowercase -} - func buildBalanceHistoryTimestamps(histories BalanceHistories) []int64 { timestamps := make([]int64, len(histories)) for i := range histories { @@ -175,20 +166,21 @@ func (w *Worker) applyFallbackTickersToBalanceHistories(histories BalanceHistori w.observeBalanceHistoryFiatDuration(pathLabel, "fallback", stats.status(), fallbackStarted) } -func (w *Worker) setFiatRateToBalanceHistories(histories BalanceHistories, currencies []string, pathLabel string) error { +// setFiatRateToBalanceHistories fills FiatRates of the passed histories. +// currenciesLowercase must already be normalized by normalizeFiatCurrencies; +// both entry points (GetBalanceHistory, GetXpubBalanceHistory) do so. +func (w *Worker) setFiatRateToBalanceHistories(histories BalanceHistories, currenciesLowercase []string, pathLabel string) { if len(histories) == 0 || w.fiatRates == nil || !w.fiatRates.Enabled { - return nil + return } pathLabel = normalizeBalanceHistoryPathLabel(pathLabel) - currenciesLowercase := normalizeCurrenciesToLowercase(currencies) timestamps := buildBalanceHistoryTimestamps(histories) tickers, batchFetchValid, reason, returnedTickers, err := w.lookupBalanceHistoryBatchTickers(timestamps, pathLabel, len(histories)) if batchFetchValid { applyBatchTickersToBalanceHistories(histories, tickers, currenciesLowercase) - return nil + return } glog.Errorf("Error finding tickers for %d timestamps (returned %d, reason %s). Error: %v", len(timestamps), returnedTickers, reason, err) w.incrementBalanceHistoryFiatFallback(pathLabel, reason) w.applyFallbackTickersToBalanceHistories(histories, currenciesLowercase, pathLabel) - return nil } diff --git a/api/fiat_balance_history_test.go b/api/fiat_balance_history_test.go index e165b4345c..fc9b68f155 100644 --- a/api/fiat_balance_history_test.go +++ b/api/fiat_balance_history_test.go @@ -37,10 +37,7 @@ func TestSetFiatRateToBalanceHistories_BatchesTickerLookup(t *testing.T) { return &tickers, nil } - err := w.setFiatRateToBalanceHistories(histories, []string{"USD", "eur", "cad"}, "address") - if err != nil { - t.Fatalf("setFiatRateToBalanceHistories returned error: %v", err) - } + w.setFiatRateToBalanceHistories(histories, []string{"usd", "eur", "cad"}, "address") if calls != 1 { t.Fatalf("expected 1 ticker lookup call, got %d", calls) } @@ -77,10 +74,7 @@ func TestSetFiatRateToBalanceHistories_AllRatesWhenCurrenciesNotSpecified(t *tes return &tickers, nil } - err := w.setFiatRateToBalanceHistories(histories, nil, "address") - if err != nil { - t.Fatalf("setFiatRateToBalanceHistories returned error: %v", err) - } + w.setFiatRateToBalanceHistories(histories, nil, "address") if !reflect.DeepEqual(histories[0].FiatRates, map[string]float32{"usd": 11, "eur": 22}) { t.Fatalf("unexpected rates for histories[0]: %v", histories[0].FiatRates) } @@ -127,10 +121,7 @@ func TestSetFiatRateToBalanceHistories_BatchFailureFallsBackToPerPoint(t *testin } } - err := w.setFiatRateToBalanceHistories(histories, []string{"usd"}, "address") - if err != nil { - t.Fatalf("setFiatRateToBalanceHistories returned error: %v", err) - } + w.setFiatRateToBalanceHistories(histories, []string{"usd"}, "address") if calls != 4 { t.Fatalf("expected 4 ticker lookup calls (1 batch + 3 point), got %d", calls) } @@ -170,10 +161,7 @@ func TestSetFiatRateToBalanceHistories_SkipsLookupForEmptyHistory(t *testing.T) return &tickers, nil } - err := w.setFiatRateToBalanceHistories(BalanceHistories{}, []string{"usd"}, "address") - if err != nil { - t.Fatalf("setFiatRateToBalanceHistories returned error: %v", err) - } + w.setFiatRateToBalanceHistories(BalanceHistories{}, []string{"usd"}, "address") if calls != 0 { t.Fatalf("expected 0 ticker lookup calls, got %d", calls) } @@ -196,10 +184,7 @@ func TestSetFiatRateToBalanceHistories_SkipsLookupWhenFiatRatesDisabled(t *testi return &tickers, nil } - err := w.setFiatRateToBalanceHistories(histories, []string{"usd"}, "address") - if err != nil { - t.Fatalf("setFiatRateToBalanceHistories returned error: %v", err) - } + w.setFiatRateToBalanceHistories(histories, []string{"usd"}, "address") if calls != 0 { t.Fatalf("expected 0 ticker lookup calls when fiat rates are disabled, got %d", calls) } diff --git a/api/fiat_rates_api.go b/api/fiat_rates_api.go index ba44dd3f74..c561cf9775 100644 --- a/api/fiat_rates_api.go +++ b/api/fiat_rates_api.go @@ -13,15 +13,40 @@ import ( // MaxFiatRatesTimestamps limits batch fiat-rate lookups to bounded request work. const MaxFiatRatesTimestamps = 1000 -// removeEmpty removes empty strings from a slice. -func removeEmpty(stringSlice []string) []string { - ret := make([]string, 0, len(stringSlice)) - for _, str := range stringSlice { - if str != "" { - ret = append(ret, str) +// MaxFiatRatesCurrencies limits explicit currency selectors. An empty list still +// means "all available currencies", so legitimate callers do not need a large +// explicit list. +const MaxFiatRatesCurrencies = 128 + +// MaxFiatRatesCurrencyCodeLength bounds each selector before it is copied into +// per-result response maps. +const MaxFiatRatesCurrencyCodeLength = 16 + +// normalizeFiatCurrencies trims, lowercases, deduplicates, and bounds explicit +// fiat currency selectors before any per-result response maps are allocated. +// The count limit applies to the raw input length so the work done here is +// bounded by it as well. +func normalizeFiatCurrencies(currencies []string) ([]string, error) { + if len(currencies) > MaxFiatRatesCurrencies { + return nil, NewAPIError(fmt.Sprintf("too many currencies, max %d", MaxFiatRatesCurrencies), true) + } + seen := make(map[string]struct{}, len(currencies)) + normalized := make([]string, 0, len(currencies)) + for _, currency := range currencies { + currency = strings.ToLower(strings.TrimSpace(currency)) + if currency == "" { + continue } + if len(currency) > MaxFiatRatesCurrencyCodeLength { + return nil, NewAPIError(fmt.Sprintf("currency code too long, max %d", MaxFiatRatesCurrencyCodeLength), true) + } + if _, found := seen[currency]; found { + continue + } + seen[currency] = struct{}{} + normalized = append(normalized, currency) } - return ret + return normalized, nil } func copyTickerRates(rates map[string]float32) map[string]float32 { @@ -33,6 +58,7 @@ func copyTickerRates(rates map[string]float32) map[string]float32 { } // getFiatRatesResult checks if CurrencyRatesTicker contains all necessary data and returns formatted result. +// currencies must already be normalized by normalizeFiatCurrencies. func (w *Worker) getFiatRatesResult(currencies []string, ticker *common.CurrencyRatesTicker, token string) (*FiatTicker, error) { if token != "" { capacity := len(currencies) @@ -51,7 +77,6 @@ func (w *Worker) getFiatRatesResult(currencies []string, ticker *common.Currency } } else { for _, currency := range currencies { - currency = strings.ToLower(currency) rate := ticker.TokenRateInCurrency(token, currency) if rate <= 0 { rate = -1 @@ -74,7 +99,6 @@ func (w *Worker) getFiatRatesResult(currencies []string, ticker *common.Currency // Check if currencies from the list are available in the ticker rates. rates := make(map[string]float32, len(currencies)) for _, currency := range currencies { - currency = strings.ToLower(currency) if rate, found := ticker.Rates[currency]; found { rates[currency] = rate } else { @@ -90,12 +114,15 @@ func (w *Worker) getFiatRatesResult(currencies []string, ticker *common.Currency // GetCurrentFiatRates returns last available fiat rates. func (w *Worker) GetCurrentFiatRates(currencies []string, token string) (*FiatTicker, error) { vsCurrency := "" - currencies = removeEmpty(currencies) + var err error + currencies, err = normalizeFiatCurrencies(currencies) + if err != nil { + return nil, err + } if len(currencies) == 1 { vsCurrency = currencies[0] } ticker := getCurrentTicker(w.fiatRates, vsCurrency, token) - var err error if ticker == nil { if token == "" { // fallback - get last fiat rate from db if not in current ticker @@ -135,7 +162,10 @@ func (w *Worker) GetFiatRatesForTimestamps(timestamps []int64, currencies []stri return nil, NewAPIError(fmt.Sprintf("too many timestamps, max %d", MaxFiatRatesTimestamps), true) } vsCurrency := "" - currencies = removeEmpty(currencies) + currencies, err := normalizeFiatCurrencies(currencies) + if err != nil { + return nil, err + } if len(currencies) == 1 { vsCurrency = currencies[0] } diff --git a/api/fiat_rates_api_test.go b/api/fiat_rates_api_test.go index 90df2b00e5..5f2edf38f7 100644 --- a/api/fiat_rates_api_test.go +++ b/api/fiat_rates_api_test.go @@ -28,11 +28,65 @@ func requireAPIError(t *testing.T, err error, wantPublic bool) *APIError { return apiErr } -func TestRemoveEmpty(t *testing.T) { - got := removeEmpty([]string{"usd", "", "eur", "", ""}) +func TestNormalizeFiatCurrencies(t *testing.T) { + got, err := normalizeFiatCurrencies([]string{" usd ", "", "USD", "eur", " Eur "}) + if err != nil { + t.Fatalf("normalizeFiatCurrencies returned error: %v", err) + } want := []string{"usd", "eur"} if !reflect.DeepEqual(got, want) { - t.Fatalf("unexpected filtered currencies: got %v, want %v", got, want) + t.Fatalf("unexpected normalized currencies: got %v, want %v", got, want) + } +} + +func TestNormalizeFiatCurrencies_LimitsCurrencyCount(t *testing.T) { + currencies := make([]string, MaxFiatRatesCurrencies+1) + for i := range currencies { + currencies[i] = fmt.Sprintf("c%03d", i) + } + _, err := normalizeFiatCurrencies(currencies) + apiErr := requireAPIError(t, err, true) + want := fmt.Sprintf("too many currencies, max %d", MaxFiatRatesCurrencies) + if apiErr.Text != want { + t.Fatalf("unexpected error text: got %q, want %q", apiErr.Text, want) + } +} + +func TestNormalizeFiatCurrencies_DeduplicatesWithinLimit(t *testing.T) { + currencies := make([]string, MaxFiatRatesCurrencies) + for i := range currencies { + currencies[i] = "USD" + } + got, err := normalizeFiatCurrencies(currencies) + if err != nil { + t.Fatalf("normalizeFiatCurrencies returned error: %v", err) + } + if !reflect.DeepEqual(got, []string{"usd"}) { + t.Fatalf("unexpected normalized currencies: got %v", got) + } +} + +func TestNormalizeFiatCurrencies_LimitAppliesToRawInputLength(t *testing.T) { + // The count cap bounds the raw input length, so even a list that would + // deduplicate to a single entry is rejected when it is too long. + currencies := make([]string, MaxFiatRatesCurrencies+1) + for i := range currencies { + currencies[i] = "USD" + } + _, err := normalizeFiatCurrencies(currencies) + apiErr := requireAPIError(t, err, true) + want := fmt.Sprintf("too many currencies, max %d", MaxFiatRatesCurrencies) + if apiErr.Text != want { + t.Fatalf("unexpected error text: got %q, want %q", apiErr.Text, want) + } +} + +func TestNormalizeFiatCurrencies_RejectsLongCode(t *testing.T) { + _, err := normalizeFiatCurrencies([]string{strings.Repeat("a", MaxFiatRatesCurrencyCodeLength+1)}) + apiErr := requireAPIError(t, err, true) + want := fmt.Sprintf("currency code too long, max %d", MaxFiatRatesCurrencyCodeLength) + if apiErr.Text != want { + t.Fatalf("unexpected error text: got %q, want %q", apiErr.Text, want) } } @@ -57,7 +111,7 @@ func TestGetFiatRatesResult_NonTokenSelectedCurrencies(t *testing.T) { }, } - got, err := w.getFiatRatesResult([]string{"USD", "gbp"}, ticker, "") + got, err := w.getFiatRatesResult([]string{"usd", "gbp"}, ticker, "") if err != nil { t.Fatalf("getFiatRatesResult returned error: %v", err) } @@ -111,7 +165,7 @@ func TestGetFiatRatesResult_TokenRates(t *testing.T) { }, } - got, err := w.getFiatRatesResult([]string{"USD", "EUR", "JPY"}, ticker, "0xToken") + got, err := w.getFiatRatesResult([]string{"usd", "eur", "jpy"}, ticker, "0xToken") if err != nil { t.Fatalf("getFiatRatesResult returned error: %v", err) } @@ -153,8 +207,8 @@ func TestGetCurrentFiatRates_UsesGetterAndCurrencyFilter(t *testing.T) { if calls != 1 { t.Fatalf("expected one ticker call, got %d", calls) } - if gotVsCurrency != "USD" { - t.Fatalf("unexpected vsCurrency: got %q, want %q", gotVsCurrency, "USD") + if gotVsCurrency != "usd" { + t.Fatalf("unexpected vsCurrency: got %q, want %q", gotVsCurrency, "usd") } if gotToken != "" { t.Fatalf("unexpected token: got %q, want empty", gotToken) @@ -203,6 +257,60 @@ func TestGetFiatRatesForTimestamps_LimitsInput(t *testing.T) { } } +func TestGetFiatRatesForTimestamps_LimitsCurrenciesBeforeLookup(t *testing.T) { + w := &Worker{fiatRates: &fiat.FiatRates{}} + originalGetter := getTickersForTimestamps + defer func() { + getTickersForTimestamps = originalGetter + }() + + calls := 0 + getTickersForTimestamps = func(_ *fiat.FiatRates, _ []int64, _, _ string) (*[]*common.CurrencyRatesTicker, error) { + calls++ + tickers := []*common.CurrencyRatesTicker{} + return &tickers, nil + } + + currencies := make([]string, MaxFiatRatesCurrencies+1) + for i := range currencies { + currencies[i] = fmt.Sprintf("c%03d", i) + } + _, err := w.GetFiatRatesForTimestamps([]int64{1}, currencies, "") + apiErr := requireAPIError(t, err, true) + want := fmt.Sprintf("too many currencies, max %d", MaxFiatRatesCurrencies) + if apiErr.Text != want { + t.Fatalf("unexpected error text: got %q, want %q", apiErr.Text, want) + } + if calls != 0 { + t.Fatalf("expected ticker lookup not to be called, got %d calls", calls) + } +} + +func TestGetFiatRatesForTimestamps_RejectsLongCurrencyBeforeLookup(t *testing.T) { + w := &Worker{fiatRates: &fiat.FiatRates{}} + originalGetter := getTickersForTimestamps + defer func() { + getTickersForTimestamps = originalGetter + }() + + calls := 0 + getTickersForTimestamps = func(_ *fiat.FiatRates, _ []int64, _, _ string) (*[]*common.CurrencyRatesTicker, error) { + calls++ + tickers := []*common.CurrencyRatesTicker{} + return &tickers, nil + } + + _, err := w.GetFiatRatesForTimestamps([]int64{1}, []string{strings.Repeat("a", MaxFiatRatesCurrencyCodeLength+1)}, "") + apiErr := requireAPIError(t, err, true) + want := fmt.Sprintf("currency code too long, max %d", MaxFiatRatesCurrencyCodeLength) + if apiErr.Text != want { + t.Fatalf("unexpected error text: got %q, want %q", apiErr.Text, want) + } + if calls != 0 { + t.Fatalf("expected ticker lookup not to be called, got %d calls", calls) + } +} + func TestGetFiatRatesForTimestamps_LenMismatchReturnsNonPublicError(t *testing.T) { w := &Worker{fiatRates: &fiat.FiatRates{}} originalGetter := getTickersForTimestamps diff --git a/api/worker.go b/api/worker.go index 2403a3db8c..d5e841d751 100644 --- a/api/worker.go +++ b/api/worker.go @@ -2044,7 +2044,11 @@ const ( // the caller supplies the transport-specific cap (WS vs REST). transport labels // the emitted metrics with the serving surface. func (w *Worker) GetBalanceHistory(address string, fromTimestamp, toTimestamp int64, currencies []string, groupBy uint32, maxTxs int, transport string) (BalanceHistories, error) { - currencies = removeEmpty(currencies) + var err error + currencies, err = normalizeFiatCurrencies(currencies) + if err != nil { + return nil, err + } bhs := make(BalanceHistories, 0) start := time.Now() addrDesc, _, err := w.getAddrDescAndNormalizeAddress(address) @@ -2101,10 +2105,7 @@ func (w *Worker) GetBalanceHistory(address string, fromTimestamp, toTimestamp in if w.metrics != nil { w.metrics.BalanceHistoryPoints.With(common.Labels{"path": "address"}).Observe(float64(len(bha))) } - err = w.setFiatRateToBalanceHistories(bha, currencies, "address") - if err != nil { - return nil, err - } + w.setFiatRateToBalanceHistories(bha, currencies, "address") glog.V(1).Info("GetBalanceHistory ", address, ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), ", ", time.Since(start)) return bha, nil } diff --git a/api/xpub.go b/api/xpub.go index b8dfb8bbb1..49d82d6a51 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -774,6 +774,11 @@ func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, e // transport-specific cap (WS vs REST). transport labels the emitted metrics with // the serving surface. func (w *Worker) GetXpubBalanceHistory(xpub string, fromTimestamp, toTimestamp int64, currencies []string, gap int, groupBy uint32, maxTxs int, transport string) (BalanceHistories, error) { + var err error + currencies, err = normalizeFiatCurrencies(currencies) + if err != nil { + return nil, err + } bhs := make(BalanceHistories, 0) start := time.Now() fromUnix, fromHeight, toUnix, toHeight := w.balanceHistoryHeightsFromTo(fromTimestamp, toTimestamp) @@ -865,10 +870,7 @@ func (w *Worker) GetXpubBalanceHistory(xpub string, fromTimestamp, toTimestamp i if w.metrics != nil { w.metrics.BalanceHistoryPoints.With(common.Labels{"path": "xpub"}).Observe(float64(len(bha))) } - err = w.setFiatRateToBalanceHistories(bha, currencies, "xpub") - if err != nil { - return nil, err - } + w.setFiatRateToBalanceHistories(bha, currencies, "xpub") glog.V(1).Info("GetUtxoBalanceHistory ", xpub[:xpubLogPrefix], ", cache ", inCache, ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), ", ", time.Since(start)) return bha, nil } diff --git a/bchain/coins/eth/contract.go b/bchain/coins/eth/contract.go index 401f1d0813..ae2e76d52c 100644 --- a/bchain/coins/eth/contract.go +++ b/bchain/coins/eth/contract.go @@ -33,6 +33,11 @@ const contractBalanceOfSignature = "0x70a08231" const ENSRegistryAddress = "0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e" // ENSRegistryAddress is the mainnet ENS registry contract address +const ( + evmWordBytes = 32 + evmWordHex = evmWordBytes * 2 +) + func addressFromPaddedHex(s string) (string, error) { var t big.Int var ok bool @@ -131,6 +136,44 @@ func processERC1155TransferSingleEvent(l *bchain.RpcLog) (transfer *bchain.Token }, nil } +func parseEVMLogWordUint64(data string, offset int) (uint64, error) { + if offset < 0 || offset > len(data) || len(data)-offset < evmWordHex { + return 0, errors.New("ERC1155 TransferBatch, invalid data length") + } + var b big.Int + _, ok := b.SetString(data[offset:offset+evmWordHex], 16) + if !ok || !b.IsUint64() { + return 0, errors.New("ERC1155 TransferBatch, not a number") + } + return b.Uint64(), nil +} + +func erc1155BatchOffsetHex(offsetBytes uint64) (int, error) { + if offsetBytes < 2*evmWordBytes { + return 0, errors.New("ERC1155 TransferBatch, invalid offset") + } + if offsetBytes%evmWordBytes != 0 { + return 0, errors.New("ERC1155 TransferBatch, invalid offset") + } + maxInt := uint64(^uint(0) >> 1) + if offsetBytes > maxInt/2 { + return 0, errors.New("ERC1155 TransferBatch, invalid offset") + } + return int(offsetBytes * 2), nil +} + +func erc1155BatchArrayEnd(offset int, count uint64) (int, error) { + if offset < 0 { + return 0, errors.New("ERC1155 TransferBatch, invalid offset") + } + maxInt := uint64(^uint(0) >> 1) + base := uint64(offset) + evmWordHex + if base > maxInt || count > (maxInt-base)/evmWordHex { + return 0, errors.New("ERC1155 TransferBatch, invalid data length") + } + return int(base + count*evmWordHex), nil +} + func processERC1155TransferBatchEvent(l *bchain.RpcLog) (transfer *bchain.TokenTransfer, err error) { defer func() { if r := recover(); r != nil { @@ -154,40 +197,63 @@ func processERC1155TransferBatchEvent(l *bchain.RpcLog) (transfer *bchain.TokenT if has0xPrefix(l.Data) { data = data[2:] } - var b big.Int - _, ok := b.SetString(data[:64], 16) - if !ok || !b.IsInt64() { - return nil, errors.New("ERC1155 TransferBatch, not a number") - } - offsetIds := int(b.Int64()) * 2 - _, ok = b.SetString(data[64:128], 16) - if !ok || !b.IsInt64() { - return nil, errors.New("ERC1155 TransferBatch, not a number") - } - offsetValues := int(b.Int64()) * 2 - _, ok = b.SetString(data[offsetIds:offsetIds+64], 16) - if !ok || !b.IsInt64() { - return nil, errors.New("ERC1155 TransferBatch, not a number") - } - countIds := int(b.Int64()) - _, ok = b.SetString(data[offsetValues:offsetValues+64], 16) - if !ok || !b.IsInt64() { - return nil, errors.New("ERC1155 TransferBatch, not a number") - } - countValues := int(b.Int64()) + if len(data) < 2*evmWordHex || len(data)%evmWordHex != 0 { + return nil, errors.New("ERC1155 TransferBatch, invalid data length") + } + offsetIdsBytes, err := parseEVMLogWordUint64(data, 0) + if err != nil { + return nil, err + } + offsetIds, err := erc1155BatchOffsetHex(offsetIdsBytes) + if err != nil { + return nil, err + } + offsetValuesBytes, err := parseEVMLogWordUint64(data, evmWordHex) + if err != nil { + return nil, err + } + offsetValues, err := erc1155BatchOffsetHex(offsetValuesBytes) + if err != nil { + return nil, err + } + countIds, err := parseEVMLogWordUint64(data, offsetIds) + if err != nil { + return nil, err + } + countValues, err := parseEVMLogWordUint64(data, offsetValues) + if err != nil { + return nil, err + } if countIds != countValues { return nil, errors.New("ERC1155 TransferBatch, count values and ids does not match") } - idValues := make([]bchain.MultiTokenValue, countValues) - for i := 0; i < countValues; i++ { + endIds, err := erc1155BatchArrayEnd(offsetIds, countValues) + if err != nil { + return nil, err + } + if endIds > len(data) { + return nil, errors.New("ERC1155 TransferBatch, invalid ids data length") + } + endValues, err := erc1155BatchArrayEnd(offsetValues, countValues) + if err != nil { + return nil, err + } + if endValues > len(data) { + return nil, errors.New("ERC1155 TransferBatch, invalid values data length") + } + // countValues cannot truncate: the endValues <= len(data) check above bounds + // it to len(data)/evmWordHex. + count := int(countValues) + idValues := make([]bchain.MultiTokenValue, count) + for i := 0; i < count; i++ { var id, value big.Int - o := offsetIds + 64 + 64*i - _, ok := id.SetString(data[o:o+64], 16) + o := offsetIds + evmWordHex + evmWordHex*i + _, ok := id.SetString(data[o:o+evmWordHex], 16) if !ok { return nil, errors.New("ERC1155 log Data id is not a number") } - o = offsetValues + 64 + 64*i - _, ok = value.SetString(data[o:o+64], 16) + o = offsetValues + evmWordHex + evmWordHex*i + _, ok = value.SetString(data[o:o+evmWordHex], 16) if !ok { return nil, errors.New("ERC1155 log Data value is not a number") } @@ -202,14 +268,17 @@ func processERC1155TransferBatchEvent(l *bchain.RpcLog) (transfer *bchain.TokenT }, nil } -func contractGetTransfersFromLog(logs []*bchain.RpcLog) (bchain.TokenTransfers, error) { +// contractGetTransfersFromLog extracts token transfers from receipt logs. +// An unparseable log is skipped with a warning so that one malformed event +// does not discard the valid transfers of the transaction. +func contractGetTransfersFromLog(logs []*bchain.RpcLog, txid string) bchain.TokenTransfers { var r bchain.TokenTransfers - var tt *bchain.TokenTransfer - var err error for _, l := range logs { tl := len(l.Topics) if tl > 0 { signature := l.Topics[0] + var tt *bchain.TokenTransfer + var err error if signature == tokenTransferEventSignature { tt, err = processTransferEvent(l) } else if signature == tokenERC1155TransferSingleEventSignature { @@ -220,14 +289,15 @@ func contractGetTransfersFromLog(logs []*bchain.RpcLog) (bchain.TokenTransfers, continue } if err != nil { - return nil, err + glog.Warningf("contractGetTransfersFromLog: skipping unparseable log of contract %s, tx %s: %v", l.Address, txid, err) + continue } if tt != nil { r = append(r, tt) } } } - return r, nil + return r } func contractGetTransfersFromTx(tx *bchain.RpcTransaction) (bchain.TokenTransfers, error) { diff --git a/bchain/coins/eth/contract_test.go b/bchain/coins/eth/contract_test.go index ca70c85878..c70c4f6607 100644 --- a/bchain/coins/eth/contract_test.go +++ b/bchain/coins/eth/contract_test.go @@ -5,6 +5,7 @@ package eth import ( "fmt" "math/big" + "runtime" "strings" "testing" @@ -14,10 +15,9 @@ import ( func Test_contractGetTransfersFromLog(t *testing.T) { tests := []struct { - name string - args []*bchain.RpcLog - want bchain.TokenTransfers - wantErr bool + name string + args []*bchain.RpcLog + want bchain.TokenTransfers }{ { name: "ERC20 transfer 1", @@ -208,11 +208,7 @@ func Test_contractGetTransfersFromLog(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := contractGetTransfersFromLog(tt.args) - if (err != nil) != tt.wantErr { - t.Errorf("contractGetTransfersFromLog error = %v, wantErr %v", err, tt.wantErr) - return - } + got := contractGetTransfersFromLog(tt.args, "0xtxid") if len(got) != len(tt.want) { t.Errorf("contractGetTransfersFromLog len not same, %+v, want %+v", got, tt.want) } @@ -227,6 +223,71 @@ func Test_contractGetTransfersFromLog(t *testing.T) { } } +func erc1155BatchWord(n uint64) string { + return fmt.Sprintf("%064x", n) +} + +func malformedERC1155TransferBatchLog(count uint64) *bchain.RpcLog { + return &bchain.RpcLog{ + Address: "0x1111111111111111111111111111111111111111", + Topics: []string{ + tokenERC1155TransferBatchEventSignature, + "0x0000000000000000000000002222222222222222222222222222222222222222", + "0x0000000000000000000000003333333333333333333333333333333333333333", + "0x0000000000000000000000004444444444444444444444444444444444444444", + }, + Data: "0x" + + erc1155BatchWord(0x40) + + erc1155BatchWord(0x60) + + erc1155BatchWord(count) + + erc1155BatchWord(count), + } +} + +func Test_processERC1155TransferBatchEventRejectsMalformedLogBeforeAllocation(t *testing.T) { + const count = 200000 + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + transfer, err := processERC1155TransferBatchEvent(malformedERC1155TransferBatchLog(count)) + runtime.ReadMemStats(&after) + if err == nil { + t.Fatal("processERC1155TransferBatchEvent returned nil error for malformed ERC1155 TransferBatch") + } + if transfer != nil { + t.Fatalf("processERC1155TransferBatchEvent transfer = %+v, want nil", transfer) + } + if strings.Contains(err.Error(), "recovered from panic") { + t.Fatalf("processERC1155TransferBatchEvent recovered from panic instead of validating input: %v", err) + } + if delta := after.TotalAlloc - before.TotalAlloc; delta > 1_000_000 { + t.Fatalf("processERC1155TransferBatchEvent allocated %d bytes for malformed ERC1155 TransferBatch count %d", delta, count) + } +} + +func Test_contractGetTransfersFromLogSkipsMalformedERC1155TransferBatch(t *testing.T) { + logs := []*bchain.RpcLog{ + malformedERC1155TransferBatchLog(200000), + { // valid ERC20 Transfer in the same receipt + Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96", + Topics: []string{ + tokenTransferEventSignature, + "0x0000000000000000000000002aacf811ac1a60081ea39f7783c0d26c500871a8", + "0x000000000000000000000000e9a5216ff992cfa01594d43501a56e12769eb9d2", + }, + Data: "0x0000000000000000000000000000000000000000000000000000000000000123", + }, + } + transfers := contractGetTransfersFromLog(logs, "0xtxid") + if len(transfers) != 1 { + t.Fatalf("contractGetTransfersFromLog transfers = %+v, want the valid transfer kept", transfers) + } + // the address could have different case + if strings.ToLower(transfers[0].Contract) != "0x76a45e8976499ab9ae223cc584019341d5a84e96" { + t.Fatalf("contractGetTransfersFromLog kept transfer of contract %s, want the valid ERC20 transfer", transfers[0].Contract) + } +} + func Test_contractGetTransfersFromTx(t *testing.T) { p := NewEthereumParser(1, false) b1 := dbtestdata.GetTestEthereumTypeBlock1(p) diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index 27e8ef740c..b32dd1ad2d 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -571,12 +571,12 @@ func (p *EthereumParser) EthereumTypeGetTokenTransfersFromTx(tx *bchain.Tx) (bch csd, ok := tx.CoinSpecificData.(bchain.EthereumSpecificData) if ok { if csd.Receipt != nil { - r, err = contractGetTransfersFromLog(csd.Receipt.Logs) + r = contractGetTransfersFromLog(csd.Receipt.Logs, tx.Txid) } else { r, err = contractGetTransfersFromTx(csd.Tx) - } - if err != nil { - return nil, err + if err != nil { + return nil, err + } } } return r, nil diff --git a/server/websocket.go b/server/websocket.go index 5f0b628743..857c047c0c 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -32,6 +32,16 @@ const defaultTimeout = 60 * time.Second const unknownMethodLabel = "unknown" const maxWebsocketMessageBytes int64 = 4 * 1024 * 1024 const maxWebsocketPendingRequests = 48 + +// maxWebsocketMempoolFiltersResponses caps per-connection getMempoolFilters +// requests over their whole lifecycle: the slot is acquired before the handler +// computes the (potentially large) response and released only after the +// response is written to the websocket (or drained on close). The point is to +// bound the peak memory held in computed-but-unwritten filter responses, so the +// cap deliberately covers compute, queueing, and write together; a client that +// pipelines more than this many requests, or reads slower than it requests, +// gets a mempool_filters_limit error instead of queueing further responses. +const maxWebsocketMempoolFiltersResponses = 4 const maxWebsocketActiveRequests = 2048 const maxWebsocketEstimateFeeBlocks = 32 const maxWebsocketSubscribeAddresses = 1000 @@ -59,6 +69,7 @@ type websocketChannel struct { conn *websocket.Conn out chan *WsRes pendingRequests chan struct{} + mempoolFiltersSlots chan struct{} // semaphore capping in-flight getMempoolFilters responses, see maxWebsocketMempoolFiltersResponses ip string ipKey string blockKey string @@ -333,16 +344,17 @@ func (s *WebsocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } conn.SetReadLimit(maxWebsocketMessageBytes) c := &websocketChannel{ - id: atomic.AddUint64(&connectionCounter, 1), - conn: conn, - out: make(chan *WsRes, outChannelSize), - pendingRequests: make(chan struct{}, maxWebsocketPendingRequests), - ip: ip, - ipKey: ipKey, - blockKey: bKey, - blockable: blockable, - requestHeader: r.Header, - alive: true, + id: atomic.AddUint64(&connectionCounter, 1), + conn: conn, + out: make(chan *WsRes, outChannelSize), + pendingRequests: make(chan struct{}, maxWebsocketPendingRequests), + mempoolFiltersSlots: make(chan struct{}, maxWebsocketMempoolFiltersResponses), + ip: ip, + ipKey: ipKey, + blockKey: bKey, + blockable: blockable, + requestHeader: r.Header, + alive: true, } if s.messageRateLimit > 0 { c.messageRate = newConnMessageRate(s.messageRateWindow) @@ -479,7 +491,7 @@ func (c *websocketChannel) CloseOut(reason string) (bool, string) { //clean out close(c.out) for len(c.out) > 0 { - <-c.out + c.finalize(<-c.out) } return true, closeReason } @@ -491,17 +503,22 @@ func (c *websocketChannel) DataOut(data *WsRes) { defer c.aliveLock.Unlock() if c.alive { if len(c.out) < outChannelSize-1 { + // Enqueued: ownership passes to the out pipeline, which finalizes it + // once written (outputLoop) or drained (CloseOut). c.out <- data - } else { - glog.Warning("Channel ", c.id, " overflow, closing") - if c.closeReason == "" { - c.closeReason = "overflow" - } - // close the connection but do not call CloseOut - would call duplicate c.aliveLock.Lock - // CloseOut will be called because the closed connection will cause break in the inputLoop - c.conn.Close() + return + } + glog.Warning("Channel ", c.id, " overflow, closing") + if c.closeReason == "" { + c.closeReason = "overflow" } + // close the connection but do not call CloseOut - would call duplicate c.aliveLock.Lock + // CloseOut will be called because the closed connection will cause break in the inputLoop + c.conn.Close() } + // Not enqueued (overflow or dead connection): the response never reaches the + // out pipeline, so release any slot it held here. + c.finalize(data) } func (c *websocketChannel) acquireRequestSlot() bool { @@ -517,6 +534,27 @@ func (c *websocketChannel) releaseRequestSlot() { <-c.pendingRequests } +func (c *websocketChannel) acquireMempoolFiltersSlot() bool { + select { + case c.mempoolFiltersSlots <- struct{}{}: + return true + default: + return false + } +} + +func (c *websocketChannel) releaseMempoolFiltersSlot() { + <-c.mempoolFiltersSlots +} + +// finalize releases any resources a response held, exactly once +func (c *websocketChannel) finalize(res *WsRes) { + if res != nil && res.release != nil { + res.release() + res.release = nil + } +} + func (s *WebsocketServer) inputLoop(c *websocketChannel) { defer func() { if r := recover(); r != nil { @@ -596,6 +634,7 @@ func (s *WebsocketServer) outputLoop(c *websocketChannel) { for m := range c.out { c.conn.SetWriteDeadline(time.Now().Add(defaultTimeout)) err := c.conn.WriteJSON(m) + c.finalize(m) if err != nil { glog.Error("Error sending message to ", c.id, ", ", err) s.closeChannel(c, "write_error") @@ -838,6 +877,8 @@ var requestHandlers = map[string]func(*WebsocketServer, *websocketChannel, *WsRe func (s *WebsocketServer) onRequest(c *websocketChannel, req *WsReq) { var err error var data interface{} + // release is non-nil while this request holds a rate-capped endpoint slot. + var release func() f, ok := requestHandlers[req.Method] methodLabel := req.Method if !ok { @@ -854,9 +895,14 @@ func (s *WebsocketServer) onRequest(c *websocketChannel, req *WsReq) { // nil data means no response if data != nil { c.DataOut(&WsRes{ - ID: req.ID, - Data: data, + ID: req.ID, + Data: data, + release: release, }) + release = nil // ownership handed to the response + } + if release != nil { + release() // no response was produced — free the slot now } s.metrics.WebsocketPendingRequests.With(common.Labels{"method": methodLabel}).Dec() }() @@ -866,6 +912,17 @@ func (s *WebsocketServer) onRequest(c *websocketChannel, req *WsReq) { s.metrics.WebsocketReqDuration.With(common.Labels{"method": methodLabel}).Observe(float64(time.Since(t)) / 1e3) // in microseconds }() if ok { + if req.Method == "getMempoolFilters" { + if !c.acquireMempoolFiltersSlot() { + e := resultError{} + e.Error.Message = "mempool_filters_limit" + data = e + s.metrics.WebsocketRequests.With(common.Labels{"method": methodLabel, "status": "failure"}).Inc() + glog.Warning("Client ", c.id, " exceeded getMempoolFilters response limit, ", c.ip) + return + } + release = c.releaseMempoolFiltersSlot + } data, err = f(s, c, req) if err == nil { glog.V(1).Info("Client ", c.id, " onRequest ", req.Method, " success") diff --git a/server/websocket_test.go b/server/websocket_test.go index f29ab9a390..cf345114c8 100644 --- a/server/websocket_test.go +++ b/server/websocket_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "net/http" + "net/http/httptest" "net/netip" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( "testing" "time" + "github.com/gorilla/websocket" "github.com/trezor/blockbook/api" "github.com/trezor/blockbook/bchain" "github.com/trezor/blockbook/bchain/coins/eth" @@ -1522,6 +1524,103 @@ func TestWebsocketTrackWorkAppliesGlobalLimit(t *testing.T) { } } +func TestWebsocketMempoolFilterResponseSlots(t *testing.T) { + c := &websocketChannel{ + mempoolFiltersSlots: make(chan struct{}, maxWebsocketMempoolFiltersResponses), + } + for i := 0; i < maxWebsocketMempoolFiltersResponses; i++ { + if !c.acquireMempoolFiltersSlot() { + t.Fatalf("acquireMempoolFiltersSlot() = false at slot %d", i) + } + } + if c.acquireMempoolFiltersSlot() { + t.Fatal("acquireMempoolFiltersSlot() = true at limit") + } + c.releaseMempoolFiltersSlot() + if !c.acquireMempoolFiltersSlot() { + t.Fatal("acquireMempoolFiltersSlot() = false after release") + } +} + +func TestWebsocketCloseOutReleasesQueuedMempoolFilterResponses(t *testing.T) { + c := &websocketChannel{ + out: make(chan *WsRes, outChannelSize), + mempoolFiltersSlots: make(chan struct{}, maxWebsocketMempoolFiltersResponses), + alive: true, + } + if !c.acquireMempoolFiltersSlot() { + t.Fatal("acquireMempoolFiltersSlot() = false") + } + c.DataOut(&WsRes{ID: "mempool", Data: struct{}{}, release: c.releaseMempoolFiltersSlot}) + if got := len(c.mempoolFiltersSlots); got != 1 { + t.Fatalf("held mempool filter slots = %d before CloseOut, want 1", got) + } + if closed, reason := c.CloseOut("test"); !closed || reason != "test" { + t.Fatalf("CloseOut() = %v, %q, want true, test", closed, reason) + } + if got := len(c.mempoolFiltersSlots); got != 0 { + t.Fatalf("held mempool filter slots = %d after CloseOut, want 0", got) + } +} + +// TestWebsocketOutputLoopReleasesMempoolFilterSlotAfterWrite exercises the +// primary slot-release path for a live connection: outputLoop's c.finalize(m) +// after a successful WriteJSON. Neither TestWebsocketMempoolFilterResponseSlots +// (raw semaphore) nor TestWebsocketCloseOutReleasesQueuedMempoolFilterResponses +// (drain-on-close) touch this line, so without this test a removal of the +// finalize call would leak a slot on every successful response and go undetected. +func TestWebsocketOutputLoopReleasesMempoolFilterSlotAfterWrite(t *testing.T) { + // Stand up a real websocket connection pair so outputLoop's WriteJSON and + // finalize run against a live *websocket.Conn. + upgrader := websocket.Upgrader{} + serverConnCh := make(chan *websocket.Conn, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade: %v", err) + return + } + serverConnCh <- conn + })) + defer ts.Close() + + clientConn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(ts.URL, "http"), nil) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer clientConn.Close() + serverConn := <-serverConnCh + defer serverConn.Close() + + c := &websocketChannel{ + conn: serverConn, + out: make(chan *WsRes, outChannelSize), + mempoolFiltersSlots: make(chan struct{}, maxWebsocketMempoolFiltersResponses), + alive: true, + } + if !c.acquireMempoolFiltersSlot() { + t.Fatal("acquireMempoolFiltersSlot() = false") + } + c.out <- &WsRes{ID: "mempool", Data: struct{}{}, release: c.releaseMempoolFiltersSlot} + close(c.out) // outputLoop returns once the single queued response is written + + s := &WebsocketServer{} + done := make(chan struct{}) + go func() { + s.outputLoop(c) + close(done) + }() + + if _, _, err := clientConn.ReadMessage(); err != nil { + t.Fatalf("client read: %v", err) + } + <-done + + if got := len(c.mempoolFiltersSlots); got != 0 { + t.Fatalf("held mempool filter slots = %d after successful write, want 0", got) + } +} + func TestWebsocketShutdownIsIdempotent(t *testing.T) { s := newShutdownTestServer() ctx, cancel := context.WithTimeout(context.Background(), time.Second) diff --git a/server/ws_types.go b/server/ws_types.go index 4792af7dcf..5c5767aca7 100644 --- a/server/ws_types.go +++ b/server/ws_types.go @@ -17,6 +17,8 @@ type WsReq struct { type WsRes struct { ID string `json:"id" ts_doc:"Corresponding request identifier."` Data interface{} `json:"data" ts_doc:"Payload of the response, structure depends on the request."` + // release, if non-nil, is invoked exactly once when this response leaves the out pipeline + release func() } type resultError struct { From 57bb05d555e65da914ad265764f0e9a2435727bf Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 9 Jul 2026 11:47:23 +0000 Subject: [PATCH 069/303] fix(eth): prevent slice-bounds panic from ENS log name-length overflow getEnsRecord read an attacker-controlled 32-byte word from an ENS NameRegistered log and computed a slice end as 194+64+(int(c)<<1). For c in [0x7FFFFFFFFFFFFF7F, 0x7FFFFFFFFFFFFFFF] the signed left shift wraps and de lands in [0, 256], slipping the `de < 0` guard, so l.Data[194+64:de] became a low>high slice expression and panicked. The panic runs in the GetBlock log-processing goroutine, which has no recover(), so it crashes the whole process before the block is committed -> deterministic crash-loop on restart. - getEnsRecord: guard the slice start (de < 194+64) instead of de < 0. - processParam: same fix (de < d) for the sibling data[d:de] slice (already under ParseInputData's recover(); correctness only). - GetBlock: recover() the log goroutine as defense-in-depth, returning an error instead of letting the process die (adds runtime/debug). - tests: regression cases for both overflow paths (de=0, de=256, and the processParam variant), verified to panic without the fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- bchain/coins/eth/dataparser.go | 14 ++++++- bchain/coins/eth/dataparser_test.go | 58 +++++++++++++++++++++++++++++ bchain/coins/eth/ethrpc.go | 13 +++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/dataparser.go b/bchain/coins/eth/dataparser.go index b8b0b525ab..89c0c0d277 100644 --- a/bchain/coins/eth/dataparser.go +++ b/bchain/coins/eth/dataparser.go @@ -169,8 +169,12 @@ func processParam(data string, index int, dataOffset int, t *abi.Type, processed dynIndex++ if t.T == abi.StringTy || t.T == abi.BytesTy { d += 64 + // count<<1 can overflow and wrap below d (same class as the + // getEnsRecord bug); the lower bound must be the slice start d, not 0, + // so that d <= de <= len(data) and data[d:de] below can never panic. + // count==0 gives de==d, which is still accepted. de := d + (count << 1) - if de > len(data) || de < 0 { + if de > len(data) || de < d { return nil, 0, false } if count == 0 { @@ -310,8 +314,14 @@ func getEnsRecord(l *rpcLogWithTxHash) *bchain.AddressAliasRecord { if err != nil { return nil } + // int(c)<<1 can overflow: c is attacker-controlled (up to 2^63-1) and Go's + // signed left shift wraps silently, so de can land anywhere in [0, 257] + // below the name start offset (194+64). The lower bound must be the slice + // start index, not 0 — a de below the start makes l.Data[194+64:de] a + // low>high slice expression that panics. Checking de < 194+64 guarantees + // 194+64 <= de <= len(l.Data), so the slice below can never panic. de := 194 + 64 + (int(c) << 1) - if de > len(l.Data) || de < 0 { + if de > len(l.Data) || de < 194+64 { return nil } b, err := hex.DecodeString(l.Data[194+64 : de]) diff --git a/bchain/coins/eth/dataparser_test.go b/bchain/coins/eth/dataparser_test.go index 12a3147aa9..05ce4a038a 100644 --- a/bchain/coins/eth/dataparser_test.go +++ b/bchain/coins/eth/dataparser_test.go @@ -457,6 +457,27 @@ func TestParseInputDataMisalignedDynamicData(t *testing.T) { } } +func TestParseInputDataOverflowDynamicLength(t *testing.T) { + parser := NewEthereumParser(1, false) + signatures := []bchain.FourByteSignature{ + {Name: "store", Parameters: []string{"bytes"}}, + } + // A "bytes" argument whose declared length word is 0x7FFFFFFFFFFFFFC0. + // In processParam count<<1 overflows int64 and wraps to -128, so + // de = d + (count<<1) = 128 - 128 = 0 while d = 128. Before the fix the old + // `de < 0` guard did not catch de==0 and data[128:0] panicked (recovered by + // ParseInputData, which then returned nil); with the `de < d` guard the field + // is rejected cleanly and ParseInputData returns the method id only. + data := "0xaabbccdd" + + "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000007fffffffffffffc0" + got := parser.ParseInputData(&signatures, data) + want := &bchain.EthereumParsedInputData{MethodId: "0xaabbccdd"} + if !reflect.DeepEqual(got, want) { + t.Errorf("ParseInputData() = %#v, want %#v", got, want) + } +} + func Test_getEnsRecord(t *testing.T) { tests := []struct { name string @@ -523,6 +544,43 @@ func Test_getEnsRecord(t *testing.T) { }, want: nil, }, + { + // Regression: c = 0x7FFFFFFFFFFFFF7F makes int(c)<<1 wrap to -258, so + // de = 194+64+(-258) = 0. Before the fix this slipped past the old + // `de < 0` guard (0 is not < 0) and l.Data[258:0] panicked, crashing the + // sync goroutine. Must be rejected cleanly. + name: "overflow name length wrapping to de=0", + log: rpcLogWithTxHash{ + RpcLog: bchain.RpcLog{ + Address: "0x283Af0B28c62C092C9727F1Ee09c02CA627EB7F5", + Topics: []string{ + "0xca6abbe9d7f11422cb6ca7629fbf6fe9efb1c621f71ce8f02b9f2a230097404f", + "0x40ce2aa8cd9ee9fef4bf3a68abab7fbcceb6bac89370518caf6a602cefe836bd", + "0x0000000000000000000000002c630b16aa53ae0189880e15c23323688acb607c", + }, + Data: "0x00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000017629245f5a86f0000000000000000000000000000000000000000000000000000000069dbb21d0000000000000000000000000000000000000000000000007fffffffffffff7f756e726176656c65640000000000000000000000000000000000000000000000", + }, + }, + want: nil, + }, + { + // Regression: c = 0x7FFFFFFFFFFFFFFF (MaxInt64) makes int(c)<<1 wrap to + // -2, so de = 256. Before the fix this passed both old guards and + // l.Data[258:256] panicked (start > end). Must be rejected cleanly. + name: "overflow name length wrapping to de=256", + log: rpcLogWithTxHash{ + RpcLog: bchain.RpcLog{ + Address: "0x283Af0B28c62C092C9727F1Ee09c02CA627EB7F5", + Topics: []string{ + "0xca6abbe9d7f11422cb6ca7629fbf6fe9efb1c621f71ce8f02b9f2a230097404f", + "0x40ce2aa8cd9ee9fef4bf3a68abab7fbcceb6bac89370518caf6a602cefe836bd", + "0x0000000000000000000000002c630b16aa53ae0189880e15c23323688acb607c", + }, + Data: "0x00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000017629245f5a86f0000000000000000000000000000000000000000000000000000000069dbb21d0000000000000000000000000000000000000000000000007fffffffffffffff756e726176656c65640000000000000000000000000000000000000000000000", + }, + }, + want: nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 32c46a8b63..45e7a6b2af 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -10,6 +10,7 @@ import ( "math/big" "net/http" "net/url" + "runtime/debug" "strconv" "strings" "sync" @@ -1626,6 +1627,18 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error logsCh := make(chan logsResult, 1) // Buffered so send won't block if we return early. internalCh := make(chan internalResult, 1) // Buffered to avoid goroutine leak on early return. go func() { + // Defense-in-depth: processEventsForBlock/getEnsRecord parse attacker-controlled + // on-chain log data. This goroutine has no other recover() on its stack, so an + // unrecovered panic here would terminate the whole process and — because the + // block is not yet committed — crash-loop on restart. Recover into an error so + // block sync surfaces/handles it instead of the process dying. + defer func() { + if r := recover(); r != nil { + glog.Error("GetBlock: recovered from panic in processEventsForBlock: ", r) + debug.PrintStack() + logsCh <- logsResult{err: fmt.Errorf("recovered from panic in processEventsForBlock: %v", r)} + } + }() logs, ens, err := b.processEventsForBlock(head.Number) logsCh <- logsResult{logs: logs, ens: ens, err: err} // Send result without shared state. }() From 6da7c565de23c63efe61afa1d945644f7c957a5a Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 06:50:09 +0000 Subject: [PATCH 070/303] fix(eth): guard offset-shift overflow in processParam, constify nameStart in getEnsRecord Address two review comments: - Add d+64 < d overflow guard at line 159 in processParam, same class as the existing de < d fix. - Replace magic 194+64 with a local const nameStart in getEnsRecord. --- bchain/coins/eth/dataparser.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bchain/coins/eth/dataparser.go b/bchain/coins/eth/dataparser.go index 89c0c0d277..92d0614651 100644 --- a/bchain/coins/eth/dataparser.go +++ b/bchain/coins/eth/dataparser.go @@ -156,7 +156,7 @@ func processParam(data string, index int, dataOffset int, t *abi.Type, processed offset <<= 1 d = int(offset) + dataOffset dynIndex := d >> 6 - if d+64 > len(data) || d < 0 { + if d+64 > len(data) || d < 0 || d+64 < d { return nil, 0, false } // get element count of dynamic type @@ -314,17 +314,18 @@ func getEnsRecord(l *rpcLogWithTxHash) *bchain.AddressAliasRecord { if err != nil { return nil } + const nameStart = 194 + 64 // int(c)<<1 can overflow: c is attacker-controlled (up to 2^63-1) and Go's // signed left shift wraps silently, so de can land anywhere in [0, 257] - // below the name start offset (194+64). The lower bound must be the slice - // start index, not 0 — a de below the start makes l.Data[194+64:de] a - // low>high slice expression that panics. Checking de < 194+64 guarantees - // 194+64 <= de <= len(l.Data), so the slice below can never panic. - de := 194 + 64 + (int(c) << 1) - if de > len(l.Data) || de < 194+64 { + // below nameStart. The lower bound must be the slice + // start index, not 0 — a de below the start makes l.Data[nameStart:de] a + // low>high slice expression that panics. Checking de < nameStart guarantees + // nameStart <= de <= len(l.Data), so the slice below can never panic. + de := nameStart + (int(c) << 1) + if de > len(l.Data) || de < nameStart { return nil } - b, err := hex.DecodeString(l.Data[194+64 : de]) + b, err := hex.DecodeString(l.Data[nameStart:de]) if err != nil { return nil } From 1236a84bac6f744f22b6861069ce148ac274a466 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 9 Jul 2026 12:17:05 +0000 Subject: [PATCH 071/303] test(fiat): raise concurrency-test drain timeout to avoid CI flake TestGetTickersForTimestamps_ConcurrentReadersAndWriters and TestLogTickersInfo_ConcurrentWithCacheWriters wait `waitTimeout` for their reader/writer/logger goroutines to drain after `stop` is closed. The deadline was 3s. That is only a deadlock safety-net, but it is too tight on a CPU-starved CI runner (the Unit Tests job compiles RocksDB in parallel), so `wg.Wait()` occasionally misses it and the test fails with "concurrent ... did not finish in time". Raise it to 30s: still fails fast on a genuine deadlock, but tolerant of scheduling delays under load. Test-only change; the concurrency exercise (driven by testDuration) is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- fiat/fiat_rates_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fiat/fiat_rates_test.go b/fiat/fiat_rates_test.go index f7643607b7..989de2729f 100644 --- a/fiat/fiat_rates_test.go +++ b/fiat/fiat_rates_test.go @@ -427,7 +427,7 @@ func TestGetTickersForTimestamps_ConcurrentReadersAndWriters(t *testing.T) { writers = 2 readers = 8 testDuration = 1200 * time.Millisecond - waitTimeout = 3 * time.Second + waitTimeout = 30 * time.Second // deadlock safety-net; kept generous so a CPU-starved CI runner does not flake ) stop := make(chan struct{}) @@ -581,7 +581,7 @@ func TestLogTickersInfo_ConcurrentWithCacheWriters(t *testing.T) { writers = 2 loggers = 4 testDuration = 300 * time.Millisecond - waitTimeout = 3 * time.Second + waitTimeout = 30 * time.Second // deadlock safety-net; kept generous so a CPU-starved CI runner does not flake ) stop := make(chan struct{}) From 084b1cc7d874b2356763e6c78ddadd0496cc90bd Mon Sep 17 00:00:00 2001 From: Or Aharonee <17099688+Aharonee@users.noreply.github.com> Date: Tue, 30 Jun 2026 18:18:50 +0300 Subject: [PATCH 072/303] perf(api): resolve tx inputs without unpacking whole prev-tx records GetTransactionFromBchainTx resolved each input's previous output by calling GetTxAddresses, which unpacks an entire TxAddresses record (all inputs and outputs) just to read a single output. For transactions with many inputs that reference large previous transactions this allocates on the order of inputs x record size, driving heap usage up sharply on high-fan-in addresses. Add db.GetTxAddressesOutput, which returns a single Outputs[vout] by skipping over the height, inputs and preceding outputs by length without allocating them, and use it in the vin-resolution loop. Allocation now scales with the number of inputs rather than inputs x record size. Adds TestGetTxAddressesOutput (both index modes) and BenchmarkTxInputResolution covering the input fan-in scaling. --- api/worker.go | 24 +++++----- api/worker_test.go | 109 +++++++++++++++++++++++++++++++++++++++++++++ db/rocksdb.go | 87 ++++++++++++++++++++++++++++++++++++ db/rocksdb_test.go | 79 ++++++++++++++++++++++++++++++++ 4 files changed, 286 insertions(+), 13 deletions(-) diff --git a/api/worker.go b/api/worker.go index d5e841d751..2d9bc38c83 100644 --- a/api/worker.go +++ b/api/worker.go @@ -361,12 +361,13 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height int, spe if w.chainType == bchain.ChainBitcoinType { // bchainVin.Txid=="" is coinbase transaction if bchainVin.Txid != "" { - // load spending addresses from TxAddresses - tas, err := w.db.GetTxAddresses(bchainVin.Txid) + // load the spending address from TxAddresses; only the spent output + // is needed, so avoid unpacking the whole previous-tx record + output, err := w.db.GetTxAddressesOutput(bchainVin.Txid, vin.Vout) if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) + return nil, errors.Annotatef(err, "GetTxAddressesOutput %v", bchainVin.Txid) } - if tas == nil { + if output == nil { // try to load from backend otx, _, err := w.txCache.GetTransaction(bchainVin.Txid) if err != nil { @@ -397,16 +398,13 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height int, spe aggregateAddresses(addresses, vin.Addresses, vin.IsAddress) } } else { - if len(tas.Outputs) > int(vin.Vout) { - output := &tas.Outputs[vin.Vout] - vin.ValueSat = (*Amount)(&output.ValueSat) - vin.AddrDesc = output.AddrDesc - vin.Addresses, vin.IsAddress, err = output.Addresses(w.chainParser) - if err != nil { - glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) - } - aggregateAddresses(addresses, vin.Addresses, vin.IsAddress) + vin.ValueSat = (*Amount)(&output.ValueSat) + vin.AddrDesc = output.AddrDesc + vin.Addresses, vin.IsAddress, err = output.Addresses(w.chainParser) + if err != nil { + glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) } + aggregateAddresses(addresses, vin.Addresses, vin.IsAddress) } if vin.ValueSat != nil { valInSat.Add(&valInSat, (*big.Int)(vin.ValueSat)) diff --git a/api/worker_test.go b/api/worker_test.go index f4a2aa12fb..3dffa79263 100644 --- a/api/worker_test.go +++ b/api/worker_test.go @@ -4,13 +4,20 @@ package api import ( "encoding/json" + "fmt" "math/big" + "os" + "strconv" "testing" "time" + "github.com/stretchr/testify/require" "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/bchain/coins/btc" "github.com/trezor/blockbook/common" + "github.com/trezor/blockbook/db" "github.com/trezor/blockbook/fiat" + "github.com/trezor/blockbook/tests/dbtestdata" ) func TestSystemInfoInSync(t *testing.T) { @@ -304,3 +311,105 @@ func TestTronBalanceHistoryOverrides(t *testing.T) { }) } } + +// BenchmarkTxInputResolution measures GetTransactionFromBchainTx as a function of +// a transaction's input fan-in, using the plain Bitcoin-testnet parser. Each input's +// previous output is resolved from the database, so this tracks how per-transaction +// allocation and time scale with the input count. The fan-in values span a wide range +// to make that scaling visible and to guard against regressions in input resolution. +// +// go test -tags unittest -run x -bench TxInputResolution -benchmem ./api/ +func BenchmarkTxInputResolution(b *testing.B) { + for _, fanIn := range []int{8, 32, 128, 512, 2048} { + b.Run("fanIn="+strconv.Itoa(fanIn), func(b *testing.B) { + worker, subjectTx, cleanup := buildHighFanInWorker(b, fanIn) + defer cleanup() + addresses := make(map[string]struct{}) + + tx, err := worker.GetTransactionFromBchainTx(subjectTx, 2, false, false, addresses) + require.NoError(b, err) + require.Len(b, tx.Vin, fanIn) + require.True(b, tx.Vin[fanIn-1].IsAddress && tx.Vin[fanIn-1].ValueSat != nil) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := worker.GetTransactionFromBchainTx(subjectTx, 2, false, false, addresses); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// buildHighFanInWorker indexes a previous transaction with fanIn outputs and a +// subject transaction whose fanIn inputs each spend one of them (all referencing the +// same previous tx, concentrating resolution on a single large record), and returns +// a Worker plus the subject tx. Only db + parser are needed: GetTransactionFromBchainTx +// for a confirmed Bitcoin-type tx (specificJSON=false) does not touch the chain. +func buildHighFanInWorker(tb testing.TB, fanIn int) (*Worker, *bchain.Tx, func()) { + tb.Helper() + parser := btc.NewBitcoinParser(btc.GetChainParams("test"), &btc.Configuration{BlockAddressesToKeep: 1}) + script := dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, parser) + + prevTxid := fanInHash(1) + prevOuts := make([]bchain.Vout, fanIn) + subjectIns := make([]bchain.Vin, fanIn) + for j := 0; j < fanIn; j++ { + prevOuts[j] = fanInVout(uint32(j), script) + subjectIns[j] = bchain.Vin{Txid: prevTxid, Vout: uint32(j), Sequence: 0xffffffff} + } + prevTx := fanInTx(prevTxid, 1, []bchain.Vin{{Coinbase: "01", Sequence: 0xffffffff}}, prevOuts) + subjectTx := fanInTx(fanInHash(2), 2, subjectIns, []bchain.Vout{fanInVout(0, script), fanInVout(1, script)}) + + blocks := []*bchain.Block{ + {BlockHeader: bchain.BlockHeader{Hash: fanInHash(1_000_001), Height: 1, Time: 1_700_000_001}, Txs: []bchain.Tx{*prevTx}}, + {BlockHeader: bchain.BlockHeader{Hash: fanInHash(1_000_002), Height: 2, Time: 1_700_000_002}, Txs: []bchain.Tx{*subjectTx}}, + } + + tmp, err := os.MkdirTemp("", "vin-resolution") + require.NoError(tb, err) + database, err := db.NewRocksDB(tmp, 100000, -1, parser, nil, false) + require.NoError(tb, err) + cleanup := func() { + require.NoError(tb, database.Close()) + require.NoError(tb, os.RemoveAll(tmp)) + } + + is, err := database.LoadInternalState(&common.Config{CoinName: "coin-unittest"}) + require.NoError(tb, err) + database.SetInternalState(is) + + bulk, err := database.InitBulkConnect() + require.NoError(tb, err) + for i, block := range blocks { + require.NoError(tb, bulk.ConnectBlock(block, i == len(blocks)-1)) + } + require.NoError(tb, bulk.Close()) + is.FinishedSync(uint32(len(blocks))) + + return &Worker{db: database, chainParser: parser, chainType: bchain.ChainBitcoinType, is: is}, subjectTx, cleanup +} + +func fanInTx(txid string, height uint32, vin []bchain.Vin, vout []bchain.Vout) *bchain.Tx { + return &bchain.Tx{ + Txid: txid, + Version: 1, + Vin: vin, + Vout: vout, + BlockHeight: height, + Confirmations: 1, + Time: int64(1_700_000_000 + height), + Blocktime: int64(1_700_000_000 + height), + } +} + +func fanInVout(n uint32, scriptHex string) bchain.Vout { + return bchain.Vout{ + ValueSat: *big.NewInt(100000), + N: n, + ScriptPubKey: bchain.ScriptPubKey{Hex: scriptHex, Addresses: []string{dbtestdata.Addr1}}, + } +} + +func fanInHash(n uint64) string { return fmt.Sprintf("%064x", n) } diff --git a/db/rocksdb.go b/db/rocksdb.go index db2ae34cd4..09bc678464 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -1117,6 +1117,49 @@ func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { return d.getTxAddresses(btxID) } +// GetTxAddressesOutput returns Outputs[vout] of the transaction's TxAddresses +// record, or nil if the record or that output is not present. Unlike +// GetTxAddresses it does not unpack the whole record: inputs and preceding +// outputs are skipped without allocating, so resolving a single previous output +// costs O(1) allocations regardless of the referenced transaction's size. +func (d *RocksDB) GetTxAddressesOutput(txid string, vout uint32) (*TxOutput, error) { + btxID, err := d.chainParser.PackTxid(txid) + if err != nil { + return nil, err + } + val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 3 is minimum length of a txAddresses record - 1 byte height, 1 byte inputs len, 1 byte outputs len + if len(buf) < 3 { + return nil, nil + } + _, l := unpackVaruint(buf) // height + if d.extendedIndex { + _, n := unpackVaruint(buf[l:]) // vsize + l += n + } + inputs, n := unpackVaruint(buf[l:]) + l += n + for i := uint(0); i < inputs; i++ { + l += d.packedTxInputLen(buf[l:]) + } + outputs, n := unpackVaruint(buf[l:]) + l += n + if uint(vout) >= outputs { + return nil, nil + } + for i := uint(0); i < uint(vout); i++ { + l += d.packedTxOutputLen(buf[l:]) + } + var output TxOutput + d.unpackTxOutput(&output, buf[l:]) + return &output, nil +} + // AddrDescForOutpoint is a function that returns address descriptor and value for given outpoint or nil if outpoint not found func (d *RocksDB) AddrDescForOutpoint(outpoint bchain.Outpoint) (bchain.AddressDescriptor, *big.Int) { ta, err := d.GetTxAddresses(outpoint.Txid) @@ -1361,6 +1404,50 @@ func (d *RocksDB) unpackTxOutput(to *TxOutput, buf []byte) int { return al } +// packedTxInputLen returns the byte length of the packed input at the start of buf +// without allocating. Keep in sync with appendTxInput/unpackTxInput. +func (d *RocksDB) packedTxInputLen(buf []byte) int { + if d.extendedIndex { + al, pos := unpackVarint(buf) + coinbase := al < 0 + if coinbase { + al = ^al + } + pos += al // addrDesc + pos += int(buf[pos]) + 1 // value (1-byte length prefix + bytes) + if !coinbase { + pos += d.chainParser.PackedTxidLen() // prev txid + _, n := unpackVaruint(buf[pos:]) // prev vout + pos += n + } + return pos + } + al, pos := unpackVaruint(buf) + pos += int(al) // addrDesc + pos += int(buf[pos]) + 1 // value + return pos +} + +// packedTxOutputLen returns the byte length of the packed output at the start of buf +// without allocating. Keep in sync with appendTxOutput/unpackTxOutput. +func (d *RocksDB) packedTxOutputLen(buf []byte) int { + al, pos := unpackVarint(buf) + spent := al < 0 + if spent { + al = ^al + } + pos += al // addrDesc + pos += int(buf[pos]) + 1 // value + if d.extendedIndex && spent { + pos += d.chainParser.PackedTxidLen() // spent txid + _, n := unpackVaruint(buf[pos:]) // spent index + pos += n + _, n = unpackVaruint(buf[pos:]) // spent height + pos += n + } + return pos +} + func (d *RocksDB) packTxIndexes(txi []txIndexes) []byte { buf := make([]byte, 0, 32) bvout := make([]byte, vlq.MaxLen32) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 16f2dffd43..217dcab0ae 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -16,6 +16,7 @@ import ( "github.com/juju/errors" "github.com/linxGnu/grocksdb" "github.com/martinboehm/btcutil/chaincfg" + "github.com/stretchr/testify/require" "github.com/trezor/blockbook/bchain" "github.com/trezor/blockbook/bchain/coins/btc" "github.com/trezor/blockbook/common" @@ -1747,3 +1748,81 @@ func TestRocksDB_packTxIndexes_unpackTxIndexes(t *testing.T) { }) } } + +// setupRocksDBWithExtendedIndex is like setupRocksDB but lets the caller choose +// the extendedIndex layout; used by TestGetTxAddressesOutput to cover both forms. +func setupRocksDBWithExtendedIndex(t *testing.T, p bchain.BlockChainParser, extendedIndex bool) *RocksDB { + t.Helper() + tmp, err := os.MkdirTemp("", "testdb") + require.NoError(t, err) + d, err := NewRocksDB(tmp, 100000, -1, p, nil, extendedIndex) + require.NoError(t, err) + is, err := d.LoadInternalState(&common.Config{CoinName: "coin-unittest"}) + require.NoError(t, err) + d.SetInternalState(is) + return d +} + +// TestGetTxAddressesOutput checks that GetTxAddressesOutput(txid, vout) returns the +// same TxOutput as GetTxAddresses(txid).Outputs[vout]: the former skips over the +// height, inputs and preceding outputs by length and unpacks only the target output, +// while the latter unpacks the whole record. It runs with extendedIndex off and on +// and uses a record with a coinbase input and both spent and unspent outputs, so +// every length-skip case in packedTxInputLen/packedTxOutputLen is covered. +func TestGetTxAddressesOutput(t *testing.T) { + for _, extendedIndex := range []bool{false, true} { + name := "basic" + if extendedIndex { + name = "extendedIndex" + } + t.Run(name, func(t *testing.T) { + d := setupRocksDBWithExtendedIndex(t, &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + }, extendedIndex) + defer closeAndDestroyRocksDB(t, d) + + ta := &TxAddresses{ + Height: 12345, + Inputs: []TxInput{ + {ValueSat: *big.NewInt(0)}, // coinbase: empty AddrDesc, no Txid + {AddrDesc: addressToAddrDesc(dbtestdata.Addr1, d.chainParser), ValueSat: *big.NewInt(1000), Txid: strings.Repeat("b2", 32), Vout: 0}, + {AddrDesc: addressToAddrDesc(dbtestdata.Addr2, d.chainParser), ValueSat: *big.NewInt(2000), Txid: strings.Repeat("c3", 32), Vout: 3}, + }, + Outputs: []TxOutput{ + {AddrDesc: addressToAddrDesc(dbtestdata.Addr3, d.chainParser), ValueSat: *big.NewInt(3000)}, + {AddrDesc: addressToAddrDesc(dbtestdata.Addr4, d.chainParser), ValueSat: *big.NewInt(4000), Spent: true, SpentTxid: strings.Repeat("d4", 32), SpentIndex: 5, SpentHeight: 12346}, + {AddrDesc: addressToAddrDesc(dbtestdata.Addr5, d.chainParser), ValueSat: *big.NewInt(5000)}, + }, + } + txid := strings.Repeat("a1", 32) + + btxID, err := d.chainParser.PackTxid(txid) + require.NoError(t, err) + wb := grocksdb.NewWriteBatch() + defer wb.Destroy() + require.NoError(t, d.storeTxAddresses(wb, map[string]*TxAddresses{string(btxID): ta})) + require.NoError(t, d.db.Write(d.wo, wb)) + + full, err := d.GetTxAddresses(txid) + require.NoError(t, err) + require.NotNil(t, full) + + // Each output read singly must equal the full-record unpack. + for vout := range full.Outputs { + got, err := d.GetTxAddressesOutput(txid, uint32(vout)) + require.NoErrorf(t, err, "vout %d", vout) + require.NotNilf(t, got, "vout %d", vout) + require.Equalf(t, full.Outputs[vout], *got, "vout %d", vout) + } + + // Out-of-range vout and missing txid both return nil. + got, err := d.GetTxAddressesOutput(txid, uint32(len(full.Outputs))) + require.NoError(t, err) + require.Nil(t, got, "out-of-range vout") + + got, err = d.GetTxAddressesOutput(strings.Repeat("ee", 32), 0) + require.NoError(t, err) + require.Nil(t, got, "missing txid") + }) + } +} From 42a69d1c465c569f26cc498ba4338d7638926c26 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 09:07:46 +0000 Subject: [PATCH 073/303] fix(ci): remove setup-node npm cache to avoid tar extraction failure on self-hosted runners The actions/setup-node built-in npm cache saves ~/.npm with paths relative to the workspace. On runners where the home directory lives outside the workspace tree, tar cannot restore these paths. The explicit actions/cache for tests/openapi/node_modules already handles caching and skips npm ci on hit, making the global npm cache redundant. --- .github/workflows/deploy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c10dc011c0..9011b89aa1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -252,8 +252,6 @@ jobs: uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - cache: npm - cache-dependency-path: tests/openapi/package-lock.json - name: Cache OpenAPI test dependencies id: openapi-node-modules-cache From d7cbc3471defe06abd1ef1652e296858ed3e30cc Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 09:11:44 +0000 Subject: [PATCH 074/303] fix(ci): bump actions to Node.js 24-compatible versions actions/checkout v4.3.1 -> v7.0.0 actions/cache v4.3.0 -> v6.1.0 actions/setup-node v4.4.0 -> v6.4.0 actions/upload-artifact v4.6.2 -> v7.0.1 All four actions were targeting Node.js 20, which is now deprecated on GitHub Actions runners running Node.js 24. The new major versions bundle a compatible runtime. --- .github/workflows/deploy.yml | 22 +++++++++++----------- .github/workflows/testing.yml | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9011b89aa1..e407da9d0e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -56,7 +56,7 @@ jobs: runner_matrix: ${{ steps.plan.outputs.runner_matrix }} steps: - name: Checkout workflow code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -66,7 +66,7 @@ jobs: run: python3 ./.github/scripts/validate_branch_or_tag.py --repo "$GITHUB_REPOSITORY" --ref "$REF_TO_VALIDATE" - name: Checkout requested branch or tag - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -89,7 +89,7 @@ jobs: test_coins_csv: ${{ steps.plan.outputs.test_coins_csv }} steps: - name: Checkout workflow code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -99,7 +99,7 @@ jobs: run: python3 ./.github/scripts/validate_branch_or_tag.py --repo "$GITHUB_REPOSITORY" --ref "$REF_TO_VALIDATE" - name: Checkout requested branch or tag - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -125,7 +125,7 @@ jobs: runs-on: ${{ fromJSON(matrix.labels_json) }} steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -157,7 +157,7 @@ jobs: runs-on: [self-hosted, bb-dev-selfhosted, "${{ matrix.runner }}"] steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -202,7 +202,7 @@ jobs: SYNC_TIMEOUT_SECONDS: "300" steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -229,7 +229,7 @@ jobs: CONNECTIVITY_REGEX: ${{ needs.prepare_deploy.outputs.connectivity_regex }} steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} persist-credentials: false @@ -249,13 +249,13 @@ jobs: run: make test-connectivity ARGS="-v" - name: Setup Node.js for OpenAPI tests - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 - name: Cache OpenAPI test dependencies id: openapi-node-modules-cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: tests/openapi/node_modules key: openapi-node-modules-node22-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/openapi/package-lock.json') }} @@ -277,7 +277,7 @@ jobs: # run is exactly when the per-test breakdown (which coin/test failed or skipped) is most useful. - name: Upload OpenAPI e2e report if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: openapi-e2e-report path: tests/openapi/reports/ diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 55d1c8dacb..cc07cf29ed 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -75,7 +75,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -101,7 +101,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false From d45d56e346354c92a5a6c9ff7436862781c875d0 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 09:21:25 +0000 Subject: [PATCH 075/303] fix(ci): use workspace-local npm cache to avoid tar extraction failure Instead of setup-node's built-in cache:npm (which saves ~/.npm with paths relative to the workspace, causing tar failures when home is outside the workspace tree), cache a local .npm-cache directory inside the workspace via explicit actions/cache, and pass --cache .npm-cache to npm ci. On node_modules cache miss, the local cache avoids re-downloading packages from the registry. The restore-keys fallback preserves partial cache benefit even across lockfile changes. --- .github/workflows/deploy.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e407da9d0e..21c0ef59dd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -253,6 +253,15 @@ jobs: with: node-version: 22 + - name: Cache npm packages + id: openapi-npm-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .npm-cache + key: openapi-npm-cache-node22-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/openapi/package-lock.json') }} + restore-keys: | + openapi-npm-cache-node22-${{ runner.os }}-${{ runner.arch }}- + - name: Cache OpenAPI test dependencies id: openapi-node-modules-cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -264,7 +273,7 @@ jobs: - name: Install OpenAPI test dependencies if: steps.openapi-node-modules-cache.outputs.cache-hit != 'true' - run: npm ci --prefix tests/openapi --prefer-offline --no-audit --no-fund + run: npm ci --prefix tests/openapi --cache .npm-cache --prefer-offline --no-audit --no-fund - name: Run OpenAPI API e2e tests env: From 08d720c63cf7aa51eca4c3efc3a8b540280e9b92 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 09:31:42 +0000 Subject: [PATCH 076/303] fix(ci): ensure .npm-cache dir exists so cache save never warns On node_modules cache hit, npm ci is skipped and .npm-cache was never created, causing the post-job cache save to warn: Path Validation Error: Path(s) specified for caching do not exist Pre-create the directory with mkdir -p so the path always exists for the save phase. On cache hit the directory is empty and the save is a no-op. --- .github/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 21c0ef59dd..12b95d3df5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -253,6 +253,9 @@ jobs: with: node-version: 22 + - name: Ensure npm cache directory exists + run: mkdir -p .npm-cache + - name: Cache npm packages id: openapi-npm-cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 From 521c56e7fbb887a7a4c18d9eef075d523feec3c9 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Fri, 10 Jul 2026 09:45:48 +0000 Subject: [PATCH 077/303] fix(ci): widen block-index search window to 20 blocks for slow-indexing chains The getSampleIndexedHeight probe searches from bestHeight-2 backward for an indexed block hash. At 12 blocks (txSearchWindow), chains with slower indexers (ethereum-classic on core-geth/blockbook-dev2) sometimes have zero indexed blocks in the window, causing flaky failures. 20 blocks covers ~4.5 minutes of ETC chain history, ample for any realistic indexing lag. The loop stops on the first hit, so healthy runs are unaffected. --- tests/openapi/src/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openapi/src/constants.ts b/tests/openapi/src/constants.ts index 3bc5ee9175..4f377ec1dd 100644 --- a/tests/openapi/src/constants.ts +++ b/tests/openapi/src/constants.ts @@ -1,6 +1,6 @@ export const wsDialTimeoutMs = 3_000; export const wsMessageTimeoutMs = 10_000; -export const txSearchWindow = 12; +export const txSearchWindow = 20; export const blockPageSize = 1; export const sampleBlockPageSize = 3; export const sampleBlockProbeMax = 3; From 5b824008275d62ec9832b0b17225ed1ef18cdc43 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek <116381722+cranycrane@users.noreply.github.com> Date: Mon, 13 Jul 2026 09:06:35 +0200 Subject: [PATCH 078/303] fix(ws): cap subscribeFiatRates tokens array length (#1623) * fix(ws): cap subscribeFiatRates tokens array to prevent DoS The subscribeFiatRates handler stored the request's tokens slice whole for the connection lifetime and re-iterated it on every fiat-rate broadcast (per subscribed currency) while holding the global fiatRatesSubscriptionsLock. It was the only WebSocket list parameter without a length cap: a single message up to the 4 MiB read limit could carry ~10^6 token strings, enabling amplified heap retention and periodic global-lock contention that stalls fiat-rate subscription handling. Reject requests whose tokens array exceeds maxWebsocketSubscribeFiatRatesTokens (1000) with a public API error, matching the existing caps on addresses, currencies, timestamps, and estimateFee blocks. Co-Authored-By: Claude Fable 5 * chore(websocket): cleanup comment --------- Co-authored-by: Claude Fable 5 --- server/public_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++ server/websocket.go | 4 +++ 2 files changed, 78 insertions(+) diff --git a/server/public_test.go b/server/public_test.go index a318407ef8..e6b221dc3d 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -2207,6 +2207,80 @@ func Test_WebsocketFiatRates_SubscribeBroadcastAndUnsubscribe(t *testing.T) { assertNoWebsocketMessage(t, ws, 300*time.Millisecond) } +func Test_WebsocketFiatRates_SubscribeTokensCap(t *testing.T) { + parser, chain := setupChain(t) + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) + defer closeAndDestroyPublicServer(t, s, dbpath) + s.ConnectFullPublicInterface() + ts := httptest.NewServer(s.https.Handler) + defer ts.Close() + + ws := connectWebsocket(t, ts) + defer ws.Close() + + makeTokens := func(n int) []string { + tokens := make([]string, n) + for i := range tokens { + tokens[i] = "0xa4dd6bc15be95af55f0447555c8b6aa3088562f3" + } + return tokens + } + + // Over the cap: the subscription is rejected with a public error and no + // token slice is retained for the connection. + if err := ws.WriteJSON(websocketReq{ + ID: "sub-over", + Method: "subscribeFiatRates", + Params: map[string]interface{}{ + "currency": "USD", + "tokens": makeTokens(maxWebsocketSubscribeFiatRatesTokens + 1), + }, + }); err != nil { + t.Fatal(err) + } + over := readWebsocketResponse(t, ws, time.Second) + if over.ID != "sub-over" { + t.Fatalf("unexpected response id: got %q, want %q", over.ID, "sub-over") + } + var overData struct { + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(over.Data, &overData); err != nil { + t.Fatal(err) + } + wantMsg := "tokens max " + strconv.Itoa(maxWebsocketSubscribeFiatRatesTokens) + if overData.Error.Message != wantMsg { + t.Fatalf("unexpected error message: got %q, want %q", overData.Error.Message, wantMsg) + } + + // Exactly at the cap: the subscription succeeds. + if err := ws.WriteJSON(websocketReq{ + ID: "sub-at", + Method: "subscribeFiatRates", + Params: map[string]interface{}{ + "currency": "USD", + "tokens": makeTokens(maxWebsocketSubscribeFiatRatesTokens), + }, + }); err != nil { + t.Fatal(err) + } + at := readWebsocketResponse(t, ws, time.Second) + if at.ID != "sub-at" { + t.Fatalf("unexpected response id: got %q, want %q", at.ID, "sub-at") + } + var atData struct { + Subscribed bool `json:"subscribed"` + } + if err := json.Unmarshal(at.Data, &atData); err != nil { + t.Fatal(err) + } + if !atData.Subscribed { + t.Fatalf("expected subscribed=true at the token cap, got false (data %s)", at.Data) + } +} + func Test_WebsocketFiatRates_GetCurrentFiatRates_TokenContractsCaseHandling_BitcoinType(t *testing.T) { parser, chain := setupChain(t) diff --git a/server/websocket.go b/server/websocket.go index 857c047c0c..bb63f70adb 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -46,6 +46,7 @@ const maxWebsocketActiveRequests = 2048 const maxWebsocketEstimateFeeBlocks = 32 const maxWebsocketSubscribeAddresses = 1000 const maxWebsocketSubscribeAddressesWithNewBlockTxs = 100 +const maxWebsocketSubscribeFiatRatesTokens = 1000 const websocketLogPreviewBytes = 256 // allRates is a special "currency" parameter that means all available currencies @@ -837,6 +838,9 @@ var requestHandlers = map[string]func(*WebsocketServer, *websocketChannel, *WsRe if err != nil { return nil, err } + if len(r.Tokens) > maxWebsocketSubscribeFiatRatesTokens { + return nil, api.NewAPIError("tokens max "+strconv.Itoa(maxWebsocketSubscribeFiatRatesTokens), true) + } r.Currency = strings.ToLower(r.Currency) return s.subscribeFiatRates(c, &r, req) From 8b8d97d3713f91986571c3e11b35ad76c3a0d010 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek <116381722+cranycrane@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:27:02 +0200 Subject: [PATCH 079/303] fix(eth): prevent duplicate ERC-721 holdings (#1624) * fix(eth): prevent duplicate ERC-721 holdings * chore(contrib-script): move the erc-721 holdings check to a gist * refactor(eth): address PR review comments on NFT self-transfer handling - derive the NFT self-transfer invariant from a shared isNFTSelfTransfer helper on both the connect and the disconnect side so they cannot drift - document that addToContract callers pass mutateHoldings=false only for ERC721 self-transfers - add a ConnectBlock -> DisconnectBlock round-trip test verifying that an ERC721 self-transfer keeps holdings untouched and tx counts balanced Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- db/rocksdb_ethereumtype.go | 57 ++++++--- db/rocksdb_ethereumtype_test.go | 218 +++++++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 18 deletions(-) diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go index cf17c39597..ce4d54c8d6 100644 --- a/db/rocksdb_ethereumtype.go +++ b/db/rocksdb_ethereumtype.go @@ -46,6 +46,9 @@ func (s *Ids) search(id big.Int) int { // insert id in ascending order func (s *Ids) insert(id big.Int) { i := s.search(id) + if i < len(*s) && (*s)[i].CmpAbs(&id) == 0 { + return + } if i == len(*s) { *s = append(*s, id) } else { @@ -399,7 +402,15 @@ func addToAddressesMapEthereumType(addresses addressesMap, strAddrDesc string, b return false } -func addToContract(c *unpackedAddrContract, contractIndex int, index int32, contract bchain.AddressDescriptor, transfer *bchain.TokenTransfer, addTxCount bool) int32 { +// isNFTSelfTransfer reports whether the transfer is an ERC721 transfer from an address to itself. +// Such a transfer must be indexed and counted, but it must not change the address's holdings - +// both processContractTransfers (connect) and disconnectAddress (rollback) derive the +// mutateHoldings behavior of addToContract from this predicate. +func isNFTSelfTransfer(from, to bchain.AddressDescriptor, standard bchain.TokenStandard) bool { + return standard == bchain.NonFungibleToken && bytes.Equal(from, to) +} + +func addToContract(c *unpackedAddrContract, contractIndex int, index int32, contract bchain.AddressDescriptor, transfer *bchain.TokenTransfer, addTxCount bool, mutateHoldings bool) int32 { var aggregate AggregateFn // index 0 is for ETH transfers, index 1 (InternalTxIndexOffset) is for internal transfers, contract indexes start with 2 (ContractIndexOffset) if index < 0 { @@ -417,6 +428,16 @@ func addToContract(c *unpackedAddrContract, contractIndex int, index int32, cont s.Add(s, v) } } + if !mutateHoldings { + // mutateHoldings is false only for ERC721 self-transfers (see isNFTSelfTransfer): + // the transfer must still be indexed in both directions and counted once, but it + // cannot change the address's holdings. Other standards must not pass false here, + // it would silently skip their value normalization and aggregation. + if addTxCount { + c.Txs++ + } + return index + } if transfer.Standard == bchain.FungibleToken { // Skip ERC20 balance aggregation; ensure a zero value is available for packing. if c.Value.Value == nil { // no decoded bigint yet; normalize before first use @@ -445,7 +466,7 @@ func addToContract(c *unpackedAddrContract, contractIndex int, index int32, cont return index } -func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, transfer *bchain.TokenTransfer, addTxCount bool, addresses addressesMap, addressContracts map[string]*unpackedAddrContracts) error { +func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, transfer *bchain.TokenTransfer, addTxCount bool, mutateHoldings bool, addresses addressesMap, addressContracts map[string]*unpackedAddrContracts) error { var err error strAddrDesc := string(addrDesc) ac, e := addressContracts[strAddrDesc] @@ -484,7 +505,7 @@ func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.Address ac.addContractIndex(contract, contractIndex) } c := &ac.Contracts[contractIndex] - index = addToContract(c, contractIndex, index, contract, transfer, addTxCount) + index = addToContract(c, contractIndex, index, contract, transfer, addTxCount, mutateHoldings) } else { if index < 0 { index = transferFrom @@ -539,7 +560,7 @@ func (d *RocksDB) processBaseTxData(blockTx *ethBlockTx, tx *bchain.Tx, addresse glog.Warningf("rocksdb: processBaseTxData: %v, tx %v, output", err, tx.Txid) } } else { - if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, transferTo, nil, nil, true, addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, transferTo, nil, nil, true, true, addresses, addressContracts); err != nil { return err } blockTx.to = to @@ -553,7 +574,7 @@ func (d *RocksDB) processBaseTxData(blockTx *ethBlockTx, tx *bchain.Tx, addresse glog.Warningf("rocksdb: processBaseTxData: %v, tx %v, input", err, tx.Txid) } } else { - if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, transferFrom, nil, nil, !bytes.Equal(from, to), addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, transferFrom, nil, nil, !bytes.Equal(from, to), true, addresses, addressContracts); err != nil { return err } blockTx.from = from @@ -599,7 +620,7 @@ func (d *RocksDB) processInternalData(blockTx *ethBlockTx, tx *bchain.Tx, id *bc return err } } - if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, internalTransferTo, nil, nil, true, addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, internalTransferTo, nil, nil, true, true, addresses, addressContracts); err != nil { return err } } @@ -622,7 +643,7 @@ func (d *RocksDB) processInternalData(blockTx *ethBlockTx, tx *bchain.Tx, id *bc return err } } - if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, internalTransferTo, nil, nil, true, addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, internalTransferTo, nil, nil, true, true, addresses, addressContracts); err != nil { return err } ito.to = to @@ -638,7 +659,7 @@ func (d *RocksDB) processInternalData(blockTx *ethBlockTx, tx *bchain.Tx, id *bc return err } } - if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, internalTransferFrom, nil, nil, !bytes.Equal(from, to), addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, internalTransferFrom, nil, nil, !bytes.Equal(from, to), true, addresses, addressContracts); err != nil { return err } ito.from = from @@ -669,11 +690,12 @@ func (d *RocksDB) processContractTransfers(blockTx *ethBlockTx, tx *bchain.Tx, a glog.Warningf("rocksdb: processContractTransfers %v, tx %v, transfer %v", err, tx.Txid, t) continue } - if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, int32(i), contract, t, true, addresses, addressContracts); err != nil { + eq := bytes.Equal(from, to) + mutateHoldings := !isNFTSelfTransfer(from, to, t.Standard) + if err = d.addToAddressesAndContractsEthereumType(to, blockTx.btxID, int32(i), contract, t, true, mutateHoldings, addresses, addressContracts); err != nil { return err } - eq := bytes.Equal(from, to) - if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, ^int32(i), contract, t, !eq, addresses, addressContracts); err != nil { + if err = d.addToAddressesAndContractsEthereumType(from, blockTx.btxID, ^int32(i), contract, t, !eq, mutateHoldings, addresses, addressContracts); err != nil { return err } bc := &blockTx.contracts[i] @@ -1254,9 +1276,11 @@ func (d *RocksDB) disconnectAddress(btxID []byte, internal bool, addrDesc bchain MultiTokenValues: btxContract.idValues, } if bytes.Equal(btxContract.from, btxContract.to) { - // Self-transfers were connected through both sides while counting one tx. - addToContract(addrContract, contractIndex, transferTo, btxContract.contract, transfer, false) - addToContract(addrContract, contractIndex, transferFrom, btxContract.contract, transfer, false) + // an NFT self-transfer did not change holdings on connect, there is nothing to reverse + if !isNFTSelfTransfer(btxContract.from, btxContract.to, btxContract.transferStandard) { + addToContract(addrContract, contractIndex, transferTo, btxContract.contract, transfer, false, true) + addToContract(addrContract, contractIndex, transferFrom, btxContract.contract, transfer, false, true) + } } else { var index int32 if bytes.Equal(addrDesc, btxContract.to) { @@ -1264,7 +1288,7 @@ func (d *RocksDB) disconnectAddress(btxID []byte, internal bool, addrDesc bchain } else { index = transferTo } - addToContract(addrContract, contractIndex, index, btxContract.contract, transfer, false) + addToContract(addrContract, contractIndex, index, btxContract.contract, transfer, false, true) } } } else { @@ -1596,6 +1620,9 @@ func (s *unpackedIds) search(id big.Int) int { // insert id in ascending order func (s *unpackedIds) insert(id big.Int) { i := s.search(id) + if i < len(*s) && (*s)[i].get().CmpAbs(&id) == 0 { + return + } if i == len(*s) { *s = append(*s, unpackedBigInt{Value: &id}) } else { diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go index 9bdf103e47..69cd6870de 100644 --- a/db/rocksdb_ethereumtype_test.go +++ b/db/rocksdb_ethereumtype_test.go @@ -3,6 +3,7 @@ package db import ( + "bytes" "encoding/hex" "fmt" "math/big" @@ -21,6 +22,24 @@ type testEthereumParser struct { *eth.EthereumParser } +type tokenTransferTestParser struct { + *eth.EthereumParser + transfers bchain.TokenTransfers +} + +func (p *tokenTransferTestParser) EthereumTypeGetTokenTransfersFromTx(*bchain.Tx) (bchain.TokenTransfers, error) { + return p.transfers, nil +} + +type txTokenTransferTestParser struct { + *eth.EthereumParser + transfers map[string]bchain.TokenTransfers +} + +func (p *txTokenTransferTestParser) EthereumTypeGetTokenTransfersFromTx(tx *bchain.Tx) (bchain.TokenTransfers, error) { + return p.transfers[tx.Txid], nil +} + func ethereumTestnetParser() *eth.EthereumParser { return eth.NewEthereumParser(1, true) } @@ -43,6 +62,199 @@ func makeTestAddrDesc(seed int) bchain.AddressDescriptor { return b } +func TestIdsInsertDoesNotDuplicate(t *testing.T) { + ids := Ids{*big.NewInt(1), *big.NewInt(2), *big.NewInt(3)} + for _, id := range []int64{1, 2, 3} { + ids.insert(*big.NewInt(id)) + } + want := Ids{*big.NewInt(1), *big.NewInt(2), *big.NewInt(3)} + if !reflect.DeepEqual(ids, want) { + t.Fatalf("Ids = %v, want %v", ids, want) + } + + unpacked := unpackedIds{ + {Value: big.NewInt(1)}, + {Value: big.NewInt(2)}, + {Value: big.NewInt(3)}, + } + for _, id := range []int64{1, 2, 3} { + unpacked.insert(*big.NewInt(id)) + } + if len(unpacked) != 3 { + t.Fatalf("unpackedIds length = %d, want 3", len(unpacked)) + } + for i, id := range []int64{1, 2, 3} { + if unpacked[i].get().Cmp(big.NewInt(id)) != 0 { + t.Fatalf("unpackedIds[%d] = %v, want %d", i, unpacked[i].get(), id) + } + } +} + +func TestERC721RepeatedTransferDoesNotLeaveStaleDuplicate(t *testing.T) { + contract := makeTestAddrDesc(0x7210) + c := &unpackedAddrContract{ + Standard: bchain.NonFungibleToken, + Contract: contract, + } + transfer := &bchain.TokenTransfer{ + Standard: bchain.NonFungibleToken, + Value: *big.NewInt(42), + } + + addToContract(c, 0, transferTo, contract, transfer, false, true) + addToContract(c, 0, transferTo, contract, transfer, false, true) + if len(c.Ids) != 1 || c.Ids[0].get().Cmp(&transfer.Value) != 0 { + t.Fatalf("Ids after repeated transfer-in = %v, want [42]", c.Ids) + } + + addToContract(c, 0, transferFrom, contract, transfer, false, true) + if len(c.Ids) != 0 { + t.Fatalf("Ids after transfer-out = %v, want empty", c.Ids) + } +} + +func TestERC721SelfTransferDoesNotMutateHoldings(t *testing.T) { + parser := ethereumTestnetParser() + address := dbtestdata.EthAddr83 + contractAddress := dbtestdata.EthAddrContractCd + addrDesc := addressToAddrDesc(address, parser) + contractDesc := addressToAddrDesc(contractAddress, parser) + transfer := &bchain.TokenTransfer{ + Standard: bchain.NonFungibleToken, + Contract: contractAddress, + From: address, + To: address, + Value: *big.NewInt(42), + } + d := &RocksDB{chainParser: &tokenTransferTestParser{ + EthereumParser: parser, + transfers: bchain.TokenTransfers{transfer}, + }} + acs := &unpackedAddrContracts{ + TotalTxs: 3, + Contracts: []unpackedAddrContract{{ + Standard: bchain.NonFungibleToken, + Contract: contractDesc, + Txs: 3, + Ids: unpackedIds{{Value: big.NewInt(42)}}, + }}, + } + addresses := addressesMap{} + addressContracts := map[string]*unpackedAddrContracts{string(addrDesc): acs} + blockTx := ðBlockTx{btxID: []byte{0x01}} + + if err := d.processContractTransfers(blockTx, &bchain.Tx{}, addresses, addressContracts); err != nil { + t.Fatal(err) + } + if acs.TotalTxs != 4 || acs.Contracts[0].Txs != 4 { + t.Fatalf("transaction counts = (%d, %d), want (4, 4)", acs.TotalTxs, acs.Contracts[0].Txs) + } + if len(acs.Contracts[0].Ids) != 1 || acs.Contracts[0].Ids[0].get().Cmp(big.NewInt(42)) != 0 { + t.Fatalf("Ids = %v, want [42]", acs.Contracts[0].Ids) + } + txIndexes := addresses[string(addrDesc)] + wantIndexes := []int32{ContractIndexOffset, ^int32(ContractIndexOffset)} + if len(txIndexes) != 1 || !reflect.DeepEqual(txIndexes[0].indexes, wantIndexes) { + t.Fatalf("address indexes = %+v, want %v for one transaction", txIndexes, wantIndexes) + } +} + +// TestERC721SelfTransferConnectDisconnectRoundTrip verifies the connect/disconnect symmetry +// of an ERC721 self-transfer: the holdings are untouched on both sides and the transaction +// counts stay balanced after the block with the self-transfer is rolled back. +func TestERC721SelfTransferConnectDisconnectRoundTrip(t *testing.T) { + parser := ethereumTestnetParser() + owner := dbtestdata.EthAddr83 + contractAddress := dbtestdata.EthAddrContractCd + tokenID := big.NewInt(42) + txidTransferIn := "80a0533b0f66e9d29aa4dbbdc8c4b90326b073e0d6b864e02c9598032ed05201" + txidSelfTransfer := "80a0533b0f66e9d29aa4dbbdc8c4b90326b073e0d6b864e02c9598032ed05202" + d := setupRocksDB(t, &txTokenTransferTestParser{ + EthereumParser: parser, + transfers: map[string]bchain.TokenTransfers{ + txidTransferIn: {{ + Standard: bchain.NonFungibleToken, + Contract: contractAddress, + From: dbtestdata.EthAddr7b, + To: owner, + Value: *tokenID, + }}, + txidSelfTransfer: {{ + Standard: bchain.NonFungibleToken, + Contract: contractAddress, + From: owner, + To: owner, + Value: *tokenID, + }}, + }, + }) + defer closeAndDestroyRocksDB(t, d) + ownerDesc := addressToAddrDesc(owner, parser) + contractDesc := addressToAddrDesc(contractAddress, parser) + + makeBlock := func(height uint32, hash, txid, from string) *bchain.Block { + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Height: height, + Hash: hash, + Time: 1534858022, + }, + Txs: []bchain.Tx{{ + Txid: txid, + Vin: []bchain.Vin{{Addresses: []string{from}}}, + Vout: []bchain.Vout{{ScriptPubKey: bchain.ScriptPubKey{Addresses: []string{contractAddress}}}}, + }}, + } + } + + ownerContracts := func(stage string, wantTotalTxs uint, wantContractTxs uint) *unpackedAddrContracts { + t.Helper() + acs, err := d.getUnpackedAddrDescContracts(ownerDesc) + if err != nil { + t.Fatalf("%s: %v", stage, err) + } + if acs == nil { + t.Fatalf("%s: owner address contracts not found", stage) + } + if acs.TotalTxs != wantTotalTxs { + t.Fatalf("%s: TotalTxs = %d, want %d", stage, acs.TotalTxs, wantTotalTxs) + } + i, found := findContractInAddressContracts(contractDesc, acs.Contracts) + if !found { + t.Fatalf("%s: contract not found in owner address contracts", stage) + } + c := &acs.Contracts[i] + if c.Txs != wantContractTxs { + t.Fatalf("%s: contract Txs = %d, want %d", stage, c.Txs, wantContractTxs) + } + if len(c.Ids) != 1 || c.Ids[0].get().Cmp(tokenID) != 0 { + t.Fatalf("%s: Ids = %v, want [42]", stage, c.Ids) + } + return acs + } + + // connect a block transferring the token to the owner + if err := d.ConnectBlock(makeBlock(4321000, "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11", txidTransferIn, dbtestdata.EthAddr7b)); err != nil { + t.Fatal(err) + } + snapshot := packUnpackedAddrContracts(ownerContracts("after connect of transfer in", 1, 1)) + + // connect a block with a self-transfer of the token: holdings must stay [42], tx counts must increase + if err := d.ConnectBlock(makeBlock(4321001, "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee", txidSelfTransfer, owner)); err != nil { + t.Fatal(err) + } + ownerContracts("after connect of self-transfer", 2, 2) + + // disconnect the self-transfer block: holdings must stay [42], tx counts must return to the previous state + if err := d.DisconnectBlockRangeEthereumType(4321001, 4321001); err != nil { + t.Fatal(err) + } + restored := packUnpackedAddrContracts(ownerContracts("after disconnect of self-transfer", 1, 1)) + if !bytes.Equal(snapshot, restored) { + t.Fatalf("owner address contracts after disconnect = %s, want %s", hex.EncodeToString(restored), hex.EncodeToString(snapshot)) + } +} + func Test_unpackedAddrContracts_findContractIndex_LazyMap(t *testing.T) { acs := &unpackedAddrContracts{} minContracts := 192 @@ -1540,7 +1752,7 @@ func Test_addToContracts(t *testing.T) { Standard: tt.args.transfer.Standard, }) } - if got := addToContract(&unpackedAddrContracts.Contracts[contractIndex], contractIndex, tt.args.index, tt.args.contract, tt.args.transfer, tt.args.addTxCount); got != tt.wantIndex { + if got := addToContract(&unpackedAddrContracts.Contracts[contractIndex], contractIndex, tt.args.index, tt.args.contract, tt.args.transfer, tt.args.addTxCount, true); got != tt.wantIndex { t.Errorf("addToContracts() = %v, want %v", got, tt.wantIndex) } // convert from partially unpacked form to final form used by API @@ -1564,7 +1776,7 @@ func Test_addToContract_ERC20ZeroesExistingValue(t *testing.T) { Contract: makeTestAddrDesc(123), Value: unpackedBigInt{Value: big.NewInt(123456)}, } - addToContract(c, 0, 1, c.Contract, transfer, false) + addToContract(c, 0, 1, c.Contract, transfer, false, true) if c.Value.Value == nil || c.Value.Value.Sign() != 0 { t.Fatalf("expected ERC20 value to be zeroed, got %v", c.Value.Value) } @@ -1577,7 +1789,7 @@ func Test_addToContract_ERC20ZeroesExistingValue(t *testing.T) { Contract: makeTestAddrDesc(124), Value: unpackedBigInt{Slice: []byte{0x1, 0x2}}, } - addToContract(c, 0, 1, c.Contract, transfer, false) + addToContract(c, 0, 1, c.Contract, transfer, false, true) if c.Value.Value == nil || c.Value.Value.Sign() != 0 { t.Fatalf("expected ERC20 value to be zeroed after slice, got %v", c.Value.Value) } From b3702820ea31a2ac5cc99035c012e51a72f06ad1 Mon Sep 17 00:00:00 2001 From: Jozef Knaperek Date: Mon, 13 Jul 2026 19:45:58 +0200 Subject: [PATCH 080/303] feat(websocket): make per-connection pending request limit env-configurable The cap on concurrently executing requests per connection is a hard-coded const (48). Internal batch clients that pipeline many requests on a single connection (e.g. bulk getAccountInfo address scans by an exchange backend) trip it and get disconnected mid-batch, with no way to tune the limit -- unlike the adjacent websocket message-rate and getAccountInfo limits, which are already env-configurable. Read the limit from _WS_PENDING_REQUESTS_LIMIT following the existing WS limiter env pattern; the default stays 48 and 0 disables the cap (the server-wide work limit remains as a backstop). Document the variable in docs/env.md and cover the override with a test. Co-Authored-By: Claude Fable 5 --- docs/env.md | 2 ++ server/public_test.go | 19 +++++++++++++++---- server/websocket.go | 25 +++++++++++++++++++++++-- server/websocket_rate_limiter.go | 17 +++++++++++++++++ 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/docs/env.md b/docs/env.md index 11e3c0df3a..56f2a5ab23 100644 --- a/docs/env.md +++ b/docs/env.md @@ -47,6 +47,8 @@ Blockbook reads these from its process environment. When installed from the Debi - `_WS_MESSAGE_RATE_WINDOW` - Trailing sliding window for `_WS_MESSAGE_RATE_LIMIT`, as a Go duration string (e.g. `10m`, `600s`). Default `10m`. +- `_WS_PENDING_REQUESTS_LIMIT` - Maximum number of requests a single WebSocket connection may have executing concurrently ("pending") before the connection is closed with `pending_requests_limit`. Accepts a non-negative integer; default `48`, `0` disables the cap. The default comfortably covers interactive clients such as Trezor Suite; internal batch clients that pipeline hundreds of requests on one connection (e.g. bulk `getAccountInfo` address scans) need a higher value or `0`. The server-wide work limit still applies as a backstop against unbounded concurrency. + - `_WS_IP_BLOCK_DURATION` - How long a client key (an IPv4 address, or a full IPv6 `/128` address) is blocked from opening new WebSocket connections after a connection trips the message rate limit, as a Go duration string (e.g. `12h`). Default `12h`; `0` disables IP blocking (an offending connection is still closed). The block key is intentionally narrower than the rate-limit key (which aggregates IPv6 to its `/64`): a hard block keys on the individual `/128` so it cannot deny service to an entire shared `/64`, while the connection rate limiter still aggregates to `/64` so an abuser cannot dodge limits by rotating addresses within an owned `/64`. The blocklist is visible at `/admin/ws-limit-exceeding-ips`, and the `blockbook_websocket_blocked_ips` / `blockbook_websocket_blocked_connections` metrics track it. Blocking keys on the same client IP attribution as the per-IP limiter. Loopback/private/link-local addresses and any configured trusted-proxy or Cloudflare edge range are never blocked, so a misconfiguration that collapses many clients onto a shared address cannot block them all. Behind Cloudflare, keep `_CLOUDFLARE_IPS` at its default (or set it to your CDN ranges) so blocks key on the real visitor IP. The default Cloudflare peer verification plus a block-safety guard (a `CF-Connecting-*`-derived address is only blocked when the peer was verified as Cloudflare or is a local proxy fronting Cloudflare) prevent a direct public peer from using a forged `CF-Connecting-IP` to block an innocent visitor; if a local proxy forwards client headers and the deployment is not behind Cloudflare, strip `CF-Connecting-*` at the proxy (see `_CLOUDFLARE_IPS`). diff --git a/server/public_test.go b/server/public_test.go index e6b221dc3d..25bbb8937d 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -1194,7 +1194,7 @@ func Test_WebsocketRejectsOversizedMessage(t *testing.T) { t.Fatalf("unexpected websocket error after oversized message: %v", err) } -func Test_WebsocketClosesWhenPendingRequestLimitExceeded(t *testing.T) { +func testWebsocketPendingRequestLimit(t *testing.T, limit int) { parser, chain := setupChain(t) s, dbpath := setupPublicHTTPServer(parser, chain, t, false) @@ -1203,7 +1203,7 @@ func Test_WebsocketClosesWhenPendingRequestLimitExceeded(t *testing.T) { releaseRequests := make(chan struct{}) defer close(releaseRequests) - startedRequests := make(chan struct{}, maxWebsocketPendingRequests) + startedRequests := make(chan struct{}, limit) originalPingHandler := requestHandlers["ping"] requestHandlers["ping"] = func(s *WebsocketServer, c *websocketChannel, req *WsReq) (interface{}, error) { startedRequests <- struct{}{} @@ -1220,12 +1220,12 @@ func Test_WebsocketClosesWhenPendingRequestLimitExceeded(t *testing.T) { ws := connectWebsocket(t, ts) defer ws.Close() - for i := 0; i < maxWebsocketPendingRequests; i++ { + for i := 0; i < limit; i++ { if err := ws.WriteJSON(websocketReq{ID: strconv.Itoa(i), Method: "ping"}); err != nil { t.Fatal(err) } } - for i := 0; i < maxWebsocketPendingRequests; i++ { + for i := 0; i < limit; i++ { select { case <-startedRequests: case <-time.After(2 * time.Second): @@ -1251,6 +1251,17 @@ func Test_WebsocketClosesWhenPendingRequestLimitExceeded(t *testing.T) { } } +func Test_WebsocketClosesWhenPendingRequestLimitExceeded(t *testing.T) { + testWebsocketPendingRequestLimit(t, defaultWsPendingRequestsLimit) +} + +func Test_WebsocketPendingRequestLimitEnvOverride(t *testing.T) { + // the test coin sets no Network, so GetNetwork() falls back to the + // CoinShortcut FAKE + t.Setenv("FAKE_WS_PENDING_REQUESTS_LIMIT", "3") + testWebsocketPendingRequestLimit(t, 3) +} + var websocketTestsBitcoinType = []websocketTest{ { name: "websocket getInfo", diff --git a/server/websocket.go b/server/websocket.go index bb63f70adb..034566edd1 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -31,7 +31,10 @@ const outChannelSize = 500 const defaultTimeout = 60 * time.Second const unknownMethodLabel = "unknown" const maxWebsocketMessageBytes int64 = 4 * 1024 * 1024 -const maxWebsocketPendingRequests = 48 +// defaultWsPendingRequestsLimit is the default per-connection cap on +// concurrently executing requests; override with +// _WS_PENDING_REQUESTS_LIMIT (0 disables), see docs/env.md. +const defaultWsPendingRequestsLimit = 48 // maxWebsocketMempoolFiltersResponses caps per-connection getMempoolFilters // requests over their whole lifecycle: the slot is acquired before the handler @@ -135,6 +138,9 @@ type WebsocketServer struct { messageRateLimit int messageRateWindow time.Duration ipBlockDuration time.Duration + // pendingRequestsLimit caps how many requests a single connection may have + // executing concurrently before it is closed; 0 disables the cap. + pendingRequestsLimit int // ipBlockEnabled gates the whole IP blocklist: true only when the rate limit can // produce breaches (messageRateLimit > 0) and blocking is configured // (ipBlockDuration > 0). When false, all block checks/sweeps are skipped. @@ -211,6 +217,9 @@ func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, mempool bchain. if err := s.configureMessageRateLimit(is.GetNetwork()); err != nil { return nil, err } + if err := s.configurePendingRequestsLimit(is.GetNetwork()); err != nil { + return nil, err + } if s.metrics != nil { s.metrics.WebsocketNewBlockTxsSubscriptions.Set(0) s.metrics.WebsocketUniqueIPs.Set(0) @@ -344,11 +353,16 @@ func (s *WebsocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } conn.SetReadLimit(maxWebsocketMessageBytes) + // a nil channel disables the per-connection pending request cap + var pendingRequests chan struct{} + if s.pendingRequestsLimit > 0 { + pendingRequests = make(chan struct{}, s.pendingRequestsLimit) + } c := &websocketChannel{ id: atomic.AddUint64(&connectionCounter, 1), conn: conn, out: make(chan *WsRes, outChannelSize), - pendingRequests: make(chan struct{}, maxWebsocketPendingRequests), + pendingRequests: pendingRequests, mempoolFiltersSlots: make(chan struct{}, maxWebsocketMempoolFiltersResponses), ip: ip, ipKey: ipKey, @@ -523,6 +537,10 @@ func (c *websocketChannel) DataOut(data *WsRes) { } func (c *websocketChannel) acquireRequestSlot() bool { + if c.pendingRequests == nil { + // pending request limit disabled + return true + } select { case c.pendingRequests <- struct{}{}: return true @@ -532,6 +550,9 @@ func (c *websocketChannel) acquireRequestSlot() bool { } func (c *websocketChannel) releaseRequestSlot() { + if c.pendingRequests == nil { + return + } <-c.pendingRequests } diff --git a/server/websocket_rate_limiter.go b/server/websocket_rate_limiter.go index d072e9e128..70d351eb50 100644 --- a/server/websocket_rate_limiter.go +++ b/server/websocket_rate_limiter.go @@ -71,6 +71,23 @@ func (s *WebsocketServer) configureMessageRateLimit(network string) error { return nil } +// configurePendingRequestsLimit reads the per-connection cap on concurrently +// executing ("pending") requests from the environment, applying the default +// (see docs/env.md); 0 disables the cap entirely. +func (s *WebsocketServer) configurePendingRequestsLimit(network string) error { + prefix := strings.ToUpper(network) + var err error + if s.pendingRequestsLimit, err = parseNonNegativeIntEnv(prefix+"_WS_PENDING_REQUESTS_LIMIT", defaultWsPendingRequestsLimit); err != nil { + return err + } + if s.pendingRequestsLimit > 0 { + glog.Infof("Websocket per-connection pending request limit: %d", s.pendingRequestsLimit) + } else { + glog.Info("Websocket per-connection pending request limit disabled") + } + return nil +} + func newWebsocketConnectionLimiter() *websocketConnectionLimiter { return &websocketConnectionLimiter{ clients: make(map[string]*websocketClientLimit), From b3db9f8e573cdf8d6ad3ff52df84abd0c351d65f Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:18:24 +0200 Subject: [PATCH 081/303] fix(eth): gate alternative-provider nonce lookups to recent private senders EthereumTypeGetNonces routed eth_getTransactionCount to the alternative send-tx provider on a bare nil-check, i.e. for every address detail request. On instances with *_ALTERNATIVE_SENDTX_URLS configured this exhausted the provider's rate quota, flooding the error log with 429s and - worse - forcing the fallback to the primary RPC exactly for the private-tx senders the feature exists for, handing them a stale pending nonce. Only senders with a privately submitted transaction in flight need the provider's nonce view. Record the sender of every transaction accepted by an alternative provider (decoded from the raw tx hex, so it works regardless of the mempool-fetch mode and survives a failed post-send fetch-back) in a TTL map, and route nonce lookups through the provider only for those senders. Everybody else is served by the primary RPC. The fallback log is demoted to a warning: with the gate in place it only fires when a provider call for an actual private sender fails. Fixes #1627 Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 64 +++++++++++ bchain/coins/eth/alternativesendtx_test.go | 119 +++++++++++++++++++++ bchain/coins/eth/ethrpc.go | 10 +- bchain/coins/eth/nonce_test.go | 68 ++++++++++++ docs/env.md | 2 +- 5 files changed, 260 insertions(+), 3 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index fc9ed68d2b..6ebc8fa7ff 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -10,6 +10,7 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "github.com/golang/glog" "github.com/juju/errors" @@ -39,6 +40,8 @@ type AlternativeSendTxProvider struct { watchMempoolTxsOnce sync.Once stop chan struct{} stopOnce sync.Once + recentSenders map[ethcommon.Address]time.Time + recentSendersMux sync.Mutex } // NewAlternativeSendTxProvider creates a new alternative send tx provider if enabled @@ -58,6 +61,7 @@ func NewAlternativeSendTxProvider(network string, rpcTimeout int, mempoolTxsTime rpcTimeout: time.Duration(rpcTimeout) * time.Second, mempoolTxsTimeout: mempoolTxsTimeout, mempoolTxs: make(map[string]storedTx), + recentSenders: make(map[ethcommon.Address]time.Time), metrics: metrics, stop: make(chan struct{}), } @@ -96,6 +100,10 @@ func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, erro } } + if retErr == nil { + p.registerSuccessfulSend(hex) + } + if p.onlyAlternative && p.fetchMempoolTx { p.handleMempoolTransaction(txid) } @@ -103,6 +111,62 @@ func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, erro return txid, retErr } +// alternativeTxSender recovers the sender address from a raw transaction hex. The chain id +// needed to derive the sender is taken from the transaction itself. +func alternativeTxSender(rawTxHex string) (ethcommon.Address, error) { + var tx types.Transaction + if err := tx.UnmarshalBinary(ethcommon.FromHex(rawTxHex)); err != nil { + return ethcommon.Address{}, err + } + return types.Sender(types.LatestSignerForChainID(tx.ChainId()), &tx) +} + +// registerSuccessfulSend records the sender of a transaction accepted by an alternative +// provider so that useForNonces routes the sender's nonce lookups to that provider while +// the transaction may still be pending there. Expired entries are swept on the way; the map +// only ever holds senders of the last mempoolTxsTimeout window, so the sweep is cheap. +func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string) { + sender, err := alternativeTxSender(rawTxHex) + if err != nil { + glog.Warningf("cannot decode sender of transaction sent to alternative provider: %v", err) + return + } + now := time.Now() + p.recentSendersMux.Lock() + defer p.recentSendersMux.Unlock() + if p.recentSenders == nil { + p.recentSenders = make(map[ethcommon.Address]time.Time) + } + for addr, t := range p.recentSenders { + if now.Sub(t) > p.mempoolTxsTimeout { + delete(p.recentSenders, addr) + } + } + p.recentSenders[sender] = now +} + +// useForNonces reports whether nonce lookups for addr should be routed to the alternative +// provider. Only addresses that recently (within mempoolTxsTimeout, the same horizon at +// which Blockbook stops surfacing the tx as pending) sent a transaction through it can have +// a pending transaction the primary RPC does not know about; for everybody else the primary +// is authoritative and the provider round-trip is pure waste of its rate-limit quota. +// Accepted limitations: a restart wipes the map (exposure bounded by mempoolTxsTimeout), a +// transaction pending longer than the timeout, and private transactions submitted outside +// this Blockbook instance. +func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { + p.recentSendersMux.Lock() + defer p.recentSendersMux.Unlock() + t, found := p.recentSenders[addr] + if !found { + return false + } + if time.Since(t) > p.mempoolTxsTimeout { + delete(p.recentSenders, addr) + return false + } + return true +} + // handleMempoolTransaction handles the transaction when using only alternative providers func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (string, error) { tx, found, err := p.getTransactionFromProviders(txid) diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index 5375ebf9a2..c1d4e58d12 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "io" + "math/big" "net/http" "net/http/httptest" "strings" @@ -12,6 +13,9 @@ import ( "time" ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/prometheus/client_golang/prometheus" "github.com/trezor/blockbook/bchain" "github.com/trezor/blockbook/common" @@ -823,3 +827,118 @@ func TestAlternativeSendTxProviderGetNonces(t *testing.T) { } }) } + +// signedTestTx builds a signed raw transaction with a throwaway key and returns its hex +// encoding together with the sender address the key derives to. +func signedTestTx(t *testing.T) (string, ethcommon.Address) { + t.Helper() + key, err := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + if err != nil { + t.Fatalf("HexToECDSA() error = %v", err) + } + to := ethcommon.HexToAddress("0x3333333333333333333333333333333333333333") + tx, err := types.SignNewTx(key, types.LatestSignerForChainID(big.NewInt(1)), &types.LegacyTx{ + Nonce: 1, + GasPrice: big.NewInt(1), + Gas: 21000, + To: &to, + Value: big.NewInt(0), + }) + if err != nil { + t.Fatalf("SignNewTx() error = %v", err) + } + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("MarshalBinary() error = %v", err) + } + return hexutil.Encode(raw), crypto.PubkeyToAddress(key.PublicKey) +} + +func TestAlternativeSendTxProviderUseForNonces(t *testing.T) { + recent := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") + expired := ethcommon.HexToAddress("0x3333333333333333333333333333333333333333") + unknown := ethcommon.HexToAddress("0x4444444444444444444444444444444444444444") + provider := &AlternativeSendTxProvider{ + mempoolTxsTimeout: time.Hour, + recentSenders: map[ethcommon.Address]time.Time{ + recent: time.Now(), + expired: time.Now().Add(-2 * time.Hour), + }, + } + + if !provider.useForNonces(recent) { + t.Error("recent sender not routed to the alternative provider") + } + if provider.useForNonces(unknown) { + t.Error("unknown address routed to the alternative provider") + } + if provider.useForNonces(expired) { + t.Error("expired sender routed to the alternative provider") + } + if _, found := provider.recentSenders[expired]; found { + t.Error("expired sender not evicted on lookup") + } +} + +func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { + rawTx, sender := signedTestTx(t) + // callHttpStringResult dials a fresh client per call, so its first request always has id 1 + sendTxResponse := `{"jsonrpc":"2.0","id":1,"result":"` + testAlternativeTxID + `"}` + + t.Run("successful send records the decoded sender", func(t *testing.T) { + server := newAlternativeTxProviderTestServer(t, sendTxResponse) + // recentSenders left nil to also cover the lazy initialization on write + provider := &AlternativeSendTxProvider{urls: []string{server.URL}, mempoolTxsTimeout: time.Hour, rpcTimeout: time.Second} + + if _, err := provider.SendRawTransaction(rawTx); err != nil { + t.Fatalf("SendRawTransaction() error = %v", err) + } + if !provider.useForNonces(sender) { + t.Error("sender not routed to the alternative provider after a successful send") + } + }) + + t.Run("failed send records nothing", func(t *testing.T) { + provider := &AlternativeSendTxProvider{urls: []string{"http://127.0.0.1:1"}, mempoolTxsTimeout: time.Hour, rpcTimeout: time.Second} + + if _, err := provider.SendRawTransaction(rawTx); err == nil { + t.Fatal("expected error from unreachable provider") + } + if provider.useForNonces(sender) { + t.Error("sender recorded despite failed send") + } + }) + + t.Run("undecodable transaction records nothing", func(t *testing.T) { + server := newAlternativeTxProviderTestServer(t, sendTxResponse) + provider := &AlternativeSendTxProvider{urls: []string{server.URL}, mempoolTxsTimeout: time.Hour, rpcTimeout: time.Second} + + if _, err := provider.SendRawTransaction("0xdeadbeef"); err != nil { + t.Fatalf("SendRawTransaction() error = %v", err) + } + if len(provider.recentSenders) != 0 { + t.Errorf("recentSenders has %d entries, want 0 after undecodable raw tx", len(provider.recentSenders)) + } + }) + + t.Run("send sweeps expired senders", func(t *testing.T) { + server := newAlternativeTxProviderTestServer(t, sendTxResponse) + stale := ethcommon.HexToAddress("0x5555555555555555555555555555555555555555") + provider := &AlternativeSendTxProvider{ + urls: []string{server.URL}, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + recentSenders: map[ethcommon.Address]time.Time{stale: time.Now().Add(-2 * time.Hour)}, + } + + if _, err := provider.SendRawTransaction(rawTx); err != nil { + t.Fatalf("SendRawTransaction() error = %v", err) + } + if _, found := provider.recentSenders[stale]; found { + t.Error("expired sender not swept on send") + } + if _, found := provider.recentSenders[sender]; !found { + t.Error("new sender not recorded") + } + }) +} diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 45e7a6b2af..a9f37e93d9 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2243,6 +2243,12 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) // EthereumTypeGetNonces returns the pending account nonce and, only when withConfirmed // is set, the confirmed (latest) nonce. // +// When an alternative send-tx provider is configured, the lookup is routed through it only +// for addresses that recently sent a transaction via that provider (see useForNonces) — +// those may have a pending transaction the primary RPC does not know about. All other +// addresses go straight to the primary RPC so that the hottest API endpoint does not burn +// the provider's rate-limit quota. +// // The pending nonce (eth_getTransactionCount at the "pending" tag) counts transactions // still queued in the mempool and is the next nonce the account will use; it is always // fetched and is required, so a failure to obtain it returns an error. The confirmed nonce @@ -2256,12 +2262,12 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, withConfirmed bool) (uint64, uint64, bool, error) { ethAddress := ethcommon.BytesToAddress(addrDesc) - if b.alternativeSendTxProvider != nil { + if b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) { pending, confirmed, confirmedOK, err := b.alternativeSendTxProvider.getNonces(ethAddress, withConfirmed) if err == nil { return pending, confirmed, confirmedOK, nil } - glog.Errorf("Alternative provider failed for eth_getTransactionCount: %v, falling back to primary RPC", err) + glog.Warningf("Alternative provider failed for eth_getTransactionCount: %v, falling back to primary RPC", err) } pending, confirmed, confirmedOK, err := b.getNoncesRPC(ethAddress, withConfirmed) diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index 33d5a5942a..8ded2543a6 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -232,6 +232,74 @@ func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedFailureIsBestEffort(t } } +// newRecentSenderProvider returns an alternative provider backed by server whose recentSenders +// map holds the given addresses, i.e. useForNonces routes exactly those to the provider. +func newRecentSenderProvider(server *nonceRPCServer, senders ...ethcommon.Address) *AlternativeSendTxProvider { + recentSenders := make(map[ethcommon.Address]time.Time) + for _, sender := range senders { + recentSenders[sender] = time.Now() + } + return &AlternativeSendTxProvider{ + urls: []string{server.URL}, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + recentSenders: recentSenders, + } +} + +// TestEthereumTypeGetNonces_AlternativeProvider_SkippedForUnknownAddress is the regression test +// for #1627: an address without a recently privately-submitted tx must be served by the primary +// RPC without any round-trip to the alternative provider. +func TestEthereumTypeGetNonces_AlternativeProvider_SkippedForUnknownAddress(t *testing.T) { + server := newNonceRPCServer(t, map[string]string{"pending": "0x9"}, nil) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: newRecentSenderProvider(server)} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending != 4 { + t.Errorf("pending = %d, want 4 from the primary RPC", pending) + } + if got := server.callCount("pending"); got != 0 { + t.Errorf("alternative provider queried %d times, want 0 for an address that did not send through it", got) + } +} + +func TestEthereumTypeGetNonces_AlternativeProvider_UsedForRecentSender(t *testing.T) { + server := newNonceRPCServer(t, map[string]string{"pending": "0x9"}, nil) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + sender := ethcommon.BytesToAddress(nonceTestAddr) + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: newRecentSenderProvider(server, sender)} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending != 9 { + t.Errorf("pending = %d, want 9 from the alternative provider", pending) + } + if len(stub.queried) != 0 { + t.Errorf("primary RPC queried tags %v, want none for a recent private sender", stub.queried) + } +} + +func TestEthereumTypeGetNonces_AlternativeProvider_FallbackToPrimaryOnProviderError(t *testing.T) { + server := newNonceRPCServer(t, nil, map[string]bool{"pending": true}) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + sender := ethcommon.BytesToAddress(nonceTestAddr) + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: newRecentSenderProvider(server, sender)} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("provider failure must fall back to the primary RPC, got error: %v", err) + } + if pending != 4 { + t.Errorf("pending = %d, want 4 from the primary RPC fallback", pending) + } +} + func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedDecodeFailureIsBestEffort(t *testing.T) { // sequential-fallback counterpart of the batched decode-failure case: an unparsable latest // result must be best-effort (pending returned, confirmedOK=false, no error) diff --git a/docs/env.md b/docs/env.md index 56f2a5ab23..45a36eace5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -87,7 +87,7 @@ Blockbook reads these from its process environment. When installed from the Debi - `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all (a whitespace-only value included), is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. +- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. - `_ALTERNATIVE_SENDTX_ONLY` - Set to `TRUE` to use only the alternative send transaction provider and avoid fallback to the normal backend RPC if alternative submission fails. From 301865bf4e38245547074d77c7b575aeea68786c Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:19:55 +0200 Subject: [PATCH 082/303] fix(eth): route nonce reads to the provider that accepted the send A broadcast to alternative providers succeeds if ANY configured URL accepts the transaction, but nonce lookups always queried urls[0]. When the first provider rejects a broadcast that a later one accepts, the sender's nonce reads would hit a provider that never saw the transaction and report a stale pending nonce. Record the accepting URL alongside the sender and route that sender's eth_getTransactionCount lookups to it, falling back to the first configured URL when unknown. Single-URL deployments are unaffected. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 67 +++++++++++++++------- bchain/coins/eth/alternativesendtx_test.go | 38 ++++++++++-- bchain/coins/eth/nonce_test.go | 4 +- 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 6ebc8fa7ff..b9ea5205ef 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -23,6 +23,13 @@ type storedTx struct { time uint32 } +// recentSender records when an address last successfully sent a transaction through an +// alternative provider and which provider URL accepted it. +type recentSender struct { + time time.Time + url string +} + const alternativeMempoolTxCheckPeriod = time.Minute // AlternativeSendTxProvider handles sending transactions to alternative providers @@ -40,7 +47,7 @@ type AlternativeSendTxProvider struct { watchMempoolTxsOnce sync.Once stop chan struct{} stopOnce sync.Once - recentSenders map[ethcommon.Address]time.Time + recentSenders map[ethcommon.Address]recentSender recentSendersMux sync.Mutex } @@ -61,7 +68,7 @@ func NewAlternativeSendTxProvider(network string, rpcTimeout int, mempoolTxsTime rpcTimeout: time.Duration(rpcTimeout) * time.Second, mempoolTxsTimeout: mempoolTxsTimeout, mempoolTxs: make(map[string]storedTx), - recentSenders: make(map[ethcommon.Address]time.Time), + recentSenders: make(map[ethcommon.Address]recentSender), metrics: metrics, stop: make(chan struct{}), } @@ -89,10 +96,14 @@ func (p *AlternativeSendTxProvider) SetupMempool(mempool *bchain.MempoolEthereum func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, error) { var txid string var retErr error + var acceptedURL string for i := range p.urls { r, err := p.callHttpStringResult(p.urls[i], "eth_sendRawTransaction", hex) glog.Infof("eth_sendRawTransaction to %s, txid %s", p.urls[i], r) + if err == nil && acceptedURL == "" { + acceptedURL = p.urls[i] + } // set success return value; or error only if there was no previous success if err == nil || len(txid) == 0 { txid = r @@ -101,7 +112,7 @@ func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, erro } if retErr == nil { - p.registerSuccessfulSend(hex) + p.registerSuccessfulSend(hex, acceptedURL) } if p.onlyAlternative && p.fetchMempoolTx { @@ -123,9 +134,11 @@ func alternativeTxSender(rawTxHex string) (ethcommon.Address, error) { // registerSuccessfulSend records the sender of a transaction accepted by an alternative // provider so that useForNonces routes the sender's nonce lookups to that provider while -// the transaction may still be pending there. Expired entries are swept on the way; the map -// only ever holds senders of the last mempoolTxsTimeout window, so the sweep is cheap. -func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string) { +// the transaction may still be pending there. A broadcast succeeds if ANY configured URL +// accepts it, so the accepting URL is recorded too - it is the one provider guaranteed to +// know the transaction (see nonceURL). Expired entries are swept on the way; the map only +// ever holds senders of the last mempoolTxsTimeout window, so the sweep is cheap. +func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acceptedURL string) { sender, err := alternativeTxSender(rawTxHex) if err != nil { glog.Warningf("cannot decode sender of transaction sent to alternative provider: %v", err) @@ -135,14 +148,14 @@ func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string) { p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() if p.recentSenders == nil { - p.recentSenders = make(map[ethcommon.Address]time.Time) + p.recentSenders = make(map[ethcommon.Address]recentSender) } - for addr, t := range p.recentSenders { - if now.Sub(t) > p.mempoolTxsTimeout { + for addr, s := range p.recentSenders { + if now.Sub(s.time) > p.mempoolTxsTimeout { delete(p.recentSenders, addr) } } - p.recentSenders[sender] = now + p.recentSenders[sender] = recentSender{time: now, url: acceptedURL} } // useForNonces reports whether nonce lookups for addr should be routed to the alternative @@ -156,11 +169,11 @@ func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string) { func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() - t, found := p.recentSenders[addr] + s, found := p.recentSenders[addr] if !found { return false } - if time.Since(t) > p.mempoolTxsTimeout { + if time.Since(s.time) > p.mempoolTxsTimeout { delete(p.recentSenders, addr) return false } @@ -553,18 +566,32 @@ func (p *AlternativeSendTxProvider) callHttpStringResult(url string, rpcMethod s return result, nil } -// getNonces returns the pending account nonce from the first configured alternative -// provider, plus the confirmed (latest) nonce when withConfirmed is set. When both are -// requested they are fetched in a single JSON-RPC batch round-trip; otherwise only the -// pending nonce is requested. The confirmed nonce is best-effort: a failed latest lookup -// yields confirmedOK=false (not an error) so the caller can omit it. An error is returned -// only when the required pending nonce cannot be obtained. +// nonceURL returns the provider URL to use for addr's nonce lookup: the URL that accepted +// the sender's most recent transaction when known (a broadcast succeeds if ANY configured +// provider accepts it, so the first URL may never have seen the transaction), falling back +// to the first configured URL. +func (p *AlternativeSendTxProvider) nonceURL(addr ethcommon.Address) string { + p.recentSendersMux.Lock() + defer p.recentSendersMux.Unlock() + if s, found := p.recentSenders[addr]; found && s.url != "" { + return s.url + } + return p.urls[0] +} + +// getNonces returns the pending account nonce from the alternative provider that accepted +// the sender's most recent transaction (see nonceURL), plus the confirmed (latest) nonce +// when withConfirmed is set. When both are requested they are fetched in a single JSON-RPC +// batch round-trip; otherwise only the pending nonce is requested. The confirmed nonce is +// best-effort: a failed latest lookup yields confirmedOK=false (not an error) so the caller +// can omit it. An error is returned only when the required pending nonce cannot be obtained. func (p *AlternativeSendTxProvider) getNonces(addr ethcommon.Address, withConfirmed bool) (uint64, uint64, bool, error) { if len(p.urls) == 0 { return 0, 0, false, errors.New("no alternative provider url configured") } + url := p.nonceURL(addr) if !withConfirmed { - pendingHex, err := p.callHttpStringResult(p.urls[0], "eth_getTransactionCount", addr, "pending") + pendingHex, err := p.callHttpStringResult(url, "eth_getTransactionCount", addr, "pending") if err != nil { return 0, 0, false, err } @@ -576,7 +603,7 @@ func (p *AlternativeSendTxProvider) getNonces(addr ethcommon.Address, withConfir } ctx, cancel := context.WithTimeout(context.Background(), p.rpcTimeout) defer cancel() - client, err := rpc.DialContext(ctx, p.urls[0]) + client, err := rpc.DialContext(ctx, url) if err != nil { return 0, 0, false, err } diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index c1d4e58d12..82045a6929 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -860,9 +860,9 @@ func TestAlternativeSendTxProviderUseForNonces(t *testing.T) { unknown := ethcommon.HexToAddress("0x4444444444444444444444444444444444444444") provider := &AlternativeSendTxProvider{ mempoolTxsTimeout: time.Hour, - recentSenders: map[ethcommon.Address]time.Time{ - recent: time.Now(), - expired: time.Now().Add(-2 * time.Hour), + recentSenders: map[ethcommon.Address]recentSender{ + recent: {time: time.Now()}, + expired: {time: time.Now().Add(-2 * time.Hour)}, }, } @@ -896,6 +896,9 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { if !provider.useForNonces(sender) { t.Error("sender not routed to the alternative provider after a successful send") } + if s := provider.recentSenders[sender]; s.url != server.URL { + t.Errorf("recorded accepting url = %q, want %q", s.url, server.URL) + } }) t.Run("failed send records nothing", func(t *testing.T) { @@ -921,6 +924,33 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { } }) + t.Run("nonce reads follow the accepting provider", func(t *testing.T) { + // urls[0] is unreachable: the broadcast succeeds only through the second provider, so + // nonce reads for the sender must go there too - urls[0] never saw the transaction. + // The nonce server answers eth_sendRawTransaction via the empty tag and + // eth_getTransactionCount via the "pending" tag. + server := newNonceRPCServer(t, map[string]string{"": testAlternativeTxID, "pending": "0x9"}, nil) + provider := &AlternativeSendTxProvider{ + urls: []string{"http://127.0.0.1:1", server.URL}, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + } + + if _, err := provider.SendRawTransaction(rawTx); err != nil { + t.Fatalf("SendRawTransaction() error = %v", err) + } + pending, _, _, err := provider.getNonces(sender, false) + if err != nil { + t.Fatalf("getNonces() error = %v", err) + } + if pending != 9 { + t.Errorf("pending = %d, want 9 from the provider that accepted the send", pending) + } + if got := server.callCount("pending"); got != 1 { + t.Errorf("accepting provider queried %d times for pending, want 1", got) + } + }) + t.Run("send sweeps expired senders", func(t *testing.T) { server := newAlternativeTxProviderTestServer(t, sendTxResponse) stale := ethcommon.HexToAddress("0x5555555555555555555555555555555555555555") @@ -928,7 +958,7 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { urls: []string{server.URL}, mempoolTxsTimeout: time.Hour, rpcTimeout: time.Second, - recentSenders: map[ethcommon.Address]time.Time{stale: time.Now().Add(-2 * time.Hour)}, + recentSenders: map[ethcommon.Address]recentSender{stale: {time: time.Now().Add(-2 * time.Hour)}}, } if _, err := provider.SendRawTransaction(rawTx); err != nil { diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index 8ded2543a6..d971639080 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -235,9 +235,9 @@ func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedFailureIsBestEffort(t // newRecentSenderProvider returns an alternative provider backed by server whose recentSenders // map holds the given addresses, i.e. useForNonces routes exactly those to the provider. func newRecentSenderProvider(server *nonceRPCServer, senders ...ethcommon.Address) *AlternativeSendTxProvider { - recentSenders := make(map[ethcommon.Address]time.Time) + recentSenders := make(map[ethcommon.Address]recentSender) for _, sender := range senders { - recentSenders[sender] = time.Now() + recentSenders[sender] = recentSender{time: time.Now(), url: server.URL} } return &AlternativeSendTxProvider{ urls: []string{server.URL}, From cb5101711d1b526c5b0ed82a03feb25feb1cc6be Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:22:28 +0200 Subject: [PATCH 083/303] fix(eth): release nonce routing when a sender's private txs settle The nonce-routing entry recorded at send time survived for the full alternativeMempoolTxTimeout even after the sender's transactions were mined, superseded or replaced. Address polling therefore kept consuming the alternative provider's rate-limit quota for minutes after nothing private remained pending. Drop the sender's entry when their last cached transaction leaves the alternative mempool cache. RemoveTransaction is the choke point every eviction funnels through (reconcile, RBF replacement, mined-tx removal via the mempool callback); the read-path TTL eviction in GetTransaction now goes through it too. The entry is kept when it is newer than the evicted transaction, so a follow-up send whose post-send fetch-back failed does not lose its routing. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 52 +++++++++++++--- bchain/coins/eth/alternativesendtx_test.go | 71 ++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index b9ea5205ef..b102cc60b2 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -163,9 +163,10 @@ func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acce // which Blockbook stops surfacing the tx as pending) sent a transaction through it can have // a pending transaction the primary RPC does not know about; for everybody else the primary // is authoritative and the provider round-trip is pure waste of its rate-limit quota. -// Accepted limitations: a restart wipes the map (exposure bounded by mempoolTxsTimeout), a -// transaction pending longer than the timeout, and private transactions submitted outside -// this Blockbook instance. +// Senders whose cached transactions have all settled are released before the timeout (see +// releaseRecentSender). Accepted limitations: a restart wipes the map (exposure bounded by +// mempoolTxsTimeout), a transaction pending longer than the timeout, and private +// transactions submitted outside this Blockbook instance. func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() @@ -180,6 +181,25 @@ func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { return true } +// releaseRecentSender drops the sender's nonce-routing entry once its last cached +// transaction left the alternative mempool cache (mined, superseded, replaced or timed +// out), so address polling stops consuming the alternative provider's quota as soon as no +// private transaction remains pending. The entry is kept when it is newer than the evicted +// transaction: the sender submitted again since then and the newer transaction may not have +// a cache entry of its own (e.g. when the post-send fetch-back failed). +func (p *AlternativeSendTxProvider) releaseRecentSender(sender ethcommon.Address, evictedTxUnix uint32) { + p.recentSendersMux.Lock() + defer p.recentSendersMux.Unlock() + s, found := p.recentSenders[sender] + if !found { + return + } + if s.time.After(time.Unix(int64(evictedTxUnix), 0).Add(time.Second)) { + return + } + delete(p.recentSenders, sender) +} + // handleMempoolTransaction handles the transaction when using only alternative providers func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (string, error) { tx, found, err := p.getTransactionFromProviders(txid) @@ -238,9 +258,7 @@ func (p *AlternativeSendTxProvider) GetTransaction(txid string) (*bchain.RpcTran if found { if time.Unix(int64(storedTx.time), 0).Before(time.Now().Add(-p.mempoolTxsTimeout)) { - p.mempoolTxsMux.Lock() - delete(p.mempoolTxs, txid) - p.mempoolTxsMux.Unlock() + p.RemoveTransaction(txid) // the same staleness timeout the reconcile loop applies, just reached on the read path // first; record it so the timeout counter and residence histogram do not undercount // entries read after expiry but before the next reconcile cycle evicts them. @@ -515,15 +533,35 @@ func (p *AlternativeSendTxProvider) removeMempoolTx(txid string) { p.RemoveTransaction(txid) } -// RemoveTransaction removes a transaction from alternative mempool cache +// RemoveTransaction removes a transaction from alternative mempool cache. When the removed +// transaction was the sender's last cached one, the sender's nonce-routing entry is released +// as well (see releaseRecentSender) so address polling stops hitting the alternative provider +// once nothing private remains pending. func (p *AlternativeSendTxProvider) RemoveTransaction(txid string) { if !p.fetchMempoolTx { return } p.mempoolTxsMux.Lock() + removedTx, found := p.mempoolTxs[txid] delete(p.mempoolTxs, txid) + senderSettled := false + var sender ethcommon.Address + if found && removedTx.tx != nil && removedTx.tx.From != "" { + sender = ethcommon.HexToAddress(removedTx.tx.From) + senderSettled = true + for _, storedTx := range p.mempoolTxs { + if storedTx.tx != nil && ethcommon.HexToAddress(storedTx.tx.From) == sender { + senderSettled = false + break + } + } + } p.mempoolTxsMux.Unlock() + + if senderSettled { + p.releaseRecentSender(sender, removedTx.time) + } } // UseOnlyAlternativeProvider returns true if only alternative providers should be used diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index 82045a6929..619bf98b0c 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -972,3 +972,74 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { } }) } + +func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) { + sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") + cachedTx := func(nonce string, added time.Time) storedTx { + return storedTx{ + tx: &bchain.RpcTransaction{ + Hash: testAlternativeTxID, + From: "0x2222222222222222222222222222222222222222", + AccountNonce: nonce, + }, + time: uint32(added.Unix()), + } + } + makeProvider := func(senderTime time.Time, txs map[string]storedTx) *AlternativeSendTxProvider { + return &AlternativeSendTxProvider{ + fetchMempoolTx: true, + mempoolTxsTimeout: time.Hour, + mempoolTxs: txs, + recentSenders: map[ethcommon.Address]recentSender{sender: {time: senderTime}}, + } + } + + t.Run("evicting the last cached tx releases the sender", func(t *testing.T) { + now := time.Now() + provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now)}) + + provider.RemoveTransaction(testAlternativeTxID) + + if provider.useForNonces(sender) { + t.Error("sender still routed to the alternative provider after its last cached tx settled") + } + }) + + t.Run("another cached tx from the sender keeps the entry", func(t *testing.T) { + now := time.Now() + provider := makeProvider(now, map[string]storedTx{ + testAlternativeTxID: cachedTx("0x1", now), + testAlternativeSecondTxID: cachedTx("0x2", now), + }) + + provider.RemoveTransaction(testAlternativeTxID) + + if !provider.useForNonces(sender) { + t.Error("sender released while another of its txs is still cached") + } + }) + + t.Run("a newer send since the evicted tx keeps the entry", func(t *testing.T) { + // the sender submitted again after the evicted tx was cached (possibly without a cache + // entry of its own, e.g. when the post-send fetch-back failed) - the entry must survive + now := time.Now() + provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now.Add(-10*time.Minute))}) + + provider.RemoveTransaction(testAlternativeTxID) + + if !provider.useForNonces(sender) { + t.Error("sender released although a newer private send may still be pending") + } + }) + + t.Run("unknown txid releases nothing", func(t *testing.T) { + now := time.Now() + provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now)}) + + provider.RemoveTransaction("0xdoesnotexist") + + if !provider.useForNonces(sender) { + t.Error("sender released by removal of an unknown txid") + } + }) +} From cfbe232df5c419d59dca21d78ea28b27b8a6f020 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:23:47 +0200 Subject: [PATCH 084/303] fix(eth): raise fallback pending nonce to the alternative mempool floor When the alternative-provider nonce lookup fails for a gated sender, the fallback answer comes from the primary RPC, which cannot see the sender's privately submitted pending transactions. Returning it as-is would let a wallet build its next transaction on a stale nonce and accidentally replace the in-flight private transaction. Raise the fallback answer to the floor implied by the alternative mempool cache (highest cached account nonce of the sender + 1), so the reported pending nonce never contradicts the pending view Blockbook itself exposes for those transactions. Without cached transactions there is nothing to raise by and the primary answer stands, which is correct for the common case of a sender whose transaction was just mined. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 26 ++++++++++++++++++++ bchain/coins/eth/alternativesendtx_test.go | 23 ++++++++++++++++++ bchain/coins/eth/ethrpc.go | 17 +++++++++++-- bchain/coins/eth/nonce_test.go | 28 ++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index b102cc60b2..8412c6f195 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -200,6 +200,32 @@ func (p *AlternativeSendTxProvider) releaseRecentSender(sender ethcommon.Address delete(p.recentSenders, sender) } +// pendingNonceFloor returns the lowest pending nonce consistent with the private +// transactions the alternative mempool cache holds for addr (highest cached account nonce +// + 1), and whether any such transaction exists. Blockbook exposes these cached txs as +// pending, so reporting a pending nonce below the floor would contradict its own view and +// lead a wallet to reuse the nonce of an in-flight private transaction. +func (p *AlternativeSendTxProvider) pendingNonceFloor(addr ethcommon.Address) (uint64, bool) { + p.mempoolTxsMux.Lock() + defer p.mempoolTxsMux.Unlock() + var floor uint64 + var found bool + for _, storedTx := range p.mempoolTxs { + if storedTx.tx == nil || ethcommon.HexToAddress(storedTx.tx.From) != addr { + continue + } + nonce, err := hexutil.DecodeUint64(storedTx.tx.AccountNonce) + if err != nil { + continue + } + if nonce+1 > floor { + floor = nonce + 1 + found = true + } + } + return floor, found +} + // handleMempoolTransaction handles the transaction when using only alternative providers func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (string, error) { tx, found, err := p.getTransactionFromProviders(txid) diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index 619bf98b0c..105f2903f5 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -1043,3 +1043,26 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) } }) } + +func TestAlternativeSendTxProviderPendingNonceFloor(t *testing.T) { + sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") + provider := &AlternativeSendTxProvider{ + mempoolTxs: map[string]storedTx{ + "0x01": {tx: &bchain.RpcTransaction{From: "0x2222222222222222222222222222222222222222", AccountNonce: "0x4"}}, + "0x02": {tx: &bchain.RpcTransaction{From: "0x2222222222222222222222222222222222222222", AccountNonce: "0x7"}}, + "0x03": {tx: &bchain.RpcTransaction{From: "0x2222222222222222222222222222222222222222", AccountNonce: "0xZZ"}}, // unparsable, skipped + "0x04": {tx: &bchain.RpcTransaction{From: "0x3333333333333333333333333333333333333333", AccountNonce: "0x9"}}, + }, + } + + floor, found := provider.pendingNonceFloor(sender) + if !found { + t.Fatal("no floor found for sender with cached txs") + } + if floor != 8 { + t.Errorf("floor = %d, want 8 (highest cached nonce 0x7 + 1)", floor) + } + if _, found := provider.pendingNonceFloor(ethcommon.HexToAddress("0x4444444444444444444444444444444444444444")); found { + t.Error("floor found for address without cached txs") + } +} diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index a9f37e93d9..155c4d0f03 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2247,7 +2247,10 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) // for addresses that recently sent a transaction via that provider (see useForNonces) — // those may have a pending transaction the primary RPC does not know about. All other // addresses go straight to the primary RPC so that the hottest API endpoint does not burn -// the provider's rate-limit quota. +// the provider's rate-limit quota. When the provider lookup for such a sender fails, the +// primary RPC answer is used as fallback, raised to the floor implied by the alternative +// mempool cache (see pendingNonceFloor) so it never contradicts Blockbook's own pending +// view of the sender's private transactions. // // The pending nonce (eth_getTransactionCount at the "pending" tag) counts transactions // still queued in the mempool and is the next nonce the account will use; it is always @@ -2262,7 +2265,8 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, withConfirmed bool) (uint64, uint64, bool, error) { ethAddress := ethcommon.BytesToAddress(addrDesc) - if b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) { + useAlternative := b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) + if useAlternative { pending, confirmed, confirmedOK, err := b.alternativeSendTxProvider.getNonces(ethAddress, withConfirmed) if err == nil { return pending, confirmed, confirmedOK, nil @@ -2275,6 +2279,15 @@ func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, w glog.Errorf("Primary RPC failed for eth_getTransactionCount: %v", err) return 0, 0, false, err } + if useAlternative { + // The primary RPC cannot see the sender's private pending transactions. Raise its + // answer to the floor implied by the alternative mempool cache so the fallback never + // reports a pending nonce that a wallet could use to accidentally replace the + // sender's in-flight private transaction. + if floor, ok := b.alternativeSendTxProvider.pendingNonceFloor(ethAddress); ok && floor > pending { + pending = floor + } + } return pending, confirmed, confirmedOK, nil } diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index d971639080..f1bfc75df6 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -286,6 +286,7 @@ func TestEthereumTypeGetNonces_AlternativeProvider_UsedForRecentSender(t *testin } func TestEthereumTypeGetNonces_AlternativeProvider_FallbackToPrimaryOnProviderError(t *testing.T) { + // without any cached private tx there is no floor to apply, so the primary answer stands server := newNonceRPCServer(t, nil, map[string]bool{"pending": true}) stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} sender := ethcommon.BytesToAddress(nonceTestAddr) @@ -300,6 +301,33 @@ func TestEthereumTypeGetNonces_AlternativeProvider_FallbackToPrimaryOnProviderEr } } +func TestEthereumTypeGetNonces_AlternativeProvider_FallbackRaisedToCacheFloor(t *testing.T) { + // the provider lookup fails and the primary answers 4, but the alternative mempool cache + // still holds a pending private tx with nonce 0x7 from the sender - the fallback must + // report at least 8, otherwise a wallet building on the primary answer would replace the + // sender's in-flight private transaction + server := newNonceRPCServer(t, nil, map[string]bool{"pending": true}) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + sender := ethcommon.BytesToAddress(nonceTestAddr) + provider := newRecentSenderProvider(server, sender) + provider.fetchMempoolTx = true + provider.mempoolTxs = map[string]storedTx{ + testAlternativeTxID: { + tx: &bchain.RpcTransaction{Hash: testAlternativeTxID, From: sender.Hex(), AccountNonce: "0x7"}, + time: uint32(time.Now().Unix()), + }, + } + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: provider} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("provider failure must fall back to the primary RPC, got error: %v", err) + } + if pending != 8 { + t.Errorf("pending = %d, want 8 (cache floor: highest cached nonce 0x7 + 1)", pending) + } +} + func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedDecodeFailureIsBestEffort(t *testing.T) { // sequential-fallback counterpart of the batched decode-failure case: an unparsable latest // result must be best-effort (pending returned, confirmedOK=false, no error) From 218fd90767856c22975829ecaa4785895664b153 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:51:37 +0200 Subject: [PATCH 085/303] fix(eth): raise successful provider nonce answers to the cache floor too The pending-nonce floor derived from the alternative mempool cache was only applied on the primary RPC fallback path. But the provider's own answer can also fall below Blockbook's advertised pending view: Blink-style relays stop counting a still-pending transaction at the pending tag while Blockbook deliberately keeps exposing it until the cache timeout (see reconcileMempoolTxs). Raise the successful provider answer to the same floor, making the invariant "never report a pending nonce below our own pending view" unconditional at the cost of one scan of the (tiny) cache per gated request. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 9 ++++++++ bchain/coins/eth/ethrpc.go | 26 ++++++++++++----------- bchain/coins/eth/nonce_test.go | 30 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 8412c6f195..803070db94 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -226,6 +226,15 @@ func (p *AlternativeSendTxProvider) pendingNonceFloor(addr ethcommon.Address) (u return floor, found } +// raiseToPendingFloor returns pending, raised to pendingNonceFloor(addr) when the cache +// holds a higher-nonce private transaction for the address. +func (p *AlternativeSendTxProvider) raiseToPendingFloor(addr ethcommon.Address, pending uint64) uint64 { + if floor, found := p.pendingNonceFloor(addr); found && floor > pending { + return floor + } + return pending +} + // handleMempoolTransaction handles the transaction when using only alternative providers func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (string, error) { tx, found, err := p.getTransactionFromProviders(txid) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 155c4d0f03..47cbd02498 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2247,10 +2247,10 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) // for addresses that recently sent a transaction via that provider (see useForNonces) — // those may have a pending transaction the primary RPC does not know about. All other // addresses go straight to the primary RPC so that the hottest API endpoint does not burn -// the provider's rate-limit quota. When the provider lookup for such a sender fails, the -// primary RPC answer is used as fallback, raised to the floor implied by the alternative -// mempool cache (see pendingNonceFloor) so it never contradicts Blockbook's own pending -// view of the sender's private transactions. +// the provider's rate-limit quota. For gated senders the pending answer - whether from the +// provider or from the primary RPC fallback - is raised to the floor implied by the +// alternative mempool cache (see pendingNonceFloor) so it never contradicts Blockbook's +// own pending view of the sender's private transactions. // // The pending nonce (eth_getTransactionCount at the "pending" tag) counts transactions // still queued in the mempool and is the next nonce the account will use; it is always @@ -2269,7 +2269,12 @@ func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, w if useAlternative { pending, confirmed, confirmedOK, err := b.alternativeSendTxProvider.getNonces(ethAddress, withConfirmed) if err == nil { - return pending, confirmed, confirmedOK, nil + // Even the provider's own answer can fall below Blockbook's advertised pending + // view: Blink-style relays stop counting a still-pending tx at the pending tag + // while Blockbook keeps exposing it until the cache timeout (see + // reconcileMempoolTxs). Raise to the cache floor so the invariant holds + // unconditionally. + return b.alternativeSendTxProvider.raiseToPendingFloor(ethAddress, pending), confirmed, confirmedOK, nil } glog.Warningf("Alternative provider failed for eth_getTransactionCount: %v, falling back to primary RPC", err) } @@ -2280,13 +2285,10 @@ func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, w return 0, 0, false, err } if useAlternative { - // The primary RPC cannot see the sender's private pending transactions. Raise its - // answer to the floor implied by the alternative mempool cache so the fallback never - // reports a pending nonce that a wallet could use to accidentally replace the - // sender's in-flight private transaction. - if floor, ok := b.alternativeSendTxProvider.pendingNonceFloor(ethAddress); ok && floor > pending { - pending = floor - } + // The primary RPC cannot see the sender's private pending transactions at all - + // without the raise, a wallet building on the fallback answer could reuse the nonce + // of an in-flight private transaction and replace it. + pending = b.alternativeSendTxProvider.raiseToPendingFloor(ethAddress, pending) } return pending, confirmed, confirmedOK, nil } diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index f1bfc75df6..e9510413ea 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -328,6 +328,36 @@ func TestEthereumTypeGetNonces_AlternativeProvider_FallbackRaisedToCacheFloor(t } } +func TestEthereumTypeGetNonces_AlternativeProvider_ProviderAnswerRaisedToCacheFloor(t *testing.T) { + // Blink-style relays stop counting a still-pending tx at the pending tag while Blockbook + // keeps exposing it until the cache timeout: the provider answers 4 although the cache + // holds a pending private tx with nonce 0x7 from the sender - the answer must be raised + // to 8 so it never contradicts Blockbook's own pending view + server := newNonceRPCServer(t, map[string]string{"pending": "0x4"}, nil) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x2"}} + sender := ethcommon.BytesToAddress(nonceTestAddr) + provider := newRecentSenderProvider(server, sender) + provider.fetchMempoolTx = true + provider.mempoolTxs = map[string]storedTx{ + testAlternativeTxID: { + tx: &bchain.RpcTransaction{Hash: testAlternativeTxID, From: sender.Hex(), AccountNonce: "0x7"}, + time: uint32(time.Now().Unix()), + }, + } + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: provider} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending != 8 { + t.Errorf("pending = %d, want 8 (cache floor over the provider answer)", pending) + } + if len(stub.queried) != 0 { + t.Errorf("primary RPC queried tags %v, want none for a successful provider lookup", stub.queried) + } +} + func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedDecodeFailureIsBestEffort(t *testing.T) { // sequential-fallback counterpart of the batched decode-failure case: an unparsable latest // result must be best-effort (pending returned, confirmedOK=false, no error) From 3c62518f30ccc6515d479bb650e61f3b0bedf154 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:51:37 +0200 Subject: [PATCH 086/303] test(eth): cover the batched nonce path against the accepting provider The accepting-provider routing test exercised only the pending-only branch of getNonces; the withConfirmed batch branch dials the URL itself rather than going through callHttpStringResult and was only ever tested against urls[0]. Add the batched variant so both branches are pinned to the provider that accepted the send. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx_test.go | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index 105f2903f5..ac8d545111 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -951,6 +951,29 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { } }) + t.Run("batched nonce read follows the accepting provider", func(t *testing.T) { + // withConfirmed=true exercises the batch branch of getNonces, which dials the url + // itself instead of going through callHttpStringResult - it must pick the accepting + // provider the same way as the pending-only branch + server := newNonceRPCServer(t, map[string]string{"": testAlternativeTxID, "pending": "0x9", "latest": "0x5"}, nil) + provider := &AlternativeSendTxProvider{ + urls: []string{"http://127.0.0.1:1", server.URL}, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + } + + if _, err := provider.SendRawTransaction(rawTx); err != nil { + t.Fatalf("SendRawTransaction() error = %v", err) + } + pending, confirmed, confirmedOK, err := provider.getNonces(sender, true) + if err != nil { + t.Fatalf("getNonces() error = %v", err) + } + if pending != 9 || confirmed != 5 || !confirmedOK { + t.Errorf("got (pending=%d confirmed=%d ok=%v), want (9 5 true) from the provider that accepted the send", pending, confirmed, confirmedOK) + } + }) + t.Run("send sweeps expired senders", func(t *testing.T) { server := newAlternativeTxProviderTestServer(t, sendTxResponse) stale := ethcommon.HexToAddress("0x5555555555555555555555555555555555555555") From 9832e46ed616b2a74e48f5ac369d576d79887a3f Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 16:51:53 +0200 Subject: [PATCH 087/303] docs(eth): document per-instance nonce routing and release edge window The nonce-routing gate lives in process memory: in deployments with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider. Websocket clients are naturally sticky to one instance, but plain REST polling may hit another instance and receive the public backend's view; a restart clears the state as well. Spell this out in the env docs for operators. Also document the accepted premature-release window in releaseRecentSender (uncached sibling transaction plus sub-second timing). Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 4 ++++ docs/env.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 803070db94..0f574d96f3 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -187,6 +187,10 @@ func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { // private transaction remains pending. The entry is kept when it is newer than the evicted // transaction: the sender submitted again since then and the newer transaction may not have // a cache entry of its own (e.g. when the post-send fetch-back failed). +// Residual risk, accepted: a sibling transaction that never got a cache entry (failed +// fetch-back) can still cause a premature release - a newer send landing within the +// one-second comparison slack, or an uncached older send outliving an evicted newer one. +// Both require that exceptional-path failure combined with unusual timing. func (p *AlternativeSendTxProvider) releaseRecentSender(sender ethcommon.Address, evictedTxUnix uint32) { p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() diff --git a/docs/env.md b/docs/env.md index 45a36eace5..7a1ea5a3e6 100644 --- a/docs/env.md +++ b/docs/env.md @@ -87,7 +87,7 @@ Blockbook reads these from its process environment. When installed from the Debi - `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all (a whitespace-only value included), is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. +- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. The routing state is kept in process memory: with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider (websocket clients are naturally sticky to one instance; REST pollers may hit another and receive the public backend's view), and a restart clears the state. - `_ALTERNATIVE_SENDTX_ONLY` - Set to `TRUE` to use only the alternative send transaction provider and avoid fallback to the normal backend RPC if alternative submission fails. From b644e70bbf7ab1524d633c5e598e115c3978cf2b Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:20:46 +0200 Subject: [PATCH 088/303] fix(eth): order sender releases by send generation instead of wall clock The release guard compared the sender's last-send time against the evicted transaction's whole-second timestamp with a one-second slack. Two sends landing within the same second were indistinguishable: when the newer one's post-send fetch-back failed (no cache entry of its own), evicting the older transaction released the sender's routing while the newer private transaction was still pending, sending subsequent nonce reads to the stale primary RPC. Replace the timestamp heuristic with a monotonic send-generation counter: every successful send bumps it, cached transactions are stamped with the sender's generation at caching time, and the routing entry is released only when its generation is not newer than the evicted transaction's. Sub-second send sequences are now ordered precisely; the only remaining (accepted, documented) gap is an uncached send OLDER than the evicted one, which mined evictions largely exclude via nonce sequencing. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 41 ++++++++++++----- bchain/coins/eth/alternativesendtx_test.go | 51 +++++++++++++++------- 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 0f574d96f3..88c97793b5 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -21,6 +21,7 @@ import ( type storedTx struct { tx *bchain.RpcTransaction time uint32 + gen uint64 // sender's send generation when the tx was cached, orders it against later sends } // recentSender records when an address last successfully sent a transaction through an @@ -28,6 +29,7 @@ type storedTx struct { type recentSender struct { time time.Time url string + gen uint64 // monotonic send generation, orders the send against cached-tx evictions } const alternativeMempoolTxCheckPeriod = time.Minute @@ -48,6 +50,7 @@ type AlternativeSendTxProvider struct { stop chan struct{} stopOnce sync.Once recentSenders map[ethcommon.Address]recentSender + sendGeneration uint64 // counts successful sends; guarded by recentSendersMux recentSendersMux sync.Mutex } @@ -155,7 +158,18 @@ func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acce delete(p.recentSenders, addr) } } - p.recentSenders[sender] = recentSender{time: now, url: acceptedURL} + p.sendGeneration++ + p.recentSenders[sender] = recentSender{time: now, url: acceptedURL, gen: p.sendGeneration} +} + +// currentSenderGeneration returns the send generation of the sender's most recent +// successful send. Cached transactions are stamped with it so releaseRecentSender can +// order evictions against later sends precisely, without relying on wall-clock resolution +// (two sends can land within the same whole-second storedTx timestamp). +func (p *AlternativeSendTxProvider) currentSenderGeneration(from string) uint64 { + p.recentSendersMux.Lock() + defer p.recentSendersMux.Unlock() + return p.recentSenders[ethcommon.HexToAddress(from)].gen } // useForNonces reports whether nonce lookups for addr should be routed to the alternative @@ -184,21 +198,22 @@ func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { // releaseRecentSender drops the sender's nonce-routing entry once its last cached // transaction left the alternative mempool cache (mined, superseded, replaced or timed // out), so address polling stops consuming the alternative provider's quota as soon as no -// private transaction remains pending. The entry is kept when it is newer than the evicted -// transaction: the sender submitted again since then and the newer transaction may not have -// a cache entry of its own (e.g. when the post-send fetch-back failed). -// Residual risk, accepted: a sibling transaction that never got a cache entry (failed -// fetch-back) can still cause a premature release - a newer send landing within the -// one-second comparison slack, or an uncached older send outliving an evicted newer one. -// Both require that exceptional-path failure combined with unusual timing. -func (p *AlternativeSendTxProvider) releaseRecentSender(sender ethcommon.Address, evictedTxUnix uint32) { +// private transaction remains pending. The entry is kept when its send generation is newer +// than the evicted transaction's: the sender submitted again after that transaction was +// cached (even within the same wall-clock second) and the newer transaction may not have a +// cache entry of its own (e.g. when the post-send fetch-back failed). +// Residual risk, accepted: an UNCACHED send OLDER than the evicted transaction cannot be +// represented and loses its routing with the release. It needs a failed fetch-back, and +// mined evictions largely exclude it anyway - the sender's nonces are sequential, so an +// older transaction cannot still be pending once a newer one mined. +func (p *AlternativeSendTxProvider) releaseRecentSender(sender ethcommon.Address, evictedTxGen uint64) { p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() s, found := p.recentSenders[sender] if !found { return } - if s.time.After(time.Unix(int64(evictedTxUnix), 0).Add(time.Second)) { + if s.gen > evictedTxGen { return } delete(p.recentSenders, sender) @@ -250,6 +265,8 @@ func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (strin return txid, bchain.ErrTxNotFound } + gen := p.currentSenderGeneration(tx.From) + p.mempoolTxsMux.Lock() // remove potential RBF transactions - with equal from and nonce var rbfTxid string @@ -261,7 +278,7 @@ func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (strin break } } - p.mempoolTxs[txid] = storedTx{tx: tx, time: uint32(time.Now().Unix())} + p.mempoolTxs[txid] = storedTx{tx: tx, time: uint32(time.Now().Unix()), gen: gen} p.mempoolTxsMux.Unlock() if rbfTxid != "" { @@ -599,7 +616,7 @@ func (p *AlternativeSendTxProvider) RemoveTransaction(txid string) { p.mempoolTxsMux.Unlock() if senderSettled { - p.releaseRecentSender(sender, removedTx.time) + p.releaseRecentSender(sender, removedTx.gen) } } diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index ac8d545111..b26def160e 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -998,28 +998,28 @@ func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) { sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") - cachedTx := func(nonce string, added time.Time) storedTx { + cachedTx := func(nonce string, gen uint64) storedTx { return storedTx{ tx: &bchain.RpcTransaction{ Hash: testAlternativeTxID, From: "0x2222222222222222222222222222222222222222", AccountNonce: nonce, }, - time: uint32(added.Unix()), + time: uint32(time.Now().Unix()), + gen: gen, } } - makeProvider := func(senderTime time.Time, txs map[string]storedTx) *AlternativeSendTxProvider { + makeProvider := func(senderGen uint64, txs map[string]storedTx) *AlternativeSendTxProvider { return &AlternativeSendTxProvider{ fetchMempoolTx: true, mempoolTxsTimeout: time.Hour, mempoolTxs: txs, - recentSenders: map[ethcommon.Address]recentSender{sender: {time: senderTime}}, + recentSenders: map[ethcommon.Address]recentSender{sender: {time: time.Now(), gen: senderGen}}, } } t.Run("evicting the last cached tx releases the sender", func(t *testing.T) { - now := time.Now() - provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now)}) + provider := makeProvider(1, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", 1)}) provider.RemoveTransaction(testAlternativeTxID) @@ -1029,10 +1029,9 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) }) t.Run("another cached tx from the sender keeps the entry", func(t *testing.T) { - now := time.Now() - provider := makeProvider(now, map[string]storedTx{ - testAlternativeTxID: cachedTx("0x1", now), - testAlternativeSecondTxID: cachedTx("0x2", now), + provider := makeProvider(2, map[string]storedTx{ + testAlternativeTxID: cachedTx("0x1", 1), + testAlternativeSecondTxID: cachedTx("0x2", 2), }) provider.RemoveTransaction(testAlternativeTxID) @@ -1044,9 +1043,10 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) t.Run("a newer send since the evicted tx keeps the entry", func(t *testing.T) { // the sender submitted again after the evicted tx was cached (possibly without a cache - // entry of its own, e.g. when the post-send fetch-back failed) - the entry must survive - now := time.Now() - provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now.Add(-10*time.Minute))}) + // entry of its own, e.g. when the post-send fetch-back failed) - the entry must survive. + // The generation counter orders the sends precisely, so this holds even when both sends + // landed within the same wall-clock second. + provider := makeProvider(2, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", 1)}) provider.RemoveTransaction(testAlternativeTxID) @@ -1056,8 +1056,7 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) }) t.Run("unknown txid releases nothing", func(t *testing.T) { - now := time.Now() - provider := makeProvider(now, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", now)}) + provider := makeProvider(1, map[string]storedTx{testAlternativeTxID: cachedTx("0x1", 1)}) provider.RemoveTransaction("0xdoesnotexist") @@ -1067,6 +1066,28 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) }) } +func TestAlternativeSendTxProviderHandleMempoolTransactionStampsGeneration(t *testing.T) { + // the cached entry must carry the sender's current send generation so that + // releaseRecentSender can order its eviction against later sends + server := newAlternativeTxProviderTestServer(t, testAlternativeKnownTxResponse) + sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") + provider := &AlternativeSendTxProvider{ + urls: []string{server.URL}, + fetchMempoolTx: true, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + mempoolTxs: map[string]storedTx{}, + recentSenders: map[ethcommon.Address]recentSender{sender: {time: time.Now(), gen: 5}}, + } + + if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err != nil { + t.Fatalf("handleMempoolTransaction() error = %v", err) + } + if got := provider.mempoolTxs[testAlternativeTxID].gen; got != 5 { + t.Errorf("cached tx generation = %d, want 5 (the sender's current send generation)", got) + } +} + func TestAlternativeSendTxProviderPendingNonceFloor(t *testing.T) { sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") provider := &AlternativeSendTxProvider{ From bd1c8bccd8cf4fdc4be965a54cc7678be6398ea0 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:22:06 +0200 Subject: [PATCH 089/303] feat(eth): add opt-out routing all nonce queries to the alternative provider The recent-sender gate lives in process memory: in a deployment with multiple load-balanced Blockbook instances, a send accepted by one instance is invisible to the others, so a REST poller without request affinity can be served the primary RPC's view, which cannot include the private pending transaction. Websocket clients are naturally sticky to one instance (send and poll share the connection), and Blockbook has no cross-instance state store to share the map through. Give such deployments an explicit choice instead of a silent trade-off: *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES=TRUE bypasses the gate and routes every address's nonce queries to the alternative provider - the pre-gating behavior, at the cost of its rate-limit quota. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 33 +++++++++++++++------- bchain/coins/eth/alternativesendtx_test.go | 7 +++++ docs/env.md | 4 ++- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 88c97793b5..818f61ac84 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -39,6 +39,7 @@ type AlternativeSendTxProvider struct { urls []string onlyAlternative bool fetchMempoolTx bool + nonceForAllAddresses bool mempoolTxs map[string]storedTx mempoolTxsMux sync.Mutex mempoolTxsTimeout time.Duration @@ -59,27 +60,32 @@ func NewAlternativeSendTxProvider(network string, rpcTimeout int, mempoolTxsTime urls := strings.Split(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_SENDTX_URLS"), ",") onlyAlternative := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_SENDTX_ONLY")) == "TRUE" fetchMempoolTx := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_FETCH_MEMPOOL_TX")) == "TRUE" + nonceForAllAddresses := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES")) == "TRUE" // Empty URL keeps the normal public RPC send path. if len(urls) == 0 || urls[0] == "" { return nil } provider := &AlternativeSendTxProvider{ - urls: urls, - onlyAlternative: onlyAlternative, - fetchMempoolTx: fetchMempoolTx, - rpcTimeout: time.Duration(rpcTimeout) * time.Second, - mempoolTxsTimeout: mempoolTxsTimeout, - mempoolTxs: make(map[string]storedTx), - recentSenders: make(map[ethcommon.Address]recentSender), - metrics: metrics, - stop: make(chan struct{}), + urls: urls, + onlyAlternative: onlyAlternative, + fetchMempoolTx: fetchMempoolTx, + nonceForAllAddresses: nonceForAllAddresses, + rpcTimeout: time.Duration(rpcTimeout) * time.Second, + mempoolTxsTimeout: mempoolTxsTimeout, + mempoolTxs: make(map[string]storedTx), + recentSenders: make(map[ethcommon.Address]recentSender), + metrics: metrics, + stop: make(chan struct{}), } glog.Infof("Using alternative send transaction providers %v. Only alternative providers %v", urls, onlyAlternative) if fetchMempoolTx { glog.Infof("Alternative fetch mempool tx %v", fetchMempoolTx) } + if nonceForAllAddresses { + glog.Infof("Alternative provider nonce routing enabled for all addresses") + } return provider } @@ -180,8 +186,15 @@ func (p *AlternativeSendTxProvider) currentSenderGeneration(from string) uint64 // Senders whose cached transactions have all settled are released before the timeout (see // releaseRecentSender). Accepted limitations: a restart wipes the map (exposure bounded by // mempoolTxsTimeout), a transaction pending longer than the timeout, and private -// transactions submitted outside this Blockbook instance. +// transactions submitted outside this Blockbook instance - which includes sends accepted +// by ANOTHER replica in a load-balanced deployment without request affinity. Deployments +// that cannot guarantee affinity can set *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES=TRUE to +// bypass the gate and route every address to the provider (the pre-gating behavior, at +// the cost of its rate-limit quota). func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { + if p.nonceForAllAddresses { + return true + } p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() s, found := p.recentSenders[addr] diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index b26def160e..998f22d84a 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -878,6 +878,13 @@ func TestAlternativeSendTxProviderUseForNonces(t *testing.T) { if _, found := provider.recentSenders[expired]; found { t.Error("expired sender not evicted on lookup") } + + // *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES bypasses the gate entirely - for load-balanced + // deployments without request affinity, where process-local sender state is insufficient + ungated := &AlternativeSendTxProvider{nonceForAllAddresses: true, mempoolTxsTimeout: time.Hour} + if !ungated.useForNonces(unknown) { + t.Error("unknown address not routed to the provider with nonceForAllAddresses set") + } } func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { diff --git a/docs/env.md b/docs/env.md index 7a1ea5a3e6..68f7fa30b9 100644 --- a/docs/env.md +++ b/docs/env.md @@ -87,12 +87,14 @@ Blockbook reads these from its process environment. When installed from the Debi - `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all (a whitespace-only value included), is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. The routing state is kept in process memory: with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider (websocket clients are naturally sticky to one instance; REST pollers may hit another and receive the public backend's view), and a restart clears the state. +- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. The routing state is kept in process memory: with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider (websocket clients are naturally sticky to one instance; REST pollers may hit another and receive the public backend's view), and a restart clears the state. Deployments that cannot guarantee request affinity can set `_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES=TRUE` to disable the gating. - `_ALTERNATIVE_SENDTX_ONLY` - Set to `TRUE` to use only the alternative send transaction provider and avoid fallback to the normal backend RPC if alternative submission fails. - `_ALTERNATIVE_FETCH_MEMPOOL_TX` - Set to `TRUE` to fetch and cache transactions submitted through the alternative provider, so Blockbook can expose them as pending even if they are not visible in the public backend mempool. Cached transactions are periodically checked with the alternative provider's `eth_getTransactionByHash`; an empty response or mined transaction removes the local pending entry. When the alternative provider is enabled, the default alternative cache timeout is 5 minutes and the default Blockbook EVM mempool timeout is 10 minutes; both can be overridden in coin config with `alternativeMempoolTxTimeout` and `mempoolTxTimeout`. +- `_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES` - Set to `TRUE` to route nonce (`eth_getTransactionCount`) queries for **all** addresses to the alternative provider instead of only recent private senders. Intended for deployments running multiple load-balanced Blockbook instances without request affinity, where the per-process sender tracking cannot see sends accepted by other instances. Costs the provider's rate-limit quota on every address request. + WebSocket `sendTransaction` can bypass the alternative provider for a single request by setting `disableAlternativeRPC` to `true`. ## Runtime settings From 50cf5808073b932934bd963911cc80ac5d367d73 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:39:08 +0200 Subject: [PATCH 090/303] revert(eth): drop the *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES escape hatch This reverts commit 31cba4114 (keeping the load-balancer limitation documented in useForNonces and docs/env.md). The flag guarded a deployment shape - multiple replicas per coin behind a non-sticky load balancer with REST-based nonce polling - that production does not run: wallet send+poll flows go over websocket and are therefore naturally affine to the instance that accepted the send. A public env var is permanent API surface and an ops footgun: setting it "to be safe" silently recreates the #1627 quota exhaustion wholesale. Should a non-affine deployment ever materialize, re-adding the opt-out is a trivial, backward-compatible change; removing a knob operators already use is not. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 33 ++++++++-------------- bchain/coins/eth/alternativesendtx_test.go | 7 ----- docs/env.md | 4 +-- 3 files changed, 12 insertions(+), 32 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 818f61ac84..06588764a6 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -39,7 +39,6 @@ type AlternativeSendTxProvider struct { urls []string onlyAlternative bool fetchMempoolTx bool - nonceForAllAddresses bool mempoolTxs map[string]storedTx mempoolTxsMux sync.Mutex mempoolTxsTimeout time.Duration @@ -60,32 +59,27 @@ func NewAlternativeSendTxProvider(network string, rpcTimeout int, mempoolTxsTime urls := strings.Split(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_SENDTX_URLS"), ",") onlyAlternative := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_SENDTX_ONLY")) == "TRUE" fetchMempoolTx := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_FETCH_MEMPOOL_TX")) == "TRUE" - nonceForAllAddresses := strings.ToUpper(os.Getenv(strings.ToUpper(network)+"_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES")) == "TRUE" // Empty URL keeps the normal public RPC send path. if len(urls) == 0 || urls[0] == "" { return nil } provider := &AlternativeSendTxProvider{ - urls: urls, - onlyAlternative: onlyAlternative, - fetchMempoolTx: fetchMempoolTx, - nonceForAllAddresses: nonceForAllAddresses, - rpcTimeout: time.Duration(rpcTimeout) * time.Second, - mempoolTxsTimeout: mempoolTxsTimeout, - mempoolTxs: make(map[string]storedTx), - recentSenders: make(map[ethcommon.Address]recentSender), - metrics: metrics, - stop: make(chan struct{}), + urls: urls, + onlyAlternative: onlyAlternative, + fetchMempoolTx: fetchMempoolTx, + rpcTimeout: time.Duration(rpcTimeout) * time.Second, + mempoolTxsTimeout: mempoolTxsTimeout, + mempoolTxs: make(map[string]storedTx), + recentSenders: make(map[ethcommon.Address]recentSender), + metrics: metrics, + stop: make(chan struct{}), } glog.Infof("Using alternative send transaction providers %v. Only alternative providers %v", urls, onlyAlternative) if fetchMempoolTx { glog.Infof("Alternative fetch mempool tx %v", fetchMempoolTx) } - if nonceForAllAddresses { - glog.Infof("Alternative provider nonce routing enabled for all addresses") - } return provider } @@ -187,14 +181,9 @@ func (p *AlternativeSendTxProvider) currentSenderGeneration(from string) uint64 // releaseRecentSender). Accepted limitations: a restart wipes the map (exposure bounded by // mempoolTxsTimeout), a transaction pending longer than the timeout, and private // transactions submitted outside this Blockbook instance - which includes sends accepted -// by ANOTHER replica in a load-balanced deployment without request affinity. Deployments -// that cannot guarantee affinity can set *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES=TRUE to -// bypass the gate and route every address to the provider (the pre-gating behavior, at -// the cost of its rate-limit quota). +// by another replica in a load-balanced deployment without request affinity (wallet +// websocket flows are naturally sticky to one instance; see docs/env.md). func (p *AlternativeSendTxProvider) useForNonces(addr ethcommon.Address) bool { - if p.nonceForAllAddresses { - return true - } p.recentSendersMux.Lock() defer p.recentSendersMux.Unlock() s, found := p.recentSenders[addr] diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index 998f22d84a..b26def160e 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -878,13 +878,6 @@ func TestAlternativeSendTxProviderUseForNonces(t *testing.T) { if _, found := provider.recentSenders[expired]; found { t.Error("expired sender not evicted on lookup") } - - // *_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES bypasses the gate entirely - for load-balanced - // deployments without request affinity, where process-local sender state is insufficient - ungated := &AlternativeSendTxProvider{nonceForAllAddresses: true, mempoolTxsTimeout: time.Hour} - if !ungated.useForNonces(unknown) { - t.Error("unknown address not routed to the provider with nonceForAllAddresses set") - } } func TestAlternativeSendTxProviderSendRecordsSender(t *testing.T) { diff --git a/docs/env.md b/docs/env.md index 68f7fa30b9..7a1ea5a3e6 100644 --- a/docs/env.md +++ b/docs/env.md @@ -87,14 +87,12 @@ Blockbook reads these from its process environment. When installed from the Debi - `_ALLOWED_EVM_CALL_METHODS` - EVM method selectors (first 4 bytes of the calldata, for example `0xdd62ed3e` for ERC-20 `allowance(address,address)`) that `rpcCall` websocket requests may invoke on any address, as a comma-separated list; the `0x` prefix is optional and matching is case-insensitive. Combines with `ALLOWED_RPC_CALL_TO`: a call is allowed when either its target address or its calldata selector is allowed. When only this variable is set, only calls with an allowed selector pass. Malformed calldata (missing `0x` prefix, invalid or odd-length hex, fewer than 4 bytes) never matches a selector. A malformed selector in the list, or a set variable that contains no selectors at all (a whitespace-only value included), is a configuration error and Blockbook fails on startup. This is the startup default of a [runtime setting](#runtime-settings): an override stored through the admin API takes precedence. -- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. The routing state is kept in process memory: with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider (websocket clients are naturally sticky to one instance; REST pollers may hit another and receive the public backend's view), and a restart clears the state. Deployments that cannot guarantee request affinity can set `_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES=TRUE` to disable the gating. +- `_ALTERNATIVE_SENDTX_URLS` - Comma-separated list of alternative EVM `eth_sendRawTransaction` providers, used for private/MEV-protected transaction submission. The prefix is the configured `network` value when present (for example `OP`, `BASE`, `POL`, `BSC`, `ARB`, `AVAX`), otherwise the coin shortcut (for example `ETH`). If omitted, Blockbook sends transactions through the normal backend RPC. Nonce (`eth_getTransactionCount`) queries are routed to the alternative provider only for addresses that recently (within `alternativeMempoolTxTimeout`) sent a transaction through it; all other addresses are served by the normal backend RPC. The routing state is kept in process memory: with multiple Blockbook instances behind a load balancer, only the instance that accepted the transaction routes the sender's nonce queries to the alternative provider (websocket clients are naturally sticky to one instance; REST pollers may hit another and receive the public backend's view), and a restart clears the state. - `_ALTERNATIVE_SENDTX_ONLY` - Set to `TRUE` to use only the alternative send transaction provider and avoid fallback to the normal backend RPC if alternative submission fails. - `_ALTERNATIVE_FETCH_MEMPOOL_TX` - Set to `TRUE` to fetch and cache transactions submitted through the alternative provider, so Blockbook can expose them as pending even if they are not visible in the public backend mempool. Cached transactions are periodically checked with the alternative provider's `eth_getTransactionByHash`; an empty response or mined transaction removes the local pending entry. When the alternative provider is enabled, the default alternative cache timeout is 5 minutes and the default Blockbook EVM mempool timeout is 10 minutes; both can be overridden in coin config with `alternativeMempoolTxTimeout` and `mempoolTxTimeout`. -- `_ALTERNATIVE_NONCE_FOR_ALL_ADDRESSES` - Set to `TRUE` to route nonce (`eth_getTransactionCount`) queries for **all** addresses to the alternative provider instead of only recent private senders. Intended for deployments running multiple load-balanced Blockbook instances without request affinity, where the per-process sender tracking cannot see sends accepted by other instances. Costs the provider's rate-limit quota on every address request. - WebSocket `sendTransaction` can bypass the alternative provider for a single request by setting `disableAlternativeRPC` to `true`. ## Runtime settings From e282fe687d78c09269263a8f065d73bacce21aae Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:46:07 +0200 Subject: [PATCH 091/303] fix(eth): stamp cached txs with their own submission's send generation handleMempoolTransaction read the sender's CURRENT generation after the fetch-back network round-trip. A concurrent send from the same sender could bump the generation during that round-trip, so the fetched transaction was cached under a generation belonging to a different submission. Concretely: transaction A (generation 1) with a slow fetch-back, concurrent transaction B (generation 2) whose own fetch-back failed - A ends up cached as generation 2, and when A is later mined and evicted, releaseRecentSender sees no newer send (2 > 2 is false) and drops the routing while B is still privately pending, sending B's nonce reads to the stale primary RPC. Capture the generation at registration time instead: registerSuccessfulSend returns the generation it assigned and SendRawTransaction carries that exact value into handleMempoolTransaction. The mutexes were never the problem - the wrong transaction was associated with the value. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 36 ++++++++++------------ bchain/coins/eth/alternativesendtx_test.go | 34 +++++++++++++------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index 06588764a6..cd9c131a63 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -21,7 +21,7 @@ import ( type storedTx struct { tx *bchain.RpcTransaction time uint32 - gen uint64 // sender's send generation when the tx was cached, orders it against later sends + gen uint64 // send generation of the submission that created this entry, orders it against later sends } // recentSender records when an address last successfully sent a transaction through an @@ -114,12 +114,13 @@ func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, erro } } + var gen uint64 if retErr == nil { - p.registerSuccessfulSend(hex, acceptedURL) + gen = p.registerSuccessfulSend(hex, acceptedURL) } if p.onlyAlternative && p.fetchMempoolTx { - p.handleMempoolTransaction(txid) + p.handleMempoolTransaction(txid, gen) } return txid, retErr @@ -141,11 +142,14 @@ func alternativeTxSender(rawTxHex string) (ethcommon.Address, error) { // accepts it, so the accepting URL is recorded too - it is the one provider guaranteed to // know the transaction (see nonceURL). Expired entries are swept on the way; the map only // ever holds senders of the last mempoolTxsTimeout window, so the sweep is cheap. -func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acceptedURL string) { +// It returns the send generation assigned to this submission (0 when the sender cannot be +// decoded); the caller must carry that exact value to the cache entry it creates for the +// transaction, so that releaseRecentSender can order evictions against later sends. +func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acceptedURL string) uint64 { sender, err := alternativeTxSender(rawTxHex) if err != nil { glog.Warningf("cannot decode sender of transaction sent to alternative provider: %v", err) - return + return 0 } now := time.Now() p.recentSendersMux.Lock() @@ -160,16 +164,7 @@ func (p *AlternativeSendTxProvider) registerSuccessfulSend(rawTxHex string, acce } p.sendGeneration++ p.recentSenders[sender] = recentSender{time: now, url: acceptedURL, gen: p.sendGeneration} -} - -// currentSenderGeneration returns the send generation of the sender's most recent -// successful send. Cached transactions are stamped with it so releaseRecentSender can -// order evictions against later sends precisely, without relying on wall-clock resolution -// (two sends can land within the same whole-second storedTx timestamp). -func (p *AlternativeSendTxProvider) currentSenderGeneration(from string) uint64 { - p.recentSendersMux.Lock() - defer p.recentSendersMux.Unlock() - return p.recentSenders[ethcommon.HexToAddress(from)].gen + return p.sendGeneration } // useForNonces reports whether nonce lookups for addr should be routed to the alternative @@ -256,8 +251,13 @@ func (p *AlternativeSendTxProvider) raiseToPendingFloor(addr ethcommon.Address, return pending } -// handleMempoolTransaction handles the transaction when using only alternative providers -func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (string, error) { +// handleMempoolTransaction handles the transaction when using only alternative providers. +// gen is the send generation registerSuccessfulSend assigned to THIS submission - it must be +// passed in rather than read from recentSenders here, because the fetch-back above is a +// network round-trip during which a concurrent send from the same sender can bump the +// sender's current generation; stamping the cache entry with that newer generation would let +// its eviction release the sender's routing while the newer transaction is still pending. +func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string, gen uint64) (string, error) { tx, found, err := p.getTransactionFromProviders(txid) if err != nil { glog.Errorf("eth_getTransactionByHash from alternative providers returned error %v", err) @@ -267,8 +267,6 @@ func (p *AlternativeSendTxProvider) handleMempoolTransaction(txid string) (strin return txid, bchain.ErrTxNotFound } - gen := p.currentSenderGeneration(tx.From) - p.mempoolTxsMux.Lock() // remove potential RBF transactions - with equal from and nonce var rbfTxid string diff --git a/bchain/coins/eth/alternativesendtx_test.go b/bchain/coins/eth/alternativesendtx_test.go index b26def160e..dd674d5ae7 100644 --- a/bchain/coins/eth/alternativesendtx_test.go +++ b/bchain/coins/eth/alternativesendtx_test.go @@ -209,7 +209,7 @@ func TestAlternativeSendTxProviderHandleMempoolTransactionFetchesFromAnyProvider provider.mempoolTxs = make(map[string]storedTx) provider.urls = append(provider.urls, knownServer.URL) - if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err != nil { + if _, err := provider.handleMempoolTransaction(testAlternativeTxID, 0); err != nil { t.Fatalf("handleMempoolTransaction() error = %v", err) } if _, found := provider.mempoolTxs[testAlternativeTxID]; !found { @@ -223,7 +223,7 @@ func TestAlternativeSendTxProviderHandleMempoolTransactionSkipsEmptyTransaction( provider := newTestAlternativeSendTxProvider(server.URL, &removed) provider.mempoolTxs = make(map[string]storedTx) - if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err == nil { + if _, err := provider.handleMempoolTransaction(testAlternativeTxID, 0); err == nil { t.Fatal("handleMempoolTransaction() error = nil, want ErrTxNotFound") } if _, found := provider.mempoolTxs[testAlternativeTxID]; found { @@ -237,7 +237,7 @@ func TestAlternativeSendTxProviderHandleMempoolTransactionSkipsTransactionWithou provider := newTestAlternativeSendTxProvider(server.URL, &removed) provider.mempoolTxs = make(map[string]storedTx) - if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err == nil { + if _, err := provider.handleMempoolTransaction(testAlternativeTxID, 0); err == nil { t.Fatal("handleMempoolTransaction() error = nil, want ErrTxNotFound") } if _, found := provider.mempoolTxs[testAlternativeTxID]; found { @@ -629,7 +629,7 @@ func TestAlternativeSendTxProviderRBFReplacementObservesMetrics(t *testing.T) { provider.RemoveTransaction(txid) } - if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err != nil { + if _, err := provider.handleMempoolTransaction(testAlternativeTxID, 0); err != nil { t.Fatalf("handleMempoolTransaction: %v", err) } @@ -1066,9 +1066,14 @@ func TestAlternativeSendTxProviderRemoveTransactionReleasesSender(t *testing.T) }) } -func TestAlternativeSendTxProviderHandleMempoolTransactionStampsGeneration(t *testing.T) { - // the cached entry must carry the sender's current send generation so that - // releaseRecentSender can order its eviction against later sends +func TestAlternativeSendTxProviderHandleMempoolTransactionStampsOwnGeneration(t *testing.T) { + // The cached entry must carry the generation of ITS OWN submission, not the sender's + // current generation: the fetch-back is a network round-trip during which a concurrent + // send can bump the sender's generation. Simulated here: transaction A (generation 1) + // finishes its slow fetch-back after a concurrent transaction B (generation 2) was + // registered but left no cache entry because B's own fetch-back failed. Stamping A with + // generation 2 would make A's eviction release the sender's routing while B is still + // privately pending. server := newAlternativeTxProviderTestServer(t, testAlternativeKnownTxResponse) sender := ethcommon.HexToAddress("0x2222222222222222222222222222222222222222") provider := &AlternativeSendTxProvider{ @@ -1077,14 +1082,21 @@ func TestAlternativeSendTxProviderHandleMempoolTransactionStampsGeneration(t *te mempoolTxsTimeout: time.Hour, rpcTimeout: time.Second, mempoolTxs: map[string]storedTx{}, - recentSenders: map[ethcommon.Address]recentSender{sender: {time: time.Now(), gen: 5}}, + recentSenders: map[ethcommon.Address]recentSender{sender: {time: time.Now(), gen: 2}}, } - if _, err := provider.handleMempoolTransaction(testAlternativeTxID); err != nil { + if _, err := provider.handleMempoolTransaction(testAlternativeTxID, 1); err != nil { t.Fatalf("handleMempoolTransaction() error = %v", err) } - if got := provider.mempoolTxs[testAlternativeTxID].gen; got != 5 { - t.Errorf("cached tx generation = %d, want 5 (the sender's current send generation)", got) + if got := provider.mempoolTxs[testAlternativeTxID].gen; got != 1 { + t.Errorf("cached tx generation = %d, want 1 (the generation of its own submission)", got) + } + + // evicting A must keep the routing alive for the uncached, possibly still pending B + provider.RemoveTransaction(testAlternativeTxID) + + if !provider.useForNonces(sender) { + t.Error("sender routing released although a newer private send (generation 2) may still be pending") } } From a2f48f25194e5bab7b6b58db5912ce41b4c7b8ea Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:59:04 +0200 Subject: [PATCH 092/303] fix(eth): apply the pending-nonce floor whenever a provider is configured The floor was gated on useForNonces, but the routing entry and the cache entry expire on different clocks: recentSender.time is stamped at send time, storedTx.time at the later fetch-back time, and the reconcile ticker adds up to a minute of granularity. In the window between the two expiries the gate is off while the cached tx is still exposed as pending, so a primary RPC answer below the floor could contradict the pending view Blockbook itself displays - exactly what the floor exists to prevent. Apply it whenever a provider is configured; the local scan of a usually-empty map costs nothing on the hot path. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/ethrpc.go | 25 +++++++++++++------------ bchain/coins/eth/nonce_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 47cbd02498..2e01d6b6df 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2247,10 +2247,10 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) // for addresses that recently sent a transaction via that provider (see useForNonces) — // those may have a pending transaction the primary RPC does not know about. All other // addresses go straight to the primary RPC so that the hottest API endpoint does not burn -// the provider's rate-limit quota. For gated senders the pending answer - whether from the -// provider or from the primary RPC fallback - is raised to the floor implied by the -// alternative mempool cache (see pendingNonceFloor) so it never contradicts Blockbook's -// own pending view of the sender's private transactions. +// the provider's rate-limit quota. Whenever a provider is configured, the pending answer - +// whether from the provider or from the primary RPC - is raised to the floor implied by +// the alternative mempool cache (see pendingNonceFloor) so it never contradicts +// Blockbook's own pending view of the sender's private transactions. // // The pending nonce (eth_getTransactionCount at the "pending" tag) counts transactions // still queued in the mempool and is the next nonce the account will use; it is always @@ -2265,15 +2265,13 @@ func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, withConfirmed bool) (uint64, uint64, bool, error) { ethAddress := ethcommon.BytesToAddress(addrDesc) - useAlternative := b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) - if useAlternative { + if b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) { pending, confirmed, confirmedOK, err := b.alternativeSendTxProvider.getNonces(ethAddress, withConfirmed) if err == nil { // Even the provider's own answer can fall below Blockbook's advertised pending // view: Blink-style relays stop counting a still-pending tx at the pending tag // while Blockbook keeps exposing it until the cache timeout (see - // reconcileMempoolTxs). Raise to the cache floor so the invariant holds - // unconditionally. + // reconcileMempoolTxs). return b.alternativeSendTxProvider.raiseToPendingFloor(ethAddress, pending), confirmed, confirmedOK, nil } glog.Warningf("Alternative provider failed for eth_getTransactionCount: %v, falling back to primary RPC", err) @@ -2284,10 +2282,13 @@ func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, w glog.Errorf("Primary RPC failed for eth_getTransactionCount: %v", err) return 0, 0, false, err } - if useAlternative { - // The primary RPC cannot see the sender's private pending transactions at all - - // without the raise, a wallet building on the fallback answer could reuse the nonce - // of an in-flight private transaction and replace it. + if b.alternativeSendTxProvider != nil { + // Applied whenever a provider is configured, not only for gated senders: the routing + // entry expires at send time + timeout while the cached tx stays exposed as pending + // until fetch-back time + timeout (plus reconcile granularity), and in that window a + // primary answer below the floor would contradict the pending tx Blockbook still + // displays. The floor is a local scan of a usually-empty map, so it costs nothing on + // the hot path. pending = b.alternativeSendTxProvider.raiseToPendingFloor(ethAddress, pending) } return pending, confirmed, confirmedOK, nil diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index e9510413ea..79cbb4c795 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -358,6 +358,36 @@ func TestEthereumTypeGetNonces_AlternativeProvider_ProviderAnswerRaisedToCacheFl } } +func TestEthereumTypeGetNonces_AlternativeProvider_FloorAppliedWithoutRouting(t *testing.T) { + // the routing entry can expire before the cached tx stops being exposed as pending + // (recentSender.time is stamped at send time, storedTx.time at the later fetch-back time, + // and the reconcile ticker adds up to a minute of granularity) - the floor must hold even + // when the gate no longer routes the sender to the provider + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + sender := ethcommon.BytesToAddress(nonceTestAddr) + provider := &AlternativeSendTxProvider{ + urls: []string{"http://127.0.0.1:1"}, + fetchMempoolTx: true, + mempoolTxsTimeout: time.Hour, + rpcTimeout: time.Second, + mempoolTxs: map[string]storedTx{ + testAlternativeTxID: { + tx: &bchain.RpcTransaction{Hash: testAlternativeTxID, From: sender.Hex(), AccountNonce: "0x7"}, + time: uint32(time.Now().Unix()), + }, + }, + } + b := &EthereumRPC{RPC: stub, Timeout: time.Second, alternativeSendTxProvider: provider} + + pending, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending != 8 { + t.Errorf("pending = %d, want 8 (floor applied although the sender is not routed)", pending) + } +} + func TestEthereumTypeGetNonces_SequentialFallback_ConfirmedDecodeFailureIsBestEffort(t *testing.T) { // sequential-fallback counterpart of the batched decode-failure case: an unparsable latest // result must be best-effort (pending returned, confirmedOK=false, no error) From fffa07e07f6a82b72f87b6d8edb97f4f7ac70cf3 Mon Sep 17 00:00:00 2001 From: Jakub Jerabek Date: Tue, 14 Jul 2026 17:59:04 +0200 Subject: [PATCH 093/303] refactor(eth): key sender registration on the accepting URL Registering on retErr == nil silently relied on callHttpStringResult never returning an empty result without an error (an empty "success" followed by a failing URL would leave retErr set although a provider accepted the tx). Key on acceptedURL != "" instead, which states the actual condition: some provider accepted this transaction. No behavior change under the current helper contract. Co-Authored-By: Claude Fable 5 --- bchain/coins/eth/alternativesendtx.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bchain/coins/eth/alternativesendtx.go b/bchain/coins/eth/alternativesendtx.go index cd9c131a63..6751973c22 100644 --- a/bchain/coins/eth/alternativesendtx.go +++ b/bchain/coins/eth/alternativesendtx.go @@ -115,7 +115,9 @@ func (p *AlternativeSendTxProvider) SendRawTransaction(hex string) (string, erro } var gen uint64 - if retErr == nil { + // keyed on acceptedURL rather than retErr, so registration does not silently depend on + // callHttpStringResult never returning an empty result without an error + if acceptedURL != "" { gen = p.registerSuccessfulSend(hex, acceptedURL) } From fcb5ce1904333c412595578d0a54d731a2a1f667 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Tue, 14 Jul 2026 18:06:25 +0000 Subject: [PATCH 094/303] feat(eth): meter alternative-provider nonce requests + Grafana panel Add eth_alternative_nonce_requests_total (counter_vec, label result= success|error) incremented only inside the useForNonces gate in EthereumTypeGetNonces, so it counts the small routed subset rather than the hot eth_getTransactionCount endpoint. A low, mostly-success rate confirms the gating keeps traffic off the provider's rate quota. The panel lives in the Sync section next to the other alternative send-tx provider metrics (eth_alternative_mempool_*), which are the same subsystem; the Fees section is fee estimation + the alternative *fee* providers, unrelated. Dashboard is generated, so the panel is added to template.json + panels.yaml (x-panel-key sync.eth_alternative_nonce_requests) and grafana.json re-rendered (render_grafana.py --check passes). Co-Authored-By: Claude Opus 4.8 (1M context) --- bchain/coins/eth/ethrpc.go | 13 +++++++ bchain/coins/eth/nonce_test.go | 67 ++++++++++++++++++++++++++++++++++ common/metrics.go | 1 + configs/grafana/panels.yaml | 7 ++++ configs/grafana/template.json | 60 ++++++++++++++++++++++++++++++ configs/metrics.yaml | 5 +++ 6 files changed, 153 insertions(+) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 2e01d6b6df..7c67dceae6 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -2049,6 +2049,17 @@ func (b *EthereumRPC) observeEip1559FeeSource(source string) { b.metrics.EthEip1559FeeSource.With(common.Labels{"source": source}).Inc() } +// observeAlternativeNonceRequest records an eth_getTransactionCount lookup routed to the alternative +// send-tx provider, labeled by result: success (provider answered) or error (provider failed and the +// lookup fell back to the primary RPC). Only recent private senders are routed here (see useForNonces), +// so this counts the gated subset rather than every address request. +func (b *EthereumRPC) observeAlternativeNonceRequest(result string) { + if b.metrics == nil || b.metrics.EthAlternativeNonceRequests == nil { + return + } + b.metrics.EthAlternativeNonceRequests.With(common.Labels{"result": result}).Inc() +} + // eip1559BaseFeeMultiplier is the headroom applied to the projected base fee when deriving // maxFeePerGas for the on-chain EIP-1559 estimate (maxFeePerGas = multiplier*baseFee + tip). // 2x is the EIP-1559-standard buffer: it keeps a transaction mineable across ~6 consecutive full @@ -2268,12 +2279,14 @@ func (b *EthereumRPC) EthereumTypeGetNonces(addrDesc bchain.AddressDescriptor, w if b.alternativeSendTxProvider != nil && b.alternativeSendTxProvider.useForNonces(ethAddress) { pending, confirmed, confirmedOK, err := b.alternativeSendTxProvider.getNonces(ethAddress, withConfirmed) if err == nil { + b.observeAlternativeNonceRequest("success") // Even the provider's own answer can fall below Blockbook's advertised pending // view: Blink-style relays stop counting a still-pending tx at the pending tag // while Blockbook keeps exposing it until the cache timeout (see // reconcileMempoolTxs). return b.alternativeSendTxProvider.raiseToPendingFloor(ethAddress, pending), confirmed, confirmedOK, nil } + b.observeAlternativeNonceRequest("error") glog.Warningf("Alternative provider failed for eth_getTransactionCount: %v, falling back to primary RPC", err) } diff --git a/bchain/coins/eth/nonce_test.go b/bchain/coins/eth/nonce_test.go index 79cbb4c795..0d63dea753 100644 --- a/bchain/coins/eth/nonce_test.go +++ b/bchain/coins/eth/nonce_test.go @@ -10,7 +10,9 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" + "github.com/prometheus/client_golang/prometheus" "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/common" ) // nonceTagFromArgs extracts the block tag ("pending"/"latest") from eth_getTransactionCount args. @@ -358,6 +360,71 @@ func TestEthereumTypeGetNonces_AlternativeProvider_ProviderAnswerRaisedToCacheFl } } +// TestEthereumTypeGetNonces_AlternativeProvider_ObservesNonceRequests asserts the +// eth_alternative_nonce_requests_total counter is incremented only for lookups actually routed to the +// alternative provider (labeled success/error), and never for a gated-out address served by the primary +// RPC - so the metric measures the small gated subset, not the hot eth_getTransactionCount endpoint. +func TestEthereumTypeGetNonces_AlternativeProvider_ObservesNonceRequests(t *testing.T) { + newMetrics := func() *common.Metrics { + return &common.Metrics{ + EthAlternativeNonceRequests: prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "test_alt_nonce_requests_total"}, []string{"result"}), + } + } + sender := ethcommon.BytesToAddress(nonceTestAddr) + + t.Run("routed provider success increments result=success", func(t *testing.T) { + server := newNonceRPCServer(t, map[string]string{"pending": "0x9"}, nil) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + m := newMetrics() + b := &EthereumRPC{RPC: stub, Timeout: time.Second, metrics: m, alternativeSendTxProvider: newRecentSenderProvider(server, sender)} + + if _, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := counterVecValue(t, m.EthAlternativeNonceRequests, "result", "success"); got != 1 { + t.Errorf("result=success = %v, want 1", got) + } + if got := counterVecValue(t, m.EthAlternativeNonceRequests, "result", "error"); got != 0 { + t.Errorf("result=error = %v, want 0", got) + } + }) + + t.Run("routed provider error increments result=error", func(t *testing.T) { + server := newNonceRPCServer(t, nil, map[string]bool{"pending": true}) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + m := newMetrics() + b := &EthereumRPC{RPC: stub, Timeout: time.Second, metrics: m, alternativeSendTxProvider: newRecentSenderProvider(server, sender)} + + if _, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false); err != nil { + t.Fatalf("provider failure must fall back to the primary RPC, got error: %v", err) + } + if got := counterVecValue(t, m.EthAlternativeNonceRequests, "result", "error"); got != 1 { + t.Errorf("result=error = %v, want 1", got) + } + if got := counterVecValue(t, m.EthAlternativeNonceRequests, "result", "success"); got != 0 { + t.Errorf("result=success = %v, want 0", got) + } + }) + + t.Run("gated-out address records nothing", func(t *testing.T) { + server := newNonceRPCServer(t, map[string]string{"pending": "0x9"}, nil) + stub := &nonceBatchStub{results: map[string]string{"pending": "0x4"}} + m := newMetrics() + // no recent senders: the address is served by the primary RPC and must not be counted + b := &EthereumRPC{RPC: stub, Timeout: time.Second, metrics: m, alternativeSendTxProvider: newRecentSenderProvider(server)} + + if _, _, _, err := b.EthereumTypeGetNonces(nonceTestAddr, false); err != nil { + t.Fatalf("unexpected error: %v", err) + } + total := counterVecValue(t, m.EthAlternativeNonceRequests, "result", "success") + + counterVecValue(t, m.EthAlternativeNonceRequests, "result", "error") + if total != 0 { + t.Errorf("nonce-request counter = %v, want 0 for a gated-out address", total) + } + }) +} + func TestEthereumTypeGetNonces_AlternativeProvider_FloorAppliedWithoutRouting(t *testing.T) { // the routing entry can expire before the cached tx stops being exposed as pending // (recentSender.time is stamped at send time, storedTx.time at the later fetch-back time, diff --git a/common/metrics.go b/common/metrics.go index 0e11783038..251b45e34e 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -67,6 +67,7 @@ type Metrics struct { EthAlternativeMempoolEvents *prometheus.CounterVec `metric:"eth_alternative_mempool_reconciliation_events_total"` EthAlternativeMempoolTxResidence *prometheus.HistogramVec `metric:"eth_alternative_mempool_tx_residence_seconds"` EthAlternativeMempoolCacheSize prometheus.Gauge `metric:"eth_alternative_mempool_cache_size"` + EthAlternativeNonceRequests *prometheus.CounterVec `metric:"eth_alternative_nonce_requests_total"` EstimatedFee *prometheus.GaugeVec `metric:"estimated_fee"` AvgBlockPeriod prometheus.Gauge `metric:"avg_block_period"` SyncBlockStats *prometheus.GaugeVec `metric:"sync_block_stats"` diff --git a/configs/grafana/panels.yaml b/configs/grafana/panels.yaml index 7496287079..350f0dfb3f 100644 --- a/configs/grafana/panels.yaml +++ b/configs/grafana/panels.yaml @@ -197,6 +197,13 @@ panels: p95: promql: histogram_quantile(0.95, sum(rate({{name:eth_alternative_mempool_tx_residence_seconds}}_bucket{job="blockbook", coin="$coin"}[$__rate_interval])) by (action, le)) legend: '{{action}} p95' + sync.eth_alternative_nonce_requests: + title: Alternative send-tx provider nonce requests (req/s) + description: 'eth_getTransactionCount lookups routed to the alternative send-tx provider, per second, split by result. Only addresses that recently sent a private transaction through the provider are routed here (see useForNonces / #1627); every other address is served by the primary RPC and never appears, so this is the small gated subset, not total nonce traffic. success = the provider answered and its pending nonce was served (raised to the mempool floor); error = the provider failed and the lookup fell back to the primary RPC. Expect a low total that tracks recent private-send activity - a sustained climb means the gating is not shedding load off the provider''s rate quota, and a rising error share means the relay is unhealthy (read alongside the Alternative mempool panels above).' + queries: + by_result: + promql: sum by (coin, result) (rate({{name:eth_alternative_nonce_requests_total}}{job="blockbook", coin="$coin"}[$__rate_interval])) + legend: '{{coin}} - {{result}}' sync.mempool_resync_throughput_p95: title: 95% quantile of mempool resync throughput description: Transactions returned by a mempool reload divided by reload duration; low values mean the backend or parser is feeding the mempool slowly. diff --git a/configs/grafana/template.json b/configs/grafana/template.json index bdc937d509..8f76ec2bab 100644 --- a/configs/grafana/template.json +++ b/configs/grafana/template.json @@ -1725,6 +1725,66 @@ "type": "timeseries", "x-panel-key": "sync.eth_alternative_mempool_tx_residence" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 349, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "range": true, + "refId": "A", + "x-query-key": "by_result" + } + ], + "type": "timeseries", + "x-panel-key": "sync.eth_alternative_nonce_requests" + }, { "fieldConfig": { "defaults": { diff --git a/configs/metrics.yaml b/configs/metrics.yaml index 55d767fd83..0e9233ad99 100644 --- a/configs/metrics.yaml +++ b/configs/metrics.yaml @@ -239,6 +239,11 @@ metrics: name: blockbook_eth_alternative_mempool_cache_size type: gauge help: Transactions currently held in the alternative send-transaction mempool cache, sampled at the end of every reconcile cycle + eth_alternative_nonce_requests_total: + name: blockbook_eth_alternative_nonce_requests_total + type: counter_vec + help: eth_getTransactionCount lookups actually routed to the alternative send-transaction provider, labeled by result - success (the provider answered and its pending nonce was served, raised to the mempool floor) or error (the provider failed and the lookup fell back to the primary RPC). Only addresses that recently sent a private transaction through the provider are routed here (see useForNonces), so this counts the small gated subset, not every address request - a low, mostly-success rate confirms the gating is keeping the hot eth_getTransactionCount endpoint off the provider's rate quota + labels: [result] estimated_fee: name: blockbook_estimated_fee type: gauge_vec From ab28c6093239604ec4b034c83291fe1effd9541a Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 15 Jul 2026 11:17:51 +0000 Subject: [PATCH 095/303] feat(xpub): make xpub expansion and cache caps configurable per coin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace three hard-coded constants (xpubCacheMaxEntries=128, xpubCacheExpirationSeconds=3600, maxXpubAddressDerivations=20002) with a per-coin configurable XpubConfig struct following the same additional_params pattern as MissingBlockRetry. Problem: the xpub cache limit of 128 entries was too small for busy instances. With hundreds of concurrent Trezor Suite users, the cache thrashes constantly — every request evicts the previous user's data and forces a full re-derive + re-sort, wasting CPU and defeating the purpose of the cache. Changes: - Add XpubConfig wire struct (bchain/types.go) with MaxCacheEntries, MaxCacheExpirationSeconds, DefaultAddressesGap, MaxAddressesGap - Add XpubConfig field + XpubConfigOverride method to BitcoinRPC only; xpub scanning is UTXO-only, so EVM chains never read it - Forward through blockChainWithMetrics - Add resolved XpubConfig type, DefaultXpubConfig and ApplyXpubConfig in api/xpub.go; raise MaxCacheEntries default from 128 to 1024 - finalizeXpubConfig computes MaxAddressDerivations and clamps DefaultAddressesGap to MaxAddressesGap on every config path - Update evictXpubCacheItemsLocked, trimXpubCacheItemsLocked, validateXpubScanLimits, and xpubScanAddresses to accept config parameters instead of reading package-level constants - Resolve config inside NewWorker via duck-typed interface assertion against chain (no call-site changes needed); log one line at startup when an override is applied - Document the knobs under additional_params in docs/config.md - Delete old constants; update tests with local constants Default config (applies when coin JSON has no xpubConfig block): MaxCacheEntries: 1024 (was 128) MaxCacheExpirationSeconds: 3600 (unchanged) DefaultAddressesGap: 20 (unchanged) MaxAddressesGap: 10000 (unchanged) To override per coin, add to additional_params in configs/coins/*.json: "xpubConfig": { "maxCacheEntries": 2048 } --- api/worker.go | 13 +++++ api/xpub.go | 102 ++++++++++++++++++++++++--------- api/xpub_test.go | 51 ++++++++++++++--- bchain/coins/blockchain.go | 11 ++++ bchain/coins/btc/bitcoinrpc.go | 13 +++++ bchain/types.go | 19 ++++++ docs/config.md | 18 ++++++ 7 files changed, 192 insertions(+), 35 deletions(-) diff --git a/api/worker.go b/api/worker.go index 2d9bc38c83..b496a2ff04 100644 --- a/api/worker.go +++ b/api/worker.go @@ -34,6 +34,7 @@ type Worker struct { is *common.InternalState fiatRates *fiat.FiatRates metrics *common.Metrics + xpubConfig XpubConfig } var getTickersForTimestamps = func(fr *fiat.FiatRates, timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) { @@ -49,6 +50,17 @@ type contractInfoCache = map[string]*bchain.ContractInfo // NewWorker creates new api worker func NewWorker(db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState, fiatRates *fiat.FiatRates) (*Worker, error) { + // Resolve per-chain xpub config override, if any. + xpubCfg := DefaultXpubConfig() + if provider, ok := chain.(interface { + XpubConfigOverride() *bchain.XpubConfig + }); ok { + if override := provider.XpubConfigOverride(); override != nil { + xpubCfg = ApplyXpubConfig(override) + glog.Infof("xpub: xpubConfig override applied: maxCacheEntries=%d maxCacheExpirationSeconds=%d defaultAddressesGap=%d maxAddressesGap=%d", + xpubCfg.MaxCacheEntries, xpubCfg.MaxCacheExpirationSeconds, xpubCfg.DefaultAddressesGap, xpubCfg.MaxAddressesGap) + } + } w := &Worker{ db: db, txCache: txCache, @@ -60,6 +72,7 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, is: is, fiatRates: fiatRates, metrics: metrics, + xpubConfig: xpubCfg, } if w.chainType == bchain.ChainBitcoinType { w.initXpubCache() diff --git a/api/xpub.go b/api/xpub.go index 49d82d6a51..ae743ed1cf 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -15,15 +15,63 @@ import ( "github.com/trezor/blockbook/db" ) -const defaultAddressesGap = 20 -const maxAddressesGap = 10000 - const txInput = 1 const txOutput = 2 -const xpubCacheExpirationSeconds = 3600 -const xpubCacheMaxEntries = 128 -const maxXpubAddressDerivations = (maxAddressesGap + 1) * 2 +// XpubConfig is the resolved xpub configuration for a single Worker instance. +type XpubConfig struct { + MaxCacheExpirationSeconds int + MaxCacheEntries int + DefaultAddressesGap int + MaxAddressesGap int + MaxAddressDerivations int // computed: (MaxAddressesGap + 1) * 2 +} + +// DefaultXpubConfig returns the built-in defaults for xpub expansion and +// caching. Operators can override these per-coin via +// `additional_params.xpubConfig` in `configs/coins/*.json`. +func DefaultXpubConfig() XpubConfig { + return finalizeXpubConfig(XpubConfig{ + MaxCacheExpirationSeconds: 3600, + MaxCacheEntries: 1024, + DefaultAddressesGap: 20, + MaxAddressesGap: 10000, + }) +} + +// finalizeXpubConfig computes MaxAddressDerivations from MaxAddressesGap and clamps DefaultAddressesGap to not exceed +// MaxAddressesGap; every path producing an XpubConfig must run through it. +func finalizeXpubConfig(cfg XpubConfig) XpubConfig { + if cfg.DefaultAddressesGap > cfg.MaxAddressesGap { + cfg.DefaultAddressesGap = cfg.MaxAddressesGap + } + cfg.MaxAddressDerivations = (cfg.MaxAddressesGap + 1) * 2 + return cfg +} + +// ApplyXpubConfig overlays the optional per-coin override onto the built-in +// defaults. Zero / missing wire fields keep the default; explicitly set but +// invalid values (negative) keep the default and log a warning. +func ApplyXpubConfig(o *bchain.XpubConfig) XpubConfig { + cfg := DefaultXpubConfig() + if o == nil { + return cfg + } + apply := func(field string, v int, set func(int)) { + if v <= 0 { + if v < 0 { + glog.Warningf("xpub: xpubConfig.%s=%d is invalid, keeping default", field, v) + } + return + } + set(v) + } + apply("maxCacheExpirationSeconds", o.MaxCacheExpirationSeconds, func(v int) { cfg.MaxCacheExpirationSeconds = v }) + apply("maxCacheEntries", o.MaxCacheEntries, func(v int) { cfg.MaxCacheEntries = v }) + apply("defaultAddressesGap", o.DefaultAddressesGap, func(v int) { cfg.DefaultAddressesGap = v }) + apply("maxAddressesGap", o.MaxAddressesGap, func(v int) { cfg.MaxAddressesGap = v }) + return finalizeXpubConfig(cfg) +} var cachedXpubs map[string]xpubData var cachedXpubsMux sync.Mutex @@ -90,7 +138,7 @@ func (w *Worker) initXpubCache() { func (w *Worker) evictXpubCacheItems() { now := time.Now().Unix() cachedXpubsMux.Lock() - count := evictXpubCacheItemsLocked(now) + count := evictXpubCacheItemsLocked(now, int64(w.xpubConfig.MaxCacheExpirationSeconds), w.xpubConfig.MaxCacheEntries) cacheSize := len(cachedXpubs) cachedXpubsMux.Unlock() @@ -98,8 +146,8 @@ func (w *Worker) evictXpubCacheItems() { glog.Info("Evicted ", count, " items from xpub cache, cache size ", cacheSize) } -func evictXpubCacheItemsLocked(now int64) int { - threshold := now - xpubCacheExpirationSeconds +func evictXpubCacheItemsLocked(now int64, expirationSeconds int64, maxEntries int) int { + threshold := now - expirationSeconds count := 0 for k, v := range cachedXpubs { if v.accessed < threshold { @@ -107,11 +155,11 @@ func evictXpubCacheItemsLocked(now int64) int { count++ } } - return count + trimXpubCacheItemsLocked() + return count + trimXpubCacheItemsLocked(maxEntries) } -func trimXpubCacheItemsLocked() int { - if len(cachedXpubs) <= xpubCacheMaxEntries { +func trimXpubCacheItemsLocked(maxEntries int) int { + if len(cachedXpubs) <= maxEntries { return 0 } type cacheEntry struct { @@ -128,20 +176,20 @@ func trimXpubCacheItemsLocked() int { } return entries[i].accessed < entries[j].accessed }) - count := len(cachedXpubs) - xpubCacheMaxEntries + count := len(cachedXpubs) - maxEntries for i := 0; i < count; i++ { delete(cachedXpubs, entries[i].key) } return count } -func validateXpubScanLimits(xd *bchain.XpubDescriptor, gap int) error { +func validateXpubScanLimits(xd *bchain.XpubDescriptor, gap int, maxDerivations int) error { if len(xd.ChangeIndexes) > bchain.MaxXpubChangeIndexes { return errors.Errorf("Xpub descriptor change index count %d exceeds limit %d", len(xd.ChangeIndexes), bchain.MaxXpubChangeIndexes) } derivations := len(xd.ChangeIndexes) * gap - if derivations > maxXpubAddressDerivations { - return errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", derivations, maxXpubAddressDerivations) + if derivations > maxDerivations { + return errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", derivations, maxDerivations) } return nil } @@ -272,9 +320,9 @@ func (w *Worker) xpubDerivedAddressBalance(data *xpubData, ad *xpubAddress) (boo return false, nil } -func (w *Worker) xpubScanAddresses(xd *bchain.XpubDescriptor, data *xpubData, addresses []xpubAddress, gap int, change uint32, minDerivedIndex int, fork bool, derivedBefore int) (int, []xpubAddress, error) { - if total := derivedBefore + len(addresses); total > maxXpubAddressDerivations { - return 0, nil, errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", total, maxXpubAddressDerivations) +func (w *Worker) xpubScanAddresses(xd *bchain.XpubDescriptor, data *xpubData, addresses []xpubAddress, gap int, change uint32, minDerivedIndex int, fork bool, derivedBefore int, maxDerivations int) (int, []xpubAddress, error) { + if total := derivedBefore + len(addresses); total > maxDerivations { + return 0, nil, errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", total, maxDerivations) } // rescan known addresses lastUsed := 0 @@ -303,8 +351,8 @@ func (w *Worker) xpubScanAddresses(xd *bchain.XpubDescriptor, data *xpubData, ad if to < minDerivedIndex { to = minDerivedIndex } - if total := derivedBefore + to; total > maxXpubAddressDerivations { - return 0, nil, errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", total, maxXpubAddressDerivations) + if total := derivedBefore + to; total > maxDerivations { + return 0, nil, errors.Errorf("Xpub descriptor scan size %d exceeds limit %d", total, maxDerivations) } descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xd, change, uint32(from), uint32(to)) if err != nil { @@ -393,14 +441,14 @@ func (w *Worker) getXpubData(xd *bchain.XpubDescriptor, page int, txsOnPage int, besthash string ) if gap <= 0 { - gap = defaultAddressesGap - } else if gap > maxAddressesGap { + gap = w.xpubConfig.DefaultAddressesGap + } else if gap > w.xpubConfig.MaxAddressesGap { // limit the maximum gap to protect against unreasonably big values that could cause high load of the server - gap = maxAddressesGap + gap = w.xpubConfig.MaxAddressesGap } // gap is increased one as there must be gap of empty addresses before the derivation is stopped gap++ - if err := validateXpubScanLimits(xd, gap); err != nil { + if err := validateXpubScanLimits(xd, gap, w.xpubConfig.MaxAddressDerivations); err != nil { return nil, 0, false, err } var processedHash string @@ -447,7 +495,7 @@ func (w *Worker) getXpubData(xd *bchain.XpubDescriptor, page int, txsOnPage int, var minDerivedIndex int totalDerived := 0 for i, change := range xd.ChangeIndexes { - minDerivedIndex, data.addresses[i], err = w.xpubScanAddresses(xd, &data, data.addresses[i], gap, change, minDerivedIndex, fork, totalDerived) + minDerivedIndex, data.addresses[i], err = w.xpubScanAddresses(xd, &data, data.addresses[i], gap, change, minDerivedIndex, fork, totalDerived, w.xpubConfig.MaxAddressDerivations) if err != nil { return nil, 0, inCache, err } @@ -479,7 +527,7 @@ func (w *Worker) getXpubData(xd *bchain.XpubDescriptor, page int, txsOnPage int, cachedXpubs = make(map[string]xpubData) } cachedXpubs[xd.XpubDescriptor] = data - trimXpubCacheItemsLocked() + trimXpubCacheItemsLocked(w.xpubConfig.MaxCacheEntries) cacheSize := len(cachedXpubs) cachedXpubsMux.Unlock() w.metrics.XPubCacheSize.Set(float64(cacheSize)) diff --git a/api/xpub_test.go b/api/xpub_test.go index a0713df1b1..3126032cdb 100644 --- a/api/xpub_test.go +++ b/api/xpub_test.go @@ -9,23 +9,58 @@ import ( "github.com/trezor/blockbook/bchain" ) +func TestDefaultXpubConfigDerivedFields(t *testing.T) { + cfg := DefaultXpubConfig() + if cfg.MaxAddressDerivations != (cfg.MaxAddressesGap+1)*2 { + t.Fatalf("DefaultXpubConfig().MaxAddressDerivations = %d, want %d", cfg.MaxAddressDerivations, (cfg.MaxAddressesGap+1)*2) + } + if cfg.MaxAddressDerivations == 0 { + t.Fatal("DefaultXpubConfig().MaxAddressDerivations must be non-zero, otherwise every xpub scan is rejected") + } + // The default config must accept a normal scan at the default gap; this is + // the value NewWorker hands to validateXpubScanLimits when a coin has no + // xpubConfig override. + if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: []uint32{0, 1}}, cfg.DefaultAddressesGap+1, cfg.MaxAddressDerivations); err != nil { + t.Fatalf("default config rejected a default-gap scan: %v", err) + } +} + +func TestApplyXpubConfigClampsAndDerives(t *testing.T) { + // A default gap larger than the max gap must be clamped down so it never + // exceeds the accepted maximum. + cfg := ApplyXpubConfig(&bchain.XpubConfig{DefaultAddressesGap: 50, MaxAddressesGap: 20}) + if cfg.DefaultAddressesGap > cfg.MaxAddressesGap { + t.Fatalf("DefaultAddressesGap = %d not clamped to MaxAddressesGap = %d", cfg.DefaultAddressesGap, cfg.MaxAddressesGap) + } + if cfg.MaxAddressDerivations != (cfg.MaxAddressesGap+1)*2 { + t.Fatalf("MaxAddressDerivations = %d, want %d", cfg.MaxAddressDerivations, (cfg.MaxAddressesGap+1)*2) + } + + // A nil override yields the finalized defaults. + if got, want := ApplyXpubConfig(nil), DefaultXpubConfig(); got != want { + t.Fatalf("ApplyXpubConfig(nil) = %+v, want %+v", got, want) + } +} + func TestValidateXpubScanLimits(t *testing.T) { - if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: []uint32{0, 1}}, maxAddressesGap+1); err != nil { + const testMaxDerivations = 20002 + if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: []uint32{0, 1}}, 10001, testMaxDerivations); err != nil { t.Fatalf("expected default change indexes at max gap to pass, got %v", err) } changes := make([]uint32, bchain.MaxXpubChangeIndexes+1) - if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: changes}, defaultAddressesGap+1); err == nil { + if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: changes}, 21, testMaxDerivations); err == nil { t.Fatal("expected change index count above limit to fail") } changes = make([]uint32, 3) - if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: changes}, maxAddressesGap+1); err == nil { + if err := validateXpubScanLimits(&bchain.XpubDescriptor{ChangeIndexes: changes}, 10001, testMaxDerivations); err == nil { t.Fatal("expected scan size above limit to fail") } } func TestTrimXpubCacheItemsLocked(t *testing.T) { + const testMaxEntries = 128 cachedXpubsMux.Lock() defer cachedXpubsMux.Unlock() @@ -34,16 +69,16 @@ func TestTrimXpubCacheItemsLocked(t *testing.T) { cachedXpubs = originalCache }() - cachedXpubs = make(map[string]xpubData, xpubCacheMaxEntries+2) - for i := 0; i < xpubCacheMaxEntries+2; i++ { + cachedXpubs = make(map[string]xpubData, testMaxEntries+2) + for i := 0; i < testMaxEntries+2; i++ { cachedXpubs[fmt.Sprintf("xpub-%03d", i)] = xpubData{accessed: int64(i)} } - if got := trimXpubCacheItemsLocked(); got != 2 { + if got := trimXpubCacheItemsLocked(testMaxEntries); got != 2 { t.Fatalf("trimXpubCacheItemsLocked() evicted %d entries, want 2", got) } - if got := len(cachedXpubs); got != xpubCacheMaxEntries { - t.Fatalf("cachedXpubs length = %d, want %d", got, xpubCacheMaxEntries) + if got := len(cachedXpubs); got != testMaxEntries { + t.Fatalf("cachedXpubs length = %d, want %d", got, testMaxEntries) } if _, ok := cachedXpubs["xpub-000"]; ok { t.Fatal("oldest cache entry was not evicted") diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 0eaa874242..b5aa8a1ae0 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -535,3 +535,14 @@ func (c *blockChainWithMetrics) MissingBlockRetryOverride() *bchain.MissingBlock } return nil } + +// XpubConfigOverride forwards the wrapped chain's per-chain xpub config +// override through the metrics wrapper; nil when none is provided. +func (c *blockChainWithMetrics) XpubConfigOverride() *bchain.XpubConfig { + if p, ok := c.b.(interface { + XpubConfigOverride() *bchain.XpubConfig + }); ok { + return p.XpubConfigOverride() + } + return nil +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index f0d5448f2c..f4505358dd 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -64,6 +64,16 @@ func (b *BitcoinRPC) MissingBlockRetryOverride() *bchain.MissingBlockRetry { return b.ChainConfig.MissingBlockRetry } +// XpubConfigOverride exposes the per-chain xpub expansion and cache config +// override (or nil to use built-in defaults). Consumed by api.NewWorker via +// a duck-typed interface assertion. +func (b *BitcoinRPC) XpubConfigOverride() *bchain.XpubConfig { + if b.ChainConfig == nil { + return nil + } + return b.ChainConfig.XpubConfig +} + // Configuration represents json config file type Configuration struct { CoinName string `json:"coin_name"` @@ -100,6 +110,9 @@ type Configuration struct { // MissingBlockRetry overrides the sync-worker missing-block retry policy // per chain. All fields are optional; missing fields use built-in defaults. MissingBlockRetry *bchain.MissingBlockRetry `json:"missingBlockRetry,omitempty"` + // XpubConfig overrides the xpub address expansion and cache limits. + // All fields are optional; missing fields use built-in defaults. + XpubConfig *bchain.XpubConfig `json:"xpubConfig,omitempty"` } // AverageBlockTimeDuration returns AverageBlockTimeMs as a time.Duration. diff --git a/bchain/types.go b/bchain/types.go index fa0c4fad0e..ca215464a9 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -448,3 +448,22 @@ type MissingBlockRetry struct { // yields errResync to the outer machinery. MaxStallMs int `json:"maxStallMs,omitempty"` } + +// XpubConfig carries per-chain overrides for xpub address expansion and +// caching. All fields are optional; zero / missing values fall back to the +// api package's built-in defaults. Operators set this under +// `additional_params.xpubConfig` in `configs/coins/*.json`. +type XpubConfig struct { + // MaxCacheExpirationSeconds is the TTL for cached xpub-to-address + // derivations (seconds). Zero / missing uses the default (3600). + MaxCacheExpirationSeconds int `json:"maxCacheExpirationSeconds,omitempty"` + // MaxCacheEntries is the hard cap on the number of cached xpubs. + // Zero / missing uses the default (1024). + MaxCacheEntries int `json:"maxCacheEntries,omitempty"` + // DefaultAddressesGap is the default BIP44 gap limit when the + // caller does not supply one (zero / missing → 20). + DefaultAddressesGap int `json:"defaultAddressesGap,omitempty"` + // MaxAddressesGap is the maximum user-supplied gap value the + // server will accept (zero / missing → 10000). + MaxAddressesGap int `json:"maxAddressesGap,omitempty"` +} diff --git a/docs/config.md b/docs/config.md index 9a8fe89717..22a5ea9811 100644 --- a/docs/config.md +++ b/docs/config.md @@ -119,6 +119,24 @@ Good examples of coin configuration are * `address_contracts_cache_min_size` – Minimum packed size (bytes) before an addressContracts entry is cached (default **300000**). * `address_contracts_cache_max_bytes` – Cache size cap in bytes used while syncing near chain tip; when exceeded, cached entries are flushed early (default **2000000000**). * `address_contracts_cache_bulk_max_bytes` – Cache size cap in bytes used during bulk connect; when exceeded, cached entries are flushed early (default **4000000000**). + * `xpubConfig` – xpub/descriptor expansion and cache tuning (BitcoinType only). Each field is optional; a missing or `<= 0` value keeps the built-in default, and a negative value is logged as invalid and ignored. When an override is present, Blockbook logs one `xpub: xpubConfig override applied: …` line at startup so you can confirm the effective values. + + | Field | Default | Semantic | + | --- | --- | --- | + | `maxCacheEntries` | **1024** | Maximum number of xpubs kept in the in-memory derivation cache before the oldest entries are evicted. Raising it reduces re-derivation on busy instances but increases resident memory, since each entry holds the xpub's derived addresses and their tx lists — size the host accordingly. | + | `maxCacheExpirationSeconds` | **3600** | Time after last access before a cached xpub entry is dropped. | + | `defaultAddressesGap` | **20** | Gap limit used when a request does not specify one. Clamped down to `maxAddressesGap` if set higher. | + | `maxAddressesGap` | **10000** | Upper bound on the caller-supplied gap limit, protecting against unreasonably large scans. | + + Example override in `configs/coins/*.json`: + + ```json + "additional_params": { + "xpubConfig": { + "maxCacheEntries": 2048 + } + } + ``` * `meta` – Common package metadata. * `package_maintainer` – Full name of package maintainer. From 6e1299a2c4c31f7d5b94a53e293d8cfb8cafdfe4 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Thu, 16 Jul 2026 11:06:18 +0000 Subject: [PATCH 096/303] 0.6.0 release changelog --- changelog.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/changelog.md b/changelog.md index aa630fa97c..31a5627502 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,11 @@ - **Faster BTC mempool resync with batching + outpoint cache** ([#1403](https://github.com/trezor/blockbook/pull/1403)): adds optional batch tx fetch with bounded concurrency and a temporary resync outpoint cache to avoid repeated parent lookups. - **Ethereum address-contract indexing micro-optimizations** ([#1417](https://github.com/trezor/blockbook/pull/1417)): adds hot-address LRU/index map to remove O(n) contract scans, bounds cache growth, and improves ERC20 aggregation hot-path overhead. - **WebSocket/API perf + fiat observability improvements** ([#1423](https://github.com/trezor/blockbook/pull/1423)): batches fiat enrichment with robust fallback reasons, improves client-facing error behavior, and reduces log noise. +- **Memory optimizations** ([#1479](https://github.com/trezor/blockbook/pull/1479)): addresses agent-discovered heap-pressure findings, reducing overall memory footprint. +- **Skip mempool tx body loading for AccountDetailsBasic** ([#1499](https://github.com/trezor/blockbook/pull/1499)): drops per-tx body fetch for `details=basic` WebSocket requests; reports `unconfirmedTxs` from cheap mempool-index length only, saving significant RPC work on mempool-heavy addresses. +- **Accumulate address-hotness hits across blocks** ([#1529](https://github.com/trezor/blockbook/pull/1529)): makes the contract-index LRU promotion counter persist across blocks instead of resetting per block, so hot-address detection works on lower-throughput chains too. +- **Websocket communication optimizations** ([#1550](https://github.com/trezor/blockbook/pull/1550)): reuses the xpub best-height within a request across derived addresses, adds a WebSocket write-buffer pool, reduces subscription lock hold time, and caches merged confirmed xpub txids for unfiltered paged requests. +- **Resolve tx inputs without unpacking full prev-tx records** ([#1595](https://github.com/trezor/blockbook/pull/1595)): reads only the single needed output per input instead of unpacking the entire previous transaction, eliminating quadratic heap allocation on high-fan-in addresses. ### Reliability and Correctness @@ -19,15 +24,58 @@ - **Reliable SIGTERM shutdown + clean RocksDB close** ([#1408](https://github.com/trezor/blockbook/pull/1408)): reworks signal fan-out so main shutdown always runs, unblocks workers, and stops periodic state writes during shutdown. - **Resync recovery on errors** ([#1409](https://github.com/trezor/blockbook/pull/1409)): detects errors in parallel/bulk sync and triggers controlled resync restarts on rollback/reorg to avoid infinite retry stalls. - **Fixed scientific notation parsing error** ([#1429](https://github.com/trezor/blockbook/pull/1429)): `AmountToBigInt` now safely handles scientific notation (`e`/`E`), keeps a fast path for plain decimals, and rejects pathological exponent expansion. +- **Fix Nillion (NIL) token decimals** ([#1443](https://github.com/trezor/blockbook/pull/1443)): adds the Nillion `NIL` ERC-20 contract to the contract-fix config with `decimals: 6`, correcting misscaled balance/transfer amounts. +- **Don't lowercase Base58 Tron token addresses for fiat rates** ([#1458](https://github.com/trezor/blockbook/pull/1458)): stops case-sensitive Base58 Tron contract addresses from being lowercased during fiat-rate lookups (which broke their rate resolution), while hex EVM tokens keep lowercase normalization. +- **Skip ZMQ initialization when message_queue_binding is empty** ([#1460](https://github.com/trezor/blockbook/pull/1460)): enables ZMQ-less operation with zebrad by skipping zmq_connect("") instead of returning EINVAL. +- **Fix WebSocket GetBlock pagination defaults** ([#1476](https://github.com/trezor/blockbook/pull/1476)): replaces the 1M-transaction default page size with a sane bound matching the REST interface, addressing security alerts. +- **Accept BTC non-standard tx versions outside int32 range** ([#1478](https://github.com/trezor/blockbook/pull/1478)): decodes `getrawtransaction` JSON version as uint32 to fix "Transaction not found" for historical bitcoin transactions whose version exceeds int32 max. +- **Retry unfinalized RPC responses** ([#1489](https://github.com/trezor/blockbook/pull/1489)): maps Avalanche "cannot query unfinalized data" errors to `ErrBlockNotFound` instead of swallowing them, preventing trace-length mismatches during sync. +- **Fix fiat rates: change platform currency from eth to usd** ([#1495](https://github.com/trezor/blockbook/pull/1495)): corrects the fiat-rate query currency for token rates; requires running `reset_eth_token_rates.sh` after deploy. +- **Stop ERC20 batch fallback from amplifying RPC calls** ([#1501](https://github.com/trezor/blockbook/pull/1501)): prevents a single bad batch response from triggering per-contract `eth_call` fallbacks that amplified failed requests into N+1 RPCs. +- **Deduplicate address descriptors and fix confirmed-block matching** ([#1508](https://github.com/trezor/blockbook/pull/1508)): deduplicates subscriber addresses before counting and restricts confirmed-block tx conversion to `publishNewBlockTxs` subscribers, reducing expensive receipt RPCs. +- **Retry transient missing-tip block fetches before ResyncIndex** ([#1517](https://github.com/trezor/blockbook/pull/1517)): retries transient `ErrBlockNotFound` inside the sequential sync loop instead of unwinding to the full `ResyncIndex` path, preventing slowdowns on load-balanced RPC tip skew. +- **Fix NFT self-transfer reorg rollback** ([#1525](https://github.com/trezor/blockbook/pull/1525)): corrects ERC-721/1155 ownership rollback when a self-transfer block is disconnected during reorg. +- **Fix config propagation duck-typing issue** ([#1528](https://github.com/trezor/blockbook/pull/1528)): fixes a structural-typing mismatch that prevented certain config values from being propagated. +- **Prevent sync stalls on load-balanced EVM RPC skew** ([#1530](https://github.com/trezor/blockbook/pull/1530)): makes EVM sync tolerate load-balanced backend tip height inconsistencies and prevents the parallel/bulk coordinator from wedging when workers report abort conditions. +- **Heal silently-stalled EVM newHeads tip feeds** ([#1533](https://github.com/trezor/blockbook/pull/1533)): adds a chain-aware staleness watchdog that detects when the push feed goes silent without raising `sub.Err()` and heals it, preventing undetected sync stalls. +- **Make CoinGecko fiat-rate fetching resilient to 429 throttling** ([#1560](https://github.com/trezor/blockbook/pull/1560)): adds a self-healing pass for missing daily rates at startup and improves resiliency of fiat rates fetching. +- **Reconcile the alternative-provider send-tx mempool cache** ([#1562](https://github.com/trezor/blockbook/pull/1562)): re-checks liveness and heals nonce-superseded entries in the self-maintained alternative/private-relay send-tx cache (which relays don't expose for sync), with new metrics; no API/wire change. +- **Keep alt-mempool txs visible when private relay stops surfacing them** ([#1572](https://github.com/trezor/blockbook/pull/1572)): defers eviction of alt-mempool txs to the absolute cache timeout when `eth_getTransactionByHash` returns empty from private relays, preventing premature disappearance. +- **Tron: polling-only tip sync fallback when ZeroMQ is unavailable** ([#1578](https://github.com/trezor/blockbook/pull/1578)): adds a polling fallback for Tron tip sync when the node runs the MongoDB event plugin instead of the native ZMQ queue. +- **Always serialize token decimals, default unresolved to coin convention** ([#1579](https://github.com/trezor/blockbook/pull/1579)): removes `omitempty` from token decimals so clients can distinguish 0-decimal tokens from missing metadata; defaults unresolved values to 18 for ERC-20. +- **Tron: keep receipt logs from unsolidified blocks** ([#1583](https://github.com/trezor/blockbook/pull/1583)): retains receipt data for not-yet-solidified Tron blocks so token transfers are indexed immediately instead of waiting for solidification. +- **Send a Blockbook user-agent on CoinGecko requests** ([#1589](https://github.com/trezor/blockbook/pull/1589)): adds a `User-Agent` header to CoinGecko fiat-rate requests to avoid being blocked/throttled, complementing the 429-resilience work in #1560. +- **Prevent panic parsing non-32-byte-aligned EVM input data** ([#1591](https://github.com/trezor/blockbook/pull/1591)): fixes an index-out-of-range panic when transaction input data is not aligned to 32-byte words. +- **Disable mempool sync on remaining L2 configs** ([#1599](https://github.com/trezor/blockbook/pull/1599)): sets `disableMempoolSync: true` for OP Stack and Arbitrum configurations where the backend does not support `newPendingTransactions` subscriptions. +- **Avoid int64 overflow in on-chain base fee conversion** ([#1605](https://github.com/trezor/blockbook/pull/1605)): uses `big.Int.SetUint64` for `baseFeePerGas` instead of int64 cast, preventing negative values on high-fee L2s. +- **Fix PIVX Sapling txid computation** ([#1607](https://github.com/trezor/blockbook/pull/1607)): computes the correct canonical txid for PIVX Sapling transactions (v>=3) by hashing the entire raw transaction bytes instead of just the decoded portion. +- **Recover GetTransaction on archive backends with pruned tx index** ([#1610](https://github.com/trezor/blockbook/pull/1610)): falls back to reconstructing the transaction from its receipt and block body when `eth_getTransactionByHash` returns null on backends with pruned transaction hash indexes. +- **Prevent slice-bounds panic from ENS log name-length overflow** ([#1613](https://github.com/trezor/blockbook/pull/1613)): guards against a crafted ENS `NameRegistered` log causing a `low > high` slice expression, fixing an unauthenticated availability DoS that could crash the EVM sync goroutine. +- **Prevent duplicate ERC-721 holdings** ([#1624](https://github.com/trezor/blockbook/pull/1624)): makes ERC-721 holding insertion idempotent so a replayed `Transfer` log can't credit the same token ID twice, preserves holdings across self-transfers and their reorg rollback, and adds a read-only RocksDB checker that reports (but never repairs) existing duplicates. +- **Gate alternative-provider nonce lookups to recent private senders** ([#1628](https://github.com/trezor/blockbook/pull/1628)): stops routing every `eth_getTransactionCount` to the alternative provider when `*_ALTERNATIVE_SENDTX_URLS` is set, restricting it to senders with a recent privately submitted tx — avoiding provider rate-quota exhaustion, 429 log floods, and stale pending nonces. ### Configuration and Deployment - **Configurable backend RPC endpoints for builds/tests** ([#1392](https://github.com/trezor/blockbook/pull/1392)): adds per-coin `BB_RPC_URL_*` overrides for non-local backends, `BB_RPC_BIND_HOST_*`/`BB_RPC_ALLOW_IP_*` for safer network exposure, plus `rpc_url_ws_template` and `BB_RPC_URL_WS_*` overrides. +- **Override Message Queue URL from environment variables** ([#1468](https://github.com/trezor/blockbook/pull/1468)): allows overriding the message queue connection URL via environment variables without modifying coin config. +- **Configurable debouncedelay; increased defaults for Polygon and Arbitrum** ([#1475](https://github.com/trezor/blockbook/pull/1475)): makes the newHeads debounce delay configurable and increases defaults for Polygon and Arbitrum to tolerate frequent short reorgs; adds explicit `trace_timeout` for heavy `debug_traceBlockByHash` calls. +- **Decrease mempool timeout for MEV-protected coins** ([#1500](https://github.com/trezor/blockbook/pull/1500)): reduces the stale-tx eviction timeout for alternative/private relay mempool entries that expire faster than the public mempool. +- **Runtime env via optional EnvironmentFile** ([#1574](https://github.com/trezor/blockbook/pull/1574)): adds an optional `EnvironmentFile=/etc/blockbook/blockbook.env` to the systemd unit so runtime config no longer requires hand-editing `/etc/systemd/system.conf`. +- **Extend rate limiting to cover all dynamic routes** ([#1588](https://github.com/trezor/blockbook/pull/1588)): replaces the `/api`-only rate-limit allowlist with a deny-all-except-static gate, covering explorer UI routes alongside REST API routes. +- **Tune REST/UI rate-limit defaults for individual user traffic** ([#1590](https://github.com/trezor/blockbook/pull/1590)): adjusts per-client rate-limit defaults to 20 req/min with burst 20, matching individual browsing patterns rather than high-throughput clients. +- **Make WebSocket per-connection pending-request limit configurable** ([#1626](https://github.com/trezor/blockbook/pull/1626)): exposes the previously hard-coded `maxWebsocketPendingRequests` (48) as a per-coin env knob so self-hosted batch consumers pipelining many requests over one connection are no longer disconnected mid-batch. +- **Configurable per-coin xpub expansion and cache caps** ([#1630](https://github.com/trezor/blockbook/pull/1630)): replaces the hard-coded xpub cache-size, expiration, and max-derivation constants with a per-coin `XpubConfig`, and raises the default cache to 1024 entries to stop thrashing under many concurrent users. ### Observability - **Syncing/caching Prometheus metrics** ([#1420](https://github.com/trezor/blockbook/pull/1420)): introduces many new metrics for syncing throughput and cache behavior. - **WebSocket/API perf + fiat observability improvements** ([#1423](https://github.com/trezor/blockbook/pull/1423)): adds Prometheus metrics for fiat enrichment and API behavior. +- **Metric for alternative fee-provider requests** ([#1485](https://github.com/trezor/blockbook/pull/1485)): adds a Prometheus counter tracking alternative fee-provider request outcomes, labeled by provider and status. +- **Graceful consensus node monitoring degradation** ([#1488](https://github.com/trezor/blockbook/pull/1488)): marks the consensus node as `"unreachable-locally"` on dev instances when not running locally, reducing maintenance overhead while preserving production visibility. +- **Observe JSON-RPC calls for block sync** ([#1527](https://github.com/trezor/blockbook/pull/1527)): adds error-request metrics for JSON-RPC calls made during block sync. +- **Auto-generated Grafana dashboard from source of truth** ([#1541](https://github.com/trezor/blockbook/pull/1541)): generates the Grafana dashboard from `metrics.yaml` and `panels.yaml` to eliminate duplication; adds four new WebSocket abuse-observability metrics (connection requests, rejections, unique IPs, per-IP connection count). +- **Correct stale EIP-1559 max-fee panel** ([#1596](https://github.com/trezor/blockbook/pull/1596)): fixes an incorrect instant-tier claim in the EIP-1559 max-fee Grafana panel description. +- **Sync RPC latency metric** ([#1601](https://github.com/trezor/blockbook/pull/1601)): adds a Prometheus latency histogram for sync-internal RPC calls. ### Fiat Pipeline @@ -36,6 +84,14 @@ ### Testing - **API-level E2E suite + deploy workflow** ([#1426](https://github.com/trezor/blockbook/pull/1426)): adds E2E tests against live Blockbook endpoints plus GitHub Actions build/deploy stages that wait for sync and run filtered E2E validation after deploy. +- **Tron E2E API tests** ([#1440](https://github.com/trezor/blockbook/pull/1440)): adds E2E tests for Tron API endpoints. +- **Ethereum Classic integration tests** ([#1454](https://github.com/trezor/blockbook/pull/1454)): adds integration tests covering Ethereum Classic RPC and API endpoints. +- **ERC-4626 E2E tests** ([#1511](https://github.com/trezor/blockbook/pull/1511)): adds E2E tests for the ERC-4626 vault enrichment protocol. +- **OpenAPI specification + swagger endpoint with TS-compiled verification** ([#1523](https://github.com/trezor/blockbook/pull/1523)): introduces an `openapi.yaml` spec served via swagger-ui, separates E2E tests from integration tests, and adds CI/CD verification that `blockbook-api.ts` is type-compatible. +- **Extend E2E suite with fiat rate and WebSocket tests** ([#1539](https://github.com/trezor/blockbook/pull/1539)): adds fiat-rate freshness and validity checks plus WebSocket block/balance-history/tx tests; produces jUnit XML test reports for CI visibility. +- **System check for local test prerequisites** ([#1564](https://github.com/trezor/blockbook/pull/1564)): adds a pre-check that validates the local environment meets testing requirements, with clear error messages for missing dependencies. +- **Testnet integration/E2E tests** ([#1575](https://github.com/trezor/blockbook/pull/1575)): adds integration and post-deploy E2E coverage for Sepolia, Hoodi, and Nile testnets; fixes a matchable-name collision that caused sibling testnets to share one subtest name. +- **Parallelize E2E test suite** ([#1601](https://github.com/trezor/blockbook/pull/1601)): runs E2E tests across coins in parallel, adds proactive sample preloading and persistent WebSocket connections. ### Security @@ -43,6 +99,13 @@ - **Security hardening: CSP + XSS fixes in templates** ([#1397](https://github.com/trezor/blockbook/pull/1397)): adds CSP headers and fixes XSS vulnerabilities in templates. - **WebSocket origin allowlist** ([#1421](https://github.com/trezor/blockbook/pull/1421)): adds optional origin checks with explicit logging to reduce cross-origin websocket exposure when not protected by a proxy. - **Request-size and template hardening** ([#1434](https://github.com/trezor/blockbook/pull/1434)): limits `/api/sendtx` body size, rejects oversized websocket messages, and avoids `template.JSStr`. +- **Limit fiat timestamp requests** ([#1520](https://github.com/trezor/blockbook/pull/1520)): adds a 1000-timestamp maximum for fiat-rate batch lookups and a global WebSocket active-request limit. +- **Limit xpub descriptor expansion** ([#1521](https://github.com/trezor/blockbook/pull/1521)): bounds xpub change-list expansion before account scans, caps the in-memory xpub cache size, and evicts LRU entries. +- **Fix XSS in xpub explorer descriptor handling** ([#1552](https://github.com/trezor/blockbook/pull/1552)): anchors the xpub descriptor regex with a terminal `$` and avoids rendering user-supplied strings through `template.JSStr` in the QR-code JavaScript expression. +- **CI/CD security hardening** ([#1553](https://github.com/trezor/blockbook/pull/1553)): pins actions and dependencies to full commit SHAs, adds dependabot, verifies Go tarball and backend artifact checksums, restricts workflow permissions, and adds opt-in TLS verification for deploy scripts. +- **Upper-cap limits on WebSocket endpoints** ([#1582](https://github.com/trezor/blockbook/pull/1582)): caps in-flight mempool-filters responses per connection, bounds fiat-rate currency selectors, and hardens ERC-1155 `TransferBatch` log parsing against OOM from malformed data. +- **Quote paths in root logrotate cleanup** ([#1602](https://github.com/trezor/blockbook/pull/1602)): prevents arbitrary recursive deletion by quoting log paths in the root cron logrotate script, fixing a local privilege-boundary issue reachable from the Blockbook service user. +- **Cap subscribeFiatRates tokens array length** ([#1623](https://github.com/trezor/blockbook/pull/1623)): adds a 1000-token maximum for `subscribeFiatRates` WebSocket requests, consistent with existing caps on other list parameters. ### New Features and Chain Support @@ -51,7 +114,44 @@ - **Tron network support** ([#1273](https://github.com/trezor/blockbook/pull/1273)): adds Tron support to Blockbook. - **Opt-in ERC-4626 vault enrichment for EVM tokens** ([#1431](https://github.com/trezor/blockbook/pull/1431)): adds REST/WS `protocols=erc4626` batched vault detection and response enrichment under `protocols.erc4626`. - **Contract metadata API with protocol enrichments**: adds REST/WS single-contract lookup so clients can fetch current contract metadata and optional protocol payloads without reloading full `accountInfo`. +- **Tron: split bandwidth into staked and free fields** ([#1444](https://github.com/trezor/blockbook/pull/1444)): separates Tron account bandwidth into `staked` and `free` fields for clearer resource accounting. +- **Tron: tx balance history maps freezing, unfreezing and voting rewards** ([#1463](https://github.com/trezor/blockbook/pull/1463)): correctly maps Tron freeze/unfreeze operations and voting rewards as incoming/outgoing in balance history. +- **Tron: recognize account activation transactions** ([#1466](https://github.com/trezor/blockbook/pull/1466)): adds `AccountCreateContract` parsing so Tron account-activation transactions are no longer shown as unparsed. +- **Tron: expose staking and voting data in account API** ([#1467](https://github.com/trezor/blockbook/pull/1467)): implements `TronStakingInfo` in `TronAccountExtraData` exposing staking and governance data via the account API. +- **EVM getContractInfo endpoint** ([#1477](https://github.com/trezor/blockbook/pull/1477)): adds a single-contract lookup endpoint so clients can refresh ERC-4626 vault data and fiat pricing without reloading full `accountInfo`; unifies protocol enrichment shape between `getAccountInfo` and `getContractInfo`. +- **WebSocket connection caps** ([#1483](https://github.com/trezor/blockbook/pull/1483)): adds per-IP connection limits with configurable trusted-proxy origin detection for more reliable client-IP attribution. +- **Stateful ERC-4626 persistent cache with multicall batching** ([#1502](https://github.com/trezor/blockbook/pull/1502)): replaces per-request sequential `eth_call`s with a block-pinned, singleflight-deduped, reorg-safe persistent cache using Multicall3 batch aggregation; eliminates virtually all RPC overhead for vault enrichment. +- **Tron: extend account data with total energy and bandwidth** ([#1547](https://github.com/trezor/blockbook/pull/1547)): adds total energy and total bandwidth to `TronAccountExtraData`. +- **Expose confirmed nonce on address API** ([#1562](https://github.com/trezor/blockbook/pull/1562)): adds the confirmed account nonce to the address API response for EVM chains. +- **Rate-limit WebSocket and REST API traffic** ([#1563](https://github.com/trezor/blockbook/pull/1563)): adds application-level abuse protection with per-connection WS message-rate limits, per-client REST request throttling, shared trusted-proxy attribution, and bounded balance-history queries per transport. +- **EVM block-gas data for EIP-1559 fee projection** ([#1586](https://github.com/trezor/blockbook/pull/1586)): adds `baseFeePerGas`, `blockGasUsed`, and `blockGasLimit` to `subscribeNewBlock` notifications so wallets can project EIP-1559 base fees deterministically. +- **Method-selector allowlist for rpcCall** ([#1597](https://github.com/trezor/blockbook/pull/1597)): adds `ALLOWED_EVM_CALL_METHODS` configuration allowing `rpcCall` to invoke whitelisted 4-byte selectors on any address; runtime-configurable via admin API. +- **Tron: expose latestWithdrawTime in staking info** ([#1604](https://github.com/trezor/blockbook/pull/1604)): adds `latestWithdrawTime` parsed from `latest_withdraw_time` to Tron account staking information. ### Backend Compatibility - **Adjusted ZebraRPC for new zebrad backend version** ([#1377](https://github.com/trezor/blockbook/pull/1377)). +- **Update Firo daemon to 0.14.15.3** ([#1432](https://github.com/trezor/blockbook/pull/1432)). +- **Upgrade Ethereum erigon/prysm to latest versions** ([#1455](https://github.com/trezor/blockbook/pull/1455)): mainnet; ([#1462](https://github.com/trezor/blockbook/pull/1462)): Sepolia testnet. +- **Zebra upgrades** ([#1504](https://github.com/trezor/blockbook/pull/1504)): 4.4.1; ([#1532](https://github.com/trezor/blockbook/pull/1532)): 4.5.0; ([#1544](https://github.com/trezor/blockbook/pull/1544)): 5.0.0. +- **Bcash 28.0.1 → 29.0.0** ([#1505](https://github.com/trezor/blockbook/pull/1505)). +- **Zcash testnet: migrate to Zebra 5.0.0 for NU6.2** ([#1548](https://github.com/trezor/blockbook/pull/1548)). +- **Tron testnet backend 4.8.2** ([#1611](https://github.com/trezor/blockbook/pull/1611)). + +### Removed + +- **Remove obsolete Socket.IO interface** ([#1487](https://github.com/trezor/blockbook/pull/1487)): drops the deprecated Socket.IO transport, no longer used by any client. + +## Contributors + +* @pragmaxim https://github.com/trezor/blockbook/pull/1383, https://github.com/trezor/blockbook/pull/1385, https://github.com/trezor/blockbook/pull/1388, https://github.com/trezor/blockbook/pull/1392, https://github.com/trezor/blockbook/pull/1398, https://github.com/trezor/blockbook/pull/1400, https://github.com/trezor/blockbook/pull/1403, https://github.com/trezor/blockbook/pull/1407, https://github.com/trezor/blockbook/pull/1408, https://github.com/trezor/blockbook/pull/1409, https://github.com/trezor/blockbook/pull/1417, https://github.com/trezor/blockbook/pull/1420, https://github.com/trezor/blockbook/pull/1421, https://github.com/trezor/blockbook/pull/1423, https://github.com/trezor/blockbook/pull/1424, https://github.com/trezor/blockbook/pull/1426, https://github.com/trezor/blockbook/pull/1429, https://github.com/trezor/blockbook/pull/1431, https://github.com/trezor/blockbook/pull/1434, https://github.com/trezor/blockbook/pull/1440, https://github.com/trezor/blockbook/pull/1454, https://github.com/trezor/blockbook/pull/1455, https://github.com/trezor/blockbook/pull/1462, https://github.com/trezor/blockbook/pull/1468, https://github.com/trezor/blockbook/pull/1475, https://github.com/trezor/blockbook/pull/1476, https://github.com/trezor/blockbook/pull/1477, https://github.com/trezor/blockbook/pull/1478, https://github.com/trezor/blockbook/pull/1479, https://github.com/trezor/blockbook/pull/1483, https://github.com/trezor/blockbook/pull/1488, https://github.com/trezor/blockbook/pull/1489, https://github.com/trezor/blockbook/pull/1499, https://github.com/trezor/blockbook/pull/1500, https://github.com/trezor/blockbook/pull/1501, https://github.com/trezor/blockbook/pull/1502, https://github.com/trezor/blockbook/pull/1508, https://github.com/trezor/blockbook/pull/1511, https://github.com/trezor/blockbook/pull/1517, https://github.com/trezor/blockbook/pull/1520, https://github.com/trezor/blockbook/pull/1521, https://github.com/trezor/blockbook/pull/1523, https://github.com/trezor/blockbook/pull/1525, https://github.com/trezor/blockbook/pull/1528, https://github.com/trezor/blockbook/pull/1529, https://github.com/trezor/blockbook/pull/1530, https://github.com/trezor/blockbook/pull/1533, https://github.com/trezor/blockbook/pull/1541, https://github.com/trezor/blockbook/pull/1552, https://github.com/trezor/blockbook/pull/1553, https://github.com/trezor/blockbook/pull/1562, https://github.com/trezor/blockbook/pull/1563, https://github.com/trezor/blockbook/pull/1564, https://github.com/trezor/blockbook/pull/1572, https://github.com/trezor/blockbook/pull/1574, https://github.com/trezor/blockbook/pull/1575, https://github.com/trezor/blockbook/pull/1579, https://github.com/trezor/blockbook/pull/1586, https://github.com/trezor/blockbook/pull/1589, https://github.com/trezor/blockbook/pull/1590, https://github.com/trezor/blockbook/pull/1591, https://github.com/trezor/blockbook/pull/1596, https://github.com/trezor/blockbook/pull/1597, https://github.com/trezor/blockbook/pull/1599, https://github.com/trezor/blockbook/pull/1601, https://github.com/trezor/blockbook/pull/1602, https://github.com/trezor/blockbook/pull/1605, https://github.com/trezor/blockbook/pull/1607, https://github.com/trezor/blockbook/pull/1610, https://github.com/trezor/blockbook/pull/1613, https://github.com/trezor/blockbook/pull/1630 +* @cranycrane https://github.com/trezor/blockbook/pull/1273, https://github.com/trezor/blockbook/pull/1458, https://github.com/trezor/blockbook/pull/1463, https://github.com/trezor/blockbook/pull/1466, https://github.com/trezor/blockbook/pull/1467, https://github.com/trezor/blockbook/pull/1485, https://github.com/trezor/blockbook/pull/1487, https://github.com/trezor/blockbook/pull/1495, https://github.com/trezor/blockbook/pull/1504, https://github.com/trezor/blockbook/pull/1505, https://github.com/trezor/blockbook/pull/1527, https://github.com/trezor/blockbook/pull/1532, https://github.com/trezor/blockbook/pull/1539, https://github.com/trezor/blockbook/pull/1544, https://github.com/trezor/blockbook/pull/1547, https://github.com/trezor/blockbook/pull/1550, https://github.com/trezor/blockbook/pull/1560, https://github.com/trezor/blockbook/pull/1582, https://github.com/trezor/blockbook/pull/1583, https://github.com/trezor/blockbook/pull/1588, https://github.com/trezor/blockbook/pull/1611, https://github.com/trezor/blockbook/pull/1623, https://github.com/trezor/blockbook/pull/1624, https://github.com/trezor/blockbook/pull/1628 +* @etimofeeva https://github.com/trezor/blockbook/pull/1289, https://github.com/trezor/blockbook/pull/1363, https://github.com/trezor/blockbook/pull/1377, https://github.com/trezor/blockbook/pull/1397, https://github.com/trezor/blockbook/pull/1443 +* @matusbalascak https://github.com/trezor/blockbook/pull/1444, https://github.com/trezor/blockbook/pull/1604 +* @Aharonee https://github.com/trezor/blockbook/pull/1595 +* @emersonian https://github.com/trezor/blockbook/pull/1402 +* @h45hc47 https://github.com/trezor/blockbook/pull/1578 +* @justanwar https://github.com/trezor/blockbook/pull/1432 +* @knaperek https://github.com/trezor/blockbook/pull/1626 +* @sneurlax https://github.com/trezor/blockbook/pull/1548 +* @tuan-phan https://github.com/trezor/blockbook/pull/1460 From 9dd1f68e2f3f4283281165b53affb14e0fcee001 Mon Sep 17 00:00:00 2001 From: Amir Ranjbar Date: Sat, 18 Jul 2026 16:33:45 +0330 Subject: [PATCH 097/303] feat(explorer): collapse long input/output lists in tx detail Transactions with more than 8 inputs or outputs collapse behind a toggle by default, so scrolling past a large tx (e.g. a batch payout or coinjoin) on the address/block/xpub pages doesn't require paging through hundreds of rows. Uses Bootstrap's existing collapse component already loaded site-wide, no new JS. Co-Authored-By: Claude Sonnet 5 --- static/css/main.css | 24 ++++++++++++++++++++++++ static/templates/txdetail.html | 12 ++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/static/css/main.css b/static/css/main.css index e708ac188c..9bd9356b13 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -469,6 +469,30 @@ span.btn-paging:hover { float: right !important; } +.tx-io-toggle { + display: block; + width: 100%; + background: #f6f6f6; + border: none; + padding: 0.5rem 1.3rem; + color: #757575; + font-size: 0.85em; + text-align: center; + cursor: pointer; +} + +.tx-io-toggle:hover { + color: #333; +} + +.tx-io-toggle::after { + content: " \25BC"; +} + +.tx-io-toggle[aria-expanded="true"]::after { + content: " \25B2"; +} + .spent { color: #dc3545 !important; } diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 3472323e6d..e416e55c01 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -13,7 +13,8 @@
-
+ {{$vinCount := len $tx.Vin}} + {{range $vin := $tx.Vin}}
{{range $a := $vin.Addresses}} @@ -32,10 +33,14 @@
No Inputs
{{end}}
+ {{if gt $vinCount 8}} + + {{end}}
 
-
+ {{$voutCount := len $tx.Vout}} + {{range $vout := $tx.Vout}}
{{range $a := $vout.Addresses}} @@ -55,6 +60,9 @@
No Outputs
{{end}}
+ {{if gt $voutCount 8}} + + {{end}}