From 90ff071e1f56d887e3bc9ddfb369b6a20516bc09 Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 12 Nov 2026 13:40:34 +0100 Subject: [PATCH 1/4] fix(llm): keep fallback calls in the caller session Co-authored-by: openhands --- examples/01_standalone_sdk/39_llm_fallback.py | 2 +- .../openhands/sdk/llm/fallback_strategy.py | 43 ++- openhands-sdk/openhands/sdk/llm/llm.py | 38 ++- .../openhands/sdk/llm/utils/metrics.py | 21 +- .../openhands/sdk/llm/utils/telemetry.py | 7 +- tests/sdk/llm/test_llm_fallback_sessions.py | 281 ++++++++++++++++++ 6 files changed, 361 insertions(+), 31 deletions(-) create mode 100644 tests/sdk/llm/test_llm_fallback_sessions.py diff --git a/examples/01_standalone_sdk/39_llm_fallback.py b/examples/01_standalone_sdk/39_llm_fallback.py index ebd891c8f37..566cc464ea1 100644 --- a/examples/01_standalone_sdk/39_llm_fallback.py +++ b/examples/01_standalone_sdk/39_llm_fallback.py @@ -3,7 +3,7 @@ When the primary LLM fails with a transient error (rate limit, timeout, etc.), FallbackStrategy automatically tries alternate LLMs in order. Fallback is per-call: each new request starts with the primary model. Token usage and -cost from fallback calls are merged into the primary LLM's metrics. +cost from fallback calls are recorded in the caller's usage session. This example: 1. Saves two fallback LLM profiles to a temporary store. diff --git a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py index faa2f43a4fc..ba653121f6b 100644 --- a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py +++ b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py @@ -21,10 +21,12 @@ is_transient_http_error, ) from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.logger import get_logger if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_response import LLMResponse from openhands.sdk.llm.utils.metrics import Metrics @@ -75,14 +77,20 @@ def try_fallback( primary_error: Exception, primary_metrics: Metrics, call_fn: Callable[[Any], LLMResponse], + *, + primary: LLM | None = None, + session: LLMSession | None = None, ) -> LLMResponse | None: - """Try fallback LLMs in order. Merges metrics into primary on success. + """Try fallback models in the caller's role and session. Args: primary_model: The primary model name (for logging). primary_error: The error from the primary model. - primary_metrics: The primary LLM's Metrics to merge fallback costs into. + primary_metrics: The caller's metrics, used directly when no session + is supplied. call_fn: A callable that takes an LLM instance and returns an LLMResponse. + primary: The caller whose usage ID and aggregate accounting are kept. + session: The caller's selected session, including telemetry callbacks. Returns: LLMResponse from the first successful fallback, or None if all fail. @@ -99,17 +107,26 @@ def try_fallback( f"{remaining} fallback(s) remaining" ) try: - # Disable nested fallbacks to prevent recursive chains - saved_strategy = fb.fallback_strategy - fb.fallback_strategy = None - metrics_before = fb.metrics.deep_copy() - try: - result = call_fn(fb) - finally: - fb.fallback_strategy = saved_strategy - # Merge fallback metrics (cost + tokens) into primary - metrics_diff = fb.metrics.diff(metrics_before) - primary_metrics.merge(metrics_diff) + fallback = fb.model_copy( + update={ + "usage_id": primary.usage_id + if primary is not None + else fb.usage_id, + "fallback_strategy": None, + } + ) + if session is None: + session = LLMSession(fallback) + session.restore_metrics(primary_metrics) + own_session = ( + primary.session if primary is not None else LLMSession(fallback) + ) + if primary is None: + own_session.restore_metrics( + primary_metrics.rolls_up_into or primary_metrics + ) + fallback._bind_call_session(session, own_session) + result = call_fn(fallback) logger.info(f"[Fallback Strategy] Fallback LLM ({fb.model}) succeeded") return result except Exception as fb_error: diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 9e77e97a103..de76e80a13a 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -16,6 +16,7 @@ Sequence, ) from contextvars import ContextVar +from functools import partial from typing import ( TYPE_CHECKING, Any, @@ -688,6 +689,7 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): # This is the one behind ``llm.metrics`` / ``llm.telemetry``; see # ``session_for`` for how it relates to a conversation's. _session: LLMSession | None = PrivateAttr(default=None) + _call_session: LLMSession | None = PrivateAttr(default=None) # Runtime-only private attrs _model_info: Any = PrivateAttr(default=None) _tokenizer: dict[str, object] | None = PrivateAttr(default=None) @@ -962,6 +964,7 @@ def telemetry(self) -> Telemetry: def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: """Return the session a call made with ``call_context`` records in. + A fallback call stays bound to its original caller's session. That is the session the conversation keeps for this LLM's role (``usage_id``). Without a conversation in the context it is the session of the registry this object was last added to, and failing that the @@ -972,17 +975,25 @@ def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: records in a second session keeps one of its own apart, and every session it records in also counts there. """ - sessions = resolve_llm_call_context(call_context).sessions - session = sessions.get(self.usage_id) if sessions is not None else None + session = self._call_session + if session is None: + sessions = resolve_llm_call_context(call_context).sessions + session = sessions.get(self.usage_id) if sessions is not None else None if session is None: session = self._registered_session() if session is None: - return self.session + session = self.session if session is not self._session: self._keep_own_session_apart() session.follow(self) return session + def _bind_call_session(self, session: LLMSession, own_session: LLMSession) -> None: + """Bind a temporary LLM to a caller's selected and aggregate sessions.""" + self._session = own_session + self._call_session = session + self._registry_session = None + def _registered_session(self) -> LLMSession | None: try: return self._registry_session @@ -1071,6 +1082,7 @@ def reset_metrics(self) -> None: instance from the session a conversation's registry opened for it. """ self._session = LLMSession(self) + self._call_session = None self._registry_session = None def _handle_error( @@ -1093,6 +1105,8 @@ def _handle_error( primary_error=error, primary_metrics=session.metrics, call_fn=fallback_call_fn, + primary=self, + session=session, ) if result is not None: return result @@ -1121,11 +1135,15 @@ async def _ahandle_error( loop = asyncio.get_running_loop() result = await loop.run_in_executor( None, - self.fallback_strategy.try_fallback, - self.model, - error, - session.metrics, - fallback_call_fn, + partial( + self.fallback_strategy.try_fallback, + self.model, + error, + session.metrics, + fallback_call_fn, + primary=self, + session=session, + ), ) if result is not None: return result @@ -1858,6 +1876,7 @@ def completion( # cached (1h TTL) and negative-cached on failure, and the probe is a # no-op for providers without runtime metadata. self.resolve_runtime_metadata() + call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -1980,6 +1999,7 @@ async def acompletion( # probe is a no-op for providers without runtime metadata, so repeated # calls are cheap. await self.aresolve_runtime_metadata() + call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -2122,6 +2142,7 @@ def responses( # Cached (1h TTL) / negative-cached; no-op for providers without runtime # metadata. self.resolve_runtime_metadata() + call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -2289,6 +2310,7 @@ async def aresponses( # the first request so context management reads runtime (not model-level) # limits. No blocking network I/O in-process. await self.aresolve_runtime_metadata() + call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 720fb27b469..e851c7d9386 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -186,24 +186,28 @@ def rolls_up_into(self) -> "Metrics | None": except AttributeError: # never attached: the slot is still empty return None - def add_cost(self, value: float) -> None: + def add_cost(self, value: float, *, model_name: str | None = None) -> None: if value < 0: raise ValueError("Added cost cannot be negative.") self.accumulated_cost += value - self.costs.append(Cost(cost=value, model=self.model_name)) + self.costs.append(Cost(cost=value, model=model_name or self.model_name)) target = self.rolls_up_into if target is not None: - target.add_cost(value) + target.add_cost(value, model_name=model_name) - def add_response_latency(self, value: float, response_id: str) -> None: + def add_response_latency( + self, value: float, response_id: str, *, model_name: str | None = None + ) -> None: self.response_latencies.append( ResponseLatency( - latency=max(0.0, value), model=self.model_name, response_id=response_id + latency=max(0.0, value), + model=model_name or self.model_name, + response_id=response_id, ) ) target = self.rolls_up_into if target is not None: - target.add_response_latency(value, response_id) + target.add_response_latency(value, response_id, model_name=model_name) def add_token_usage( self, @@ -214,13 +218,15 @@ def add_token_usage( context_window: int, response_id: str, reasoning_tokens: int = 0, + *, + model_name: str | None = None, ) -> None: """Add a single usage record.""" # Token each turn for calculating context usage. per_turn_token = prompt_tokens + completion_tokens usage = TokenUsage( - model=self.model_name, + model=model_name or self.model_name, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cache_read_tokens=cache_read_tokens, @@ -259,6 +265,7 @@ def add_token_usage( context_window=context_window, response_id=response_id, reasoning_tokens=reasoning_tokens, + model_name=model_name, ) def merge(self, other: "Metrics") -> None: diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index d2d7c71ce28..a1616e2beef 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -227,13 +227,15 @@ def on_response( # 1) latency self._last_latency = time.time() - (self._req_start or time.time()) response_id = resp.id - self.metrics.add_response_latency(self._last_latency, response_id) + self.metrics.add_response_latency( + self._last_latency, response_id, model_name=self.model_name + ) # 2) cost cost = self._compute_cost(resp, provider_info=provider_info) # A confirmed zero charge is different from unavailable pricing. if cost is not None: - self.metrics.add_cost(cost) + self.metrics.add_cost(cost, model_name=self.model_name) # 3) tokens - normalize the provider's usage shape exactly once usage = normalize_usage(_response_usage(resp)) @@ -315,6 +317,7 @@ def _record_usage( reasoning_tokens=usage.reasoning_tokens, context_window=context_window, response_id=response_id, + model_name=self.model_name, ) # ---------- Observability span ---------- diff --git a/tests/sdk/llm/test_llm_fallback_sessions.py b/tests/sdk/llm/test_llm_fallback_sessions.py new file mode 100644 index 00000000000..5fbece2566b --- /dev/null +++ b/tests/sdk/llm/test_llm_fallback_sessions.py @@ -0,0 +1,281 @@ +"""Fallback requests keep the caller's role and accounting session.""" + +import json +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Literal +from unittest.mock import AsyncMock + +import pytest +from litellm.exceptions import APIConnectionError +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + FallbackStrategy, + LLMCallContext, + LLMRegistry, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope +from openhands.sdk.llm.llm_profile_store import LLMProfileStore + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,role,context_mode,collision,shared", + [ + ("completion", "default", "explicit", False, False), + ("acompletion", "default", "scoped", False, True), + ("responses", "condenser", "explicit", True, False), + ("aresponses", "condenser", "scoped", True, False), + ("completion", "condenser", "explicit", False, False), + ("acompletion", "condenser", "scoped", False, False), + ("completion", "condenser", "standalone", False, False), + ("aresponses", "default", "standalone", False, False), + ], +) +async def test_fallback_preserves_caller_accounting( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + method: Literal["completion", "acompletion", "responses", "aresponses"], + role: str, + context_mode: Literal["explicit", "scoped", "standalone"], + collision: bool, + shared: bool, +): + fallback = LLM( + model="gpt-4o-mini", + api_key=SecretStr("test-key"), + num_retries=0, + input_cost_per_token=0.01, + output_cost_per_token=0.02, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + ) + LLMProfileStore(base_dir=tmp_path / "profiles").save( + "backup", fallback, include_secrets=True + ) + primary = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id=role, + num_retries=0, + input_cost_per_token=0.1, + output_cost_per_token=0.2, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + fallback_strategy=FallbackStrategy( + fallback_llms=["backup"], profile_store_dir=tmp_path / "profiles" + ), + ) + registry = LLMRegistry() + if context_mode != "standalone": + registry.add(primary) + if collision: + registry.add(LLM(model="gpt-4o-mini")) + other_registry = LLMRegistry() + if shared: + other_registry.add(primary) + metrics = ( + primary.metrics + if context_mode == "standalone" + else registry.sessions[role].metrics + ) + notifications: list[tuple[int, float]] = [] + logs: list[tuple[str, str, dict[str, Any]]] = [] + + def on_stats() -> None: + usage = metrics.accumulated_token_usage + assert usage is not None + notifications.append((usage.prompt_tokens, metrics.accumulated_cost)) + + def on_log(filename: str, data: str, usage_id: str, model: str) -> None: + logs.append((usage_id, model, json.loads(data))) + + if context_mode == "standalone": + primary.telemetry.set_stats_update_callback(on_stats) + primary.telemetry.set_log_completions_callback( + lambda filename, data: on_log( + filename, data, role, primary.telemetry.model_name + ) + ) + else: + registry.set_stats_update_callback(on_stats) + registry.set_log_completions_callback(on_log) + + context = LLMCallContext( + prompt_cache_key="caller-cache", + session_id="caller-conversation", + sessions=registry.sessions if context_mode != "standalone" else None, + ) + requests: list[dict[str, Any]] = [] + primary_attempts = 0 + + def respond(**kwargs: Any) -> ModelResponse | ResponsesAPIResponse: + nonlocal primary_attempts + requests.append(kwargs) + model = kwargs["model"] + if model == primary.model: + primary_attempts += 1 + if primary_attempts <= 2: + raise APIConnectionError( + message="primary unavailable", llm_provider="openai", model=model + ) + if method in ("responses", "aresponses"): + return ResponsesAPIResponse( + id=f"response-{len(requests)}", + created_at=1, + model=model, + object="response", + output=[ + { + "type": "message", + "id": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "ok", "annotations": []} + ], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage( + input_tokens=10, output_tokens=5, total_tokens=15 + ), + ) + return ModelResponse( + id=f"response-{len(requests)}", + model=model, + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_completion", respond) + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_responses", respond) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_acompletion", AsyncMock(side_effect=respond) + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_aresponses", AsyncMock(side_effect=respond) + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + supplied_context = context if context_mode == "explicit" else None + scope = ( + llm_call_context_scope(context) if context_mode == "scoped" else nullcontext() + ) + with scope: + for count, cost in enumerate((0.2, 0.4, 2.4), start=1): + if method == "completion": + result = primary.completion(messages, call_context=supplied_context) + elif method == "acompletion": + result = await primary.acompletion( + messages, call_context=supplied_context + ) + elif method == "responses": + result = primary.responses(messages, call_context=supplied_context) + else: + result = await primary.aresponses( + messages, call_context=supplied_context + ) + assert result.metrics.accumulated_token_usage is not None + assert result.metrics.accumulated_token_usage.prompt_tokens == 10 * count + assert result.metrics.accumulated_cost == pytest.approx(cost) + assert metrics.accumulated_cost == pytest.approx(cost) + assert primary.metrics.accumulated_cost == pytest.approx(cost) + assert [tokens for tokens, _ in notifications] == [ + 10 * n for n in range(1, count + 1) + ] + assert [total for _, total in notifications] == pytest.approx( + (0.2, 0.4, 2.4)[:count] + ) + + assert [record.model for record in metrics.token_usages] == [ + "gpt-4o-mini", + "gpt-4o-mini", + "gpt-4o", + ] + assert [(uid, model) for uid, model, data in logs if "response" in data] == [ + (role, "gpt-4o-mini"), + (role, "gpt-4o-mini"), + (role, "gpt-4o"), + ] + if context_mode != "standalone": + for request in requests: + assert request["prompt_cache_key"] == "caller-cache" + assert ( + request["extra_headers"]["x-litellm-session-id"] + == "caller-conversation" + ) + if collision: + assert registry.sessions["default"].metrics.accumulated_cost == 0 + if shared: + assert other_registry.sessions[role].metrics.accumulated_cost == 0 + + +@pytest.mark.parametrize("with_context", [False, True]) +def test_direct_fallback_preserves_supplied_metrics( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, with_context: bool +): + primary = LLM(model="gpt-4o", usage_id="condenser") + registry, other_registry = LLMRegistry(), LLMRegistry() + registry.add(primary) + other_registry.add(primary) + registry.add(LLM(model="gpt-4o")) + metrics = registry.sessions["condenser"].metrics + context = LLMCallContext(sessions=registry.sessions) if with_context else None + LLMProfileStore(base_dir=tmp_path).save( + "backup", LLM(model="gpt-4o-mini", api_key=SecretStr("test-key")) + ) + strategy = FallbackStrategy(fallback_llms=["backup"], profile_store_dir=tmp_path) + response = ModelResponse( + id="fallback-response", + model="gpt-4o-mini", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_completion", lambda **kwargs: response + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + + result = strategy.try_fallback( + primary.model, + APIConnectionError( + message="primary unavailable", llm_provider="openai", model=primary.model + ), + metrics, + lambda fallback: fallback.completion(messages, call_context=context), + ) + + assert result is not None + assert metrics.accumulated_token_usage is not None + assert primary.metrics.accumulated_token_usage is not None + assert metrics.accumulated_token_usage.prompt_tokens == 10 + assert primary.metrics.accumulated_token_usage.prompt_tokens == 10 + assert len(metrics.token_usages) == 1 + assert registry.sessions["default"].metrics.token_usages == [] + assert other_registry.sessions["condenser"].metrics.token_usages == [] From 8447adcf64383f37b0426c8328a1be40b7a7a371 Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 12 Nov 2026 13:44:28 +0100 Subject: [PATCH 2/4] docs(.pr): show fallback session accounting before and after Co-authored-by: openhands --- .pr/design.html | 147 +++++++++++++++++++ .pr/fallback-session-evidence.json | 227 +++++++++++++++++++++++++++++ .pr/fallback-session-repro.py | 165 +++++++++++++++++++++ 3 files changed, 539 insertions(+) create mode 100644 .pr/design.html create mode 100644 .pr/fallback-session-evidence.json create mode 100644 .pr/fallback-session-repro.py diff --git a/.pr/design.html b/.pr/design.html new file mode 100644 index 00000000000..4b1bb398cbe --- /dev/null +++ b/.pr/design.html @@ -0,0 +1,147 @@ + + + + + +Fallbacks keep the caller's accounting session + + + +
+ +
+
+
Stacked fix for SDK PR #6690
+

Fallbacks keep the caller's accounting session.

+

A fallback changes the model serving a request. Its tokens, cost and telemetry still belong to the agent, condenser or ask-agent role that made the request.

+

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix binds a temporary fallback LLM to the caller's selected session and removes that second accounting path.

+

Before: fallback_strategy.py:99 · After: fallback_strategy.py:110

+
★ One successful provider request writes once to the caller's ledger. The individual records and completion logs identify the model that actually answered.
+
+
+

1. Bind the request before calling the fallback

+
+ +An ask-agent fallback before and after the fix +Before, the fallback profile selects the default role and its metrics are merged into ask-agent, counting one response twice. After, a temporary fallback object is explicitly bound to ask-agent's session, which records one hundred tokens once. + + + + + + + + + +Before · profile role wins + + +try_fallback() → call_fn(fb) +Saved fallback keeps usage_id="default" +fallback_strategy.py:99 + + + +session_for() → sessions["default"] +Agent ledger receives +100 tokens +llm.py:975 + + + +primary_metrics.merge(metrics_diff) +Ask-agent ledger receives another +100 +fallback_strategy.py:110 +100 provider tokens → 200 recorded +Same-role fallback also doubles its single ledger. + + + +After · caller role stays + + +fb.model_copy(usage_id=caller.role) +Temporary copy disables nested fallbacks +fallback_strategy.py:110 + + + +_bind_call_session(session, own) +Selected role + caller's aggregate ledger +llm.py:991 + + + +session_for() → caller's session +Ask-agent records +100; its callbacks fire +llm.py:978 +100 provider tokens → 100 recorded +There is no post-call metrics merge. + + +
Fig. 1 · The fallback profile supplies model configuration; the caller supplies accounting ownership. Orange arrows mark the changed path.
+
+
+

Context survives async fallback

All four LLM entry points resolve the call context before fallback can move into an executor thread. The fallback receives the same conversation identity and prompt-cache key.

llm.py:1999 · acompletion()
+

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()
+
+
+
+

2. Preserve existing call signatures

+

The original positional arguments remain valid. New session bindings are runtime-only; the serialized LLM and profile shapes are unchanged. llm.py:692

+
+

Before

try_fallback(model, error, metrics, call_fn)
+metrics.add_cost(value)
+metrics.add_response_latency(value, response_id)
+metrics.add_token_usage(...)
+

After · optional keyword additions

try_fallback(model, error, metrics, call_fn,
+             *, primary=None, session=None)
+metrics.add_cost(value, *, model_name=None)
+metrics.add_response_latency(..., model_name=None)
+metrics.add_token_usage(..., model_name=None)
+
+

fallback_strategy.py:74 · metrics.py:189

+

Direct callers of try_fallback() still accumulate into their supplied Metrics, including an existing aggregate destination. Native LLM calls also preserve the selected session's callbacks. The fallback example's accounting description is updated to match. fallback_strategy.py:118

+
+
+

3. Five operations, five corrected ledgers

+

The same script ran on PR #6690's 0a68ab94 and on this fix. Each operation sent one request that received HTTP 503, then one successful fallback request reporting 100 prompt tokens.

+ + + + + + + + + +
Public operationBefore: token increaseAfter: token increase
run(), first turndefault +200default +100
run(), second turndefault +200default +100
run(), third turndefault +200default +100
ask_agent()default +100; ask-agent-llm +100ask-agent-llm +100
condense()default +100; condenser +100condenser +100
+

Evidence: fallback-session-evidence.json · fallback-session-repro.py

+

macOS · Python 3.13.11 · real settings construction, profile save/load, conversation lifecycle and HTTP transport. The provider runs on loopback and supplies deterministic responses. These observations establish token totals and role attribution; they do not measure callback counts, actual provider billing or remote-service behavior.

+
Reproduce with the checkout's dependencies
OH_PERSISTENCE_DIR=/tmp/fallback-repro-state \
+OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
+uv run python .pr/fallback-session-repro.py --assert-fixed

The final flag makes incorrect accounting fail with exit code 1. The script starts and shuts down its own local HTTP server and uses temporary profile and conversation directories.

+
+
+

4. Regression checks cover the other boundaries

+

1,792 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks were rerun successfully without the temporary persistence-directory override.

+

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. Separate cases exercise the original direct try_fallback() API.

+

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:231

+
Focused regression command
uv run pytest tests/sdk/llm/test_llm_fallback_sessions.py \
+  tests/sdk/llm/test_llm_fallback.py tests/sdk/llm/test_llm_session.py -q

These regression tests replace provider responses at the LiteLLM boundary. The loopback reproduction above separately exercises real HTTP transport.

+
+
Temporary reviewer context in .pr/ · Base: 0a68ab94 · Source links identify the code revision reviewed.
+
+
+ + diff --git a/.pr/fallback-session-evidence.json b/.pr/fallback-session-evidence.json new file mode 100644 index 00000000000..53cd5574de5 --- /dev/null +++ b/.pr/fallback-session-evidence.json @@ -0,0 +1,227 @@ +{ + "base_commit": "0a68ab94cb3f447254bdc68ce778048ce517be22", + "fixed_source_sha256": { + "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "a129313e3ec36fdc2d505a30ab2404e6f4824b2a80cd3f829384c3777f01eee5", + "openhands-sdk/openhands/sdk/llm/llm.py": "a9a7de5fa0fa7a549828a325eb1b33944d641468976a5ca9bc5ed72e076cec67", + "openhands-sdk/openhands/sdk/llm/utils/metrics.py": "c8b6977296c1274fdd8f63a2e4556d9be2303469e9b559065048536e8996f93b", + "openhands-sdk/openhands/sdk/llm/utils/telemetry.py": "bed4636f931f6dd95f3275ac63cca3d77e533e2d4f1648d4d56a864eee5e73b4" + }, + "runtime": "macOS, Python 3.13.11, PR checkout editable dependencies", + "entry_points": [ + "OpenHandsAgentSettings.create_agent", + "LLMProfileStore.save/load", + "Conversation.run", + "Conversation.ask_agent", + "Conversation.condense" + ], + "provider": "Loopback HTTP provider: gpt-4o returns 503; gpt-4o-mini returns 200 with 100 prompt tokens and 5 completion tokens.", + "limitations": "Provider responses are simulated; SDK execution, profile persistence, HTTP transport and accounting are real. This artifact checks token totals and role attribution. It does not measure costs or callback invocation counts.", + "reproduce": "OH_PERSISTENCE_DIR=/tmp/fallback-repro-state OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True uv run python .pr/fallback-session-repro.py --assert-fixed", + "before": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "ask-agent-llm": 100 + }, + "correct": false + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "condenser": 100 + }, + "correct": false + } + ], + "after": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "correct": true + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "condenser": 100 + }, + "correct": true + } + ], + "fixed_commit": "90ff071e1f56d887e3bc9ddfb369b6a20516bc09" +} diff --git a/.pr/fallback-session-repro.py b/.pr/fallback-session-repro.py new file mode 100644 index 00000000000..88c67297f2e --- /dev/null +++ b/.pr/fallback-session-repro.py @@ -0,0 +1,165 @@ +"""Exercise fallback accounting through real SDK conversations and loopback HTTP. + +Run with an isolated OH_PERSISTENCE_DIR and the checkout's Python environment: + OH_PERSISTENCE_DIR=/tmp/fallback-repro-state uv run python \ + .pr/fallback-session-repro.py + +The provider is simulated: primary requests receive HTTP 503, while successful +fallback requests report exactly 100 input and 5 output tokens. SDK execution, +settings, profile loading, HTTP transport, telemetry and accounting are real. +""" + +import argparse +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from openhands.sdk import LLM, Conversation +from openhands.sdk.llm.fallback_strategy import FallbackStrategy +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.settings import OpenHandsAgentSettings + + +def token_totals(conversation: Conversation) -> dict[str, int]: + return { + role: metrics.accumulated_token_usage.prompt_tokens + if metrics.accumulated_token_usage is not None + else 0 + for role, metrics in conversation.conversation_stats.usage_to_metrics.items() + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assert-fixed", action="store_true") + args = parser.parse_args() + if not os.environ.get("OH_PERSISTENCE_DIR"): + parser.error("Set OH_PERSISTENCE_DIR to an isolated temporary directory") + + requests: list[dict[str, Any]] = [] + + class Provider(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model = body["model"] + failed = model == "gpt-4o" + status = 503 if failed else 200 + requests.append({"model": model, "status": status}) + if failed: + payload = {"error": {"message": "Simulated outage", "type": "server"}} + else: + payload = { + "id": f"chatcmpl-{len(requests)}", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Done."}, + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + }, + } + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + evidence: list[dict[str, Any]] = [] + try: + with TemporaryDirectory(prefix="fallback-repro-") as directory: + root = Path(directory) + common = { + "base_url": f"http://127.0.0.1:{server.server_port}/v1", + "api_key": "loopback-only", + "num_retries": 1, + "stream": False, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + profiles = root / "profiles" + LLMProfileStore(profiles).save( + "fallback", + LLM(model="openai/gpt-4o-mini", **common), + include_secrets=True, + ) + primary = LLM( + model="openai/gpt-4o", + fallback_strategy=FallbackStrategy( + fallback_llms=["fallback"], profile_store_dir=profiles + ), + **common, + ) + settings = OpenHandsAgentSettings(llm=primary, tools=[]) + conversation = Conversation( + agent=settings.create_agent(), + workspace=root / "workspace", + persistence_dir=root / "conversations", + visualizer=None, + ) + try: + for operation, expected_role in [ + ("run-1", "default"), + ("run-2", "default"), + ("run-3", "default"), + ("ask_agent", "ask-agent-llm"), + ("condense", "condenser"), + ]: + before = token_totals(conversation) + start = len(requests) + if operation.startswith("run"): + conversation.send_message(f"Reply briefly for {operation}.") + conversation.run() + elif operation == "ask_agent": + conversation.ask_agent("What are you doing?") + else: + conversation.condense() + after = token_totals(conversation) + deltas = { + role: total - before.get(role, 0) + for role, total in after.items() + if total != before.get(role, 0) + } + receipts = requests[start:] + successes = sum(item["status"] == 200 for item in receipts) + expected = {expected_role: 100 * successes} + evidence.append( + { + "operation": operation, + "http_requests": receipts, + "expected_prompt_token_deltas": expected, + "actual_prompt_token_deltas": deltas, + "correct": successes == 1 and deltas == expected, + } + ) + finally: + conversation.close() + finally: + server.shutdown() + server.server_close() + worker.join() + print(json.dumps(evidence, indent=2)) + if args.assert_fixed and not all(item["correct"] for item in evidence): + raise SystemExit(1) + + +if __name__ == "__main__": + main() From 6bfdc18917a3ec2847029f3b22cf1861bf8b99ec Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 12 Nov 2026 15:14:37 +0100 Subject: [PATCH 3/4] refactor(llm): reuse existing sessions for fallback calls Co-authored-by: openhands --- .../openhands/sdk/llm/fallback_strategy.py | 23 ++++------ openhands-sdk/openhands/sdk/llm/llm.py | 42 +++++-------------- tests/sdk/llm/test_llm_fallback_sessions.py | 8 +--- 3 files changed, 20 insertions(+), 53 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py index ba653121f6b..c9ca0c9794c 100644 --- a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py +++ b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py @@ -21,7 +21,6 @@ is_transient_http_error, ) from openhands.sdk.llm.llm_profile_store import LLMProfileStore -from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.logger import get_logger @@ -79,18 +78,15 @@ def try_fallback( call_fn: Callable[[Any], LLMResponse], *, primary: LLM | None = None, - session: LLMSession | None = None, ) -> LLMResponse | None: """Try fallback models in the caller's role and session. Args: primary_model: The primary model name (for logging). primary_error: The error from the primary model. - primary_metrics: The caller's metrics, used directly when no session - is supplied. + primary_metrics: The primary LLM's metrics for legacy direct callers. call_fn: A callable that takes an LLM instance and returns an LLMResponse. primary: The caller whose usage ID and aggregate accounting are kept. - session: The caller's selected session, including telemetry callbacks. Returns: LLMResponse from the first successful fallback, or None if all fail. @@ -115,18 +111,15 @@ def try_fallback( "fallback_strategy": None, } ) - if session is None: - session = LLMSession(fallback) - session.restore_metrics(primary_metrics) - own_session = ( - primary.session if primary is not None else LLMSession(fallback) + metrics_before = ( + fallback.metrics.deep_copy() if primary is None else None ) - if primary is None: - own_session.restore_metrics( - primary_metrics.rolls_up_into or primary_metrics - ) - fallback._bind_call_session(session, own_session) + if primary is not None: + fallback._session = primary.session + fallback._registry_session = primary._registered_session() result = call_fn(fallback) + if metrics_before is not None: + primary_metrics.merge(fallback.metrics.diff(metrics_before)) logger.info(f"[Fallback Strategy] Fallback LLM ({fb.model}) succeeded") return result except Exception as fb_error: diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index de76e80a13a..6a65bcef11c 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -16,7 +16,6 @@ Sequence, ) from contextvars import ContextVar -from functools import partial from typing import ( TYPE_CHECKING, Any, @@ -689,7 +688,6 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): # This is the one behind ``llm.metrics`` / ``llm.telemetry``; see # ``session_for`` for how it relates to a conversation's. _session: LLMSession | None = PrivateAttr(default=None) - _call_session: LLMSession | None = PrivateAttr(default=None) # Runtime-only private attrs _model_info: Any = PrivateAttr(default=None) _tokenizer: dict[str, object] | None = PrivateAttr(default=None) @@ -964,7 +962,6 @@ def telemetry(self) -> Telemetry: def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: """Return the session a call made with ``call_context`` records in. - A fallback call stays bound to its original caller's session. That is the session the conversation keeps for this LLM's role (``usage_id``). Without a conversation in the context it is the session of the registry this object was last added to, and failing that the @@ -975,10 +972,8 @@ def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: records in a second session keeps one of its own apart, and every session it records in also counts there. """ - session = self._call_session - if session is None: - sessions = resolve_llm_call_context(call_context).sessions - session = sessions.get(self.usage_id) if sessions is not None else None + sessions = resolve_llm_call_context(call_context).sessions + session = sessions.get(self.usage_id) if sessions is not None else None if session is None: session = self._registered_session() if session is None: @@ -988,12 +983,6 @@ def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: session.follow(self) return session - def _bind_call_session(self, session: LLMSession, own_session: LLMSession) -> None: - """Bind a temporary LLM to a caller's selected and aggregate sessions.""" - self._session = own_session - self._call_session = session - self._registry_session = None - def _registered_session(self) -> LLMSession | None: try: return self._registry_session @@ -1082,7 +1071,6 @@ def reset_metrics(self) -> None: instance from the session a conversation's registry opened for it. """ self._session = LLMSession(self) - self._call_session = None self._registry_session = None def _handle_error( @@ -1106,7 +1094,6 @@ def _handle_error( primary_metrics=session.metrics, call_fn=fallback_call_fn, primary=self, - session=session, ) if result is not None: return result @@ -1125,25 +1112,20 @@ async def _ahandle_error( The *fallback_call_fn* is synchronous (it calls the fallback LLM's sync ``completion``/``responses``), so the fallback attempt is - offloaded to a thread via :func:`asyncio.loop.run_in_executor` to + offloaded to a thread via :func:`asyncio.to_thread` to avoid blocking the event loop. """ if session is None: session = self.session session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, - partial( - self.fallback_strategy.try_fallback, - self.model, - error, - session.metrics, - fallback_call_fn, - primary=self, - session=session, - ), + result = await asyncio.to_thread( + self.fallback_strategy.try_fallback, + self.model, + error, + session.metrics, + fallback_call_fn, + primary=self, ) if result is not None: return result @@ -1876,7 +1858,6 @@ def completion( # cached (1h TTL) and negative-cached on failure, and the probe is a # no-op for providers without runtime metadata. self.resolve_runtime_metadata() - call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -1999,7 +1980,6 @@ async def acompletion( # probe is a no-op for providers without runtime metadata, so repeated # calls are cheap. await self.aresolve_runtime_metadata() - call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -2142,7 +2122,6 @@ def responses( # Cached (1h TTL) / negative-cached; no-op for providers without runtime # metadata. self.resolve_runtime_metadata() - call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() @@ -2310,7 +2289,6 @@ async def aresponses( # the first request so context management reads runtime (not model-level) # limits. No blocking network I/O in-process. await self.aresolve_runtime_metadata() - call_context = resolve_llm_call_context(call_context) session = self.session_for(call_context) _caller_kwargs = kwargs.copy() diff --git a/tests/sdk/llm/test_llm_fallback_sessions.py b/tests/sdk/llm/test_llm_fallback_sessions.py index 5fbece2566b..b2b713571f9 100644 --- a/tests/sdk/llm/test_llm_fallback_sessions.py +++ b/tests/sdk/llm/test_llm_fallback_sessions.py @@ -227,17 +227,14 @@ def respond(**kwargs: Any) -> ModelResponse | ResponsesAPIResponse: assert other_registry.sessions[role].metrics.accumulated_cost == 0 -@pytest.mark.parametrize("with_context", [False, True]) def test_direct_fallback_preserves_supplied_metrics( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, with_context: bool + monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): primary = LLM(model="gpt-4o", usage_id="condenser") registry, other_registry = LLMRegistry(), LLMRegistry() registry.add(primary) other_registry.add(primary) - registry.add(LLM(model="gpt-4o")) metrics = registry.sessions["condenser"].metrics - context = LLMCallContext(sessions=registry.sessions) if with_context else None LLMProfileStore(base_dir=tmp_path).save( "backup", LLM(model="gpt-4o-mini", api_key=SecretStr("test-key")) ) @@ -268,7 +265,7 @@ def test_direct_fallback_preserves_supplied_metrics( message="primary unavailable", llm_provider="openai", model=primary.model ), metrics, - lambda fallback: fallback.completion(messages, call_context=context), + lambda fallback: fallback.completion(messages), ) assert result is not None @@ -277,5 +274,4 @@ def test_direct_fallback_preserves_supplied_metrics( assert metrics.accumulated_token_usage.prompt_tokens == 10 assert primary.metrics.accumulated_token_usage.prompt_tokens == 10 assert len(metrics.token_usages) == 1 - assert registry.sessions["default"].metrics.token_usages == [] assert other_registry.sessions["condenser"].metrics.token_usages == [] From f0cd7d944ce37501a5cf22248a02b4540aacdf19 Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 12 Nov 2026 15:17:47 +0100 Subject: [PATCH 4/4] docs(.pr): align evidence with the simplified fallback fix Co-authored-by: openhands --- .pr/design.html | 40 +++++++++++++++--------------- .pr/fallback-session-evidence.json | 6 ++--- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.pr/design.html b/.pr/design.html index 4b1bb398cbe..8f0437cb972 100644 --- a/.pr/design.html +++ b/.pr/design.html @@ -25,8 +25,8 @@
Stacked fix for SDK PR #6690

Fallbacks keep the caller's accounting session.

A fallback changes the model serving a request. Its tokens, cost and telemetry still belong to the agent, condenser or ask-agent role that made the request.

-

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix binds a temporary fallback LLM to the caller's selected session and removes that second accounting path.

-

Before: fallback_strategy.py:99 · After: fallback_strategy.py:110

+

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix gives a temporary fallback LLM the caller's existing session references and removes the second accounting path for SDK calls.

+

Before: fallback_strategy.py:99 · After: fallback_strategy.py:106

★ One successful provider request writes once to the caller's ledger. The individual records and completion logs identify the model that actually answered.
@@ -68,50 +68,50 @@

1. Bind the request before calling the fallback

After · caller role stays - + fb.model_copy(usage_id=caller.role) Temporary copy disables nested fallbacks -fallback_strategy.py:110 +fallback_strategy.py:106 - + -_bind_call_session(session, own) -Selected role + caller's aggregate ledger -llm.py:991 +Reuse _session + _registry_session +Existing role and aggregate ownership +fallback_strategy.py:117 - + session_for() → caller's session Ask-agent records +100; its callbacks fire -llm.py:978 +llm.py:975 100 provider tokens → 100 recorded -There is no post-call metrics merge. +SDK calls skip the post-call metrics merge.
Fig. 1 · The fallback profile supplies model configuration; the caller supplies accounting ownership. Orange arrows mark the changed path.
-

Context survives async fallback

All four LLM entry points resolve the call context before fallback can move into an executor thread. The fallback receives the same conversation identity and prompt-cache key.

llm.py:1999 · acompletion()
-

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()
+

Context survives async fallback

asyncio.to_thread() carries the scoped call context into async fallback work. Explicit contexts already travel with the callback, so both paths preserve the conversation identity and prompt-cache key.

llm.py:1122 · _ahandle_error()
+

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()

2. Preserve existing call signatures

-

The original positional arguments remain valid. New session bindings are runtime-only; the serialized LLM and profile shapes are unchanged. llm.py:692

+

The original positional arguments remain valid. The fallback reuses existing session references; serialized LLM and profile shapes are unchanged. fallback_strategy.py:117

Before

try_fallback(model, error, metrics, call_fn)
 metrics.add_cost(value)
 metrics.add_response_latency(value, response_id)
 metrics.add_token_usage(...)

After · optional keyword additions

try_fallback(model, error, metrics, call_fn,
-             *, primary=None, session=None)
+             *, primary=None)
 metrics.add_cost(value, *, model_name=None)
 metrics.add_response_latency(..., model_name=None)
 metrics.add_token_usage(..., model_name=None)
-

fallback_strategy.py:74 · metrics.py:189

-

Direct callers of try_fallback() still accumulate into their supplied Metrics, including an existing aggregate destination. Native LLM calls also preserve the selected session's callbacks. The fallback example's accounting description is updated to match. fallback_strategy.py:118

+

fallback_strategy.py:73 · metrics.py:189

+

The original four-argument try_fallback() API keeps its metrics-diff merge. SDK calls pass primary=self, reuse the caller's sessions and callbacks, and skip that merge. The fallback example's accounting description is updated to match. fallback_strategy.py:117

3. Five operations, five corrected ledgers

@@ -134,9 +134,9 @@

3. Five operations, five corrected ledgers

4. Regression checks cover the other boundaries

-

1,792 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks were rerun successfully without the temporary persistence-directory override.

-

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. Separate cases exercise the original direct try_fallback() API.

-

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:231

+

1,791 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks ran separately without the temporary persistence-directory override. The two fallback regression files account for 58 passing cases.

+

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. A separate case exercises the original direct try_fallback() API.

+

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:230

Focused regression command
uv run pytest tests/sdk/llm/test_llm_fallback_sessions.py \
   tests/sdk/llm/test_llm_fallback.py tests/sdk/llm/test_llm_session.py -q

These regression tests replace provider responses at the LiteLLM boundary. The loopback reproduction above separately exercises real HTTP transport.

diff --git a/.pr/fallback-session-evidence.json b/.pr/fallback-session-evidence.json index 53cd5574de5..65bffa51193 100644 --- a/.pr/fallback-session-evidence.json +++ b/.pr/fallback-session-evidence.json @@ -1,8 +1,8 @@ { "base_commit": "0a68ab94cb3f447254bdc68ce778048ce517be22", "fixed_source_sha256": { - "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "a129313e3ec36fdc2d505a30ab2404e6f4824b2a80cd3f829384c3777f01eee5", - "openhands-sdk/openhands/sdk/llm/llm.py": "a9a7de5fa0fa7a549828a325eb1b33944d641468976a5ca9bc5ed72e076cec67", + "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "c7f268068411b524ab403d53326f41ec68ae1f17b6be39f95ef2e7085845c48c", + "openhands-sdk/openhands/sdk/llm/llm.py": "a16bc975d4b126839dede49c84e0401f9236a48014e27c7c3ee3aa4639c82c05", "openhands-sdk/openhands/sdk/llm/utils/metrics.py": "c8b6977296c1274fdd8f63a2e4556d9be2303469e9b559065048536e8996f93b", "openhands-sdk/openhands/sdk/llm/utils/telemetry.py": "bed4636f931f6dd95f3275ac63cca3d77e533e2d4f1648d4d56a864eee5e73b4" }, @@ -223,5 +223,5 @@ "correct": true } ], - "fixed_commit": "90ff071e1f56d887e3bc9ddfb369b6a20516bc09" + "fixed_commit": "6bfdc18917a3ec2847029f3b22cf1861bf8b99ec" }