diff --git a/.pr/design.html b/.pr/design.html new file mode 100644 index 00000000000..8f0437cb972 --- /dev/null +++ b/.pr/design.html @@ -0,0 +1,147 @@ + + + + + +Fallbacks keep the caller's accounting session + + + +
+ +
+
+
Stacked fix for SDK PR #6690
+

Fallbacks keep the caller's accounting session.

+

A fallback changes the model serving a request. Its tokens, cost and telemetry still belong to the agent, condenser or ask-agent role that made the request.

+

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix gives a temporary fallback LLM the caller's existing session references and removes the second accounting path for SDK calls.

+

Before: fallback_strategy.py:99 · After: fallback_strategy.py:106

+
★ One successful provider request writes once to the caller's ledger. The individual records and completion logs identify the model that actually answered.
+
+
+

1. Bind the request before calling the fallback

+
+ +An ask-agent fallback before and after the fix +Before, the fallback profile selects the default role and its metrics are merged into ask-agent, counting one response twice. After, a temporary fallback object is explicitly bound to ask-agent's session, which records one hundred tokens once. + + + + + + + + + +Before · profile role wins + + +try_fallback() → call_fn(fb) +Saved fallback keeps usage_id="default" +fallback_strategy.py:99 + + + +session_for() → sessions["default"] +Agent ledger receives +100 tokens +llm.py:975 + + + +primary_metrics.merge(metrics_diff) +Ask-agent ledger receives another +100 +fallback_strategy.py:110 +100 provider tokens → 200 recorded +Same-role fallback also doubles its single ledger. + + + +After · caller role stays + + +fb.model_copy(usage_id=caller.role) +Temporary copy disables nested fallbacks +fallback_strategy.py:106 + + + +Reuse _session + _registry_session +Existing role and aggregate ownership +fallback_strategy.py:117 + + + +session_for() → caller's session +Ask-agent records +100; its callbacks fire +llm.py:975 +100 provider tokens → 100 recorded +SDK calls skip the post-call metrics merge. + + +
Fig. 1 · The fallback profile supplies model configuration; the caller supplies accounting ownership. Orange arrows mark the changed path.
+
+
+

Context survives async fallback

asyncio.to_thread() carries the scoped call context into async fallback work. Explicit contexts already travel with the callback, so both paths preserve the conversation identity and prompt-cache key.

llm.py:1122 · _ahandle_error()
+

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()
+
+
+
+

2. Preserve existing call signatures

+

The original positional arguments remain valid. The fallback reuses existing session references; serialized LLM and profile shapes are unchanged. fallback_strategy.py:117

+
+

Before

try_fallback(model, error, metrics, call_fn)
+metrics.add_cost(value)
+metrics.add_response_latency(value, response_id)
+metrics.add_token_usage(...)
+

After · optional keyword additions

try_fallback(model, error, metrics, call_fn,
+             *, primary=None)
+metrics.add_cost(value, *, model_name=None)
+metrics.add_response_latency(..., model_name=None)
+metrics.add_token_usage(..., model_name=None)
+
+

fallback_strategy.py:73 · metrics.py:189

+

The original four-argument try_fallback() API keeps its metrics-diff merge. SDK calls pass primary=self, reuse the caller's sessions and callbacks, and skip that merge. The fallback example's accounting description is updated to match. fallback_strategy.py:117

+
+
+

3. Five operations, five corrected ledgers

+

The same script ran on PR #6690's 0a68ab94 and on this fix. Each operation sent one request that received HTTP 503, then one successful fallback request reporting 100 prompt tokens.

+ + + + + + + + + +
Public operationBefore: token increaseAfter: token increase
run(), first turndefault +200default +100
run(), second turndefault +200default +100
run(), third turndefault +200default +100
ask_agent()default +100; ask-agent-llm +100ask-agent-llm +100
condense()default +100; condenser +100condenser +100
+

Evidence: fallback-session-evidence.json · fallback-session-repro.py

+

macOS · Python 3.13.11 · real settings construction, profile save/load, conversation lifecycle and HTTP transport. The provider runs on loopback and supplies deterministic responses. These observations establish token totals and role attribution; they do not measure callback counts, actual provider billing or remote-service behavior.

+
Reproduce with the checkout's dependencies
OH_PERSISTENCE_DIR=/tmp/fallback-repro-state \
+OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
+uv run python .pr/fallback-session-repro.py --assert-fixed

The final flag makes incorrect accounting fail with exit code 1. The script starts and shuts down its own local HTTP server and uses temporary profile and conversation directories.

+
+
+

4. Regression checks cover the other boundaries

+

1,791 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks ran separately without the temporary persistence-directory override. The two fallback regression files account for 58 passing cases.

+

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. A separate case exercises the original direct try_fallback() API.

+

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:230

+
Focused regression command
uv run pytest tests/sdk/llm/test_llm_fallback_sessions.py \
+  tests/sdk/llm/test_llm_fallback.py tests/sdk/llm/test_llm_session.py -q

These regression tests replace provider responses at the LiteLLM boundary. The loopback reproduction above separately exercises real HTTP transport.

+
+ +
+
+ + diff --git a/.pr/fallback-session-evidence.json b/.pr/fallback-session-evidence.json new file mode 100644 index 00000000000..65bffa51193 --- /dev/null +++ b/.pr/fallback-session-evidence.json @@ -0,0 +1,227 @@ +{ + "base_commit": "0a68ab94cb3f447254bdc68ce778048ce517be22", + "fixed_source_sha256": { + "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "c7f268068411b524ab403d53326f41ec68ae1f17b6be39f95ef2e7085845c48c", + "openhands-sdk/openhands/sdk/llm/llm.py": "a16bc975d4b126839dede49c84e0401f9236a48014e27c7c3ee3aa4639c82c05", + "openhands-sdk/openhands/sdk/llm/utils/metrics.py": "c8b6977296c1274fdd8f63a2e4556d9be2303469e9b559065048536e8996f93b", + "openhands-sdk/openhands/sdk/llm/utils/telemetry.py": "bed4636f931f6dd95f3275ac63cca3d77e533e2d4f1648d4d56a864eee5e73b4" + }, + "runtime": "macOS, Python 3.13.11, PR checkout editable dependencies", + "entry_points": [ + "OpenHandsAgentSettings.create_agent", + "LLMProfileStore.save/load", + "Conversation.run", + "Conversation.ask_agent", + "Conversation.condense" + ], + "provider": "Loopback HTTP provider: gpt-4o returns 503; gpt-4o-mini returns 200 with 100 prompt tokens and 5 completion tokens.", + "limitations": "Provider responses are simulated; SDK execution, profile persistence, HTTP transport and accounting are real. This artifact checks token totals and role attribution. It does not measure costs or callback invocation counts.", + "reproduce": "OH_PERSISTENCE_DIR=/tmp/fallback-repro-state OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True uv run python .pr/fallback-session-repro.py --assert-fixed", + "before": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "ask-agent-llm": 100 + }, + "correct": false + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "condenser": 100 + }, + "correct": false + } + ], + "after": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "correct": true + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "condenser": 100 + }, + "correct": true + } + ], + "fixed_commit": "6bfdc18917a3ec2847029f3b22cf1861bf8b99ec" +} diff --git a/.pr/fallback-session-repro.py b/.pr/fallback-session-repro.py new file mode 100644 index 00000000000..88c67297f2e --- /dev/null +++ b/.pr/fallback-session-repro.py @@ -0,0 +1,165 @@ +"""Exercise fallback accounting through real SDK conversations and loopback HTTP. + +Run with an isolated OH_PERSISTENCE_DIR and the checkout's Python environment: + OH_PERSISTENCE_DIR=/tmp/fallback-repro-state uv run python \ + .pr/fallback-session-repro.py + +The provider is simulated: primary requests receive HTTP 503, while successful +fallback requests report exactly 100 input and 5 output tokens. SDK execution, +settings, profile loading, HTTP transport, telemetry and accounting are real. +""" + +import argparse +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from openhands.sdk import LLM, Conversation +from openhands.sdk.llm.fallback_strategy import FallbackStrategy +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.settings import OpenHandsAgentSettings + + +def token_totals(conversation: Conversation) -> dict[str, int]: + return { + role: metrics.accumulated_token_usage.prompt_tokens + if metrics.accumulated_token_usage is not None + else 0 + for role, metrics in conversation.conversation_stats.usage_to_metrics.items() + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assert-fixed", action="store_true") + args = parser.parse_args() + if not os.environ.get("OH_PERSISTENCE_DIR"): + parser.error("Set OH_PERSISTENCE_DIR to an isolated temporary directory") + + requests: list[dict[str, Any]] = [] + + class Provider(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model = body["model"] + failed = model == "gpt-4o" + status = 503 if failed else 200 + requests.append({"model": model, "status": status}) + if failed: + payload = {"error": {"message": "Simulated outage", "type": "server"}} + else: + payload = { + "id": f"chatcmpl-{len(requests)}", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Done."}, + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + }, + } + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + evidence: list[dict[str, Any]] = [] + try: + with TemporaryDirectory(prefix="fallback-repro-") as directory: + root = Path(directory) + common = { + "base_url": f"http://127.0.0.1:{server.server_port}/v1", + "api_key": "loopback-only", + "num_retries": 1, + "stream": False, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + profiles = root / "profiles" + LLMProfileStore(profiles).save( + "fallback", + LLM(model="openai/gpt-4o-mini", **common), + include_secrets=True, + ) + primary = LLM( + model="openai/gpt-4o", + fallback_strategy=FallbackStrategy( + fallback_llms=["fallback"], profile_store_dir=profiles + ), + **common, + ) + settings = OpenHandsAgentSettings(llm=primary, tools=[]) + conversation = Conversation( + agent=settings.create_agent(), + workspace=root / "workspace", + persistence_dir=root / "conversations", + visualizer=None, + ) + try: + for operation, expected_role in [ + ("run-1", "default"), + ("run-2", "default"), + ("run-3", "default"), + ("ask_agent", "ask-agent-llm"), + ("condense", "condenser"), + ]: + before = token_totals(conversation) + start = len(requests) + if operation.startswith("run"): + conversation.send_message(f"Reply briefly for {operation}.") + conversation.run() + elif operation == "ask_agent": + conversation.ask_agent("What are you doing?") + else: + conversation.condense() + after = token_totals(conversation) + deltas = { + role: total - before.get(role, 0) + for role, total in after.items() + if total != before.get(role, 0) + } + receipts = requests[start:] + successes = sum(item["status"] == 200 for item in receipts) + expected = {expected_role: 100 * successes} + evidence.append( + { + "operation": operation, + "http_requests": receipts, + "expected_prompt_token_deltas": expected, + "actual_prompt_token_deltas": deltas, + "correct": successes == 1 and deltas == expected, + } + ) + finally: + conversation.close() + finally: + server.shutdown() + server.server_close() + worker.join() + print(json.dumps(evidence, indent=2)) + if args.assert_fixed and not all(item["correct"] for item in evidence): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/01_standalone_sdk/39_llm_fallback.py b/examples/01_standalone_sdk/39_llm_fallback.py index ebd891c8f37..566cc464ea1 100644 --- a/examples/01_standalone_sdk/39_llm_fallback.py +++ b/examples/01_standalone_sdk/39_llm_fallback.py @@ -3,7 +3,7 @@ When the primary LLM fails with a transient error (rate limit, timeout, etc.), FallbackStrategy automatically tries alternate LLMs in order. Fallback is per-call: each new request starts with the primary model. Token usage and -cost from fallback calls are merged into the primary LLM's metrics. +cost from fallback calls are recorded in the caller's usage session. This example: 1. Saves two fallback LLM profiles to a temporary store. diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index dc3c81c940f..d72e4f3a074 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1130,32 +1130,30 @@ def locked_on_event(): partial(_log_emit_failure, type(event).__name__), ) - def _setup_llm_log_streaming(self, agent: AgentBase) -> None: - """Configure LLM log callbacks to stream logs via events.""" - for llm in agent.get_all_llms(): - if not llm.log_completions: - continue + def _setup_llm_log_streaming(self, conversation: LocalConversation) -> None: + """Configure LLM log callbacks to stream logs via events. - # Capture variables for closure - usage_id = llm.usage_id - model_name = llm.model + Set on the conversation's registry, so it covers every LLM session of + the conversation, including those opened by a later ``switch_llm`` / + ``switch_profile``. Only LLMs with ``log_completions`` on produce logs. + """ - def log_callback( - filename: str, log_data: str, uid=usage_id, model=model_name - ) -> None: - """Callback to emit LLM completion logs as events.""" - try: - event = LLMCompletionLogEvent( - filename=filename, - log_data=log_data, - model_name=model, - usage_id=uid, - ) - self._emit_event_from_thread(event) - except Exception: - logger.exception("Failed to emit LLM completion log event") + def log_callback( + filename: str, log_data: str, usage_id: str, model_name: str + ) -> None: + """Callback to emit LLM completion logs as events.""" + try: + event = LLMCompletionLogEvent( + filename=filename, + log_data=log_data, + model_name=model_name, + usage_id=usage_id, + ) + self._emit_event_from_thread(event) + except Exception: + logger.exception("Failed to emit LLM completion log event") - llm.telemetry.set_log_completions_callback(log_callback) + conversation.llm_registry.set_log_completions_callback(log_callback) def _setup_acp_activity_heartbeat(self, agent: AgentBase) -> None: """Wire ACP activity heartbeat to the idle timer. @@ -1207,8 +1205,12 @@ def callback(event: Event) -> None: return callback - def _setup_stats_streaming(self, agent: AgentBase) -> None: - """Configure stats update callbacks to stream stats changes via events.""" + def _setup_stats_streaming(self, conversation: LocalConversation) -> None: + """Configure stats update callbacks to stream stats changes via events. + + Set on the conversation's registry, so it keeps firing for the LLM a + later ``switch_llm`` / ``switch_profile`` installs. + """ def stats_callback() -> None: """Callback to emit stats updates. @@ -1236,8 +1238,7 @@ def stats_callback() -> None: ) self._emit_event_from_thread(event) - for llm in agent.get_all_llms(): - llm.telemetry.set_stats_update_callback(stats_callback) + conversation.llm_registry.set_stats_update_callback(stats_callback) @staticmethod def _ensure_workspace_is_git_repo(working_dir: Path) -> None: @@ -1441,10 +1442,10 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: ) # Setup LLM log streaming for remote execution - self._setup_llm_log_streaming(self._conversation.agent) + self._setup_llm_log_streaming(self._conversation) # Setup stats streaming for remote execution - self._setup_stats_streaming(self._conversation.agent) + self._setup_stats_streaming(self._conversation) # Wire ACP activity heartbeat so ACP tool calls (which run inside # the subprocess and never hit HTTP endpoints) still reset the diff --git a/openhands-sdk/openhands/sdk/agent/acp_agent.py b/openhands-sdk/openhands/sdk/agent/acp_agent.py index 2c3824e7fb6..1d19f6fdf2a 100644 --- a/openhands-sdk/openhands/sdk/agent/acp_agent.py +++ b/openhands-sdk/openhands/sdk/agent/acp_agent.py @@ -2257,6 +2257,11 @@ def _record_usage( elapsed: Wall-clock seconds for this prompt round-trip (optional). usage_update: The synchronized ACP UsageUpdate for this turn, if any. """ + # The conversation's session for the sentinel LLM: its registry opened + # it, so this resolves on the portal thread as well as on the caller's. + session = self.llm.session_for() + metrics = session.metrics + # -- Cost recording --------------------------------------------------- # claude-agent-acp, codex-acp: report cost via UsageUpdate notification # gemini-cli: does not send UsageUpdate (cost derived from tokens below) @@ -2265,7 +2270,7 @@ def _record_usage( last_cost = self._client._last_cost_by_session.get(session_id, 0.0) delta = usage_update.cost.amount - last_cost if delta > 0: - self.llm.metrics.add_cost(delta) + metrics.add_cost(delta) cost_recorded = True self._client._last_cost_by_session[session_id] = usage_update.cost.amount self._client._last_cost = usage_update.cost.amount @@ -2275,7 +2280,7 @@ def _record_usage( _extract_token_usage(response) ) if input_tokens or output_tokens: - self.llm.metrics.add_token_usage( + metrics.add_token_usage( prompt_tokens=input_tokens, completion_tokens=output_tokens, cache_read_tokens=cache_read, @@ -2296,7 +2301,7 @@ def _record_usage( self.acp_model, input_tokens, output_tokens ) if cost > 0: - self.llm.metrics.add_cost(cost) + metrics.add_cost(cost) if not cost_recorded and not input_tokens and not output_tokens: # gemini-cli currently returns response.usage=None and @@ -2308,13 +2313,9 @@ def _record_usage( ) if elapsed is not None: - self.llm.metrics.add_response_latency(elapsed, session_id) + metrics.add_response_latency(elapsed, session_id) - if self.llm.telemetry._stats_update_callback is not None: - try: - self.llm.telemetry._stats_update_callback() - except Exception: - logger.debug("Stats update callback failed", exc_info=True) + session.telemetry.notify_stats_update() # -- Capability helpers ------------------------------------------------ @@ -4865,9 +4866,12 @@ def set_acp_model(self, model: str) -> None: # authoritative current model is persisted by # :meth:`LocalConversation.switch_acp_model` via an agent ``model_copy``. self.llm.model = model - self.llm.metrics.model_name = model - if self.llm.metrics.accumulated_token_usage is not None: - self.llm.metrics.accumulated_token_usage.model = model + # Both ledgers carry the model name: the LLM's own and the one the + # conversation keeps for it (the same object outside a conversation). + for metrics in (self.llm.metrics, self.llm.session_for().metrics): + metrics.model_name = model + if metrics.accumulated_token_usage is not None: + metrics.accumulated_token_usage.model = model # Refresh the surfaced model state so the chip/picker # (``ConversationInfo.current_model_id``) reflects the switch instead # of the stale session-start value. ``_current_model_id`` is a diff --git a/openhands-sdk/openhands/sdk/conversation/conversation_stats.py b/openhands-sdk/openhands/sdk/conversation/conversation_stats.py index e574c7caf7e..1bee51cd437 100644 --- a/openhands-sdk/openhands/sdk/conversation/conversation_stats.py +++ b/openhands-sdk/openhands/sdk/conversation/conversation_stats.py @@ -68,18 +68,19 @@ def get_metrics_for_usage(self, usage_id: str) -> Metrics: return self.usage_to_metrics[usage_id] def register_llm(self, event: RegistryEvent): - # Listen for LLM creations and track their metrics - llm = event.llm - usage_id = llm.usage_id + # Listen for LLM registrations and track the metrics of their session: + # the one the registry keeps for the usage ID, else the LLM's own. + usage_id = event.llm.usage_id + session = event.session if event.session is not None else event.llm.session # Usage costs exist but have not been restored yet if ( usage_id in self.usage_to_metrics and usage_id not in self._restored_usage_ids ): - llm.restore_metrics(self.usage_to_metrics[usage_id]) + session.restore_metrics(self.usage_to_metrics[usage_id]) self._restored_usage_ids.add(usage_id) # Usage is new, track its metrics - if usage_id not in self.usage_to_metrics and llm.metrics: - self.usage_to_metrics[usage_id] = llm.metrics + if usage_id not in self.usage_to_metrics: + self.usage_to_metrics[usage_id] = session.metrics diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index 01337808b71..a2e2c751269 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -368,11 +368,16 @@ def __init__( # Create-or-resume: factory inspects BASE_STATE to decide desired_id = conversation_id or uuid.uuid4() + # The registry owns this conversation's LLM sessions (usage metrics and + # telemetry per usage_id); the call context carries them to every LLM + # call. Agent LLMs are registered later, in _ensure_agent_ready(). + self.llm_registry = LLMRegistry() self._llm_call_context = ( _parent_llm_call_context or LLMCallContext() ).for_conversation( str(desired_id), prompt_cache_key=prompt_cache_key, + sessions=self.llm_registry.sessions, ) # Resolve client-defined tools, then register them and inject the matching @@ -583,7 +588,6 @@ def _flush_pending_reactions(e): # Agent initialization is deferred to _ensure_agent_ready() for lazy loading # This ensures plugins are loaded before agent initialization - self.llm_registry = LLMRegistry() self._profile_store = LLMProfileStore(profile_store_dir) self._cipher = cipher @@ -1907,7 +1911,6 @@ def _condenser_for_switched_llm( condenser_llm = new_llm.model_copy( update={"usage_id": condenser.llm.usage_id}, ) - condenser_llm.reset_metrics() return condenser.model_copy(update={"llm": condenser_llm}) def _reject_llm_switch_on_acp(self) -> None: @@ -1926,8 +1929,8 @@ def switch_llm(self, llm: LLM) -> None: entry with that key already exists and has the same config, the cached LLM is reused. If its config differs (e.g. ``default``, which agent initialization registers, posted with another model), the - passed ``llm`` replaces the entry and takes over the metrics already - recorded under that ``usage_id``. + passed ``llm`` replaces the entry. Usage keeps accumulating in the + session the registry holds for that ``usage_id``. Args: llm: LLM to install on the agent. @@ -1948,10 +1951,6 @@ def switch_llm(self, llm: LLM) -> None: if registered is not None: self.llm_registry.remove(usage_id) self.llm_registry.add(new_llm) - if registered is not None: - # Keep one metrics object per usage_id: ConversationStats - # tracks the registered LLM's, so accrued usage is retained. - new_llm.restore_metrics(registered.metrics) # A switch_llm tool runs on a worker thread while run()/arun() holds the # state lock across the agent step on another thread, blocked awaiting # this very tool. Re-acquiring _state here would deadlock, so skip it: diff --git a/openhands-sdk/openhands/sdk/hooks/executor.py b/openhands-sdk/openhands/sdk/hooks/executor.py index 8f75ace7f99..830b5bf1a8f 100644 --- a/openhands-sdk/openhands/sdk/hooks/executor.py +++ b/openhands-sdk/openhands/sdk/hooks/executor.py @@ -244,8 +244,8 @@ def _execute_agent_hook( "timeout": hook.timeout, } ) - # Isolate Metrics so hook spend doesn't accrue to the parent's bucket. - hook_llm.reset_metrics() + # Its own usage_id gives the copy its own session, so hook spend + # doesn't accrue to the parent's bucket. # Never hand the parent's already-initialized visualizer instance to the # sub-conversation: LocalConversation.__init__ calls initialize() on it, @@ -328,7 +328,6 @@ def _execute_prompt_hook( "stream": False, } ) - hook_llm.reset_metrics() messages = [ Message( diff --git a/openhands-sdk/openhands/sdk/llm/__init__.py b/openhands-sdk/openhands/sdk/llm/__init__.py index 8cc1a58871c..53e3e94f399 100644 --- a/openhands-sdk/openhands/sdk/llm/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/__init__.py @@ -34,6 +34,7 @@ ) from openhands.sdk.llm.llm_registry import LLMRegistry, RegistryEvent from openhands.sdk.llm.llm_response import LLMResponse +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.llm.message import ( ImageContent, Message, @@ -95,6 +96,7 @@ "LLMCallContext", "LLM_PROFILE_SCHEMA_VERSION", "LLMRegistry", + "LLMSession", "LLMProfileLoader", "LLMProfileMutator", "LLMProfileStore", diff --git a/openhands-sdk/openhands/sdk/llm/call_context.py b/openhands-sdk/openhands/sdk/llm/call_context.py index a073e4a457d..7dad7fd8c4c 100644 --- a/openhands-sdk/openhands/sdk/llm/call_context.py +++ b/openhands-sdk/openhands/sdk/llm/call_context.py @@ -2,11 +2,15 @@ from __future__ import annotations -from collections.abc import Iterator, MutableMapping +from collections.abc import Iterator, Mapping, MutableMapping from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass, replace -from typing import Any +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from openhands.sdk.llm.llm_session import LLMSession @dataclass(frozen=True) @@ -21,23 +25,33 @@ class LLMCallContext: prompt_cache_key: str | None = None session_id: str | None = None + # The conversation's LLM sessions by role (``usage_id``). A call records its + # usage in the session of the calling LLM's role, when the conversation has + # one. It is a live view owned by the conversation's ``LLMRegistry``, so it + # takes no part in equality or hashing. + sessions: Mapping[str, LLMSession] | None = field( + default=None, compare=False, repr=False + ) def for_conversation( self, conversation_id: str, *, prompt_cache_key: str | None = None, + sessions: Mapping[str, LLMSession] | None = None, ) -> LLMCallContext: """Derive context for a conversation while preserving inheritable fields. - Conversation identity is always replaced. Any additional fields added to - this dataclass are inherited automatically unless this method explicitly - gives them conversation-local semantics. + Conversation identity is always replaced, and so are the sessions: a + child conversation never records into its parent's. Any additional + fields added to this dataclass are inherited automatically unless this + method explicitly gives them conversation-local semantics. """ return replace( self, prompt_cache_key=prompt_cache_key or conversation_id, session_id=conversation_id, + sessions=sessions, ) diff --git a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py index faa2f43a4fc..c9ca0c9794c 100644 --- a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py +++ b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_response import LLMResponse from openhands.sdk.llm.utils.metrics import Metrics @@ -75,14 +76,17 @@ def try_fallback( primary_error: Exception, primary_metrics: Metrics, call_fn: Callable[[Any], LLMResponse], + *, + primary: LLM | None = None, ) -> LLMResponse | None: - """Try fallback LLMs in order. Merges metrics into primary on success. + """Try fallback models in the caller's role and session. Args: primary_model: The primary model name (for logging). primary_error: The error from the primary model. - primary_metrics: The primary LLM's Metrics to merge fallback costs into. + primary_metrics: The primary LLM's metrics for legacy direct callers. call_fn: A callable that takes an LLM instance and returns an LLMResponse. + primary: The caller whose usage ID and aggregate accounting are kept. Returns: LLMResponse from the first successful fallback, or None if all fail. @@ -99,17 +103,23 @@ def try_fallback( f"{remaining} fallback(s) remaining" ) try: - # Disable nested fallbacks to prevent recursive chains - saved_strategy = fb.fallback_strategy - fb.fallback_strategy = None - metrics_before = fb.metrics.deep_copy() - try: - result = call_fn(fb) - finally: - fb.fallback_strategy = saved_strategy - # Merge fallback metrics (cost + tokens) into primary - metrics_diff = fb.metrics.diff(metrics_before) - primary_metrics.merge(metrics_diff) + fallback = fb.model_copy( + update={ + "usage_id": primary.usage_id + if primary is not None + else fb.usage_id, + "fallback_strategy": None, + } + ) + metrics_before = ( + fallback.metrics.deep_copy() if primary is None else None + ) + if primary is not None: + fallback._session = primary.session + fallback._registry_session = primary._registered_session() + result = call_fn(fallback) + if metrics_before is not None: + primary_metrics.merge(fallback.metrics.diff(metrics_before)) logger.info(f"[Fallback Strategy] Fallback LLM ({fb.model}) succeeded") return result except Exception as fb_error: diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index b836a7265df..6a65bcef11c 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -121,6 +121,8 @@ token_counter, ) +# OpenHands utilities +from openhands.sdk.llm.call_context import resolve_llm_call_context from openhands.sdk.llm.exceptions import ( LLMContextWindowTooSmallError, LLMNoResponseError, @@ -128,9 +130,8 @@ looks_like_auth_error, map_provider_exception, ) - -# OpenHands utilities from openhands.sdk.llm.llm_response import LLMResponse +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.llm.message import ( Message, TextContent, @@ -277,6 +278,11 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): ``` """ + # The session of the conversation whose LLMRegistry this object was added + # to (see ``session_for``). It says where the object is used, not what it + # is, so it sits in a slot: pydantic neither compares nor copies it. + __slots__ = ("_registry_session",) + # ========================================================================= # Config fields # ========================================================================= @@ -678,12 +684,14 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): default=None, exclude=True, ) - _metrics: Metrics | None = PrivateAttr(default=None) + # Usage metrics and telemetry live in a session, not on the configuration. + # This is the one behind ``llm.metrics`` / ``llm.telemetry``; see + # ``session_for`` for how it relates to a conversation's. + _session: LLMSession | None = PrivateAttr(default=None) # Runtime-only private attrs _model_info: Any = PrivateAttr(default=None) _tokenizer: dict[str, object] | None = PrivateAttr(default=None) _chat_template_tokenizer: ChatTemplateTokenizer | None = PrivateAttr(default=None) - _telemetry: Telemetry | None = PrivateAttr(default=None) _is_subscription: bool = PrivateAttr(default=False) _subscription_credential_store: Any = PrivateAttr(default=None) _subscription_credentials: Any = PrivateAttr(default=None) @@ -781,22 +789,12 @@ def _post_init(self): # OpenRouter ``HTTP-Referer`` / ``X-Title`` headers flow per-call # through ``_openrouter_headers()``. - # Metrics + Telemetry wiring. Guard both: this validator re-runs whenever - # the LLM is passed into another Pydantic model (e.g. RegistryEvent), - # and replacing _telemetry would silently drop any callback callers + # Guarded: this validator re-runs whenever the LLM is passed into + # another Pydantic model (e.g. RegistryEvent), and a new session would + # silently drop the usage recorded so far and any callback callers # have attached via telemetry.set_*_callback(). - if self._metrics is None: - self._metrics = Metrics(model_name=self.model) - - if self._telemetry is None: - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self._metrics, - ) + if self._session is None: + self._session = LLMSession(self) # Tokenizer if self.custom_tokenizer: @@ -846,6 +844,14 @@ def model_copy( if route_changed: copied._refresh_litellm_metadata() copied._reset_runtime_metadata_for_key() + if copied.usage_id != self.usage_id: + # Sessions follow the role. A copy made for another role is a + # different user of the model, so it starts its own accounting. + copied.reset_metrics() + elif not deep: + # Same role, same sessions: the shallow copy shares ``_session`` + # already, and stays with the conversation the original is in. + copied._registry_session = self._registered_session() return copied def _openrouter_headers(self) -> dict[str, str]: @@ -908,10 +914,26 @@ def _serialize_secrets(self, v: SecretStr | None, info): # ========================================================================= # Public API # ========================================================================= + @property + def session(self) -> LLMSession: + """The session of this LLM instance itself. + + It counts everything recorded through this object. Once a conversation + registers the LLM this is that conversation's session for the LLM's + role (see :meth:`session_for`). + """ + if self._session is None: + self._session = LLMSession(self) + return self._session + @property def metrics(self) -> Metrics: """Get usage metrics for this LLM instance. + These cover all the usage of this object. A conversation reports its + own usage through ``conversation.conversation_stats``; the two differ + when the same LLM object is used in more than one conversation. + Returns: Metrics object containing token usage, costs, and other statistics. @@ -921,9 +943,7 @@ def metrics(self) -> Metrics: print(f"Total cost: ${cost}") ``` """ - if self._metrics is None: - self._metrics = Metrics(model_name=self.model) - return self._metrics + return self.session.metrics @property def telemetry(self) -> Telemetry: @@ -937,16 +957,66 @@ def telemetry(self) -> Telemetry: llm.telemetry.set_log_completions_callback(my_callback) ``` """ - if self._telemetry is None: - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self.metrics, - ) - return self._telemetry + return self.session.telemetry + + def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: + """Return the session a call made with ``call_context`` records in. + + That is the session the conversation keeps for this LLM's role + (``usage_id``). Without a conversation in the context it is the session + of the registry this object was last added to, and failing that the + object's own. + + The first conversation to register an LLM takes the LLM's own session, + so there ``llm.metrics`` is the conversation's ledger. An object that + records in a second session keeps one of its own apart, and every + session it records in also counts there. + """ + sessions = resolve_llm_call_context(call_context).sessions + session = sessions.get(self.usage_id) if sessions is not None else None + if session is None: + session = self._registered_session() + if session is None: + session = self.session + if session is not self._session: + self._keep_own_session_apart() + session.follow(self) + return session + + def _registered_session(self) -> LLMSession | None: + try: + return self._registry_session + except AttributeError: # never added to a registry: the slot is empty + return None + + def _join_registry_session(self, session: LLMSession) -> None: + """Record in ``session``, which a registry keeps for this LLM's role.""" + registered = self._registered_session() + if registered is session: + return + if registered is None: + # Not part of a conversation yet: its session is this object's own + # from now on, as if the usage had always been recorded there. + self._session = session + else: + self._keep_own_session_apart() + self._registry_session = session + session.follow(self) + + def _keep_own_session_apart(self) -> None: + """Give this object a session of its own if a registry holds its current one. + + Needed before it records in another session: usage of one conversation + must not land in the ledger of another. The new session starts from + what the shared one holds, so ``llm.metrics`` keeps its totals. + """ + registered = self._registered_session() + if registered is None or self._session is not registered: + return + own = LLMSession(self) + own.metrics.merge(registered.metrics) + self._session = own + registered.follow(self) @computed_field( return_type=bool, @@ -988,55 +1058,42 @@ def _restore_is_subscription(cls, data, handler): _serialized_is_subscription.reset(token) def restore_metrics(self, metrics: Metrics) -> None: - # Only used by ConversationStats to seed metrics - self._metrics = metrics - # Keep telemetry in sync so post-resume LLM calls record into - # the restored metrics object, not the stale one from __init__. - if self._telemetry is not None: - self._telemetry.metrics = metrics + """Make ``metrics`` the usage metrics of this LLM instance.""" + self.session.restore_metrics(metrics) def reset_metrics(self) -> None: - """Reset metrics and telemetry to fresh instances. - - This is used by the LLMRegistry to ensure each registered LLM has - independent metrics, preventing metrics from being shared between - LLMs that were created via model_copy(). - - When an LLM is copied (e.g., to create a condenser LLM from an agent LLM), - Pydantic's model_copy() does a shallow copy of private attributes by default, - causing the original and copied LLM to share the same Metrics object. - This method allows the registry to fix this by creating fresh metrics and - telemetry immediately, so the copied LLM is ready for the next completion - call even if callers do not access ``metrics``/``telemetry`` first. + """Give this LLM instance a fresh session of its own. + + A copy made with ``model_copy()`` shares the session of the LLM it was + copied from unless it is given another ``usage_id``, so both count into + the same ``llm.metrics``. Call this on a copy that should be counted + apart, such as the LLM handed to a sub-agent. It also detaches the + instance from the session a conversation's registry opened for it. """ - self._metrics = Metrics(model_name=self.model) - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self._metrics, - ) + self._session = LLMSession(self) + self._registry_session = None def _handle_error( self, error: Exception, fallback_call_fn: Callable[[LLM], LLMResponse], + session: LLMSession | None = None, ) -> LLMResponse: """Handle an error from completion/responses: try fallback, then map and raise. Must be called from within an except block. Either returns an LLMResponse (fallback succeeded) or re-raises (mapped or original). """ - assert self._telemetry is not None - self._telemetry.on_error(error) + if session is None: + session = self.session + session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): result = self.fallback_strategy.try_fallback( primary_model=self.model, primary_error=error, - primary_metrics=self.metrics, + primary_metrics=session.metrics, call_fn=fallback_call_fn, + primary=self, ) if result is not None: return result @@ -1049,25 +1106,26 @@ async def _ahandle_error( self, error: Exception, fallback_call_fn: Callable[[LLM], LLMResponse], + session: LLMSession | None = None, ) -> LLMResponse: """Async variant of :meth:`_handle_error`. The *fallback_call_fn* is synchronous (it calls the fallback LLM's sync ``completion``/``responses``), so the fallback attempt is - offloaded to a thread via :func:`asyncio.loop.run_in_executor` to + offloaded to a thread via :func:`asyncio.to_thread` to avoid blocking the event loop. """ - assert self._telemetry is not None - self._telemetry.on_error(error) + if session is None: + session = self.session + session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, + result = await asyncio.to_thread( self.fallback_strategy.try_fallback, self.model, error, - self.metrics, + session.metrics, fallback_call_fn, + primary=self, ) if result is not None: return result @@ -1226,22 +1284,30 @@ async def _aiter_with_idle_timeout( return yield item - def _build_completion_result(self, resp: ModelResponse) -> LLMResponse: + def _build_completion_result( + self, resp: ModelResponse, session: LLMSession | None = None + ) -> LLMResponse: """Convert a raw :class:`ModelResponse` into an :class:`LLMResponse`.""" + if session is None: + session = self.session first_choice = resp["choices"][0] message = Message.from_llm_chat_message(first_choice["message"]) return LLMResponse( message=message, - metrics=self.metrics.get_snapshot(), + metrics=session.metrics.get_snapshot(), raw_response=resp, ) - def _build_responses_result(self, resp: ResponsesAPIResponse) -> LLMResponse: + def _build_responses_result( + self, resp: ResponsesAPIResponse, session: LLMSession | None = None + ) -> LLMResponse: """Convert a raw :class:`ResponsesAPIResponse` into an :class:`LLMResponse`.""" + if session is None: + session = self.session message = Message.from_llm_responses_output(resp.output) return LLMResponse( message=message, - metrics=self.metrics.get_snapshot(), + metrics=session.metrics.get_snapshot(), raw_response=resp, ) @@ -1330,6 +1396,7 @@ def _finalize_stream_response( self, completed_response: ResponseCompletedEvent | None, collected_output_items: list[Any], + session: LLMSession | None = None, ) -> ResponsesAPIResponse: """Validate and patch the completed response from a Responses stream. @@ -1351,8 +1418,9 @@ def _finalize_stream_response( if not completed_resp.output and collected_output_items: completed_resp.output = collected_output_items - assert self._telemetry is not None - self._telemetry.on_response( + if session is None: + session = self.session + session.telemetry.on_response( completed_resp, provider_info=self._provider_info, ) @@ -1486,7 +1554,7 @@ def _finalize_completion_params( # 4) request context for telemetry (always include context_window for metrics) # Always pass context_window so metrics are tracked even when # logging is disabled. - telemetry = self.telemetry + telemetry = self.session_for(call_context).telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0 } @@ -1622,7 +1690,7 @@ def _finalize_responses_params( # Request context for telemetry (always include context_window for metrics) # Always pass context_window so metrics are tracked even when # logging is disabled. - telemetry = self.telemetry + telemetry = self.session_for(call_context).telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0 } @@ -1647,6 +1715,7 @@ def _validate_chat_response( formatted_messages: list[dict[str, Any]], cc_tools: list[ChatCompletionToolParam], add_security_risk_prediction: bool, + session: LLMSession | None = None, ) -> ModelResponse: """Post-process a chat completion response inside the retry boundary. @@ -1669,8 +1738,9 @@ def _validate_chat_response( ) # 6) telemetry - assert self._telemetry is not None - self._telemetry.on_response( + if session is None: + session = self.session + session.telemetry.on_response( resp, raw_resp=raw_resp, provider_info=self._provider_info, @@ -1788,6 +1858,7 @@ def completion( # cached (1h TTL) and negative-cached on failure, and the probe is a # no-op for providers without runtime metadata. self.resolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -1819,8 +1890,7 @@ def completion( @self._make_retry_decorator() def _one_attempt(**retry_kwargs: Any) -> ModelResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} resp = self._transport_call( messages=formatted_messages, @@ -1834,11 +1904,12 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: formatted_messages=formatted_messages, cc_tools=cc_tools, add_security_risk_prediction=add_security_risk_prediction, + session=session, ) return resp try: - return self._build_completion_result(_one_attempt()) + return self._build_completion_result(_one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -1854,6 +1925,7 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -1881,6 +1953,7 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -1907,6 +1980,7 @@ async def acompletion( # probe is a no-op for providers without runtime metadata, so repeated # calls are cheap. await self.aresolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -1939,8 +2013,7 @@ async def acompletion( @self._make_retry_decorator() @self._async_hard_timeout_decorator() async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} resp = await self._atransport_call( messages=formatted_messages, @@ -1954,11 +2027,12 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: formatted_messages=formatted_messages, cc_tools=cc_tools, add_security_risk_prediction=add_security_risk_prediction, + session=session, ) return resp try: - return self._build_completion_result(await _one_attempt()) + return self._build_completion_result(await _one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -1974,6 +2048,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2007,6 +2082,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -2046,6 +2122,7 @@ def responses( # Cached (1h TTL) / negative-cached; no-op for providers without runtime # metadata. self.resolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -2081,8 +2158,7 @@ def responses( @self._make_retry_decorator() def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} litellm_kwargs = self._build_responses_call_kwargs( input_items, instructions, resp_tools, final_kwargs @@ -2096,7 +2172,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: "provider returned a non-streaming response; " "no on_token deltas will be emitted." ) - self._telemetry.on_response( + session.telemetry.on_response( ret, provider_info=self._provider_info, ) @@ -2131,13 +2207,13 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: completed_response = get_completed_response(ret, completed_response) return self._finalize_stream_response( - completed_response, collected_output_items + completed_response, collected_output_items, session ) raise AssertionError(f"Expected ResponsesAPIResponse, got {type(ret)}") try: - return self._build_responses_result(_one_attempt()) + return self._build_responses_result(_one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -2155,6 +2231,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: store, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2186,6 +2263,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -2211,6 +2289,7 @@ async def aresponses( # the first request so context management reads runtime (not model-level) # limits. No blocking network I/O in-process. await self.aresolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -2249,8 +2328,7 @@ async def aresponses( async def _one_attempt( **retry_kwargs: Any, ) -> ResponsesAPIResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} auth_values = await self._aget_litellm_auth_values() litellm_kwargs = self._build_responses_call_kwargs( @@ -2269,7 +2347,7 @@ async def _one_attempt( "provider returned a non-streaming response; " "no on_token deltas will be emitted." ) - self._telemetry.on_response( + session.telemetry.on_response( ret, provider_info=self._provider_info, ) @@ -2323,13 +2401,13 @@ async def _one_attempt( completed_response = get_completed_response(ret, completed_response) return self._finalize_stream_response( - completed_response, collected_output_items + completed_response, collected_output_items, session ) raise AssertionError(f"Expected ResponsesAPIResponse, got {type(ret)}") try: - return self._build_responses_result(await _one_attempt()) + return self._build_responses_result(await _one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -2347,6 +2425,7 @@ async def _one_attempt( store, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2382,6 +2461,7 @@ async def _one_attempt( call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= diff --git a/openhands-sdk/openhands/sdk/llm/llm_registry.py b/openhands-sdk/openhands/sdk/llm/llm_registry.py index 7d1460b85b3..59ae7f991d8 100644 --- a/openhands-sdk/openhands/sdk/llm/llm_registry.py +++ b/openhands-sdk/openhands/sdk/llm/llm_registry.py @@ -3,17 +3,28 @@ from typing import ClassVar from uuid import uuid4 -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from openhands.sdk.llm.llm import LLM +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.logger import get_logger logger = get_logger(__name__) +# (log file name, log data, usage ID, model name) +LogCompletionsCallback = Callable[[str, str, str, str], None] + class RegistryEvent(BaseModel): llm: LLM + session: LLMSession | None = Field( + default=None, + description=( + "The session the registry keeps for the LLM's usage ID. Without " + "one the event is about the LLM's own session." + ), + ) model_config: ClassVar[ConfigDict] = ConfigDict( arbitrary_types_allowed=True, @@ -21,15 +32,16 @@ class RegistryEvent(BaseModel): class LLMRegistry: - """A minimal LLM registry for managing LLM instances by usage ID. + """The LLMs of one conversation and their sessions, by role (usage ID). - This registry provides a simple way to manage multiple LLM instances, + The registry maps each usage ID to the LLM that currently fills that role, avoiding the need to recreate LLMs with the same configuration. - The registry also ensures that each registered LLM has independent metrics, - preventing metrics from being shared between LLMs that were created via - model_copy(). This is important for scenarios like creating a condenser LLM - from an agent LLM, where each should track its own usage independently. + It also keeps one :class:`LLMSession` per usage ID. A session holds what the + role accumulates (usage metrics, telemetry callbacks), so it outlives the + LLM object in the slot: replacing or re-adding the LLM for a usage ID keeps + the usage recorded so far. Two LLMs under different usage IDs never share a + session, including when one was created from the other via model_copy(). """ registry_id: str @@ -47,8 +59,9 @@ def __init__( self.registry_id = str(uuid4()) self.retry_listener = retry_listener self._usage_to_llm: dict[str, LLM] = {} - # Track metrics object IDs to detect shared metrics - self._metrics_ids: set[int] = set() + self._sessions: dict[str, LLMSession] = {} + self._stats_update_callback: Callable[[], None] | None = None + self._log_completions_callback: LogCompletionsCallback | None = None self.subscriber: Callable[[RegistryEvent], None] | None = None def subscribe(self, callback: Callable[[RegistryEvent], None]) -> None: @@ -77,41 +90,61 @@ def usage_to_llm(self) -> MappingProxyType[str, LLM]: return MappingProxyType(self._usage_to_llm) - def _ensure_independent_metrics(self, llm: LLM) -> None: - """Ensure the LLM has independent metrics not shared with other LLMs. + @property + def sessions(self) -> MappingProxyType[str, LLMSession]: + """The sessions by usage ID (read-only live view). - When LLMs are created via model_copy(), Pydantic does a shallow copy of - private attributes by default, causing the original and copied LLM to - share the same Metrics object. This method detects such sharing and - resets the metrics to ensure each LLM tracks its own usage independently. + A usage ID keeps its session for the life of the registry, also while + no LLM is registered under it. + """ + return MappingProxyType(self._sessions) - Args: - llm: The LLM instance to check and potentially reset metrics for. + def set_stats_update_callback(self, callback: Callable[[], None] | None) -> None: + """Call ``callback`` whenever any session of this registry records usage. + + It applies to the sessions the registry has now and to those it opens + later, so it does not need setting again when an LLM is switched. + """ + self._stats_update_callback = callback + for session in self._sessions.values(): + session.telemetry.set_stats_update_callback(callback) + + def set_log_completions_callback( + self, callback: LogCompletionsCallback | None + ) -> None: + """Send completion logs to ``callback`` instead of writing them to disk. + + ``callback`` receives the log file name, the log data, the usage ID and + the name of the model that was called. Like + :meth:`set_stats_update_callback`, it covers present and future + sessions; LLMs with ``log_completions`` off produce no logs. """ - # Access the metrics to trigger lazy initialization if needed - metrics = llm.metrics - metrics_id = id(metrics) + self._log_completions_callback = callback + for usage_id, session in self._sessions.items(): + self._apply_log_completions_callback(usage_id, session) - # Check if this metrics object is already tracked by another LLM - if metrics_id in self._metrics_ids: - logger.debug( - f"[LLM registry {self.registry_id}]: Detected shared metrics for " - f"usage '{llm.usage_id}', resetting to independent metrics" - ) - llm.reset_metrics() - # Get the new metrics ID after reset - metrics_id = id(llm.metrics) + def _apply_log_completions_callback( + self, usage_id: str, session: LLMSession + ) -> None: + callback = self._log_completions_callback + telemetry = session.telemetry + if callback is None: + telemetry.set_log_completions_callback(None) + return + + def log_completion(filename: str, log_data: str) -> None: + callback(filename, log_data, usage_id, telemetry.model_name) - # Track this metrics object ID - self._metrics_ids.add(metrics_id) + telemetry.set_log_completions_callback(log_completion) def add(self, llm: LLM) -> None: """Add an LLM instance to the registry. - This method ensures that the LLM has independent metrics before - registering it. If the LLM's metrics are shared with another - registered LLM (e.g., due to model_copy()), fresh metrics will - be created automatically. + The LLM records in the session of its usage ID from then on. The first + LLM added for a usage ID brings its own session along (a new one if it + already serves another conversation). The registry keeps that session + afterwards, so an LLM added later for the same usage ID continues from + the metrics already recorded. Args: llm: The LLM instance to register. @@ -128,11 +161,27 @@ def add(self, llm: LLM) -> None: ) raise ValueError(message) - # Ensure this LLM has independent metrics before registering - self._ensure_independent_metrics(llm) + session = self._sessions.get(usage_id) + if session is None: + session = llm.session + # One session per role: an LLM whose session is spoken for (it + # serves another conversation, or shares it with a registered LLM + # it was copied from) gets a new one here. + if llm._registered_session() is not None or any( + session is other for other in self._sessions.values() + ): + session = LLMSession(llm) + self._sessions[usage_id] = session + # Callbacks the LLM already carries stay unless the registry has + # its own to install. + if self._stats_update_callback is not None: + session.telemetry.set_stats_update_callback(self._stats_update_callback) + if self._log_completions_callback is not None: + self._apply_log_completions_callback(usage_id, session) + llm._join_registry_session(session) self._usage_to_llm[usage_id] = llm - self.notify(RegistryEvent(llm=llm)) + self.notify(RegistryEvent(llm=llm, session=session)) logger.debug( f"[LLM registry {self.registry_id}]: Added LLM for usage {usage_id}" ) @@ -165,18 +214,14 @@ def remove(self, usage_id: str) -> None: Unlike :meth:`add` and :meth:`get`, this is a no-op when ``usage_id`` is absent, so callers can invalidate a lazily-cached slot without - first checking for it. + first checking for it. The session of the usage ID stays, so the usage + it recorded is not lost. Args: usage_id: Usage slot to drop from the registry. """ llm = self._usage_to_llm.pop(usage_id, None) if llm is not None: - # The slot no longer holds this instance, so its metrics are no - # longer shared with anyone here. Keeping the id would make a later - # add() mistake the same instance for a shared-metrics copy and - # reset the usage it has already accumulated. - self._metrics_ids.discard(id(llm.metrics)) logger.debug( f"[LLM registry {self.registry_id}]: Removed LLM for usage {usage_id}" ) diff --git a/openhands-sdk/openhands/sdk/llm/llm_session.py b/openhands-sdk/openhands/sdk/llm/llm_session.py new file mode 100644 index 00000000000..6947cf7f4f6 --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/llm_session.py @@ -0,0 +1,102 @@ +"""Runtime state of an LLM, kept apart from its configuration.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from openhands.sdk.llm.utils.metrics import Metrics +from openhands.sdk.llm.utils.telemetry import Telemetry + + +if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM + + +class LLMSession: + """What accumulates while an LLM is used: usage metrics and telemetry. + + An :class:`LLM` describes how to call a model. A session holds what calling + it produces, so one configuration can serve several users without their + usage, callbacks or in-flight request state running together. + + There are two kinds of owner: + + * every ``LLM`` has a session of its own, behind ``llm.metrics`` and + ``llm.telemetry``. It counts all the usage of that object. + * a conversation's :class:`LLMRegistry` keeps one session per role + (``usage_id``). A call made with that conversation's + :class:`LLMCallContext` records there, whichever ``LLM`` object fills the + role at the time. + + The two are the same session while an LLM serves a single conversation: + the registry takes over the session of the first LLM it gets for a role. + They come apart when one LLM object is used by a second conversation, so + that neither conversation sees the other's usage. + """ + + __slots__ = ("_telemetry",) + + def __init__(self, llm: LLM) -> None: + self._telemetry: Telemetry = Telemetry( + model_name=llm.model, + log_enabled=llm.log_completions, + log_dir=_log_dir(llm), + input_cost_per_token=llm.input_cost_per_token, + output_cost_per_token=llm.output_cost_per_token, + metrics=Metrics(model_name=llm.model), + ) + + def __eq__(self, other: object) -> bool: + # By value, as the metrics and telemetry it holds compare: two LLMs + # with equal configuration and equal usage are equal. + if not isinstance(other, LLMSession): + return NotImplemented + return self._telemetry == other._telemetry + + @property + def metrics(self) -> Metrics: + """Usage recorded in this session.""" + return self._telemetry.metrics + + @property + def telemetry(self) -> Telemetry: + """Telemetry handler that records into :attr:`metrics`.""" + return self._telemetry + + def follow(self, llm: LLM) -> None: + """Serve ``llm``, the LLM now calling in this session's role. + + The metrics and the telemetry callbacks belong to the role and stay. + The settings telemetry takes from an LLM follow the caller, and unless + this is the LLM's own session, ``llm.metrics`` also counts what is + recorded here. + """ + telemetry = self._telemetry + telemetry.model_name = llm.model + telemetry.log_enabled = llm.log_completions + telemetry.log_dir = _log_dir(llm) + telemetry.input_cost_per_token = llm.input_cost_per_token + telemetry.output_cost_per_token = llm.output_cost_per_token + + own = llm.session.metrics + self.metrics.roll_up_into(None if own is self.metrics else own) + + def restore_metrics(self, metrics: Metrics) -> None: + """Continue from ``metrics``, e.g. those persisted with a conversation. + + The session records into the given object from now on. What that object + already holds is added once to the ledger this session rolls up into. + """ + current = self.metrics + if metrics is current: + return + target = current.rolls_up_into + current.roll_up_into(None) + if target is not None and target is not metrics: + target.merge(metrics) + metrics.roll_up_into(target) + self._telemetry.metrics = metrics + + +def _log_dir(llm: LLM) -> str | None: + return llm.log_completions_folder if llm.log_completions else None diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 732af4c6213..e851c7d9386 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -117,8 +117,15 @@ class Metrics(MetricsSnapshot): - max_budget_per_task (budget limit) - A list of ResponseLatency - A list of TokenUsage (one per call). + + A ledger can roll up into another one (see :meth:`roll_up_into`): what a + conversation records for a role is also counted on the ``LLM`` it used. """ + # The roll-up target is wiring, not data. A slot keeps it out of what + # pydantic serializes, compares and copies, so a copy is a plain snapshot. + __slots__ = ("_rollup",) + costs: list[Cost] = Field( default_factory=list, description="List of individual costs" ) @@ -162,18 +169,45 @@ def get_snapshot(self) -> MetricsSnapshot: else None, ) - def add_cost(self, value: float) -> None: + def roll_up_into(self, target: "Metrics | None") -> None: + """Also record into ``target`` whatever is recorded here from now on. + + What this ledger already holds is not copied over. ``None`` detaches. + """ + if target is self: + raise ValueError("Metrics cannot roll up into itself.") + self._rollup = target + + @property + def rolls_up_into(self) -> "Metrics | None": + """The ledger that also records what this one does, if any.""" + try: + return self._rollup + except AttributeError: # never attached: the slot is still empty + return None + + def add_cost(self, value: float, *, model_name: str | None = None) -> None: if value < 0: raise ValueError("Added cost cannot be negative.") self.accumulated_cost += value - self.costs.append(Cost(cost=value, model=self.model_name)) + self.costs.append(Cost(cost=value, model=model_name or self.model_name)) + target = self.rolls_up_into + if target is not None: + target.add_cost(value, model_name=model_name) - def add_response_latency(self, value: float, response_id: str) -> None: + def add_response_latency( + self, value: float, response_id: str, *, model_name: str | None = None + ) -> None: self.response_latencies.append( ResponseLatency( - latency=max(0.0, value), model=self.model_name, response_id=response_id + latency=max(0.0, value), + model=model_name or self.model_name, + response_id=response_id, ) ) + target = self.rolls_up_into + if target is not None: + target.add_response_latency(value, response_id, model_name=model_name) def add_token_usage( self, @@ -184,13 +218,15 @@ def add_token_usage( context_window: int, response_id: str, reasoning_tokens: int = 0, + *, + model_name: str | None = None, ) -> None: """Add a single usage record.""" # Token each turn for calculating context usage. per_turn_token = prompt_tokens + completion_tokens usage = TokenUsage( - model=self.model_name, + model=model_name or self.model_name, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cache_read_tokens=cache_read_tokens, @@ -219,8 +255,25 @@ def add_token_usage( else: self.accumulated_token_usage = self.accumulated_token_usage + new_usage + target = self.rolls_up_into + if target is not None: + target.add_token_usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_read_tokens=cache_read_tokens, + cache_write_tokens=cache_write_tokens, + context_window=context_window, + response_id=response_id, + reasoning_tokens=reasoning_tokens, + model_name=model_name, + ) + def merge(self, other: "Metrics") -> None: """Merge 'other' metrics into this one.""" + target = self.rolls_up_into + if target is not None: + target.merge(other) + self.accumulated_cost += other.accumulated_cost # Keep the max_budget_per_task from other if it's set and this one isn't diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index 3a0c61eafd0..a1616e2beef 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -199,6 +199,15 @@ def set_stats_update_callback(self, callback: Callable[[], None] | None) -> None """ self._stats_update_callback = callback + def notify_stats_update(self) -> None: + """Tell the stats-update callback, if any, that the metrics changed.""" + if self._stats_update_callback is None: + return + try: + self._stats_update_callback() + except Exception: + logger.exception("Stats update callback failed", exc_info=True) + def on_request(self, telemetry_ctx: dict | None) -> None: self._req_start = time.time() self._req_ctx = _redact_request_credentials(telemetry_ctx or {}) @@ -218,13 +227,15 @@ def on_response( # 1) latency self._last_latency = time.time() - (self._req_start or time.time()) response_id = resp.id - self.metrics.add_response_latency(self._last_latency, response_id) + self.metrics.add_response_latency( + self._last_latency, response_id, model_name=self.model_name + ) # 2) cost cost = self._compute_cost(resp, provider_info=provider_info) # A confirmed zero charge is different from unavailable pricing. if cost is not None: - self.metrics.add_cost(cost) + self.metrics.add_cost(cost, model_name=self.model_name) # 3) tokens - normalize the provider's usage shape exactly once usage = normalize_usage(_response_usage(resp)) @@ -243,11 +254,7 @@ def on_response( self._close_span() # 6) notify about stats update - if self._stats_update_callback is not None: - try: - self._stats_update_callback() - except Exception: - logger.exception("Stats update callback failed", exc_info=True) + self.notify_stats_update() return self.metrics.deep_copy() @@ -310,6 +317,7 @@ def _record_usage( reasoning_tokens=usage.reasoning_tokens, context_window=context_window, response_id=response_id, + model_name=self.model_name, ) # ---------- Observability span ---------- diff --git a/openhands-sdk/openhands/sdk/settings/model.py b/openhands-sdk/openhands/sdk/settings/model.py index 2c5864cdcc2..46e2cd01f31 100644 --- a/openhands-sdk/openhands/sdk/settings/model.py +++ b/openhands-sdk/openhands/sdk/settings/model.py @@ -318,7 +318,6 @@ def build_condenser(self, llm: LLM) -> LLMSummarizingCondenser | None: from openhands.sdk.context.condenser import LLMSummarizingCondenser condenser_llm = llm.model_copy(update={"usage_id": "condenser"}) - condenser_llm.reset_metrics() condenser_kwargs = self.model_dump( exclude={"enabled", "condenser_kind"}, exclude_none=True, diff --git a/openhands-sdk/openhands/sdk/subagent/registry.py b/openhands-sdk/openhands/sdk/subagent/registry.py index 0f0c99fa2d1..350a06fd65c 100644 --- a/openhands-sdk/openhands/sdk/subagent/registry.py +++ b/openhands-sdk/openhands/sdk/subagent/registry.py @@ -256,14 +256,14 @@ def _factory(llm: "LLM") -> "Agent": # Sub-agents get a summarizing condenser by default (parity with the # top-level agent) so deep runs auto-compact instead of erroring on context - # overflow. The condenser LLM needs a distinct usage_id or its tokens get - # deduped out of conversation stats. A NoOpCondenser disables condensation. + # overflow. The condenser LLM needs a distinct usage_id or its tokens are + # reported together with the agent's. A NoOpCondenser disables condensation. if agent_def.condenser is not None: condenser = agent_def.condenser if isinstance(condenser, LLMSummarizingCondenser): # Freshen per spawn: a distinct LLM with its own metrics (so # sub-agents don't share/double-count condenser cost) and a - # usage_id distinct from the agent's, else its tokens get deduped. + # usage_id distinct from the agent's, so its tokens show apart. cond_llm = condenser.llm usage_id = ( cond_llm.usage_id diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 1fa76bce8d3..2b0be7c8a68 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -3596,15 +3596,8 @@ def _make_service_with_callback(self): service._emit_event_from_thread = MagicMock(name="_emit_event_from_thread") callbacks: list = [] - service._setup_stats_streaming( - MagicMock( - get_all_llms=lambda: [ - MagicMock( - telemetry=MagicMock(set_stats_update_callback=callbacks.append) - ) - ] - ) - ) + conversation.llm_registry.set_stats_update_callback = callbacks.append + service._setup_stats_streaming(conversation) assert len(callbacks) == 1, "stats_callback must be registered exactly once" return service, state, callbacks[0] @@ -3919,8 +3912,8 @@ def test_llm_log_callback_swallows_emit_failures( event_service: EventService, caplog ) -> None: callbacks = [] - llm = MagicMock(log_completions=True, usage_id="test-usage", model="gpt-4o") - llm.telemetry.set_log_completions_callback.side_effect = callbacks.append + conversation = MagicMock() + conversation.llm_registry.set_log_completions_callback = callbacks.append object.__setattr__( event_service, @@ -3929,8 +3922,8 @@ def test_llm_log_callback_swallows_emit_failures( ) with caplog.at_level("ERROR"): - event_service._setup_llm_log_streaming(MagicMock(get_all_llms=lambda: [llm])) - callbacks[0]("completion.json", "{}") + event_service._setup_llm_log_streaming(conversation) + callbacks[0]("completion.json", "{}", "test-usage", "gpt-4o") emit_mock = cast(MagicMock, event_service._emit_event_from_thread) emit_mock.assert_called_once() diff --git a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py index 06099406b30..ecaace6e400 100644 --- a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py +++ b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py @@ -93,8 +93,7 @@ def create_completion_result(content: str) -> LLMResponse: mock_llm.input_cost_per_token = None mock_llm.output_cost_per_token = None - mock_llm._metrics = None - mock_llm._telemetry = None + mock_llm._session = None # Helper method to set mock response content def set_mock_response_content(content: str): diff --git a/tests/sdk/conversation/local/test_llm_session_isolation.py b/tests/sdk/conversation/local/test_llm_session_isolation.py new file mode 100644 index 00000000000..7f851d66789 --- /dev/null +++ b/tests/sdk/conversation/local/test_llm_session_isolation.py @@ -0,0 +1,186 @@ +"""A conversation's LLM usage and callbacks belong to it, per usage_id. + +The usage metrics and telemetry callbacks used to sit on the ``LLM`` object, so +they went wherever that object went: into a second conversation that reused +the LLM, and away from the conversation when the LLM was switched. +""" + +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch + +import pytest +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.agent import Agent +from openhands.sdk.context.condenser import LLMSummarizingCondenser +from openhands.sdk.conversation.impl.local_conversation import ( + ASK_AGENT_LLM_USAGE_ID, + LocalConversation, +) +from openhands.sdk.llm import LLM, Message, TextContent + + +PROMPT_TOKENS = 100 + + +def _llm(model: str = "gpt-4o") -> LLM: + return LLM(model=model, api_key=SecretStr("k"), usage_id="agent") + + +def _conversation(tmp_path: Path, llm: LLM, name: str = "conv") -> LocalConversation: + condenser = LLMSummarizingCondenser( + llm=llm.model_copy(update={"usage_id": "condenser"}), max_size=80 + ) + return LocalConversation( + agent=Agent(llm=llm, tools=[], condenser=condenser), + workspace=tmp_path / name, + visualizer=None, + ) + + +def _run(conversation: LocalConversation, text: str = "hi") -> None: + conversation.send_message(text) + conversation.run() + + +def _prompt_tokens(conversation: LocalConversation, usage_id: str = "agent") -> int: + usage = conversation.conversation_stats.usage_to_metrics[ + usage_id + ].accumulated_token_usage + assert usage is not None + return usage.prompt_tokens + + +@pytest.fixture(autouse=True) +def fake_completion() -> Iterator[None]: + def respond(*args, **kwargs) -> ModelResponse: + return ModelResponse( + id="resp", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="done"), + ) + ], + model="gpt-4o", + usage=Usage( + prompt_tokens=PROMPT_TOKENS, completion_tokens=1, total_tokens=101 + ), + ) + + with patch("openhands.sdk.llm.llm.litellm_completion", side_effect=respond): + yield + + +def test_conversations_sharing_an_llm_keep_their_usage_apart(tmp_path): + llm = _llm() + first = _conversation(tmp_path, llm, "first") + second = _conversation(tmp_path, llm, "second") + + _run(first) + _run(second) + _run(second) + + assert _prompt_tokens(first) == PROMPT_TOKENS + assert _prompt_tokens(second) == 2 * PROMPT_TOKENS + # The LLM object still reports everything that went through it. + assert llm.metrics.accumulated_token_usage is not None + assert llm.metrics.accumulated_token_usage.prompt_tokens == 3 * PROMPT_TOKENS + + +def test_conversations_sharing_an_llm_keep_their_callbacks_apart(tmp_path): + llm = _llm() + first = _conversation(tmp_path, llm, "first") + second = _conversation(tmp_path, llm, "second") + fired: list[str] = [] + first.llm_registry.set_stats_update_callback(lambda: fired.append("first")) + second.llm_registry.set_stats_update_callback(lambda: fired.append("second")) + + _run(first) + _run(second) + + assert fired == ["first", "second"] + + +def test_single_conversation_reports_through_the_llm_object(tmp_path): + """The common case is unchanged: ``llm.metrics`` is the conversation's ledger.""" + llm = _llm() + conversation = _conversation(tmp_path, llm) + + _run(conversation) + + assert conversation.conversation_stats.usage_to_metrics["agent"] is llm.metrics + assert _prompt_tokens(conversation) == PROMPT_TOKENS + + +def test_stats_callback_keeps_firing_after_switch_llm(tmp_path): + conversation = _conversation(tmp_path, _llm()) + fired: list[str] = [] + conversation.llm_registry.set_stats_update_callback(lambda: fired.append("stats")) + _run(conversation) + + conversation.switch_llm(_llm("gpt-4o-mini")) + _run(conversation, "after the switch") + + assert fired == ["stats", "stats"] + assert _prompt_tokens(conversation) == 2 * PROMPT_TOKENS + + +def test_condenser_usage_is_counted_after_switch_llm(tmp_path): + conversation = _conversation(tmp_path, _llm()) + _run(conversation) + assert _prompt_tokens(conversation, "condenser") == 0 + + conversation.switch_llm(_llm("gpt-4o-mini")) + condenser = conversation.agent.condenser + assert isinstance(condenser, LLMSummarizingCondenser) + assert condenser.llm.model == "gpt-4o-mini" + condenser.llm.completion( + [Message(role="user", content=[TextContent(text="summarise")])], + call_context=conversation.get_llm_call_context(), + ) + + assert _prompt_tokens(conversation, "condenser") == PROMPT_TOKENS + + +def test_ask_agent_usage_is_counted_after_every_switch(tmp_path): + conversation = _conversation(tmp_path, _llm()) + _run(conversation) + + counted = [] + for model in ("gpt-4o", "gpt-4o-mini", "gpt-4o"): + conversation.switch_llm(_llm(model)) + conversation.ask_agent("what are you doing?") + counted.append(_prompt_tokens(conversation, ASK_AGENT_LLM_USAGE_ID)) + + assert counted == [PROMPT_TOKENS, 2 * PROMPT_TOKENS, 3 * PROMPT_TOKENS] + + +def test_resumed_conversation_continues_from_its_persisted_usage(tmp_path): + persistence = tmp_path / "persist" + workspace = tmp_path / "ws" + first = LocalConversation( + agent=Agent(llm=_llm(), tools=[]), + workspace=workspace, + persistence_dir=persistence, + visualizer=None, + ) + _run(first) + conversation_id = first.id + first.close() + + llm = _llm() + resumed = LocalConversation( + agent=Agent(llm=llm, tools=[]), + workspace=workspace, + persistence_dir=persistence, + conversation_id=conversation_id, + visualizer=None, + ) + _run(resumed, "again") + + assert _prompt_tokens(resumed) == 2 * PROMPT_TOKENS + assert llm.metrics is resumed.conversation_stats.usage_to_metrics["agent"] diff --git a/tests/sdk/conversation/test_switch_model.py b/tests/sdk/conversation/test_switch_model.py index 8df3f26d8da..36b78bced69 100644 --- a/tests/sdk/conversation/test_switch_model.py +++ b/tests/sdk/conversation/test_switch_model.py @@ -536,7 +536,7 @@ def test_switch_llm_refreshes_llm_condenser_credentials( assert state_condenser_llm.model == condenser_llm.model assert state_condenser_llm.api_key == condenser_llm.api_key assert condenser_llm.metrics is not conv.agent.llm.metrics - assert condenser_llm._telemetry is not None + assert condenser_llm.telemetry.metrics is condenser_llm.metrics async def _fake_acompletion(**kwargs): return create_mock_litellm_response( diff --git a/tests/sdk/llm/test_llm.py b/tests/sdk/llm/test_llm.py index 6279575d2e8..b3888786963 100644 --- a/tests/sdk/llm/test_llm.py +++ b/tests/sdk/llm/test_llm.py @@ -1799,8 +1799,7 @@ def test_llm_reset_metrics(): llm.reset_metrics() # Verify new metrics are created - assert llm._metrics is not None - assert llm._telemetry is not None + assert llm.telemetry.metrics is llm.metrics assert llm.metrics is not original_metrics assert llm.telemetry is not original_telemetry assert llm.metrics.accumulated_cost == 0.0 @@ -1867,14 +1866,14 @@ def test_stale_metrics_not_updated(llm): assert llm.metrics.accumulated_cost == 2.75 -def test_restore_metrics_telemetry_none(): - """restore_metrics() must not crash when telemetry has not been initialized.""" +def test_restore_metrics_session_none(): + """restore_metrics() must not crash when the session has not been created.""" llm = LLM( model="openai/gpt-4o", api_key=SecretStr("test-key"), usage_id="test-service", ) - llm._telemetry = None + llm._session = None restored = Metrics(model_name="openai/gpt-4o") restored.add_cost(1.00) @@ -1926,13 +1925,12 @@ def test_telemetry_callback_preserved_across_revalidation(): ) fired: list[bool] = [] llm.telemetry.set_stats_update_callback(lambda: fired.append(True)) - telemetry_before = llm._telemetry + telemetry_before = llm.telemetry RegistryEvent(llm=llm) - assert llm._telemetry is telemetry_before - assert llm.telemetry._stats_update_callback is not None - llm.telemetry._stats_update_callback() + assert llm.telemetry is telemetry_before + llm.telemetry.notify_stats_update() assert fired == [True] diff --git a/tests/sdk/llm/test_llm_fallback_sessions.py b/tests/sdk/llm/test_llm_fallback_sessions.py new file mode 100644 index 00000000000..b2b713571f9 --- /dev/null +++ b/tests/sdk/llm/test_llm_fallback_sessions.py @@ -0,0 +1,277 @@ +"""Fallback requests keep the caller's role and accounting session.""" + +import json +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Literal +from unittest.mock import AsyncMock + +import pytest +from litellm.exceptions import APIConnectionError +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + FallbackStrategy, + LLMCallContext, + LLMRegistry, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope +from openhands.sdk.llm.llm_profile_store import LLMProfileStore + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,role,context_mode,collision,shared", + [ + ("completion", "default", "explicit", False, False), + ("acompletion", "default", "scoped", False, True), + ("responses", "condenser", "explicit", True, False), + ("aresponses", "condenser", "scoped", True, False), + ("completion", "condenser", "explicit", False, False), + ("acompletion", "condenser", "scoped", False, False), + ("completion", "condenser", "standalone", False, False), + ("aresponses", "default", "standalone", False, False), + ], +) +async def test_fallback_preserves_caller_accounting( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + method: Literal["completion", "acompletion", "responses", "aresponses"], + role: str, + context_mode: Literal["explicit", "scoped", "standalone"], + collision: bool, + shared: bool, +): + fallback = LLM( + model="gpt-4o-mini", + api_key=SecretStr("test-key"), + num_retries=0, + input_cost_per_token=0.01, + output_cost_per_token=0.02, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + ) + LLMProfileStore(base_dir=tmp_path / "profiles").save( + "backup", fallback, include_secrets=True + ) + primary = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id=role, + num_retries=0, + input_cost_per_token=0.1, + output_cost_per_token=0.2, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + fallback_strategy=FallbackStrategy( + fallback_llms=["backup"], profile_store_dir=tmp_path / "profiles" + ), + ) + registry = LLMRegistry() + if context_mode != "standalone": + registry.add(primary) + if collision: + registry.add(LLM(model="gpt-4o-mini")) + other_registry = LLMRegistry() + if shared: + other_registry.add(primary) + metrics = ( + primary.metrics + if context_mode == "standalone" + else registry.sessions[role].metrics + ) + notifications: list[tuple[int, float]] = [] + logs: list[tuple[str, str, dict[str, Any]]] = [] + + def on_stats() -> None: + usage = metrics.accumulated_token_usage + assert usage is not None + notifications.append((usage.prompt_tokens, metrics.accumulated_cost)) + + def on_log(filename: str, data: str, usage_id: str, model: str) -> None: + logs.append((usage_id, model, json.loads(data))) + + if context_mode == "standalone": + primary.telemetry.set_stats_update_callback(on_stats) + primary.telemetry.set_log_completions_callback( + lambda filename, data: on_log( + filename, data, role, primary.telemetry.model_name + ) + ) + else: + registry.set_stats_update_callback(on_stats) + registry.set_log_completions_callback(on_log) + + context = LLMCallContext( + prompt_cache_key="caller-cache", + session_id="caller-conversation", + sessions=registry.sessions if context_mode != "standalone" else None, + ) + requests: list[dict[str, Any]] = [] + primary_attempts = 0 + + def respond(**kwargs: Any) -> ModelResponse | ResponsesAPIResponse: + nonlocal primary_attempts + requests.append(kwargs) + model = kwargs["model"] + if model == primary.model: + primary_attempts += 1 + if primary_attempts <= 2: + raise APIConnectionError( + message="primary unavailable", llm_provider="openai", model=model + ) + if method in ("responses", "aresponses"): + return ResponsesAPIResponse( + id=f"response-{len(requests)}", + created_at=1, + model=model, + object="response", + output=[ + { + "type": "message", + "id": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "ok", "annotations": []} + ], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage( + input_tokens=10, output_tokens=5, total_tokens=15 + ), + ) + return ModelResponse( + id=f"response-{len(requests)}", + model=model, + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_completion", respond) + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_responses", respond) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_acompletion", AsyncMock(side_effect=respond) + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_aresponses", AsyncMock(side_effect=respond) + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + supplied_context = context if context_mode == "explicit" else None + scope = ( + llm_call_context_scope(context) if context_mode == "scoped" else nullcontext() + ) + with scope: + for count, cost in enumerate((0.2, 0.4, 2.4), start=1): + if method == "completion": + result = primary.completion(messages, call_context=supplied_context) + elif method == "acompletion": + result = await primary.acompletion( + messages, call_context=supplied_context + ) + elif method == "responses": + result = primary.responses(messages, call_context=supplied_context) + else: + result = await primary.aresponses( + messages, call_context=supplied_context + ) + assert result.metrics.accumulated_token_usage is not None + assert result.metrics.accumulated_token_usage.prompt_tokens == 10 * count + assert result.metrics.accumulated_cost == pytest.approx(cost) + assert metrics.accumulated_cost == pytest.approx(cost) + assert primary.metrics.accumulated_cost == pytest.approx(cost) + assert [tokens for tokens, _ in notifications] == [ + 10 * n for n in range(1, count + 1) + ] + assert [total for _, total in notifications] == pytest.approx( + (0.2, 0.4, 2.4)[:count] + ) + + assert [record.model for record in metrics.token_usages] == [ + "gpt-4o-mini", + "gpt-4o-mini", + "gpt-4o", + ] + assert [(uid, model) for uid, model, data in logs if "response" in data] == [ + (role, "gpt-4o-mini"), + (role, "gpt-4o-mini"), + (role, "gpt-4o"), + ] + if context_mode != "standalone": + for request in requests: + assert request["prompt_cache_key"] == "caller-cache" + assert ( + request["extra_headers"]["x-litellm-session-id"] + == "caller-conversation" + ) + if collision: + assert registry.sessions["default"].metrics.accumulated_cost == 0 + if shared: + assert other_registry.sessions[role].metrics.accumulated_cost == 0 + + +def test_direct_fallback_preserves_supplied_metrics( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + primary = LLM(model="gpt-4o", usage_id="condenser") + registry, other_registry = LLMRegistry(), LLMRegistry() + registry.add(primary) + other_registry.add(primary) + metrics = registry.sessions["condenser"].metrics + LLMProfileStore(base_dir=tmp_path).save( + "backup", LLM(model="gpt-4o-mini", api_key=SecretStr("test-key")) + ) + strategy = FallbackStrategy(fallback_llms=["backup"], profile_store_dir=tmp_path) + response = ModelResponse( + id="fallback-response", + model="gpt-4o-mini", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_completion", lambda **kwargs: response + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + + result = strategy.try_fallback( + primary.model, + APIConnectionError( + message="primary unavailable", llm_provider="openai", model=primary.model + ), + metrics, + lambda fallback: fallback.completion(messages), + ) + + assert result is not None + assert metrics.accumulated_token_usage is not None + assert primary.metrics.accumulated_token_usage is not None + assert metrics.accumulated_token_usage.prompt_tokens == 10 + assert primary.metrics.accumulated_token_usage.prompt_tokens == 10 + assert len(metrics.token_usages) == 1 + assert other_registry.sessions["condenser"].metrics.token_usages == [] diff --git a/tests/sdk/llm/test_llm_metrics.py b/tests/sdk/llm/test_llm_metrics.py index 95bb6192d36..ecbb4936307 100644 --- a/tests/sdk/llm/test_llm_metrics.py +++ b/tests/sdk/llm/test_llm_metrics.py @@ -642,3 +642,53 @@ def test_cache_hit_rate_is_not_serialized(): snapshot = Metrics().get_snapshot() assert "cache_hit_rate" not in snapshot.model_dump() assert "cache_hit_rate" not in MetricsSnapshot.model_json_schema()["properties"] + + +def test_metrics_roll_up_into_records_in_both_ledgers(): + """What a ledger records after it is attached is also added to its target.""" + total = Metrics(model_name="gpt-4o") + total.add_cost(1.0) + ledger = Metrics(model_name="gpt-4o") + ledger.add_cost(0.5) # recorded before attaching: stays local + + ledger.roll_up_into(total) + ledger.add_cost(2.0) + ledger.add_token_usage(100, 50, 10, 5, 4096, "resp-1") + ledger.add_response_latency(1.5, "resp-1") + other = Metrics(model_name="gpt-4o") + other.add_cost(4.0) + ledger.merge(other) + + assert ledger.rolls_up_into is total + assert ledger.accumulated_cost == 6.5 + assert total.accumulated_cost == 7.0 + assert total.accumulated_token_usage is not None + assert total.accumulated_token_usage.prompt_tokens == 100 + assert [latency.response_id for latency in total.response_latencies] == ["resp-1"] + + # One direction only, and it can be detached again. + total.add_cost(1.0) + assert ledger.accumulated_cost == 6.5 + ledger.roll_up_into(None) + ledger.add_cost(1.0) + assert total.accumulated_cost == 8.0 + + +def test_metrics_roll_up_target_is_wiring_not_data(): + """The target is not serialized, compared or carried over to copies.""" + total = Metrics(model_name="gpt-4o") + ledger = Metrics(model_name="gpt-4o") + ledger.roll_up_into(total) + ledger.add_cost(1.0) + + same_data = Metrics.model_validate(ledger.model_dump()) + assert same_data.rolls_up_into is None + assert ledger == same_data + + for copied in (ledger.deep_copy(), ledger.model_copy()): + assert copied.rolls_up_into is None + copied.add_cost(5.0) + assert total.accumulated_cost == 1.0 + + with pytest.raises(ValueError, match="roll up into itself"): + ledger.roll_up_into(ledger) diff --git a/tests/sdk/llm/test_llm_registry.py b/tests/sdk/llm/test_llm_registry.py index 39fc1388cc6..b359f94c307 100644 --- a/tests/sdk/llm/test_llm_registry.py +++ b/tests/sdk/llm/test_llm_registry.py @@ -1,12 +1,16 @@ from __future__ import annotations import unittest -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, patch from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_registry import LLMRegistry, RegistryEvent +def _llm(usage_id: str) -> LLM: + return LLM(model="gpt-4o", usage_id=usage_id) + + class TestLLMRegistry(unittest.TestCase): def setUp(self): """Set up test environment before each test.""" @@ -23,19 +27,17 @@ def callback(event: RegistryEvent): # Subscribe to events self.registry.subscribe(callback) - # Create a mock LLM and add it to trigger notification - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "notify-service" + # Add an LLM to trigger notification + llm = _llm("notify-service") + self.registry.add(llm) - # Mock the RegistryEvent to avoid LLM attribute access - with patch( - "openhands.sdk.llm.llm_registry.RegistryEvent" - ) as mock_registry_event: - mock_registry_event.return_value = Mock() - self.registry.add(mock_llm) - - # Should receive notification for the newly added LLM + # Should receive notification for the newly added LLM, with the + # session the registry keeps for its usage ID self.assertEqual(len(events_received), 1) + self.assertIs(events_received[0].llm, llm) + self.assertIs( + events_received[0].session, self.registry.sessions["notify-service"] + ) # Test that the subscriber is set correctly self.assertIsNotNone(self.registry.subscriber) @@ -67,7 +69,7 @@ def failing_subscriber(event): # Mock the logger to capture warning messages with patch("openhands.sdk.llm.llm_registry.logger") as mock_logger: # Create a mock event - mock_event = Mock() + mock_event = MagicMock() # This should handle the exception and log a warning (lines 146-147) registry.notify(mock_event) @@ -82,26 +84,16 @@ def test_llm_registry_list_usage_ids(): registry = LLMRegistry() - # Create mock LLM objects - mock_llm1 = Mock(spec=LLM) - mock_llm1.usage_id = "service1" - mock_llm2 = Mock(spec=LLM) - mock_llm2.usage_id = "service2" - - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() + # Add some LLMs using the new API + registry.add(_llm("service1")) + registry.add(_llm("service2")) - # Add some LLMs using the new API - registry.add(mock_llm1) - registry.add(mock_llm2) + # Test list_usage_ids + usage_ids = registry.list_usage_ids() - # Test list_usage_ids - usage_ids = registry.list_usage_ids() - - assert "service1" in usage_ids - assert "service2" in usage_ids - assert len(usage_ids) == 2 + assert "service1" in usage_ids + assert "service2" in usage_ids + assert len(usage_ids) == 2 def test_llm_registry_remove_method(): @@ -109,12 +101,7 @@ def test_llm_registry_remove_method(): registry = LLMRegistry() - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "service1" - - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - registry.add(mock_llm) + registry.add(_llm("service1")) assert "service1" in registry.list_usage_ids() @@ -144,28 +131,19 @@ def test_llm_registry_add_method(): """Test the new add() method for LLMRegistry.""" registry = LLMRegistry() - # Create a mock LLM - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "test-service" - service_id = mock_llm.usage_id - - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() + llm = _llm("test-service") + service_id = llm.usage_id - # Test adding an LLM - registry.add(mock_llm) - - # Verify the LLM was added - assert service_id in registry.usage_to_llm - assert registry.usage_to_llm[service_id] is mock_llm + # Test adding an LLM + registry.add(llm) - # Verify RegistryEvent was called - mock_registry_event.assert_called_once_with(llm=mock_llm) + # Verify the LLM was added + assert service_id in registry.usage_to_llm + assert registry.usage_to_llm[service_id] is llm # Test that adding the same usage_id raises ValueError with unittest.TestCase().assertRaises(ValueError) as context: - registry.add(mock_llm) + registry.add(llm) assert "already exists in registry" in str(context.exception) @@ -174,21 +152,15 @@ def test_llm_registry_get_method(): """Test the new get() method for LLMRegistry.""" registry = LLMRegistry() - # Create a mock LLM - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "test-service" - service_id = mock_llm.usage_id + llm = _llm("test-service") + service_id = llm.usage_id - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - - # Add the LLM first - registry.add(mock_llm) + # Add the LLM first + registry.add(llm) - # Test getting the LLM - retrieved_llm = registry.get(service_id) - assert retrieved_llm is mock_llm + # Test getting the LLM + retrieved_llm = registry.get(service_id) + assert retrieved_llm is llm # Test getting non-existent service raises KeyError with unittest.TestCase().assertRaises(KeyError) as context: @@ -201,33 +173,26 @@ def test_llm_registry_add_get_workflow(): """Test the complete add/get workflow.""" registry = LLMRegistry() - # Create mock LLMs - llm1 = Mock(spec=LLM) - llm1.usage_id = "service1" - llm2 = Mock(spec=LLM) - llm2.usage_id = "service2" + llm1 = _llm("service1") + llm2 = _llm("service2") - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - - # Add multiple LLMs - registry.add(llm1) - registry.add(llm2) + # Add multiple LLMs + registry.add(llm1) + registry.add(llm2) - # Verify we can retrieve them - assert registry.get("service1") is llm1 - assert registry.get("service2") is llm2 + # Verify we can retrieve them + assert registry.get("service1") is llm1 + assert registry.get("service2") is llm2 - # Verify list_usage_ids works - usage_ids = registry.list_usage_ids() - assert "service1" in usage_ids - assert "service2" in usage_ids - assert len(usage_ids) == 2 + # Verify list_usage_ids works + usage_ids = registry.list_usage_ids() + assert "service1" in usage_ids + assert "service2" in usage_ids + assert len(usage_ids) == 2 - # Verify usage_id is set correctly - assert llm1.usage_id == "service1" - assert llm2.usage_id == "service2" + # Verify usage_id is set correctly + assert llm1.usage_id == "service1" + assert llm2.usage_id == "service2" def test_llm_registry_ensures_independent_metrics_for_copied_llms(): @@ -251,21 +216,18 @@ def test_llm_registry_ensures_independent_metrics_for_copied_llms(): usage_id="original-llm", ) - # Copy with updated usage_id (simulates creating a condenser LLM) - # Note: model_copy() does a shallow copy of private attributes by default, - # so the copied LLM shares the same metrics object as the original + # Copy with updated usage_id (simulates creating a condenser LLM). A copy + # made for another usage ID starts its own session, so it never shares the + # original's metrics, registered or not. copied = original.model_copy(update={"usage_id": "copied-llm"}) + assert original.metrics is not copied.metrics - # Before registering, they share the same metrics (this is the bug we're fixing) - assert original.metrics is copied.metrics - - # Register both LLMs - the registry should detect and fix shared metrics registry.add(original) registry.add(copied) - # After registering, they should have different metrics objects + # They still have different metrics objects after registering assert original.metrics is not copied.metrics - assert id(original.metrics) != id(copied.metrics) + assert registry.sessions["original-llm"] is not registry.sessions["copied-llm"] # Verify metrics are independent - changes to one don't affect the other original.metrics.add_cost(1.0) @@ -294,19 +256,37 @@ def test_llm_registry_ensures_independent_telemetry_for_copied_llms(): usage_id="original-llm", ) - # Copy with updated usage_id + # Copy with updated usage_id: its own session means its own telemetry copied = original.model_copy(update={"usage_id": "copied-llm"}) - - # Before registering, they share the same telemetry - assert original.telemetry is copied.telemetry + assert original.telemetry is not copied.telemetry + assert copied.telemetry.metrics is copied.metrics # Register both LLMs registry.add(original) registry.add(copied) - # After registering, they should have different telemetry objects + # They still have different telemetry objects after registering assert original.telemetry is not copied.telemetry - assert id(original.telemetry) != id(copied.telemetry) + + +def test_llm_registry_separates_llms_that_share_a_session(): + """Two usage IDs never get one session, however the LLMs came to share it. + + A plain copy keeps the session of its source. Renaming it afterwards is + the one way left to bring a shared session to the registry under two IDs. + """ + registry = LLMRegistry() + original = _llm("original-llm") + renamed = original.model_copy() + renamed.usage_id = "renamed-llm" + assert renamed.metrics is original.metrics + + registry.add(original) + registry.add(renamed) + + assert registry.sessions["original-llm"] is not registry.sessions["renamed-llm"] + original.metrics.add_cost(1.0) + assert renamed.metrics.accumulated_cost == 0.0 def test_llm_registry_does_not_reset_metrics_for_independent_llms(): diff --git a/tests/sdk/llm/test_llm_serialization.py b/tests/sdk/llm/test_llm_serialization.py index a5fd013a1e0..b5ebab4846c 100644 --- a/tests/sdk/llm/test_llm_serialization.py +++ b/tests/sdk/llm/test_llm_serialization.py @@ -115,7 +115,7 @@ def test_llm_private_attributes_not_serialized() -> None: # Private attributes should not be present assert "_model_info" not in llm_dict assert "_tokenizer" not in llm_dict - assert "_telemetry" not in llm_dict + assert "_session" not in llm_dict # Serialize to JSON and deserialize llm_json = llm.model_dump_json() @@ -127,8 +127,8 @@ def test_llm_private_attributes_not_serialized() -> None: assert deserialized_llm._tokenizer is None assert deserialized_llm.native_tool_calling is True assert ( - deserialized_llm._telemetry is not None - ) # LLM creates telemetry automatically + deserialized_llm._session is not None + ) # LLM creates its session (metrics + telemetry) automatically assert deserialized_llm.model_dump() == llm.model_dump() diff --git a/tests/sdk/llm/test_llm_session.py b/tests/sdk/llm/test_llm_session.py new file mode 100644 index 00000000000..0a506dccd6a --- /dev/null +++ b/tests/sdk/llm/test_llm_session.py @@ -0,0 +1,284 @@ +"""Tests for LLMSession: where an LLM's usage metrics and telemetry live.""" + +from collections.abc import Iterator +from unittest.mock import patch + +import pytest +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + LLMCallContext, + LLMRegistry, + LLMSession, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope + + +PROMPT_TOKENS = 100 + + +def _llm(usage_id: str = "agent", model: str = "gpt-4o", **kwargs) -> LLM: + return LLM(model=model, api_key=SecretStr("k"), usage_id=usage_id, **kwargs) + + +def _context(registry: LLMRegistry) -> LLMCallContext: + return LLMCallContext().for_conversation("conv", sessions=registry.sessions) + + +def _call(llm: LLM, call_context: LLMCallContext | None = None) -> None: + llm.completion( + [Message(role="user", content=[TextContent(text="hi")])], + call_context=call_context, + ) + + +def _prompt_tokens(session: LLMSession) -> int: + usage = session.metrics.accumulated_token_usage + assert usage is not None + return usage.prompt_tokens + + +@pytest.fixture(autouse=True) +def fake_completion() -> Iterator[None]: + def respond(*args, **kwargs) -> ModelResponse: + return ModelResponse( + id="resp", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + model="gpt-4o", + usage=Usage( + prompt_tokens=PROMPT_TOKENS, completion_tokens=1, total_tokens=101 + ), + ) + + with patch("openhands.sdk.llm.llm.litellm_completion", side_effect=respond): + yield + + +# ── The LLM's own session ────────────────────────────────────────────── + + +def test_llm_metrics_and_telemetry_are_its_session(): + llm = _llm() + + assert llm.metrics is llm.session.metrics + assert llm.telemetry is llm.session.telemetry + assert llm.session_for() is llm.session + + _call(llm) + assert _prompt_tokens(llm.session) == PROMPT_TOKENS + + +def test_copy_keeps_the_session_unless_the_usage_id_changes(): + llm = _llm() + + assert llm.model_copy(update={"stream": False}).session is llm.session + + other_role = llm.model_copy(update={"usage_id": "condenser"}) + assert other_role.session is not llm.session + _call(other_role) + assert _prompt_tokens(other_role.session) == PROMPT_TOKENS + assert _prompt_tokens(llm.session) == 0 + + +def test_equal_llms_stay_equal(): + """Sessions compare by value, so LLM equality is unchanged.""" + assert _llm() == _llm() + used = _llm() + used.metrics.add_cost(1.0) + assert used != _llm() + + +# ── Sessions of a registry ───────────────────────────────────────────── + + +def test_registry_takes_over_the_session_of_the_first_llm_of_a_role(): + registry = LLMRegistry() + llm = _llm() + llm.metrics.add_cost(1.0) # usage from before the conversation is kept + + registry.add(llm) + + assert registry.sessions["agent"] is llm.session + assert llm.metrics.accumulated_cost == 1.0 + _call(llm, _context(registry)) + assert _prompt_tokens(registry.sessions["agent"]) == PROMPT_TOKENS + + +def test_session_stays_with_the_role_when_the_llm_is_replaced(): + registry = LLMRegistry() + first = _llm() + registry.add(first) + session = registry.sessions["agent"] + _call(first, _context(registry)) + + registry.remove("agent") + assert registry.sessions["agent"] is session # kept while the slot is empty + second = _llm(model="gpt-4o-mini") + registry.add(second) + + assert registry.sessions["agent"] is session + assert second.metrics is session.metrics + assert session.telemetry.model_name == "gpt-4o-mini" + _call(second, _context(registry)) + assert _prompt_tokens(session) == 2 * PROMPT_TOKENS + + +def test_call_records_in_the_session_of_its_role_whichever_object_calls(): + """A copy that never went through ``add()`` still counts for its role.""" + registry = LLMRegistry() + registry.add(_llm(usage_id="condenser")) + unregistered = _llm(model="gpt-4o-mini").model_copy( + update={"usage_id": "condenser"} + ) + + _call(unregistered, _context(registry)) + + session = registry.sessions["condenser"] + assert _prompt_tokens(session) == PROMPT_TOKENS + assert session.telemetry.model_name == "gpt-4o-mini" + # ...and on the object that made the call. + assert _prompt_tokens(unregistered.session) == PROMPT_TOKENS + + +def test_call_without_a_registered_role_records_on_the_llm_itself(): + registry = LLMRegistry() + registry.add(_llm()) + helper = _llm(usage_id="helper") + + _call(helper, _context(registry)) + + assert "helper" not in registry.sessions + assert _prompt_tokens(helper.session) == PROMPT_TOKENS + assert _prompt_tokens(registry.sessions["agent"]) == 0 + + +def test_registered_llm_records_in_its_conversation_without_a_context(): + registry = LLMRegistry() + llm = _llm() + registry.add(llm) + + _call(llm) + + assert _prompt_tokens(registry.sessions["agent"]) == PROMPT_TOKENS + + +def test_scoped_context_selects_the_session_like_an_explicit_one(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + first.add(llm) + second.add(llm) + + with llm_call_context_scope(_context(first)): + _call(llm) + + assert _prompt_tokens(first.sessions["agent"]) == PROMPT_TOKENS + assert _prompt_tokens(second.sessions["agent"]) == 0 + + +def test_one_llm_in_two_registries_gets_a_session_in_each(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + first.add(llm) + _call(llm, _context(first)) + + second.add(llm) + _call(llm, _context(second)) + _call(llm, _context(second)) + _call(llm, _context(first)) + + assert first.sessions["agent"] is not second.sessions["agent"] + assert _prompt_tokens(first.sessions["agent"]) == 2 * PROMPT_TOKENS + assert _prompt_tokens(second.sessions["agent"]) == 2 * PROMPT_TOKENS + # The object itself keeps counting everything it did. + assert _prompt_tokens(llm.session) == 4 * PROMPT_TOKENS + + +def test_child_conversation_context_does_not_inherit_sessions(): + registry = LLMRegistry() + parent = _context(registry) + + assert parent.for_conversation("child").sessions is None + assert parent == LLMCallContext(prompt_cache_key="conv", session_id="conv") + + +# ── Registry-wide callbacks ──────────────────────────────────────────── + + +def test_registry_stats_callback_covers_present_and_future_sessions(): + registry = LLMRegistry() + agent_llm = _llm() + registry.add(agent_llm) + fired: list[str] = [] + registry.set_stats_update_callback(lambda: fired.append("stats")) + + _call(agent_llm, _context(registry)) + late = _llm(usage_id="late") + registry.add(late) + _call(late, _context(registry)) + registry.remove("agent") + replacement = _llm(model="gpt-4o-mini") + registry.add(replacement) + _call(replacement, _context(registry)) + + assert fired == ["stats"] * 3 + + +def test_registry_callbacks_do_not_cross_registries(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + fired: list[str] = [] + first.set_stats_update_callback(lambda: fired.append("first")) + second.set_stats_update_callback(lambda: fired.append("second")) + first.add(llm) + second.add(llm) + + _call(llm, _context(first)) + _call(llm, _context(second)) + + assert fired == ["first", "second"] + + +def test_registry_log_callback_names_the_role_and_the_model_called(tmp_path): + registry = LLMRegistry() + logs: list[tuple[str, str]] = [] + registry.set_log_completions_callback( + lambda filename, data, usage_id, model: logs.append((usage_id, model)) + ) + quiet = _llm() + registry.add(quiet) + _call(quiet, _context(registry)) + assert logs == [] # log_completions is off for this LLM + + registry.remove("agent") + logged = _llm( + model="gpt-4o-mini", + log_completions=True, + log_completions_folder=str(tmp_path), + ) + registry.add(logged) + _call(logged, _context(registry)) + + assert logs == [("agent", "gpt-4o-mini")] + assert list(tmp_path.iterdir()) == [] # sent to the callback, not to disk + + +def test_callback_set_on_the_llm_still_fires_in_a_conversation(): + registry = LLMRegistry() + llm = _llm() + fired: list[str] = [] + llm.telemetry.set_stats_update_callback(lambda: fired.append("llm")) + + registry.add(llm) + _call(llm, _context(registry)) + + assert fired == ["llm"] diff --git a/tests/sdk/llm/test_responses_parsing_and_kwargs.py b/tests/sdk/llm/test_responses_parsing_and_kwargs.py index 35692e08c27..bd209887814 100644 --- a/tests/sdk/llm/test_responses_parsing_and_kwargs.py +++ b/tests/sdk/llm/test_responses_parsing_and_kwargs.py @@ -246,7 +246,7 @@ def test_llm_responses_end_to_end(mock_responses_call): "ok" ] # Telemetry should have recorded usage (one entry) - assert len(llm._telemetry.metrics.token_usages) == 1 # type: ignore[attr-defined] + assert len(llm.telemetry.metrics.token_usages) == 1 @pytest.mark.parametrize(