From 7ae829318d1dbb516327a2ace449c146d70d9acd Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Wed, 18 Nov 2026 00:51:58 +0100 Subject: [PATCH 1/5] refactor(judge): call the System One judgement Decision Every page, message, document, setting and name says Decision: the DECISION_PROVIDER, DECISION_PROTOCOL and DECISION_MODEL settings, the decision lane, /settings/decision/models and docs/settings/decision.md. Settings saved as SYSTEM_ONE_* are still read, and saving the new name from the page takes the old line out of the file. Judgements stay in the same folder. TypeSafe's protocol keeps its id and path, systemone. Without a provider, the judge says it needs a model, not the TypeSafe key: OpenRouter and OpenAI serve decision models too. --- README.md | 2 +- app/.env.example | 18 ++--- app/nullius/judge/__init__.py | 2 +- app/nullius/judge/batch.py | 2 +- app/nullius/judge/client.py | 80 ++++++++++---------- app/nullius/judge/store.py | 12 +-- app/nullius/onboarding.py | 4 +- app/nullius/providers.py | 4 +- app/nullius/settings.py | 39 ++++++---- app/site/mkdocs.yml | 2 +- app/static/settings/autosave.js | 2 +- app/static/settings/judges.js | 4 +- app/templates/compare/compare.html | 2 +- app/templates/runs/feasibility.html | 2 +- app/templates/runs/job.html | 2 +- app/templates/settings/settings.html | 18 ++--- app/templates/traces/_judgement.html | 4 +- app/templates/traces/trial.html | 2 +- app/tests/test_judge.py | 4 +- app/tests/test_judge_decisions.py | 65 +++++++++------- app/tests/test_judge_openrouter.py | 22 +++--- app/tests/test_provider_registry.py | 4 +- app/tests/test_settings.py | 10 ++- app/ui_tests/test_every_route.py | 4 +- app/ui_tests/test_judge_cards.py | 6 +- app/ui_tests/test_routes.py | 2 +- app/ui_tests/test_settings_page.py | 14 ++-- app/web/judge.py | 18 ++--- app/web/settings.py | 14 ++-- app/web/traces.py | 2 +- docs/judgement.md | 6 +- docs/settings/README.md | 4 +- docs/settings/{system-one.md => decision.md} | 27 +++---- docs/settings/llm-judge.md | 6 +- docs/settings/model-providers.md | 8 +- 35 files changed, 227 insertions(+), 190 deletions(-) rename docs/settings/{system-one.md => decision.md} (63%) diff --git a/README.md b/README.md index 0c587b99..77440b41 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ site](app/site/README.md). | [Judgement](docs/judgement.md) | What a judge model is asked | | [The Nullius standard](docs/standard.md) | Ports, kits, setups, controls, proofs; importing | | [Architecture](docs/how-it-works.md) | Execution paths, a run on disk, the pages | -| [Settings](docs/settings/README.md) | Providers, System One, LLM judge, hosts, keys | +| [Settings](docs/settings/README.md) | Providers, Decision, LLM judge, hosts, keys | | [Benchmarks](docs/benchmarks/supported/README.md), [Agents](docs/agents/supported/README.md), [Capabilities](docs/capabilities/supported/README.md) | What is supported; adding one | | [CLI](docs/cli/README.md) | Every command, by topic | | [Metric field guide](docs/field-guide.md) | Every metric and outcome | diff --git a/app/.env.example b/app/.env.example index 4fb3b953..feda3fbc 100644 --- a/app/.env.example +++ b/app/.env.example @@ -33,20 +33,20 @@ OPENAI_API_KEY= OPENROUTER_API_KEY= # An Anthropic key, for a run of the Nullius standard whose agent speaks Anthropic's Messages, and for the LLM judge: the calls go to Anthropic's own base with this key. Only the relay and the judge hold it, never an agent, a recipe or a log. ANTHROPIC_API_KEY= -# A TypeSafe key, for the System One judgement on TypeSafe's own base: sent as a bearer header there and nowhere else. +# A TypeSafe key, for the Decision judgement on TypeSafe's own base: sent as a bearer header there and nowhere else. # e.g. ts-… TYPESAFE_API_KEY= -# The provider the System One judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is. -SYSTEM_ONE_PROVIDER= -# The decision protocol System One is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves. -SYSTEM_ONE_PROTOCOL= -# The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default. -SYSTEM_ONE_MODEL= -# The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last. +# The provider the Decision judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is. +DECISION_PROVIDER= +# The decision protocol Decision is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves. +DECISION_PROTOCOL= +# The Decision model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default. +DECISION_MODEL= +# The provider of the LLM judgement, the second judgement beside Decision: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last. LLM_JUDGE_PROVIDER= # The protocol the LLM judge is asked in, on a provider that serves both: chat (OpenAI Chat Completions) or messages (Anthropic Messages). Unset, or one the provider does not serve: the first the provider serves, chat before messages. LLM_JUDGE_PROTOCOL= -# The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence. +# The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as Decision and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence. LLM_JUDGE_MODEL= # Additions to the OpenRouter provider, as a JSON object without the key: endpoints of protocols the preset does not list, {"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one. OPENROUTER= diff --git a/app/nullius/judge/__init__.py b/app/nullius/judge/__init__.py index 8299ed5f..0ec9c22d 100644 --- a/app/nullius/judge/__init__.py +++ b/app/nullius/judge/__init__.py @@ -1,5 +1,5 @@ """Judgement: structured, confidence-scored answers about a recorded trial -from a judge model (TypeSafe.ai System One), shown beside the deterministic +from a judge model (TypeSafe.ai Decision), shown beside the deterministic evidence and never in its place. The contract: the question diff --git a/app/nullius/judge/batch.py b/app/nullius/judge/batch.py index 4b207638..6410e3be 100644 --- a/app/nullius/judge/batch.py +++ b/app/nullius/judge/batch.py @@ -116,7 +116,7 @@ def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, rec = ask_states(states, key, post=post, calibration=calibration) if rec is not None: # a call over some of the kinds keeps the earlier readings for the others, as the answers are kept - lane = getattr(key, "lane", "system-one") # the LLM judge's records apart from System One's + lane = getattr(key, "lane", "decision") # the LLM judge's records apart from Decision's earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane, started=started) or {} rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})} store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started) diff --git a/app/nullius/judge/client.py b/app/nullius/judge/client.py index 7208038e..496b4d6e 100644 --- a/app/nullius/judge/client.py +++ b/app/nullius/judge/client.py @@ -3,7 +3,7 @@ one Settings chose); each asks a provider of the registry (``nullius.providers``) that serves its protocol: -- **System One**, a decision model on a decision protocol: ``systemone`` +- **Decision**, a decision model on a decision protocol: ``systemone`` (TypeSafe's Jev, on TypeSafe or OpenRouter), ``decisions`` (OpenRouter's, for many makers' models) or ``openai-decisions``. Its answers are the model's own: a Noul's probability, a Choice's distribution over its @@ -43,37 +43,42 @@ CALIBRATION_MIN_CHARS = 4_000 # a stored call this small says nothing about a large request's ratio: it is kept out of the band -LANES = ("system-one", "llm") # the two judgements a trial can have, each with its own records +LANES = ("decision", "llm") # the two judgements a trial can have, each with its own records # The protocols each judge speaks, in its order of preference; the providers come from the registry. LLM_PROTOCOLS = ("chat", "messages") -SYSTEM_ONE_PROTOCOLS = ("systemone", "decisions", "openai-decisions") -# System One's models as each provider names them: the first is its default. Another is typed in Settings. -SYSTEM_ONE_MODELS = {"typesafe": [MODEL], +DECISION_PROTOCOLS = ("systemone", "decisions", "openai-decisions") +# Decision's models as each provider names them: the first is its default. Another is typed in Settings. +DECISION_MODELS = {"typesafe": [MODEL], "openrouter": ["~typesafe/jev-latest", "typesafe/jev-1.14", "typesafe/jev-1.13", "typesafe/jev-router", "jaredpalmer/kev-4b"], "openai": ["gpt-6-luna"]} -def system_one_models(choice: str, fetch: bool = False) -> list: - """System One's models on a provider, its default first: on OpenRouter, its decision models +def decision_judge_models(choice: str, fetch: bool = False) -> list: + """Decision's models on a provider, its default first: on OpenRouter, its decision models (``providers.decision_models``, cached; asked only when ``fetch``), TypeSafe's next; else, or with - none, ``SYSTEM_ONE_MODELS``.""" - listed = SYSTEM_ONE_MODELS.get(choice, []) + none, ``DECISION_MODELS``.""" + listed = DECISION_MODELS.get(choice, []) live = providers.decision_models(fetch=fetch) if choice == "openrouter" else [] if not live: return list(listed) return [listed[0]] + sorted(set(live) - {listed[0]}, key=lambda m: ("typesafe/" not in m, m)) -def system_one_protocol(choice: str, model: str, env=os.environ) -> str: - """The decision protocol System One speaks on a provider: ``SYSTEM_ONE_PROTOCOL`` when the provider +def _setting(env, suffix: str) -> str: + """``DECISION_``, else its name before the rename (``settings.RENAMED``), stripped.""" + return (env.get(f"DECISION_{suffix}") or env.get(f"SYSTEM_ONE_{suffix}") or "").strip() + + +def decision_judge_protocol(choice: str, model: str, env=os.environ) -> str: + """The decision protocol Decision speaks on a provider: ``DECISION_PROTOCOL`` when the provider lists it; else, on one that serves both ``systemone`` and ``decisions`` (OpenRouter), ``systemone`` for TypeSafe's models and the ones listed for it, and ``decisions`` for another maker's; else the first the provider lists.""" rec = providers.record(choice, env) - listed = [p for p in SYSTEM_ONE_PROTOCOLS if providers.serves(rec, p)] - if (env.get("SYSTEM_ONE_PROTOCOL") or "").strip() in listed: - return env["SYSTEM_ONE_PROTOCOL"].strip() - typesafe = "typesafe/" in model or model.startswith("jev") or model in SYSTEM_ONE_MODELS.get(choice, []) + listed = [p for p in DECISION_PROTOCOLS if providers.serves(rec, p)] + if _setting(env, "PROTOCOL") in listed: + return _setting(env, "PROTOCOL") + typesafe = "typesafe/" in model or model.startswith("jev") or model in DECISION_MODELS.get(choice, []) if "systemone" in listed and "decisions" in listed: return "systemone" if typesafe else "decisions" return listed[0] if listed else "systemone" @@ -107,11 +112,11 @@ class Judge: def __init__(self, provider: str, key: str, model: str, url: str = "", lane: str = "", name: str = "", protocol: str = ""): self.provider, self.key, self.model, self.url, self.name = provider, key, model, url, name - self.lane = lane or ("system-one" if provider == "typesafe" else "llm") - self.protocol = protocol or ("systemone" if self.lane == "system-one" else "chat") + self.lane = lane or ("decision" if provider == "typesafe" else "llm") + self.protocol = protocol or ("systemone" if self.lane == "decision" else "chat") def __bool__(self) -> bool: - if self.lane == "system-one": + if self.lane == "decision": return bool(self.key) and bool(self.model) return bool(self.model) and bool(self.url) and (bool(self.key) or self.provider.startswith("custom")) @@ -129,9 +134,8 @@ def label(self) -> str: @property def missing(self) -> str: """What Settings still needs for this judge, in a phrase; '' when it can be asked.""" - if self.lane == "system-one": - return " and ".join(([f"the {self.who} key"] if not self.key else []) - + (["a model"] if not self.model else [])) + if self.lane == "decision": + return "" if self.key and self.model else "a model" need = [] if not self.url: need.append("an endpoint (MODEL_API_URL)") @@ -142,31 +146,31 @@ def missing(self) -> str: return " and ".join(need) -def system_one_providers(env=os.environ) -> dict: - """System One's providers: every one that lists one of its protocols, each with its models - (``system_one_models``, never asked here; the first, ``model``, its default; ``live`` when the +def decision_judge_providers(env=os.environ) -> dict: + """Decision's providers: every one that lists one of its protocols, each with its models + (``decision_judge_models``, never asked here; the first, ``model``, its default; ``live`` when the provider has a list to ask) and the protocol it would be asked in.""" - out = {k: dict(d, models=system_one_models(k), model=(SYSTEM_ONE_MODELS.get(k) or [""])[0], live=k == "openrouter") - for k, d in _table(SYSTEM_ONE_PROTOCOLS, env, known=True).items()} + out = {k: dict(d, models=decision_judge_models(k), model=(DECISION_MODELS.get(k) or [""])[0], live=k == "openrouter") + for k, d in _table(DECISION_PROTOCOLS, env, known=True).items()} for k, d in out.items(): - d["protocol"] = system_one_protocol(k, (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"], env) + d["protocol"] = decision_judge_protocol(k, _setting(env, "MODEL") or d["model"], env) return out -def system_one(env=os.environ) -> Judge: - """The System One judgement: on ``SYSTEM_ONE_PROVIDER`` (unset, or one - that serves no System One protocol: TypeSafe when its key is set, else +def decision_judge(env=os.environ) -> Judge: + """The Decision judgement: on ``DECISION_PROVIDER`` (unset, or one + that serves no Decision protocol: TypeSafe when its key is set, else OpenRouter when its is), with that provider's key, and the model - ``SYSTEM_ONE_MODEL`` names or the provider's default.""" - table = system_one_providers(env) - chosen = _choice(env.get("SYSTEM_ONE_PROVIDER")) + ``DECISION_MODEL`` names or the provider's default.""" + table = decision_judge_providers(env) + chosen = _choice(_setting(env, "PROVIDER")) if chosen not in table: chosen = "openrouter" if not env.get("TYPESAFE_API_KEY") and env.get("OPENROUTER_API_KEY") else "typesafe" d = table[chosen] - model = (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"] - protocol = system_one_protocol(chosen, model, env) + model = _setting(env, "MODEL") or d["model"] + protocol = decision_judge_protocol(chosen, model, env) return Judge(chosen, env.get(d["key"], ""), model, providers.url(providers.record(chosen, env), protocol), - lane="system-one", name=d["label"], protocol=protocol) + lane="decision", name=d["label"], protocol=protocol) def llm_providers(env=os.environ) -> dict: @@ -203,7 +207,7 @@ def settings_placeholder() -> str: def judge_for(lane: str, env=os.environ) -> Judge: - return llm_judge(env) if lane == "llm" else system_one(env) + return llm_judge(env) if lane == "llm" else decision_judge(env) class JudgeError(Exception): @@ -426,7 +430,7 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla key = key.key if not key: raise JudgeError(401, "no TypeSafe.ai key: set TYPESAFE_API_KEY in Settings, Judge" if provider == PROVIDER - else f"no {who} key for System One: set it in Settings, Model providers") + else f"no {who} key for Decision: set it in Settings, Model providers") asked = questions if questions is not None else Q.request_questions(ids) openai = protocol == "openai-decisions" body = {"model": model, "input": state, "questions": openai_questions(asked)} if openai else {"state": state, "model": model, "questions": asked} diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index 93e012b5..fe35cb87 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -21,14 +21,14 @@ _SAFE = re.compile(r"[^A-Za-z0-9._@+-]") -ROOTS = {"system-one": "judgements", "llm": "judgements-llm"} # one directory per judgement +ROOTS = {"decision": "judgements", "llm": "judgements-llm"} # one directory per judgement def _root(ws_dir: str, lane: str) -> str: return os.path.join(ws_dir, ROOTS.get(lane, "judgements")) -def _path(ws_dir: str, job_id: str, trial_name: str, lane: str = "system-one") -> str: +def _path(ws_dir: str, job_id: str, trial_name: str, lane: str = "decision") -> str: return os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id), _SAFE.sub("_", trial_name) + ".json") @@ -52,7 +52,7 @@ def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> return None -def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one", +def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "decision", started: Optional[str] = None) -> Optional[dict]: """The record under `version`; with none named, the current version or the newest earlier one that has a record. `started`: the trial's own @@ -69,7 +69,7 @@ def is_current(record: Optional[dict]) -> bool: return bool(record) and record.get("questions") == Q.VERSION -def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one", +def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "decision", started: Optional[str] = None) -> None: """The record under its question version; other versions kept. A record's answers merge over the version's earlier answers, so judging @@ -94,7 +94,7 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "s os.replace(tmp, path) -def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one", +def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "decision", started: Optional[dict] = None) -> dict: """Every judged trial of a job: {trial_name: record}, each the current version's record or the newest earlier one, as `load` picks it. @@ -129,7 +129,7 @@ def count(ws_dir: str) -> int: return sum(n.endswith(".json") for lane in ROOTS for _, _, names in os.walk(_root(ws_dir, lane)) for n in names) -def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "system-one"): +def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "decision"): """Every stored record, every version: yields (job_id, trial_name, version, record). One job's when `job_id` is given.""" root = _root(ws_dir, lane) diff --git a/app/nullius/onboarding.py b/app/nullius/onboarding.py index 92510219..1b4047c1 100644 --- a/app/nullius/onboarding.py +++ b/app/nullius/onboarding.py @@ -30,7 +30,7 @@ {"id": "where", "text": "Choose where runs happen", "fact": "where", "target": [["#hosts", "Add a host, or run on this machine with Docker or Podman"]]}, {"id": "judge", "text": "Pick a judge", "fact": "judge", "optional": True, - "target": [["#judge", "Choose System One or an LLM judge"]]}, + "target": [["#judge", "Choose Decision or an LLM judge"]]}, ]}, {"id": "bench", "name": "Pick a benchmark", "page": "benchmarks_page", "where": "Benchmarks", "subs": [ {"id": "browse", "text": "Browse the catalogue", "visit": r"^/benchmarks/?$", @@ -86,7 +86,7 @@ def facts(env, jobs: Iterable[dict] = (), judged: bool = False, docker: Optional return { "provider": bool(providers.available(env)), # a run provider: a TypeSafe key serves the judge only "where": bool((env.get("HOSTS") or "").strip()) or found, - "judge": bool(client.system_one(env)) or bool(client.llm_judge(env)), + "judge": bool(client.decision_judge(env)) or bool(client.llm_judge(env)), "control_run": any(a in CONTROLS for a in agents), "agent_run": any(a and a not in CONTROLS for a in agents), "judged": bool(judged), diff --git a/app/nullius/providers.py b/app/nullius/providers.py index f2f80427..4e05b26b 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -1,7 +1,7 @@ """Where model calls go: the one definition of each model provider, and the builder's steps. Every feature reads the registry below (``record``, ``records``, ``serves``) and asks for the -protocol it speaks: a run what its agent speaks, the LLM judge ``chat`` or ``messages``, System One +protocol it speaks: a run what its agent speaks, the LLM judge ``chat`` or ``messages``, Decision a decision protocol. A provider is a preset (``PRESETS``) or a custom provider, ``custom``: the OpenAI-compatible endpoint at MODEL_API_URL, and one per id in Settings (``settings.endpoint_ids``). A recipe names one by ``provider`` (``PROVIDERS``, the ones a run is sent to) and, for a custom one, @@ -38,7 +38,7 @@ "chat": {"label": "OpenAI Chat Completions", "path": "/chat/completions", "auth": "authorization"}, "responses": {"label": "OpenAI Responses", "path": "/responses", "auth": "authorization"}, "messages": {"label": "Anthropic Messages", "path": "/messages", "auth": "x-api-key"}, - "systemone": {"label": "TypeSafe System One", "path": "/systemone", "auth": "authorization"}, + "systemone": {"label": "TypeSafe Decision", "path": "/systemone", "auth": "authorization"}, "decisions": {"label": "OpenRouter Decisions", "path": "/decisions", "auth": "authorization"}, "openai-decisions": {"label": "OpenAI Decisions", "path": "/decisions", "auth": "authorization"}, } diff --git a/app/nullius/settings.py b/app/nullius/settings.py index f241a09a..08663b21 100644 --- a/app/nullius/settings.py +++ b/app/nullius/settings.py @@ -41,13 +41,13 @@ _TEXT_OK = re.compile(r"^[^\x00-\x1f]{1,400}$") _MODEL_OK = re.compile(r"^~?[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,159}$") # an OpenRouter model id: [~]vendor/model[:variant] # a provider choice each judge can use: a preset that serves its protocols, or custom (local, its old name); -# System One's custom provider is a named one, the only kind whose record lists a decision protocol +# Decision's custom provider is a named one, the only kind whose record lists a decision protocol _ID = r"[A-Z0-9]+(_[A-Z0-9]+)*" -_S1_OK = re.compile(rf"^(typesafe|openrouter|openai|(custom|local):{_ID})$") +_DECISION_PROVIDER_OK = re.compile(rf"^(typesafe|openrouter|openai|(custom|local):{_ID})$") _LLM_JUDGE_OK = re.compile(rf"^(openrouter|openai|anthropic|(custom|local)(:{_ID})?)$") -# the protocols each judge speaks (``judge.client``: LLM_PROTOCOLS, SYSTEM_ONE_PROTOCOLS) +# the protocols each judge speaks (``judge.client``: LLM_PROTOCOLS, DECISION_PROTOCOLS) _LLM_PROTOCOL_OK = re.compile(r"^(chat|messages)$") -_S1_PROTOCOL_OK = re.compile(r"^(systemone|decisions|openai-decisions)$") +_DECISION_PROTOCOL_OK = re.compile(r"^(systemone|decisions|openai-decisions)$") _ENVNAME_OK = re.compile(r"^[A-Z_][A-Z0-9_]*$") _BOOL = {"0", "1", "true", "false", "yes", "no"} @@ -149,7 +149,7 @@ class Variable: name: str group: str purpose: str - kind: str # secret | localkey | endpoint | mirror | provider | path | paths | url | repo | name | bool | text | envname | hosts | constraints | model | llmprovider | s1provider | llmprotocol | s1protocol | envnames + kind: str # secret | localkey | endpoint | mirror | provider | path | paths | url | repo | name | bool | text | envname | hosts | constraints | model | llmprovider | decisionprovider | llmprotocol | decisionprotocol | envnames example: str = "" # shown as a placeholder, written to .env.example as a comment @property @@ -182,9 +182,9 @@ def validate(self, value: str) -> Optional[str]: "constraints": _constraints, "model": _check(_MODEL_OK, "a model id as OpenRouter lists it, vendor/model"), "llmprovider": _check(_LLM_JUDGE_OK, "a provider: openrouter, openai, anthropic, custom or custom:"), - "s1provider": _check(_S1_OK, "a provider: typesafe, openrouter, openai or custom:"), + "decisionprovider": _check(_DECISION_PROVIDER_OK, "a provider: typesafe, openrouter, openai or custom:"), "llmprotocol": _check(_LLM_PROTOCOL_OK, "a protocol: chat or messages"), - "s1protocol": _check(_S1_PROTOCOL_OK, "a protocol: systemone, decisions or openai-decisions"), + "decisionprotocol": _check(_DECISION_PROTOCOL_OK, "a protocol: systemone, decisions or openai-decisions"), "envnames": _agent_key_names, }[self.kind](value) @@ -213,13 +213,13 @@ def validate(self, value: str) -> Optional[str]: Variable("OPENAI_API_KEY", GROUPS[0], "An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), Variable("OPENROUTER_API_KEY", GROUPS[0], "An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), Variable("ANTHROPIC_API_KEY", GROUPS[0], "An Anthropic key, for a run of the Nullius standard whose agent speaks Anthropic's Messages, and for the LLM judge: the calls go to Anthropic's own base with this key. Only the relay and the judge hold it, never an agent, a recipe or a log.", "secret"), - Variable("TYPESAFE_API_KEY", GROUPS[0], "A TypeSafe key, for the System One judgement on TypeSafe's own base: sent as a bearer header there and nowhere else.", "secret", "ts-…"), - Variable("SYSTEM_ONE_PROVIDER", GROUPS[0], "The provider the System One judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "s1provider"), - Variable("SYSTEM_ONE_PROTOCOL", GROUPS[0], "The decision protocol System One is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves.", "s1protocol"), - Variable("SYSTEM_ONE_MODEL", GROUPS[0], "The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default.", "model"), - Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last.", "llmprovider"), + Variable("TYPESAFE_API_KEY", GROUPS[0], "A TypeSafe key, for the Decision judgement on TypeSafe's own base: sent as a bearer header there and nowhere else.", "secret", "ts-…"), + Variable("DECISION_PROVIDER", GROUPS[0], "The provider the Decision judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "decisionprovider"), + Variable("DECISION_PROTOCOL", GROUPS[0], "The decision protocol Decision is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves.", "decisionprotocol"), + Variable("DECISION_MODEL", GROUPS[0], "The Decision model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default.", "model"), + Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside Decision: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last.", "llmprovider"), Variable("LLM_JUDGE_PROTOCOL", GROUPS[0], "The protocol the LLM judge is asked in, on a provider that serves both: chat (OpenAI Chat Completions) or messages (Anthropic Messages). Unset, or one the provider does not serve: the first the provider serves, chat before messages.", "llmprotocol"), - Variable("LLM_JUDGE_MODEL", GROUPS[0], "The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.", "model"), + Variable("LLM_JUDGE_MODEL", GROUPS[0], "The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as Decision and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.", "model"), *(Variable(name, GROUPS[0], f"Additions to the {title} provider, as a JSON object without the key: endpoints of protocols the preset does not list, " '{"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one.', "provider") for name, title in (("OPENROUTER", "OpenRouter"), ("OPENAI", "OpenAI"), ("ANTHROPIC", "Anthropic"), ("TYPESAFE", "TypeSafe"))), @@ -260,7 +260,11 @@ def validate(self, value: str) -> Optional[str]: # says which of its own names was meant - for HOST and PORT, a start-up # option, read at start. OPENAI_API_KEY and OPENROUTER_API_KEY are its own: # the keys a recipe on OpenAI or on OpenRouter is sent with. -INSTEAD: Dict[str, str] = {"API_TOKEN": "MODEL_API_KEY", "API_URL": "MODEL_API_URL", "OPENAI_BASE_URL": "MODEL_API_URL", +# The Decision judge's settings before it had that name: still read (``judge.client``), and taken out of +# the file when the page saves the new one, so a saved choice survives the rename. +RENAMED: Dict[str, str] = {"SYSTEM_ONE_PROVIDER": "DECISION_PROVIDER", "SYSTEM_ONE_PROTOCOL": "DECISION_PROTOCOL", + "SYSTEM_ONE_MODEL": "DECISION_MODEL"} +INSTEAD: Dict[str, str] = {**RENAMED, "API_TOKEN": "MODEL_API_KEY", "API_URL": "MODEL_API_URL", "OPENAI_BASE_URL": "MODEL_API_URL", "API_KEY_TYPESAFE": "TYPESAFE_API_KEY", "JUDGE_MODEL": "LLM_JUDGE_MODEL", "HOST": "APP_HOST", "PORT": "APP_PORT"} @@ -641,8 +645,15 @@ def save_many(path: str, changes: Dict[str, Optional[str]]) -> None: problems.append(f"{name} must be {why}.") if problems: raise ValueError(" ".join(problems)) + # a setting saved under its new name takes its old one out of the file + old = {o for o, n in RENAMED.items() if n in changes and o not in changes} lines, done = [], set() for line, key, _ in parse(path): + if key in old: + if key in LOADED_KEYS: + os.environ.pop(key, None) + LOADED_KEYS.discard(key) + continue if key in changes: value = changes[key] if value is not None and key not in done: diff --git a/app/site/mkdocs.yml b/app/site/mkdocs.yml index 82ac2df5..ce75b8cd 100644 --- a/app/site/mkdocs.yml +++ b/app/site/mkdocs.yml @@ -109,7 +109,7 @@ nav: - Settings: - Overview: settings/README.md - Model providers: settings/model-providers.md - - System One provider: settings/system-one.md + - Decision provider: settings/decision.md - LLM judge provider: settings/llm-judge.md - Custom keys: settings/agent-keys.md - Model prices: settings/model-prices.md diff --git a/app/static/settings/autosave.js b/app/static/settings/autosave.js index 72c2809c..7cd43b8f 100644 --- a/app/static/settings/autosave.js +++ b/app/static/settings/autosave.js @@ -33,7 +33,7 @@ } else if (key.startsWith('value:')) names.add(key.slice(6)); } // A model edit always carries the displayed provider, including a sole auto-selection. - for (const prefix of ['SYSTEM_ONE', 'LLM_JUDGE']) { + for (const prefix of ['DECISION', 'LLM_JUDGE']) { if (body.has('value:' + prefix + '_MODEL') || body.has('clear:' + prefix + '_MODEL')) { const provider = current.get('value:' + prefix + '_PROVIDER'); if (provider) body.set('value:' + prefix + '_PROVIDER', provider); diff --git a/app/static/settings/judges.js b/app/static/settings/judges.js index 7f0864a6..c01f5b58 100644 --- a/app/static/settings/judges.js +++ b/app/static/settings/judges.js @@ -47,13 +47,13 @@ if (model) model.placeholder = on?.dataset.default || ''; } // A provider with no list of its own here: its catalogue, asked once when the field is first used; a live list - // (System One on OpenRouter) replaces the one drawn with the page. Typing a model still works. + // (Decision on OpenRouter) replaces the one drawn with the page. Typing a model still works. const catalogues = new Map(); async function offerCatalogue(card, model) { const on = chosen(card); if (!on || (on.dataset.models && !('live' in on.dataset)) || !model.list) return; const provider = on.value.replace(/^local(?=$|:)/, 'custom'); // local:, a named endpoint, is the builder's custom: - const url = 'live' in on.dataset ? '/settings/system-one/models' : '/new-run/models'; + const url = 'live' in on.dataset ? '/settings/decision/models' : '/new-run/models'; if (!catalogues.has(url + provider)) { catalogues.set(url + provider, nulliusFetch(url, {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': document.querySelector('#settings-form [name=csrf]').value}, body: JSON.stringify({provider})}) diff --git a/app/templates/compare/compare.html b/app/templates/compare/compare.html index ee3fe3fb..bdaaa627 100644 --- a/app/templates/compare/compare.html +++ b/app/templates/compare/compare.html @@ -144,7 +144,7 @@

Task by task

- {% elif judgement.unjudged %}System One not configured — set its key in Settings{% endif %} + {% elif judgement.unjudged %}Decision not configured: needs a model — choose one in Settings{% endif %} {% endif %}
diff --git a/app/templates/runs/feasibility.html b/app/templates/runs/feasibility.html index 1ba7cd28..3508c1c2 100644 --- a/app/templates/runs/feasibility.html +++ b/app/templates/runs/feasibility.html @@ -68,7 +68,7 @@ - {% elif triage.unjudged %}System One not configured — set its key in Settings{% endif %} + {% elif triage.unjudged %}Decision not configured: needs a model — choose one in Settings{% endif %}
{% endif %} diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 97a571b7..c240c199 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -298,7 +298,7 @@ {% if (judgement.unjudged) or (judgement.llm.unjudged) or (tq and tq.unjudged) %}
{# one button per judge: each judges apart, so a person asks for the one they want #} - {% for lane, name, n, keyed in [('system-one', 'System One', judgement.unjudged, judgement.keyed), ('llm', 'LLM judge', judgement.llm.unjudged, judgement.llm.keyed)] if n %} + {% for lane, name, n, keyed in [('decision', 'Decision', judgement.unjudged, judgement.keyed), ('llm', 'LLM judge', judgement.llm.unjudged, judgement.llm.keyed)] if n %} {% if keyed %}
diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index 042e6418..9ad2d793 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -52,7 +52,7 @@

Settings

{{ c.title }}{{ ': set' if c.ready else ': not set' }} {% endfor %} Custom{{ ': set' if ns.custom else ': not set' }} - {% for jj, name in [(judges[0], 'System One'), (judges[1], 'LLM judge')] %} {{ name }}{{ ': set' if jj else ': not set' }}{% endfor %} + {% for jj, name in [(judges[0], 'Decision'), (judges[1], 'LLM judge')] %} {{ name }}{{ ': set' if jj else ': not set' }}{% endfor %} {% for item in chosen %} {{ item.split('=', 1)[0] if '=' in item.split('@', 1)[0] else item.split(':/')[0].split(':~')[0] }}{% else %} no host{% endfor %}
@@ -80,7 +80,7 @@

Settings

{% for b in benchmark_access %}{% set access_search.words = access_search.words ~ ' ' ~ b.title %}{% for s in b.needs %}{% set access_search.words = access_search.words ~ ' ' ~ s.name ~ ' ' ~ s.summary %}{% endfor %}{% endfor %} {{ section_link('data-providers', 'Data providers', 'cloud-down', 'hugging face huggingface hf_token docker hub login registry token dataset pull limit') }} {{ section_link('custom-keys', 'Custom keys', 'key', access_search.words) }} - {{ section_link('judge', 'Judges', 'scale', 'typesafe system one llm model judgement') }} + {{ section_link('judge', 'Judges', 'scale', 'typesafe decision llm model judgement') }} {{ section_link('prices', 'Model prices', 'dollar', 'cost price estimate usd tokens list update') }}
@@ -186,7 +186,7 @@

Add a provider

Judges

Choose the judges that evaluate trial evidence.

- {% set s1, llm = judges %} + {% set decision, llm = judges %} {#- Provider, then Protocol when the provider lists two this judge speaks (else a quiet line), then Model. -#} {% macro judge_rows(prefix, choices, judge, label) %} @@ -218,12 +218,12 @@

Judges

{% endmacro %} - {% set srow = by.get('SYSTEM_ONE_PROVIDER') %} - {% if srow and by.get('SYSTEM_ONE_MODEL') %} -
- System One - {{ judge_rows('SYSTEM_ONE', s1_providers, s1, 'Where System One is asked') }} - {{ field(by['SYSTEM_ONE_MODEL'], 'Model', writable, s1_providers[s1.provider].models) }} + {% set srow = by.get('DECISION_PROVIDER') %} + {% if srow and by.get('DECISION_MODEL') %} +
+ Decision + {{ judge_rows('DECISION', decision_providers, decision, 'Where Decision is asked') }} + {{ field(by['DECISION_MODEL'], 'Model', writable, decision_providers[decision.provider].models) }}
{% endif %} {% set prow = by.get('LLM_JUDGE_PROVIDER') %} diff --git a/app/templates/traces/_judgement.html b/app/templates/traces/_judgement.html index 9f06c5e3..e3780b0f 100644 --- a/app/templates/traces/_judgement.html +++ b/app/templates/traces/_judgement.html @@ -15,7 +15,7 @@ {% if j.keyed %} - {% else %}{{ 'LLM judge' if llm else 'System One' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %} + {% else %}{{ 'LLM judge' if llm else 'Decision' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %} {% if j.record and (j.record.refusals or j.earlier or j.record.truncated) %}{% set note = namespace(sep='') %}

{%- if j.record.refusals %}{{ j.record.refusals|length }} refused{% set note.sep = ' · ' %}{% endif -%} @@ -64,7 +64,7 @@

- {% for name, label in [('overview', 'Overview'), ('judgement', 'System One judge'), ('llm-judge', 'LLM judge'), ('task', 'Task instruction'), ('trace', 'Trace'), ('conversation', 'Conversation'), ('artifacts', 'Artifacts')] if not (replaced and name in ('judgement', 'llm-judge')) %} + {% for name, label in [('overview', 'Overview'), ('judgement', 'Decision judge'), ('llm-judge', 'LLM judge'), ('task', 'Task instruction'), ('trace', 'Trace'), ('conversation', 'Conversation'), ('artifacts', 'Artifacts')] if not (replaced and name in ('judgement', 'llm-judge')) %} {% endfor %} diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 8ca868ad..bd717689 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -298,7 +298,7 @@ def post(url, body, key): self.assertIsNone(store.load(str(self.tmp), "job", "empty")) - def test_the_llm_judge_keeps_its_records_apart_from_system_one(self): + def test_the_llm_judge_keeps_its_records_apart_from_decision_judge(self): import json as _json def chat(url, body, key): asked = [q for q in ("task_completed",) if f'"{q}"' in body["messages"][1]["content"]] @@ -308,7 +308,7 @@ def chat(url, body, key): rec = batch.ask_and_store(str(self.tmp), "job", "t", {"trial": (["task_completed"], "the trial state", False)}, llm, post=chat) self.assertEqual(rec["judge"], {"provider": "openrouter", "model": "vendor/m"}) self.assertEqual(store.load(str(self.tmp), "job", "t", lane="llm")["answers"], {"task_completed": {"noul": 0.3}}) - self.assertIsNone(store.load(str(self.tmp), "job", "t")) # System One's is its own + self.assertIsNone(store.load(str(self.tmp), "job", "t")) # Decision's is its own self.assertTrue((self.tmp / "judgements-llm" / "job" / "t.json").is_file()) self.assertEqual(store.for_job(str(self.tmp), "job"), {}); self.assertEqual(list(store.for_job(str(self.tmp), "job", lane="llm")), ["t"]) diff --git a/app/tests/test_judge_decisions.py b/app/tests/test_judge_decisions.py index 46bd174f..ad7d1e2d 100644 --- a/app/tests/test_judge_decisions.py +++ b/app/tests/test_judge_decisions.py @@ -1,4 +1,4 @@ -"""System One in each decision protocol, and the LLM judge on Anthropic's +"""Decision in each decision protocol, and the LLM judge on Anthropic's Messages. Every provider is a fake ``post``: it records what it was sent and answers with a fixture reply.""" from __future__ import annotations @@ -28,7 +28,7 @@ def post(url, body, key, timeout=120.0, auth=None): def judge(env): - return client.system_one(env) + return client.decision_judge(env) class Protocols(unittest.TestCase): @@ -36,25 +36,38 @@ def test_each_provider_and_model_gets_its_decision_protocol(self): got = lambda env: (judge(env).protocol, judge(env).url) self.assertEqual(got({"TYPESAFE_API_KEY": "k"}), ("systemone", "https://api.typesafe.ai/v1/systemone")) self.assertEqual(got({"OPENROUTER_API_KEY": "k"}), ("systemone", "https://openrouter.ai/api/v1/systemone")) - self.assertEqual(got({"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_MODEL": "microsoft/microsoft-decision-1"}), + self.assertEqual(got({"OPENROUTER_API_KEY": "k", "DECISION_MODEL": "microsoft/microsoft-decision-1"}), ("decisions", "https://openrouter.ai/api/alpha/decisions")) # another maker's model - self.assertEqual(got({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}), ("openai-decisions", "https://api.openai.com/v1/decisions")) - self.assertEqual(judge({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}).model, "gpt-6-luna") + self.assertEqual(got({"OPENAI_API_KEY": "k", "DECISION_PROVIDER": "openai"}), ("openai-decisions", "https://api.openai.com/v1/decisions")) + self.assertEqual(judge({"OPENAI_API_KEY": "k", "DECISION_PROVIDER": "openai"}).model, "gpt-6-luna") custom = {"MODEL_GW": '{"base": "https://gw/v1", "endpoints": {"decisions": "/alpha/decisions"}}', "MODEL_GW_API_KEY": "k", - "SYSTEM_ONE_PROVIDER": "custom:GW", "SYSTEM_ONE_MODEL": "m"} + "DECISION_PROVIDER": "custom:GW", "DECISION_MODEL": "m"} self.assertEqual(got(custom), ("decisions", "https://gw/v1/alpha/decisions")) - self.assertNotIn("custom:LAB", client.system_one_providers({"MODEL_LAB": '{"base": "http://lab/v1"}'})) # lists none: not offered + self.assertNotIn("custom:LAB", client.decision_judge_providers({"MODEL_LAB": '{"base": "http://lab/v1"}'})) # lists none: not offered + + def test_a_setting_saved_before_the_rename_is_still_read_and_the_new_name_wins(self): + old = {"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openrouter", "SYSTEM_ONE_MODEL": "typesafe/jev-1.14", + "SYSTEM_ONE_PROTOCOL": "decisions"} + j = judge(old) + self.assertEqual((j.lane, j.provider, j.model, j.protocol), ("decision", "openrouter", "typesafe/jev-1.14", "decisions")) + self.assertEqual(judge(dict(old, DECISION_MODEL="typesafe/jev-2")).model, "typesafe/jev-2") + + def test_without_a_provider_for_it_the_decision_judge_needs_a_model_on_any_of_them(self): + """Not "the TypeSafe key": OpenRouter and OpenAI serve decision models too.""" + self.assertEqual(judge({}).missing, "a model") + self.assertFalse(judge({})) + self.assertEqual(judge({"OPENROUTER_API_KEY": "k"}).missing, "") def test_a_chosen_protocol_wins_where_the_provider_serves_it_else_the_automatic_one(self): from nullius import settings got = lambda env: (judge(env).protocol, judge(env).url) router = {"OPENROUTER_API_KEY": "k"} - self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="decisions")), ("decisions", "https://openrouter.ai/api/alpha/decisions")) # over TypeSafe's model - self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="systemone", SYSTEM_ONE_MODEL="microsoft/microsoft-decision-1")), + self.assertEqual(got(dict(router, DECISION_PROTOCOL="decisions")), ("decisions", "https://openrouter.ai/api/alpha/decisions")) # over TypeSafe's model + self.assertEqual(got(dict(router, DECISION_PROTOCOL="systemone", DECISION_MODEL="microsoft/microsoft-decision-1")), ("systemone", "https://openrouter.ai/api/v1/systemone")) - self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="openai-decisions")), ("systemone", "https://openrouter.ai/api/v1/systemone")) # not served: automatic - self.assertEqual(client.system_one_providers(dict(router, SYSTEM_ONE_PROTOCOL="decisions"))["openrouter"]["protocol"], "decisions") - self.assertEqual(client.system_one_providers(router)["openrouter"]["listed"], ["systemone", "decisions"]) + self.assertEqual(got(dict(router, DECISION_PROTOCOL="openai-decisions")), ("systemone", "https://openrouter.ai/api/v1/systemone")) # not served: automatic + self.assertEqual(client.decision_judge_providers(dict(router, DECISION_PROTOCOL="decisions"))["openrouter"]["protocol"], "decisions") + self.assertEqual(client.decision_judge_providers(router)["openrouter"]["listed"], ["systemone", "decisions"]) both = {"ANTHROPIC_API_KEY": "k", "ANTHROPIC": '{"endpoints": {"chat": "/chat/completions"}}', "LLM_JUDGE_PROVIDER": "anthropic", "LLM_JUDGE_MODEL": "m"} llm = lambda env: (client.llm_judge(env).protocol, client.llm_judge(env).url) self.assertEqual(llm(both), ("chat", "https://api.anthropic.com/v1/chat/completions")) # unset: chat before messages @@ -62,13 +75,13 @@ def test_a_chosen_protocol_wins_where_the_provider_serves_it_else_the_automatic_ self.assertEqual(client.llm_providers(both)["anthropic"]["listed"], ["chat", "messages"]) self.assertEqual(llm({"OPENROUTER_API_KEY": "k", "LLM_JUDGE_MODEL": "m", "LLM_JUDGE_PROTOCOL": "messages"}), ("chat", "https://openrouter.ai/api/v1/chat/completions")) # not served: automatic - for name, spoken in (("LLM_JUDGE_PROTOCOL", client.LLM_PROTOCOLS), ("SYSTEM_ONE_PROTOCOL", client.SYSTEM_ONE_PROTOCOLS)): + for name, spoken in (("LLM_JUDGE_PROTOCOL", client.LLM_PROTOCOLS), ("DECISION_PROTOCOL", client.DECISION_PROTOCOLS)): var = settings.BY_NAME[name] self.assertEqual([p for p in client.providers.PROTOCOLS if var.validate(p) is None], list(spoken)) # what that judge speaks, no more self.assertIsNotNone(var.validate("Chat")) -class SystemOne(unittest.TestCase): +class Decision(unittest.TestCase): def test_systemone_keeps_the_providers_cost_and_records_a_refusal(self): post, _ = fake(200, {"model": "~typesafe/jev-latest", "answers": { "task_completed": {"noul": 0.8}, "failure_cause": {"type": "refusal", "refusal": "not enough evidence"}}, @@ -88,7 +101,7 @@ def test_a_402_says_what_it_means(self): self.assertIn("credit is spent", str(e.exception)); self.assertIn("Insufficient credits", str(e.exception)) def test_decisions_reads_confidence_and_its_error_shape(self): - env = {"OPENROUTER_API_KEY": "sk-or-1", "SYSTEM_ONE_MODEL": "inception/mercury-decide"} + env = {"OPENROUTER_API_KEY": "sk-or-1", "DECISION_MODEL": "inception/mercury-decide"} post, sent = fake(200, {"model": "inception/mercury-decide", "answers": {"task_completed": {"noul": 0.3, "confidence": 0.9}}, "usage": {"input_tokens": 10, "output_tokens": 0, "cost": 0.001}}) rec = client.ask("state", IDS, judge(env), post=post) @@ -111,8 +124,8 @@ def test_a_reply_of_another_shape_is_a_judge_error(self): for env, data in (({"OPENROUTER_API_KEY": "k"}, {"answers": [1, 2]}), ({"OPENROUTER_API_KEY": "k"}, {"answers": {"failure_cause": {"choice": "a", "probabilities": {"a": "x"}}}}), ({"OPENROUTER_API_KEY": "k"}, ["not", "an", "object"]), - ({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}, {"answers": [{"name": ["task_completed"]}]}), - ({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}, + ({"OPENAI_API_KEY": "k", "DECISION_PROVIDER": "openai"}, {"answers": [{"name": ["task_completed"]}]}), + ({"OPENAI_API_KEY": "k", "DECISION_PROVIDER": "openai"}, {"answers": [{"name": "failure_cause", "choice": next(iter(Q.QUESTIONS["failure_cause"]["criteria"])), "probabilities": 5}]})): post, _ = fake(200, data) with self.assertRaises(client.JudgeError, msg=data) as e: @@ -220,27 +233,27 @@ def do_GET(self): def test_the_suggestions_are_the_list_with_the_alias_first_or_the_short_list(self): with mock.patch.object(providers, "decision_models", return_value=["z/one", "typesafe/jev-1.14", "a/two:free", "~typesafe/jev-latest"]) as live: - self.assertEqual(client.system_one_models("openrouter"), ["~typesafe/jev-latest", "typesafe/jev-1.14", "a/two:free", "z/one"]) + self.assertEqual(client.decision_judge_models("openrouter"), ["~typesafe/jev-latest", "typesafe/jev-1.14", "a/two:free", "z/one"]) live.assert_called_with(fetch=False) - client.system_one_models("openrouter", fetch=True); live.assert_called_with(fetch=True) - self.assertEqual(client.system_one_models("openai"), ["gpt-6-luna"]) - self.assertEqual(client.system_one_providers({"OPENROUTER_API_KEY": "k"})["openrouter"]["model"], "~typesafe/jev-latest") + client.decision_judge_models("openrouter", fetch=True); live.assert_called_with(fetch=True) + self.assertEqual(client.decision_judge_models("openai"), ["gpt-6-luna"]) + self.assertEqual(client.decision_judge_providers({"OPENROUTER_API_KEY": "k"})["openrouter"]["model"], "~typesafe/jev-latest") with mock.patch.object(providers, "decision_models", return_value=["b/x"]): - self.assertEqual(client.system_one_models("openrouter"), ["~typesafe/jev-latest", "b/x"]) + self.assertEqual(client.decision_judge_models("openrouter"), ["~typesafe/jev-latest", "b/x"]) with mock.patch.object(providers, "decision_models", return_value=[]): - self.assertEqual(client.system_one_models("openrouter"), client.SYSTEM_ONE_MODELS["openrouter"]) + self.assertEqual(client.decision_judge_models("openrouter"), client.DECISION_MODELS["openrouter"]) def test_the_protocol_follows_the_model_not_the_list(self): - on = lambda model, **env: client.system_one({"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_MODEL": model, **env}).protocol + on = lambda model, **env: client.decision_judge({"OPENROUTER_API_KEY": "k", "DECISION_MODEL": model, **env}).protocol with mock.patch.object(providers, "decision_models", return_value=["google/gemini-decision-1", "typesafe/jev-1.14"]): self.assertEqual(on("typesafe/jev-1.14"), "systemone") self.assertEqual(on("google/gemini-decision-1"), "decisions") self.assertEqual(on("someone/typed-by-hand"), "decisions") - self.assertEqual(on("google/gemini-decision-1", SYSTEM_ONE_PROTOCOL="systemone"), "systemone") + self.assertEqual(on("google/gemini-decision-1", DECISION_PROTOCOL="systemone"), "systemone") class OpenAIDecisions(unittest.TestCase): - ENV = {"OPENAI_API_KEY": "sk-oa-1", "SYSTEM_ONE_PROVIDER": "openai"} + ENV = {"OPENAI_API_KEY": "sk-oa-1", "DECISION_PROVIDER": "openai"} def test_the_request_is_openais_shape(self): post, sent = fake(200, {"model": "gpt-6-luna", "answers": [], "usage": {}}) diff --git a/app/tests/test_judge_openrouter.py b/app/tests/test_judge_openrouter.py index 11a21031..a66ba3d2 100644 --- a/app/tests/test_judge_openrouter.py +++ b/app/tests/test_judge_openrouter.py @@ -26,9 +26,9 @@ def post(url, body, key, timeout=120.0): class Configured(unittest.TestCase): - def test_system_one_and_the_llm_judge_are_set_apart(self): - self.assertFalse(client.system_one({})); self.assertTrue(client.system_one({"TYPESAFE_API_KEY": "ts-1"})) - self.assertEqual(client.system_one({}).lane, "system-one") + def test_decision_judge_and_the_llm_judge_are_set_apart(self): + self.assertFalse(client.decision_judge({})); self.assertTrue(client.decision_judge({"TYPESAFE_API_KEY": "ts-1"})) + self.assertEqual(client.decision_judge({}).lane, "decision") j = client.llm_judge({"OPENROUTER_API_KEY": "sk-or-1"}) self.assertEqual((j.provider, j.lane, j.url), ("openrouter", "llm", "https://openrouter.ai/api/v1/chat/completions")) self.assertFalse(j); self.assertEqual(j.missing, "a model") # no model: not ready @@ -55,12 +55,12 @@ def test_the_llm_judge_on_a_named_endpoint_uses_its_url_and_its_key(self): self.assertEqual((gone.provider, gone.key), ("custom", "default-key")) -class SystemOneOnOpenRouter(unittest.TestCase): - def test_system_one_is_asked_where_settings_say_with_the_model_named(self): - self.assertEqual(client.system_one({"OPENROUTER_API_KEY": "sk-or-1"}).provider, "openrouter") # the only key: OpenRouter - self.assertEqual(client.system_one({"OPENROUTER_API_KEY": "sk-or-1", "TYPESAFE_API_KEY": "ts-1"}).provider, "typesafe") - j = client.system_one({"OPENROUTER_API_KEY": "sk-or-1", "SYSTEM_ONE_PROVIDER": "openrouter", "SYSTEM_ONE_MODEL": "jaredpalmer/kev-4b"}) - self.assertEqual((j.lane, j.url, j.model), ("system-one", "https://openrouter.ai/api/v1/systemone", "jaredpalmer/kev-4b")) +class DecisionOnOpenRouter(unittest.TestCase): + def test_decision_judge_is_asked_where_settings_say_with_the_model_named(self): + self.assertEqual(client.decision_judge({"OPENROUTER_API_KEY": "sk-or-1"}).provider, "openrouter") # the only key: OpenRouter + self.assertEqual(client.decision_judge({"OPENROUTER_API_KEY": "sk-or-1", "TYPESAFE_API_KEY": "ts-1"}).provider, "typesafe") + j = client.decision_judge({"OPENROUTER_API_KEY": "sk-or-1", "DECISION_PROVIDER": "openrouter", "DECISION_MODEL": "jaredpalmer/kev-4b"}) + self.assertEqual((j.lane, j.url, j.model), ("decision", "https://openrouter.ai/api/v1/systemone", "jaredpalmer/kev-4b")) sent = [] def post(url, body, key): sent.append((url, body, key)) @@ -68,10 +68,10 @@ def post(url, body, key): rec = client.ask("the state", ["task_completed"], j, post=post) url, body, key = sent[0] self.assertEqual((url, key, body["model"]), ("https://openrouter.ai/api/v1/systemone", "sk-or-1", "jaredpalmer/kev-4b")) - self.assertIn("questions", body); self.assertEqual(body["state"], "the state") # the System One contract, not a chat + self.assertIn("questions", body); self.assertEqual(body["state"], "the state") # the Decision contract, not a chat self.assertEqual(rec["judge"], {"provider": "openrouter", "model": "jaredpalmer/kev-4b"}) self.assertEqual(rec["answers"], {"task_completed": {"noul": 0.81}}) - self.assertEqual(client.system_one({"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openrouter"}).model, "~typesafe/jev-latest") + self.assertEqual(client.decision_judge({"OPENROUTER_API_KEY": "k", "DECISION_PROVIDER": "openrouter"}).model, "~typesafe/jev-latest") class OpenRouterJudge(unittest.TestCase): diff --git a/app/tests/test_provider_registry.py b/app/tests/test_provider_registry.py index 4203cb20..5bee316e 100644 --- a/app/tests/test_provider_registry.py +++ b/app/tests/test_provider_registry.py @@ -107,7 +107,7 @@ def test_a_consumer_gets_the_providers_that_may_serve_its_protocol(self): self.assertEqual(providers.offered(["responses"], {}), ["openrouter", "openai", "custom"]) env = dict(KEYS, MODEL_GW=json.dumps({"base": "https://gw/v1", "endpoints": {"decisions": "/d"}}), MODEL_LAB=json.dumps({"base": "http://lab/v1"})) self.assertEqual(list(client._table(["decisions"], env, known=False)), ["openrouter", "custom:GW", "custom:LAB"]) - self.assertEqual(list(client.system_one_providers(env)), ["openrouter", "openai", "typesafe", "custom:GW"]) + self.assertEqual(list(client.decision_judge_providers(env)), ["openrouter", "openai", "typesafe", "custom:GW"]) px = dict(env, MODEL_PX=json.dumps({"name": "PX", "base": "https://px/v1", "endpoints": {"messages": "/messages"}})) self.assertEqual(client.llm_providers(px)["custom:PX"]["protocol"], "messages") # the one it lists, before chat self.assertEqual(providers.protocol_for("anthropic", ["chat", "messages"], env), "messages") @@ -117,7 +117,7 @@ def test_a_consumer_gets_the_providers_that_may_serve_its_protocol(self): run.key_header("anthropic", {"key": "reference", "model": {"apis": ["chat"]}}, env) # as recipe.compile refuses it def test_each_judge_setting_takes_only_a_provider_that_judge_can_use(self): - s1, llm = settings.BY_NAME["SYSTEM_ONE_PROVIDER"].validate, settings.BY_NAME["LLM_JUDGE_PROVIDER"].validate + s1, llm = settings.BY_NAME["DECISION_PROVIDER"].validate, settings.BY_NAME["LLM_JUDGE_PROVIDER"].validate for ok in ("typesafe", "openrouter", "openai", "custom:GW", "local:GW"): self.assertIsNone(s1(ok), ok) for ok in ("openrouter", "openai", "anthropic", "custom", "custom:GW", "local"): diff --git a/app/tests/test_settings.py b/app/tests/test_settings.py index b5975def..1c400410 100644 --- a/app/tests/test_settings.py +++ b/app/tests/test_settings.py @@ -43,7 +43,7 @@ def test_every_variable_the_application_reads_is_in_the_table(self): def test_the_page_offers_what_a_person_sets_and_no_more(self): self.assertEqual([v.name for v in settings.VARIABLES], - ["MODEL_API_KEY", "MODEL_API_URL", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "TYPESAFE_API_KEY", "SYSTEM_ONE_PROVIDER", "SYSTEM_ONE_PROTOCOL", "SYSTEM_ONE_MODEL", "LLM_JUDGE_PROVIDER", "LLM_JUDGE_PROTOCOL", "LLM_JUDGE_MODEL", + ["MODEL_API_KEY", "MODEL_API_URL", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "TYPESAFE_API_KEY", "DECISION_PROVIDER", "DECISION_PROTOCOL", "DECISION_MODEL", "LLM_JUDGE_PROVIDER", "LLM_JUDGE_PROTOCOL", "LLM_JUDGE_MODEL", "OPENROUTER", "OPENAI", "ANTHROPIC", "TYPESAFE", "AGENT_KEYS", "HOSTS", "HF_ENDPOINT", "HOST_CONSTRAINTS"]) self.assertEqual(sorted(v.name for v in settings.VARIABLES if v.secret), ["ANTHROPIC_API_KEY", "MODEL_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "TYPESAFE_API_KEY"]) self.assertEqual(settings.BY_NAME["MODEL_API_URL"].kind, "endpoint") # shown, so it may hold no credential @@ -255,6 +255,14 @@ def test_named_endpoints_are_settings_of_their_own_and_the_environment_wins(self with self.assertRaises(ValueError): settings.save_many(self.path, {"LLM_JUDGE_PROVIDER": "local:gpu"}) + def test_saving_a_decision_setting_takes_its_old_name_out_of_the_file(self): + self.write("SYSTEM_ONE_MODEL=typesafe/jev-1.14\nSYSTEM_ONE_PROVIDER=openrouter\nHOSTS=bench\n") + settings.load(self.path) + self.assertEqual([u["instead"] for u in settings.unread(self.path)], ["DECISION_MODEL", "DECISION_PROVIDER"]) + settings.save_many(self.path, {"DECISION_MODEL": "typesafe/jev-2"}) + self.assertEqual(settings.in_file(self.path), {"SYSTEM_ONE_PROVIDER": "openrouter", "HOSTS": "bench", "DECISION_MODEL": "typesafe/jev-2"}) + self.assertNotIn("SYSTEM_ONE_MODEL", os.environ); self.assertEqual(os.environ["DECISION_MODEL"], "typesafe/jev-2") + def test_an_endpoint_url_is_shown_so_it_holds_no_credential(self): """An endpoint's URL is shown on the page: an http(s) URL with a host, without a user, a password or a key-like query parameter. One set diff --git a/app/ui_tests/test_every_route.py b/app/ui_tests/test_every_route.py index 409a9d74..6cc636ae 100644 --- a/app/ui_tests/test_every_route.py +++ b/app/ui_tests/test_every_route.py @@ -240,7 +240,7 @@ def test_the_batch_judge_routes_judge_on_a_post(self): self.assertTrue(asked, "no batch route asked the judge anything") def test_the_job_page_offers_each_judge_its_own_button(self): - """System One and the LLM judge judge apart, so the job page has a + """Decision and the LLM judge judge apart, so the job page has a button for each, and the one a post names is the one asked.""" judge = web.judge job = web.shared._load_jobs()[0] # the fixtures are controls: read as an agent's job here @@ -248,7 +248,7 @@ def test_the_job_page_offers_each_judge_its_own_button(self): env = dict(TYPESAFE_API_KEY="ts-fake", OPENROUTER_API_KEY="sk-or-fake", LLM_JUDGE_PROVIDER="openrouter", LLM_JUDGE_MODEL="vendor/judge-1") with patch.dict(os.environ, env): page = self.get(f"/job/{job.job_id}").get_data(as_text=True) - for lane in ("system-one", "llm"): + for lane in ("decision", "llm"): self.assertIn(f'id="judge-all-{lane}"', page) asked = [] with patch.object(judge, "judge_trial", lambda j, t, k: asked.append(k.lane) or {}): diff --git a/app/ui_tests/test_judge_cards.py b/app/ui_tests/test_judge_cards.py index fc995bc9..9669a753 100644 --- a/app/ui_tests/test_judge_cards.py +++ b/app/ui_tests/test_judge_cards.py @@ -46,7 +46,7 @@ process.stdout.write(JSON.stringify(out)); ''' -# System One on OpenRouter: the list drawn with the page, then the live one asked when the field is first used. +# Decision on OpenRouter: the list drawn with the page, then the live one asked when the field is first used. LIVE = PROGRAM.split("require(process.argv[1]);")[0].replace( "['openrouter', {available: 'true', protocol: 'chat', protocols: 'chat'}, 'OpenRouter']", "['openrouter', {available: 'true', protocol: 'systemone', protocols: 'systemone|decisions', models: '~typesafe/jev-latest', live: ''}, 'OpenRouter']" @@ -83,9 +83,9 @@ def test_the_protocol_follows_the_provider_and_is_saved_only_once_chosen(self): self.assertEqual(got[6], {"provider": "", "hidden": True, "protocol": "messages", "named": "", "protocolRow": False, "via": "", "none": True, "model": False}) # none left: Add a provider - def test_system_one_on_openrouter_asks_the_live_list_once_when_the_field_is_used(self): + def test_decision_judge_on_openrouter_asks_the_live_list_once_when_the_field_is_used(self): got = json.loads(subprocess.run([self.node, "-e", LIVE, str(SCRIPT)], capture_output=True, text=True, check=True).stdout) - self.assertEqual(got["asked"], [["/settings/system-one/models", {"provider": "openrouter"}]]) # once, for both uses + self.assertEqual(got["asked"], [["/settings/decision/models", {"provider": "openrouter"}]]) # once, for both uses self.assertEqual(got["shown"], [["~typesafe/jev-latest", "b/decider:free"]] * 2) diff --git a/app/ui_tests/test_routes.py b/app/ui_tests/test_routes.py index fb073cf3..fa0e15f2 100644 --- a/app/ui_tests/test_routes.py +++ b/app/ui_tests/test_routes.py @@ -292,7 +292,7 @@ def test_conversation_returns_a_bounded_page_with_full_text_search_reachable(sel def test_a_trial_has_two_judgements_in_two_tabs(self): with patch.dict(os.environ, {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": "sk-or-route-test", "LLM_JUDGE_MODEL": ""}): html = self.get_ok("/trial/active-run/task-001").get_data(as_text=True) - self.assertIn('id="tab-judgement" role="tab" type="button" data-tab="judgement"', html); self.assertIn(">System One judge", html) + self.assertIn('id="tab-judgement" role="tab" type="button" data-tab="judgement"', html); self.assertIn(">Decision judge", html) self.assertIn('id="tab-llm-judge"', html); self.assertIn(">LLM judge", html) self.assertIn('id="judgement"', html); self.assertIn('id="llm-judgement"', html) self.assertIn("LLM judge not configured: needs a model", html) # the OpenRouter key is set, the model is not diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index 5abe6c7b..acbb99d9 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -75,7 +75,7 @@ def test_a_file_edited_by_hand_is_seen_on_the_next_request(self): html = self.page() seen = re.search(r']*id="TYPESAFE_API_KEY".*?', html, re.S).group(0) self.assertIn('data-source="file"', seen); self.assertIn("********", seen); self.assertNotIn("characters", seen); self.assertNotIn("not set", seen) - self.assertNotIn('System One needs the TypeSafe key', html) + self.assertNotIn('Decision needs the TypeSafe key', html) self.assertEqual(os.environ["TYPESAFE_API_KEY"], "ts-fake-0123456789") # the judge sees it too, not only the page self.assertEqual(os.environ["MODEL_API_KEY"], "sk-fake-9876543210"); self.assertNotIn("HOSTS", os.environ) notice = html[html.index('id="settings-unread"'):] @@ -135,7 +135,7 @@ def test_each_judge_card_starts_on_its_provider_and_holds_each_provider_s_key_ro settings.load(self.env_file) html = self.page() judge_section = html.split('id="judge"')[1].split('id="custom-keys"')[0] - for prefix in ("SYSTEM_ONE", "LLM_JUDGE"): # Provider, Protocol, Model, each labelled + for prefix in ("DECISION", "LLM_JUDGE"): # Provider, Protocol, Model, each labelled labels = [judge_section.index(f'for="value-{prefix}_{part.upper()}">{part}') for part in ("Provider", "Protocol", "Model")] self.assertEqual(labels, sorted(labels)) self.assertIn(f' - {% elif judgement.unjudged %}Decision not configured: needs a model — choose one in Settings{% endif %} + {% elif judgement.unjudged %}Decision model not configured: needs a model — choose one in Settings{% endif %} {% endif %}
diff --git a/app/templates/runs/feasibility.html b/app/templates/runs/feasibility.html index 3508c1c2..af04743d 100644 --- a/app/templates/runs/feasibility.html +++ b/app/templates/runs/feasibility.html @@ -68,7 +68,7 @@ - {% elif triage.unjudged %}Decision not configured: needs a model — choose one in Settings{% endif %} + {% elif triage.unjudged %}Decision model not configured: needs a model — choose one in Settings{% endif %}
{% endif %} diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index c240c199..4a6128a0 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -298,7 +298,7 @@ {% if (judgement.unjudged) or (judgement.llm.unjudged) or (tq and tq.unjudged) %}
{# one button per judge: each judges apart, so a person asks for the one they want #} - {% for lane, name, n, keyed in [('decision', 'Decision', judgement.unjudged, judgement.keyed), ('llm', 'LLM judge', judgement.llm.unjudged, judgement.llm.keyed)] if n %} + {% for lane, name, n, keyed in [('decision', 'Decision model', judgement.unjudged, judgement.keyed), ('llm', 'LLM judge', judgement.llm.unjudged, judgement.llm.keyed)] if n %} {% if keyed %}
diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index d37f2b77..4a1b73dd 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -52,7 +52,7 @@

Settings

{{ c.title }}{{ ': set' if c.ready else ': not set' }} {% endfor %} Custom{{ ': set' if ns.custom else ': not set' }} - {% for jj, name in [(judges[0], 'Decision'), (judges[1], 'LLM judge')] %} {{ name }}{{ ': set' if jj else ': not set' }}{% endfor %} + {% for jj, name in [(judges[0], 'Decision model'), (judges[1], 'LLM')] %} {{ name }}{{ ': set' if jj else ': not set' }}{% endfor %} {% for item in chosen %} {{ item.split('=', 1)[0] if '=' in item.split('@', 1)[0] else item.split(':/')[0].split(':~')[0] }}{% else %} no host{% endfor %}
@@ -222,7 +222,7 @@

Judges

{% if srow and by.get('DECISION_MODEL') %}
Decision model - {{ judge_rows('DECISION', decision_providers, decision, 'Where Decision is asked') }} + {{ judge_rows('DECISION', decision_providers, decision, 'Where the Decision model is asked') }} {{ field(by['DECISION_MODEL'], 'Model', writable, decision_providers[decision.provider].models) }}
{% endif %} diff --git a/app/templates/traces/_judgement.html b/app/templates/traces/_judgement.html index e3780b0f..0c85e0b4 100644 --- a/app/templates/traces/_judgement.html +++ b/app/templates/traces/_judgement.html @@ -15,7 +15,7 @@ {% if j.keyed %} - {% else %}{{ 'LLM judge' if llm else 'Decision' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %} + {% else %}{{ 'LLM judge' if llm else 'Decision model' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %} {% if j.record and (j.record.refusals or j.earlier or j.record.truncated) %}{% set note = namespace(sep='') %}

{%- if j.record.refusals %}{{ j.record.refusals|length }} refused{% set note.sep = ' · ' %}{% endif -%} diff --git a/app/templates/traces/trial.html b/app/templates/traces/trial.html index 2b10480c..1220eb97 100644 --- a/app/templates/traces/trial.html +++ b/app/templates/traces/trial.html @@ -73,7 +73,7 @@

{{ trial.task_base }}

{% endif %} diff --git a/app/ui_tests/test_routes.py b/app/ui_tests/test_routes.py index fa0e15f2..3df718f0 100644 --- a/app/ui_tests/test_routes.py +++ b/app/ui_tests/test_routes.py @@ -292,7 +292,7 @@ def test_conversation_returns_a_bounded_page_with_full_text_search_reachable(sel def test_a_trial_has_two_judgements_in_two_tabs(self): with patch.dict(os.environ, {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": "sk-or-route-test", "LLM_JUDGE_MODEL": ""}): html = self.get_ok("/trial/active-run/task-001").get_data(as_text=True) - self.assertIn('id="tab-judgement" role="tab" type="button" data-tab="judgement"', html); self.assertIn(">Decision judge", html) + self.assertIn('id="tab-judgement" role="tab" type="button" data-tab="judgement"', html); self.assertIn(">Decision model", html) self.assertIn('id="tab-llm-judge"', html); self.assertIn(">LLM judge", html) self.assertIn('id="judgement"', html); self.assertIn('id="llm-judgement"', html) self.assertIn("LLM judge not configured: needs a model", html) # the OpenRouter key is set, the model is not diff --git a/app/web/judge.py b/app/web/judge.py index ac455336..95ab677d 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -301,7 +301,7 @@ def judge_job_route(job_id): k = key(lane) done, failed = run_batch(todo, lambda t: judge_trial(job, t, k), lambda t: t.trial_name) from flask import flash - flash(f"{'LLM judge' if lane == 'llm' else 'Decision'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") + flash(f"{'LLM judge' if lane == 'llm' else 'Decision model'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("job", job_id=job_id) + "#judgement") diff --git a/docs/judgement.md b/docs/judgement.md index 7db3a660..b7ef3368 100644 --- a/docs/judgement.md +++ b/docs/judgement.md @@ -9,7 +9,7 @@ Two judgements per trial, a tab each, records in `judgements/` and `judgements-l | [Decision](settings/decision.md) | A decision model's answers: a yes/no's probability, a choice's distribution, a score, each with the model's confidence when it gives one; one call per state | | [LLM judge](settings/llm-judge.md) | What a chat model states, one JSON object per state, no confidence | -The job, compare, feasibility and task-instruction pages ask Decision. A record names the provider and the model that answered, and the questions the model refused. The cost estimate for TypeSafe is calibrated on its own stored calls. +The job, compare, feasibility and task-instruction pages ask the Decision model. A record names the provider and the model that answered, and the questions the model refused. The cost estimate for TypeSafe is calibrated on its own stored calls. !!! note "A judge never changes an outcome, a count or a verdict" diff --git a/docs/settings/decision.md b/docs/settings/decision.md index f8ce9fe0..9c0098d2 100644 --- a/docs/settings/decision.md +++ b/docs/settings/decision.md @@ -56,7 +56,7 @@ your existing `settings.json` ([location and precedence](README.md#save)). ## Request a judgement -A trial's or a job's Decision tab. +A trial's or a job's Decision model tab. !!! warning "A judgement is a paid call" Configuration alone calls nothing; a batch button says how many calls it will make.