diff --git a/AGENTS.md b/AGENTS.md index 8a336c0c..be935542 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,7 +56,7 @@ and its resources. The source layout stays under `app/`. a Harbor benchmark added from the Benchmarks page's form; `citation`, an entry's BibTeX read into the paper its page shows; `storage`, whole-folder user overlays and restoring bundled entries; `sources`, where each dataset's data comes from for each way it runs), `recipes/` - (`launch`, the builder's vocabulary), `providers` (where a run's model calls go, and the builder's steps), + (`launch`, the builder's vocabulary), `providers` (the model provider registry: protocols, presets and each provider's record, read by every consumer; and the builder's steps), `harbor/` (`source`: the Harbor a recipe pins - built, kept, scanned for its agents; `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), diff --git a/app/.env.example b/app/.env.example index 7086609b..4fb3b953 100644 --- a/app/.env.example +++ b/app/.env.example @@ -24,26 +24,38 @@ # The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word 'local' as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log. # e.g. sk-… MODEL_API_KEY= -# The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL__NAME, MODEL__API_URL and MODEL__API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL. +# The base URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More custom providers, each with a name, are a record MODEL_ and a key MODEL__API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL. # e.g. https://…/v1 MODEL_API_URL= -# An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log. +# An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log. OPENAI_API_KEY= -# An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own endpoint, https://openrouter.ai/api/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log. +# An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log. OPENROUTER_API_KEY= -# An Anthropic key, for a run of the Nullius standard whose provider is Anthropic: its relay sends the calls to Anthropic's own endpoint, https://api.anthropic.com/v1, with this key. Only the relay holds it - never an agent, a recipe or a log. +# An Anthropic key, for a run of the Nullius standard whose agent speaks Anthropic's Messages, and for the LLM judge: the calls go to Anthropic's own base with this key. Only the relay and the judge hold it, never an agent, a recipe or a log. ANTHROPIC_API_KEY= -# TypeSafe.ai key for the System One judgement on TypeSafe's own endpoint, POST https://api.typesafe.ai/v1/systemone — sent as a bearer header there and nowhere else. +# A TypeSafe key, for the System One judgement on TypeSafe's own base: sent as a bearer header there and nowhere else. # e.g. ts-… TYPESAFE_API_KEY= -# Where the System One judgement is asked: typesafe (its own endpoint, with TYPESAFE_API_KEY) or openrouter (https://openrouter.ai/api/v1/systemone, the same contract, with OPENROUTER_API_KEY). Unset: TypeSafe when its key is set, else OpenRouter when its key is. +# The provider the System One judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is. SYSTEM_ONE_PROVIDER= -# The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev. +# The decision protocol System One is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves. +SYSTEM_ONE_PROTOCOL= +# The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default. SYSTEM_ONE_MODEL= -# The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local: (the endpoint MODEL__API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set. +# The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last. LLM_JUDGE_PROVIDER= +# The protocol the LLM judge is asked in, on a provider that serves both: chat (OpenAI Chat Completions) or messages (Anthropic Messages). Unset, or one the provider does not serve: the first the provider serves, chat before messages. +LLM_JUDGE_PROTOCOL= # The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence. LLM_JUDGE_MODEL= +# Additions to the OpenRouter provider, as a JSON object without the key: endpoints of protocols the preset does not list, {"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one. +OPENROUTER= +# Additions to the OpenAI provider, as a JSON object without the key: endpoints of protocols the preset does not list, {"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one. +OPENAI= +# Additions to the Anthropic provider, as a JSON object without the key: endpoints of protocols the preset does not list, {"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one. +ANTHROPIC= +# Additions to the TypeSafe provider, as a JSON object without the key: endpoints of protocols the preset does not list, {"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one. +TYPESAFE= # --- Run hosts --- # Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are. diff --git a/app/nullius/execution.py b/app/nullius/execution.py index c38811fa..283d7315 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -122,7 +122,7 @@ from datetime import datetime, timezone from typing import Callable, Dict, List, Optional -from nullius import hosts, paths, prices as price_list, progress, redact, secrets, settings +from nullius import hosts, paths, prices as price_list, progress, providers, redact, secrets, settings from nullius.harbor import source as harbor_source from nullius.standard import container, payload as standard_payload @@ -339,7 +339,7 @@ def withheld() -> List[str]: Custom keys - and GitHub tokens from saved settings or the shell Nullius was started in. Every named endpoint's settings too, so a run gets its own endpoint's key alone. A data provider's token name is not one.""" - secret = {v.name for v in settings.VARIABLES if v.secret or v.kind == "endpoint"} | set(settings.endpoint_names()) + secret = {v.name for v in settings.VARIABLES if v.secret or v.kind in ("endpoint", "provider")} | set(settings.endpoint_names()) return sorted(secret | {old for old, new in settings.INSTEAD.items() if new in secret} | set(KEY_NAMES + URL_NAMES) | {"GITHUB_TOKEN", "GH_TOKEN"} | set(settings.agent_key_names()) | set(_licence_names())) @@ -390,7 +390,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl self._run = run or (lambda argv, timeout=600.0, stdin=None: subprocess_run(argv, timeout, stdin, stop=self._stop)) # an endpoint of the run's own (url_var: an OpenAI-compatible one) may check no key: it is sent the placeholder self._key = (key if key is not None else env.get(plan.key_var, "")) or (settings.MODEL_KEY_PLACEHOLDER if plan.url_var else "") - self._url = url if url is not None else (env.get(plan.url_var, "") if plan.url_var else plan.base_url) + self._url = url if url is not None else (providers.base_of(plan.url_var, env) if plan.url_var else plan.base_url) if plan.keyless: # a run that calls no model is given nothing it could leak self._key, self._url = "", "" # the agent keys Settings holds (CURSOR_API_KEY, ...): a Harbor run's agent may need its own vendor's; the standard's never gets one diff --git a/app/nullius/judge/client.py b/app/nullius/judge/client.py index 8227da5f..7208038e 100644 --- a/app/nullius/judge/client.py +++ b/app/nullius/judge/client.py @@ -1,15 +1,17 @@ """The judge call: a state and typed questions, the answer as the record the -pages store. Two judges answer the same questions (:func:`configured` says -which one Settings chose): - -- **TypeSafe** (``POST https://api.typesafe.ai/v1/systemone``, model - jev-latest), whose answers are its own: a Noul's probability, a Choice's - distribution over its options, a Score with the judge's confidence. -- **OpenRouter** (``POST https://openrouter.ai/api/v1/chat/completions``), - any chat model named in Settings, asked to answer every question in one - JSON object. Its numbers are what the model states, not a calibrated - distribution, so a Choice or a Score carries no confidence; the record - names the provider and the model, and the pages say which judge spoke. +pages store. Two judges answer the same questions (``judge_for`` says which +one Settings chose); each asks a provider of the registry (``nullius.providers``) +that serves its protocol: + +- **System One**, a decision model on a decision protocol: ``systemone`` + (TypeSafe's Jev, on TypeSafe or OpenRouter), ``decisions`` (OpenRouter's, + for many makers' models) or ``openai-decisions``. Its answers are the + model's own: a Noul's probability, a Choice's distribution over its + options, a Score with the model's confidence. +- **The LLM judge**, any chat model named in Settings (``chat``, or Anthropic's + ``messages``), asked to answer every question in one JSON object. Its numbers + are what the model states, so a Choice or a Score carries no confidence; the + record names the provider and the model, and the pages say which judge spoke. The key goes in the header and nowhere else - not in argv, not in a log, not in the record. The transport is a call-time seam, so the suite asks a @@ -25,9 +27,10 @@ from datetime import datetime, timezone from typing import Callable, Optional +from nullius import providers from . import questions as Q -URL = "https://api.typesafe.ai/v1/systemone" +URL = providers.join(providers.PRESETS["typesafe"]["base"], providers.PRESETS["typesafe"]["endpoints"]["systemone"]) MODEL = "jev-latest" PROVIDER = "typesafe.ai" PRICE_PER_MTOK = 0.042 # USD per million input tokens; output tokens are free (docs.typesafe.ai/models) @@ -40,103 +43,157 @@ CALIBRATION_MIN_CHARS = 4_000 # a stored call this small says nothing about a large request's ratio: it is kept out of the band -# The LLM judge's providers: the chat endpoint and the Settings variables of its key (and, for an -# OpenAI-compatible endpoint, of its URL) - the keys and endpoints the model providers already use. -# A named endpoint is one more, local: (``llm_providers``). -LLM_PROVIDERS = { - "openrouter": {"label": "OpenRouter", "url": "https://openrouter.ai/api/v1/chat/completions", "key": "OPENROUTER_API_KEY"}, - "openai": {"label": "OpenAI", "url": "https://api.openai.com/v1/chat/completions", "key": "OPENAI_API_KEY"}, - "local": {"label": "OpenAI-compatible", "url_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, -} LANES = ("system-one", "llm") # the two judgements a trial can have, each with its own records -# Where System One is asked: TypeSafe's own endpoint, or OpenRouter's, which serves the same /v1/systemone contract -# (state and typed questions in, calibrated typed answers out) for TypeSafe's Jev models and others built to it. -SYSTEM_ONE_PROVIDERS = { - "typesafe": {"label": "TypeSafe", "url": URL, "key": "TYPESAFE_API_KEY", "model": MODEL, - "models": [MODEL]}, - "openrouter": {"label": "OpenRouter", "url": "https://openrouter.ai/api/v1/systemone", "key": "OPENROUTER_API_KEY", - "model": "~typesafe/jev-latest", - "models": ["~typesafe/jev-latest", "typesafe/jev-1.13", "typesafe/jev-router", "jaredpalmer/kev-4b"]}, -} +# The protocols each judge speaks, in its order of preference; the providers come from the registry. +LLM_PROTOCOLS = ("chat", "messages") +SYSTEM_ONE_PROTOCOLS = ("systemone", "decisions", "openai-decisions") +# System One's models as each provider names them: the first is its default. Another is typed in Settings. +SYSTEM_ONE_MODELS = {"typesafe": [MODEL], + "openrouter": ["~typesafe/jev-latest", "typesafe/jev-1.14", "typesafe/jev-1.13", "typesafe/jev-router", "jaredpalmer/kev-4b"], + "openai": ["gpt-6-luna"]} + + +def system_one_models(choice: str, fetch: bool = False) -> list: + """System One's models on a provider, its default first: on OpenRouter, its decision models + (``providers.decision_models``, cached; asked only when ``fetch``), TypeSafe's next; else, or with + none, ``SYSTEM_ONE_MODELS``.""" + listed = SYSTEM_ONE_MODELS.get(choice, []) + live = providers.decision_models(fetch=fetch) if choice == "openrouter" else [] + if not live: + return list(listed) + return [listed[0]] + sorted(set(live) - {listed[0]}, key=lambda m: ("typesafe/" not in m, m)) + + +def system_one_protocol(choice: str, model: str, env=os.environ) -> str: + """The decision protocol System One speaks on a provider: ``SYSTEM_ONE_PROTOCOL`` when the provider + lists it; else, on one that serves both ``systemone`` and ``decisions`` (OpenRouter), ``systemone`` + for TypeSafe's models and the ones listed for it, and ``decisions`` for another maker's; else the + first the provider lists.""" + rec = providers.record(choice, env) + listed = [p for p in SYSTEM_ONE_PROTOCOLS if providers.serves(rec, p)] + if (env.get("SYSTEM_ONE_PROTOCOL") or "").strip() in listed: + return env["SYSTEM_ONE_PROTOCOL"].strip() + typesafe = "typesafe/" in model or model.startswith("jev") or model in SYSTEM_ONE_MODELS.get(choice, []) + if "systemone" in listed and "decisions" in listed: + return "systemone" if typesafe else "decisions" + return listed[0] if listed else "systemone" + + +def _choice(raw) -> str: + """A provider choice as Settings saved it: ``local`` and ``local:`` are the old names of ``custom``.""" + kind, colon, endpoint = str(raw or "").strip().partition(":") + kind = "custom" if kind.lower() == "local" else kind.lower() + return kind + colon + endpoint.upper() + + +def _table(protocols, env, known: bool) -> dict: + """{choice: {label, url, key, ready, protocol, fit, listed}} of every provider that may serve one of + ``protocols``, ready or not (Settings shows one once its key is saved): ``fit`` the ones it may serve, + ``listed`` the ones it lists, ``protocol`` the first of them. ``known``: only those that list one.""" + out = {} + for rec in providers.records(env): + fit = [p for p in protocols if providers.serves(rec, p) or (providers.serves(rec, p) is None and not known)] + fit.sort(key=lambda p: not providers.serves(rec, p)) # a protocol it lists before one it may serve + if fit: + out[rec["id"]] = {"label": rec["name"], "url": providers.url(rec, fit[0]) if rec["base"] else "", + "key": rec["key"], "ready": providers.ready(rec, env), "protocol": fit[0], + "fit": fit, "listed": [p for p in fit if providers.serves(rec, p)]} + return out class Judge: """One judge: who answers, with what key, model and endpoint; true when it can be asked. Its repr never shows the key.""" - def __init__(self, provider: str, key: str, model: str, url: str = "", lane: str = "", name: str = ""): + def __init__(self, provider: str, key: str, model: str, url: str = "", lane: str = "", name: str = "", protocol: str = ""): self.provider, self.key, self.model, self.url, self.name = provider, key, model, url, name self.lane = lane or ("system-one" if provider == "typesafe" else "llm") + self.protocol = protocol or ("systemone" if self.lane == "system-one" else "chat") def __bool__(self) -> bool: if self.lane == "system-one": return bool(self.key) and bool(self.model) - return bool(self.model) and bool(self.url) and (bool(self.key) or self.provider.startswith("local")) + return bool(self.model) and bool(self.url) and (bool(self.key) or self.provider.startswith("custom")) def __repr__(self) -> str: return f"Judge({self.provider!r}, model={self.model!r}, key={'set' if self.key else 'unset'})" + @property + def who(self) -> str: + return self.name or providers.label(self.provider) + @property def label(self) -> str: - names = SYSTEM_ONE_PROVIDERS if self.lane == "system-one" else LLM_PROVIDERS - return f"{self.name or names.get(self.provider, {}).get('label', self.provider)} · {self.model or 'no model chosen'}" + return f"{self.who} · {self.model or 'no model chosen'}" @property def missing(self) -> str: """What Settings still needs for this judge, in a phrase; '' when it can be asked.""" if self.lane == "system-one": - return " and ".join(([f"the {SYSTEM_ONE_PROVIDERS[self.provider]['label']} key"] if not self.key else []) + return " and ".join(([f"the {self.who} key"] if not self.key else []) + (["a model"] if not self.model else [])) need = [] if not self.url: need.append("an endpoint (MODEL_API_URL)") - if not self.key and not self.provider.startswith("local"): - need.append(f"the {LLM_PROVIDERS[self.provider]['label']} key") + if not self.key and not self.provider.startswith("custom"): + need.append(f"the {self.who} key") if not self.model: need.append("a model") return " and ".join(need) +def system_one_providers(env=os.environ) -> dict: + """System One's providers: every one that lists one of its protocols, each with its models + (``system_one_models``, never asked here; the first, ``model``, its default; ``live`` when the + provider has a list to ask) and the protocol it would be asked in.""" + out = {k: dict(d, models=system_one_models(k), model=(SYSTEM_ONE_MODELS.get(k) or [""])[0], live=k == "openrouter") + for k, d in _table(SYSTEM_ONE_PROTOCOLS, env, known=True).items()} + for k, d in out.items(): + d["protocol"] = system_one_protocol(k, (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"], env) + return out + + def system_one(env=os.environ) -> Judge: - """The System One judgement: on ``SYSTEM_ONE_PROVIDER`` (typesafe, or - openrouter, which serves the same contract; unset, TypeSafe when its key - is set, else OpenRouter when its is), with that provider's key, and the - model ``SYSTEM_ONE_MODEL`` names or the provider's Jev.""" - chosen = (env.get("SYSTEM_ONE_PROVIDER") or "").strip().lower() - if chosen not in SYSTEM_ONE_PROVIDERS: + """The System One judgement: on ``SYSTEM_ONE_PROVIDER`` (unset, or one + that serves no System One protocol: TypeSafe when its key is set, else + OpenRouter when its is), with that provider's key, and the model + ``SYSTEM_ONE_MODEL`` names or the provider's default.""" + table = system_one_providers(env) + chosen = _choice(env.get("SYSTEM_ONE_PROVIDER")) + if chosen not in table: chosen = "openrouter" if not env.get("TYPESAFE_API_KEY") and env.get("OPENROUTER_API_KEY") else "typesafe" - d = SYSTEM_ONE_PROVIDERS[chosen] - return Judge(chosen, env.get(d["key"], ""), (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"], d["url"], lane="system-one") + d = table[chosen] + model = (env.get("SYSTEM_ONE_MODEL") or "").strip() or d["model"] + protocol = system_one_protocol(chosen, model, env) + return Judge(chosen, env.get(d["key"], ""), model, providers.url(providers.record(chosen, env), protocol), + lane="system-one", name=d["label"], protocol=protocol) def llm_providers(env=os.environ) -> dict: - """``LLM_PROVIDERS`` and one ``local:`` per named OpenAI-compatible endpoint whose URL is set.""" - from nullius import providers - out = dict(LLM_PROVIDERS) - for e in providers.endpoints(env): - if e["id"]: - url_var, key_var = providers.endpoint_vars(e["id"]) - out["local:" + e["id"]] = {"label": e["name"], "url_var": url_var, "key": key_var} + """The LLM judge's providers: every one that may serve ``chat`` or ``messages``, each with the protocol + it would be asked in: ``LLM_JUDGE_PROTOCOL`` when the provider may serve it, else the first it lists.""" + out = _table(LLM_PROTOCOLS, env, known=False) + explicit = (env.get("LLM_JUDGE_PROTOCOL") or "").strip() + for k, d in out.items(): + if explicit in d["fit"] and explicit != d["protocol"]: + d["protocol"], d["url"] = explicit, providers.url(providers.record(k, env), explicit) if d["url"] else "" return out def llm_judge(env=os.environ) -> Judge: """The LLM judgement: the model ``LLM_JUDGE_MODEL`` names, on the - provider ``LLM_JUDGE_PROVIDER`` names (unset, or an endpoint that is gone: - the first of OpenRouter, OpenAI, the OpenAI-compatible endpoints whose key - or endpoint is set), with that provider's key and endpoint from the model - providers' settings.""" + provider ``LLM_JUDGE_PROVIDER`` names (unset, or a provider that is gone: + the first that is set up, Anthropic last: a key set for runs does not move + a judge on a custom provider to it), with that provider's key and endpoint + from the registry.""" table = llm_providers(env) - kind, colon, endpoint = (env.get("LLM_JUDGE_PROVIDER") or "").strip().partition(":") - chosen = kind.lower() + colon + endpoint.upper() + chosen = _choice(env.get("LLM_JUDGE_PROVIDER")) if chosen not in table: - chosen = next((p for p, d in table.items() if env.get(d.get("url_var") or d["key"])), "openrouter") + chosen = next((p for p in sorted(table, key=lambda p: p == "anthropic") if table[p]["ready"]), "openrouter") d = table[chosen] - from nullius import settings - at = env.get(d.get("url_var", "")) or "" - url = d.get("url") or (at.rstrip("/") + "/chat/completions" if at and not settings.endpoint_url_problem(at) else "") - key = env.get(d["key"], "") or (settings_placeholder() if chosen.startswith("local") else "") - return Judge(chosen, key, (env.get("LLM_JUDGE_MODEL") or "").strip(), url, name=d["label"] if ":" in chosen else "") + url = d["url"] if d["ready"] or chosen in providers.PRESETS else "" # a custom base that is refused is no endpoint + key = env.get(d["key"], "") or (settings_placeholder() if chosen.startswith("custom") else "") + return Judge(chosen, key, (env.get("LLM_JUDGE_MODEL") or "").strip(), url, name=d["label"], protocol=d["protocol"]) def settings_placeholder() -> str: @@ -229,15 +286,20 @@ def backoff(attempt: int, data, rand: Callable = random.random) -> float: return 1.5 * (2 ** attempt) * (0.5 + rand()) -def http_post(url: str, body: dict, key: str, timeout: float = 120.0) -> tuple: - """(status, json) - the one place the key is used.""" +def http_post(url: str, body: dict, key: str, timeout: float = 120.0, auth: Optional[dict] = None) -> tuple: + """(status, json) - the one place the key is used: as a bearer header, or as ``auth``, the headers of + the protocol (``providers.auth``). No redirect is followed, so the key stays at its endpoint.""" data = json.dumps(body).encode("utf-8") req = urllib.request.Request(url, data=data, method="POST", - headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json", "Accept": "application/json"}) + headers={**(auth or {"Authorization": f"Bearer {key}"}), "Content-Type": "application/json", "Accept": "application/json"}) try: - with urllib.request.urlopen(req, timeout=timeout) as resp: + with urllib.request.build_opener(providers._NoRedirect()).open(req, timeout=timeout) as resp: return resp.status, json.loads(resp.read().decode("utf-8") or "{}") except urllib.error.HTTPError as e: + if 300 <= e.code < 400: + e.close() + raise JudgeError(e.code, f"the judge's endpoint redirects ({e.code}), and the key is not sent on: " + "set the endpoint's own URL in Settings") from None try: detail = json.loads(e.read().decode("utf-8") or "{}") except ValueError: @@ -251,26 +313,123 @@ def http_post(url: str, body: dict, key: str, timeout: float = 120.0) -> tuple: raise JudgeError(0, f"no answer from the judge: {getattr(e, 'reason', e)}") from e +def _said(data) -> str: + """A provider's own error message: ``{"error": {"code", "message"}}`` (OpenRouter, OpenAI) or ``{"error": "..."}``.""" + e = data.get("error") if isinstance(data, dict) else None + return str((e.get("message") if isinstance(e, dict) else e) or "")[:300] + + +def _number(v) -> Optional[float]: + return float(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None + + +def _refusal(a) -> Optional[str]: + """Why a model refused a question (a ``refusal`` answer), or None when ``a`` is an answer. OpenAI-style + replies send ``"refusal": null`` beside an answer: that is no refusal.""" + if isinstance(a, dict) and (a.get("type") == "refusal" or a.get("refusal")): + return str(a.get("refusal") or a.get("reason") or "refused")[:300] + return None + + +def _typesafe_answers(data: dict, asked: dict) -> tuple: + """(answers, refusals) of a ``systemone`` or ``decisions`` reply: ``answers`` keyed by question id.""" + answers, refusals = {}, {} + for qid, a in (data.get("answers") or {}).items(): + if qid not in asked or not isinstance(a, dict): + continue + if _refusal(a) is not None: + refusals[qid] = _refusal(a) + continue + qtype = asked[qid]["type"] + if qtype == "noul" and _number(a.get("noul")) is not None: + answers[qid] = {"noul": float(a["noul"]), **({"confidence": _number(a["confidence"])} if _number(a.get("confidence")) is not None else {})} + elif qtype == "choice" and isinstance(a.get("choice"), str): + answers[qid] = {"choice": a["choice"], "probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()}, + "confidence": _number(a.get("confidence"))} + elif qtype == "score" and _number(a.get("score")) is not None: + legend = a.get("legend") + answers[qid] = {"score": float(a["score"]), "legend": list(asked[qid]["criteria"]) if not isinstance(legend, dict) else [legend[k] for k in sorted(legend, key=lambda x: int(x))], + "probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()}, + "confidence": _number(a.get("confidence"))} + return answers, refusals + + +def openai_questions(asked: dict) -> list: + """The questions as OpenAI's Decisions takes them: an array, each with its ``name``. A yes/no question + is a ``predicate``, which has no field for criteria: they fold into its instructions. A choice lists + ``choices`` and a score ``levels``. A box question has no form there: it is not sent.""" + out = [] + for qid, q in asked.items(): + crit = q.get("criteria") + if q["type"] == "noul": + fold = f" True means: {crit.get('true', '')}. False means: {crit.get('false', '')}." if isinstance(crit, dict) else "" + out.append({"name": qid, "type": "predicate", "instructions": q["instructions"] + fold}) + elif q["type"] == "choice" and isinstance(crit, dict): + out.append({"name": qid, "type": "choice", "instructions": q["instructions"], + "choices": [{"value": k, "description": v} for k, v in crit.items()]}) + elif q["type"] == "score" and isinstance(crit, list): + out.append({"name": qid, "type": "score", "instructions": q["instructions"], + "levels": [{"label": str(i), "description": v} for i, v in enumerate(crit)]}) + return out + + +def _listed(rows, key: str) -> dict: + """``[{key: k, "probability": p}]`` as {k: p}, or a dict as it is.""" + if isinstance(rows, dict): + return {str(k): float(v) for k, v in rows.items() if _number(v) is not None} + return {str(r[key]): float(r["probability"]) for r in rows or [] if isinstance(r, dict) and key in r and _number(r.get("probability")) is not None} + + +def _openai_answers(data: dict, asked: dict) -> tuple: + """(answers, refusals) of an ``openai-decisions`` reply: an array of answers by ``name``, probabilities as + lists, read into the record's shape. An answer outside its question's choices or levels is left out.""" + answers, refusals = {}, {} + for a in data.get("answers") or []: + qid = a.get("name") if isinstance(a, dict) else None + if qid not in asked: + continue + if _refusal(a) is not None: + refusals[qid] = _refusal(a) + continue + q, crit = asked[qid], asked[qid].get("criteria") + p = _number(a.get("probability")) + if q["type"] == "noul" and p is not None and 0 <= p <= 1: + answers[qid] = {"noul": p} + elif q["type"] == "choice" and isinstance(crit, dict) and a.get("choice") in crit: + answers[qid] = {"choice": a["choice"], "probabilities": _listed(a.get("probabilities"), "value"), "confidence": _number(a.get("confidence"))} + elif q["type"] == "score" and isinstance(crit, list): + level = a.get("level", a.get("score")) + index = int(level) if str(level).isdigit() else None + if index is not None and 0 <= index < len(crit): + answers[qid] = {"score": float(index), "legend": list(crit), "probabilities": _listed(a.get("probabilities"), "label"), + "confidence": _number(a.get("confidence"))} + return answers, refusals + + def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Callable] = None, model: str = MODEL, retries: int = 3, sleep: Callable = time.sleep, kind: Optional[str] = None, questions: Optional[dict] = None) -> dict: """One judge call over one state for the questions named; the record. - 429, 529 and 503 are retried with backoff; 401 and 422 are said plainly. + 429, 529 and 503 are retried with backoff; 401, 402 and 422 are said plainly. `kind` names the state's kind in the record's `states`. `questions` is a prepared map in the API shape (type, instructions, criteria), sent instead of the vocabulary's entries for `ids`; each answer is parsed by the type its question declares. `key` is a TypeSafe key, or - the :class:`Judge` Settings chose - an OpenRouter one is asked there.""" - url, provider = URL, PROVIDER + the :class:`Judge` Settings chose, asked in its protocol: ``systemone`` + and ``decisions`` share one shape, ``openai-decisions`` has its own. A + refused question is recorded in ``refusals``, never as an answer; the + provider's own cost (``usage.cost``) is kept when it gives one.""" + url, provider, protocol, who = URL, PROVIDER, "systemone", "TypeSafe" if isinstance(key, Judge): if key.lane == "llm": return ask_chat(state, ids, key, truncated=truncated, post=post, retries=retries, sleep=sleep, kind=kind, questions=questions) - url, model, provider = key.url, key.model, (PROVIDER if key.provider == "typesafe" else key.provider) + url, model, provider, protocol, who = key.url, key.model, (PROVIDER if key.provider == "typesafe" else key.provider), key.protocol, key.who key = key.key if not key: raise JudgeError(401, "no TypeSafe.ai key: set TYPESAFE_API_KEY in Settings, Judge" if provider == PROVIDER - else f"no {provider} key for System One: set it in Settings, Model providers") + else f"no {who} key for System One: set it in Settings, Model providers") asked = questions if questions is not None else Q.request_questions(ids) - body = {"state": state, "model": model, "questions": asked} + openai = protocol == "openai-decisions" + body = {"model": model, "input": state, "questions": openai_questions(asked)} if openai else {"state": state, "model": model, "questions": asked} post = post or http_post status, data = 0, {} for attempt in range(retries + 1): @@ -279,33 +438,28 @@ def ask(state: str, ids, key: str, truncated: bool = False, post: Optional[Calla sleep(backoff(attempt, data)) continue break + said = _said(data) if status == 401: raise JudgeError(401, "the judge refused the key (401)") + if status == 402: + raise JudgeError(402, f"{who} asks for payment (402): the key's credit is spent, or the model is not free" + (f": {said}" if said else "")) if status == 422: - raise JudgeError(422, "the judge refused the request (422): " + json.dumps(data)[:300]) + raise JudgeError(422, "the judge refused the request (422): " + (said or json.dumps(data)[:300])) if status in (429, 503, 529): raise JudgeError(status, f"the judge is busy ({status}); try again in a moment") if status != 200: - raise JudgeError(status, f"the judge answered {status}") - answers = {} - for qid, a in (data.get("answers") or {}).items(): - if qid not in asked or not isinstance(a, dict): - continue - qtype = asked[qid]["type"] - if qtype == "noul" and isinstance(a.get("noul"), (int, float)): - answers[qid] = {"noul": float(a["noul"])} - elif qtype == "choice" and isinstance(a.get("choice"), str): - answers[qid] = {"choice": a["choice"], "probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()}, - "confidence": float(a["confidence"]) if isinstance(a.get("confidence"), (int, float)) else None} - elif qtype == "score" and isinstance(a.get("score"), (int, float)): - legend = a.get("legend") - answers[qid] = {"score": float(a["score"]), "legend": list(asked[qid]["criteria"]) if not isinstance(legend, dict) else [legend[k] for k in sorted(legend, key=lambda x: int(x))], - "probabilities": {k: float(v) for k, v in (a.get("probabilities") or {}).items()}, - "confidence": float(a["confidence"]) if isinstance(a.get("confidence"), (int, float)) else None} - usage = data.get("usage") or {} - rec = {"questions": Q.VERSION, "judge": {"provider": provider, "model": str(data.get("model") or model)}, - "asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated), - "answers": answers, "usage": {"input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens")}} + raise JudgeError(status, f"the judge answered {status}" + (f": {said}" if said else "")) + try: + answers, refusals = (_openai_answers if openai else _typesafe_answers)(data, asked) + usage = data.get("usage") or {} + rec = {"questions": Q.VERSION, "judge": {"provider": provider, "model": str(data.get("model") or model)}, + "asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated), + "answers": answers, "usage": {"input_tokens": usage.get("input_tokens"), "output_tokens": usage.get("output_tokens"), + **({"cost_usd": float(usage["cost"])} if _number(usage.get("cost")) is not None else {})}} + except (AttributeError, KeyError, TypeError, ValueError) as e: # a reply of another shape: said, never a crash of the batch + raise JudgeError(502, f"the judge's reply could not be read: {e.__class__.__name__}") from e + if refusals: + rec["refusals"] = refusals if kind is not None: # `request_chars` is what the judge tokenised: the state and the questions' JSON. A calibration reads it, # so a tiny state whose request is mostly questions does not read as a low characters-per-token ratio. @@ -354,21 +508,26 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio record, in the same shape as TypeSafe's. Each answer is checked against its question: a probability in 0..1, a choice among its options, a score among its levels; one that is not is left out, as an unanswered question is.""" - who = judge.name or LLM_PROVIDERS.get(judge.provider, {}).get("label", judge.provider) + who = judge.who if not judge: raise JudgeError(401 if not judge.key else 400, f"the LLM judge is not ready: it needs {judge.missing} (Settings, Judge)") asked = questions if questions is not None else Q.request_questions(ids) - body = {"model": judge.model, "messages": openrouter_prompt(state, asked), "temperature": 0, - "response_format": {"type": "json_object"}, **({"usage": {"include": True}} if judge.provider == "openrouter" else {})} + prompt = openrouter_prompt(state, asked) + messages = judge.protocol == "messages" + if messages: # Anthropic's Messages: the rules as its system prompt, no JSON mode + body = {"model": judge.model, "max_tokens": 4096, "temperature": 0, "system": prompt[0]["content"], "messages": prompt[1:]} + else: + body = {"model": judge.model, "messages": prompt, "temperature": 0, + "response_format": {"type": "json_object"}, **({"usage": {"include": True}} if judge.provider == "openrouter" else {})} post = post or http_post status, data = 0, {} for attempt in range(retries + 1): - status, data = post(judge.url, body, judge.key) + status, data = post(judge.url, body, judge.key, auth=providers.auth("messages", judge.key)) if messages else post(judge.url, body, judge.key) if status in (429, 502, 503, 529) and attempt < retries: sleep(backoff(attempt, data)) continue break - said = ((data.get("error") or {}).get("message") if isinstance(data.get("error"), dict) else "") or "" + said = _said(data) if status == 401: raise JudgeError(401, f"{who} refused the key (401)") if status == 402: @@ -380,7 +539,7 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio if status != 200: raise JudgeError(status, f"{who} answered {status}" + (f": {said}" if said else "")) try: - content = data["choices"][0]["message"]["content"] + content = "".join(b.get("text", "") for b in data["content"] if b.get("type") == "text") if messages else data["choices"][0]["message"]["content"] got = _json_of(content).get("answers") or {} except (KeyError, IndexError, TypeError, ValueError, AttributeError) as e: raise JudgeError(502, f"{judge.model} did not answer in the JSON asked for ({e.__class__.__name__})") from e @@ -400,13 +559,14 @@ def ask_chat(state: str, ids, judge: Judge, truncated: bool = False, post: Optio elif q["type"] == "score" and isinstance(crit, list) and isinstance(a.get("score"), (int, float)) and 0 <= a["score"] <= len(crit) - 1: answers[qid] = {"score": float(a["score"]), "legend": list(crit), "probabilities": {}, "confidence": None} usage = data.get("usage") or {} + tokens_in, tokens_out = ("input_tokens", "output_tokens") if messages else ("prompt_tokens", "completion_tokens") rec = {"questions": Q.VERSION, "judge": {"provider": judge.provider, "model": str(data.get("model") or judge.model)}, "asked_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "state_chars": len(state), "truncated": bool(truncated), - "answers": answers, "usage": {"input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), + "answers": answers, "usage": {"input_tokens": usage.get(tokens_in), "output_tokens": usage.get(tokens_out), **({"cost_usd": usage["cost"]} if isinstance(usage.get("cost"), (int, float)) else {})}} if kind is not None: - rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get("prompt_tokens"), - "request_chars": sum(len(m["content"]) for m in body["messages"])}} + rec["states"] = {kind: {"chars": len(state), "truncated": bool(truncated), "input_tokens": usage.get(tokens_in), + "request_chars": sum(len(m["content"]) for m in prompt)}} return rec @@ -426,6 +586,8 @@ def merge(records) -> Optional[dict]: merged["truncated"] = merged["truncated"] or rec["truncated"] merged["states"].update(rec.get("states") or {}) merged["hygiene"].update(rec.get("hygiene") or {}) # per state kind, like `states` + if rec.get("refusals"): + merged["refusals"] = {**(merged.get("refusals") or {}), **rec["refusals"]} merged["asked_at"] = rec["asked_at"]; merged["judge"] = rec["judge"] for k in ("input_tokens", "output_tokens", "cost_usd"): if rec["usage"].get(k) is not None: diff --git a/app/nullius/migrate.py b/app/nullius/migrate.py index 7ccf3ded..99316158 100644 --- a/app/nullius/migrate.py +++ b/app/nullius/migrate.py @@ -238,12 +238,57 @@ def mkdir(parent): os.unlink(temporary) +def provider_records(values: dict) -> dict: + """The changes that move each custom provider's old name and URL settings (MODEL__NAME, + MODEL__API_URL) into its record, MODEL_: {setting: new value or None}. A field the record + already holds wins, and the old setting goes. Keys keep their names.""" + changes = {} + for i in settings.endpoint_ids(values): + old = {f: values.get(f"MODEL_{i}_{part}") for f, part in (("name", "NAME"), ("base", "API_URL"))} + if not any(v is not None for v in old.values()): + continue + rec = settings.json_object(values.get(f"MODEL_{i}")) or {} + rec = {**{f: v for f, v in old.items() if v}, **rec} + changes[f"MODEL_{i}"] = json.dumps(rec, ensure_ascii=False) + changes.update({f"MODEL_{i}_{part}": None for part in ("NAME", "API_URL") if f"MODEL_{i}_{part}" in values}) + return changes + + +def migrate_providers(apply_it: bool) -> int: + """``--providers``: preview, or with ``apply_it`` write, the rewrite of the old provider settings in the + user's settings.json. The environment is the user's: it is never read or changed here.""" + target = paths.settings_file() + try: + changes = provider_records(settings.in_file(target)) + bad = [f"{n} must be {why}." for n, v in changes.items() if v and (why := settings.endpoint_variable(n).validate(v))] + if bad: + raise ValueError(" ".join(bad)) + if apply_it and changes: + settings.save_many(target, changes) + except (OSError, ValueError) as error: + print(f"Migration stopped: {error}", file=sys.stderr) + return 1 + print("Rewritten." if apply_it else "Preview only. Add --apply to rewrite.") + for name, value in changes.items(): + print(f"{name}: {'removed' if value is None else 'written'}") + if not changes: + print("No old provider settings.") + print(f"settings: {target}") + return 0 + + def main(argv=None) -> int: parser = argparse.ArgumentParser(description="Copy legacy settings, custom catalogue entries and workspaces; preview by default.") - parser.add_argument("--from", dest="checkout", type=Path, required=True, help="existing Nullius checkout (left unchanged)") + parser.add_argument("--from", dest="checkout", type=Path, help="existing Nullius checkout (left unchanged)") parser.add_argument("--secrets", type=Path, help="also copy a legacy benchmark licence directory") + parser.add_argument("--providers", action="store_true", + help="rewrite the old provider settings of the user's settings.json into provider records instead") parser.add_argument("--apply", action="store_true", help="write the previewed copy; refuse conflicting destination data") args = parser.parse_args(argv) + if args.providers: + return migrate_providers(args.apply) + if args.checkout is None: + parser.error("--from is required, unless --providers is given") try: pending, counts = prepare(args.checkout, args.secrets) if args.apply: diff --git a/app/nullius/onboarding.py b/app/nullius/onboarding.py index 5af93cfd..92510219 100644 --- a/app/nullius/onboarding.py +++ b/app/nullius/onboarding.py @@ -22,7 +22,6 @@ from nullius.standard import container CONTROLS = ("nop", "oracle") -PROVIDER_KEYS = ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "MODEL_API_URL") SECTIONS = [ {"id": "setup", "name": "Set up", "page": "settings_page", "where": "Settings", "subs": [ @@ -85,7 +84,7 @@ def facts(env, jobs: Iterable[dict] = (), judged: bool = False, docker: Optional agents = [(j.get("agent") or "").lower() for j in jobs] found = (docker or docker_answers)() return { - "provider": any((env.get(k) or "").strip() for k in PROVIDER_KEYS) or bool(providers.endpoints(env)), + "provider": bool(providers.available(env)), # a run provider: a TypeSafe key serves the judge only "where": bool((env.get("HOSTS") or "").strip()) or found, "judge": bool(client.system_one(env)) or bool(client.llm_judge(env)), "control_run": any(a in CONTROLS for a in agents), diff --git a/app/nullius/providers.py b/app/nullius/providers.py index 22c65bf3..e26eb167 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -1,12 +1,12 @@ -"""Where a run's model calls go, and the builder's steps. - -The builder's three providers (command-builder-core.js, ``PROVIDERS``), named -the same here: what each is called, where it is served, the setting its key is -read from, and what makes it one a run can start on here. The third, -``custom``, is any number of OpenAI-compatible endpoints: the one at -MODEL_API_URL, and one per id in Settings (``settings.endpoint_ids``). A -recipe names one by ``provider: custom`` and its ``endpoint`` id; the builder -and the judge pickers by a choice, ``custom:`` (``choice``, ``split``). Also the builder's +"""Where model calls go: the one definition of each model provider, and the builder's steps. + +Every feature reads the registry below (``record``, ``records``, ``serves``) and asks for the +protocol it speaks: a run what its agent speaks, the LLM judge ``chat`` or ``messages``, System One +a decision protocol. A provider is a preset (``PRESETS``) or a custom provider, ``custom``: the +OpenAI-compatible endpoint at MODEL_API_URL, and one per id in Settings (``settings.endpoint_ids``). +A recipe names one by ``provider`` (``PROVIDERS``, the ones a run is sent to) and, for a custom one, +its ``endpoint`` id; the builder and the judge pickers by a choice, ``custom:`` (``choice``, +``split``). Also the builder's step numbers, so an error from a compiled recipe names the step that edits it. Settings' Test (``test``) asks a provider with free GETs only, and keeps the last result per choice (``tests``, ``remember``) for its card and Review. @@ -24,10 +24,201 @@ import urllib.error import urllib.parse import urllib.request -from typing import List +from typing import List, Optional from nullius import paths, settings +# --- the registry: protocols, presets and each provider's record ------------------------------------ +# +# A protocol is a wire format, defined once here: its default path below a provider's base, how the key is +# sent (``auth``: a bearer ``authorization`` header, or ``x-api-key`` with ``anthropic-version``) and what +# it is called. The consumer picks the protocol it speaks; a provider never enforces one. +PROTOCOLS = { + "chat": {"label": "OpenAI Chat Completions", "path": "/chat/completions", "auth": "authorization"}, + "responses": {"label": "OpenAI Responses", "path": "/responses", "auth": "authorization"}, + "messages": {"label": "Anthropic Messages", "path": "/messages", "auth": "x-api-key"}, + "systemone": {"label": "TypeSafe System One", "path": "/systemone", "auth": "authorization"}, + "decisions": {"label": "OpenRouter Decisions", "path": "/decisions", "auth": "authorization"}, + "openai-decisions": {"label": "OpenAI Decisions", "path": "/decisions", "auth": "authorization"}, +} +# A preset is a provider record whose name, base, key and known endpoints are fixed here. The user adds +# only endpoints of other protocols, in the setting ``setting`` (``settings.is_record``); its key is +# the secret setting ``key``. Its ``endpoints`` are also the protocols it serves. +PRESETS = { + "openrouter": {"name": "OpenRouter", "base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY", + "endpoints": {"chat": "/chat/completions", "responses": "/responses", "systemone": "/systemone", + "decisions": "https://openrouter.ai/api/alpha/decisions"}}, + "openai": {"name": "OpenAI", "base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY", + "endpoints": {"chat": "/chat/completions", "responses": "/responses", "openai-decisions": "/decisions"}}, + "anthropic": {"name": "Anthropic", "base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", + "endpoints": {"messages": "/messages"}}, + "typesafe": {"name": "TypeSafe", "base": "https://api.typesafe.ai/v1", "key": "TYPESAFE_API_KEY", + "endpoints": {"systemone": "/systemone"}}, +} +FIELDS = ("name", "base", "key", "endpoints") + + +def setting_of(choice: str) -> str: + """The setting that holds a provider's record: a preset's name in capitals, ``MODEL_`` for a custom + provider, "" for the OpenAI-compatible endpoint at MODEL_API_URL, which has its old settings only.""" + p, e = split(choice) + return p.upper() if p in PRESETS else f"MODEL_{e}" if p == "custom" and e else "" + + +def key_setting(choice: str) -> str: + """The secret setting a provider's key is read from. Never the key.""" + p, e = split(choice) + return PRESETS[p]["key"] if p in PRESETS else endpoint_vars(e)[1] + + +def _entry_problem(entry, host: str = "") -> Optional[str]: + """Why ``entry`` cannot be an endpoint: a path starting with one slash, without a ``..`` segment, or a + full URL that passes the checks a base gets; on ``host`` only, when one is given (a preset's own).""" + if not isinstance(entry, str): + return "a path or a URL" + if entry.startswith(("http://", "https://")): + if host and urllib.parse.urlsplit(entry).hostname != host: + return f"a path, or a URL on {host}: the preset's key is sent there" + return settings.endpoint_url_problem(entry) + if not entry.startswith("/") or entry.startswith("//") or any(c.isspace() or ord(c) < 32 for c in entry): + return "a path that starts with / or an http(s) URL" + if ".." in entry.split("/"): + return "a path without a .. segment" + return settings.endpoint_url_problem("http://x" + entry) + + +def record_problem(name: str, value: str) -> Optional[str]: + """Why ``value`` cannot be the record setting ``name``, or None. A JSON object of ``FIELDS``: a name, a + base the checks of an endpoint's URL pass, the key's setting name (only its own: never a key), and + endpoints by protocol, each a path, a full URL or null. A preset's fields and known endpoints are fixed; + a full URL it adds is on its own host, the one its key is for.""" + rec = settings.json_object(value) + if rec is None: + return 'a JSON object, such as {"base": "https://…/v1"}' + preset = next((p for p in PRESETS if p.upper() == name), "") + if set(rec) - set(FIELDS): + return "an object of " + ", ".join(FIELDS) + fixed = [k for k in ("name", "base", "key") if preset and k in rec] + if fixed: + return f"a record without {', '.join(fixed)}: they are fixed for {PRESETS[preset]['name']}" + if rec.get("name") is not None and (not isinstance(rec["name"], str) or not 0 < len(rec["name"]) <= 400 # as MODEL__NAME was + or any(ord(c) < 32 for c in rec["name"])): + return "a record whose name is one line of 400 characters at most" + if rec.get("base") is not None and (not isinstance(rec["base"], str) or settings.endpoint_url_problem(rec["base"])): + return f"a record whose base is {settings.endpoint_url_problem(str(rec['base']))}" + if rec.get("key") is not None and rec["key"] != f"{name}_API_KEY": + return f"a record whose key is the name of its key setting, {name}_API_KEY, or none" + ends = rec.get("endpoints") + if ends is None: + return None + if not isinstance(ends, dict): + return "a record whose endpoints are an object by protocol" + for proto, entry in ends.items(): + if proto not in PROTOCOLS: + return f"a record whose endpoints are some of {', '.join(PROTOCOLS)}" + if preset and proto in PRESETS[preset]["endpoints"]: + return f"a record without the {proto} endpoint: it is fixed for {PRESETS[preset]['name']}" + why = entry is not None and _entry_problem(entry, urllib.parse.urlsplit(PRESETS[preset]["base"]).hostname if preset else "") + if why: + return f"a record whose {proto} endpoint is {why}" + return None + + +def join(base: str, entry: str) -> str: + """An endpoint's URL: a full URL as it is, a path appended to ``base`` with exactly one slash between + them (never resolved as a relative URL, which would drop the base's own path).""" + if entry.startswith(("http://", "https://")): + return entry + return base.rstrip("/") + "/" + entry.lstrip("/") if base else "" + + +def record(choice: str, env=None) -> dict: + """A provider's record, the three layers merged field by field (the later wins): the preset in code, + the file's and then the environment's record setting (``settings.load`` merged those two). A custom + provider reads its old settings where its record says nothing (a null says: none): MODEL__NAME as ``name``, + MODEL__API_URL (MODEL_API_URL, unnamed) as ``base``; ``ignored`` names an old setting the record + overrides. ``{id, name, base, key, endpoints, preset, ignored}``, where ``key`` is a setting name only.""" + env = os.environ if env is None else env + p, e = split(choice) + preset = PRESETS.get(p) + own = settings.json_object(env.get(setting_of(choice))) if setting_of(choice) else None + if own is not None and record_problem(setting_of(choice), json.dumps(own)): + own = None # a record that does not validate is no record: Settings says why + rec = settings.merge_patch({k: v for k, v in (preset or {}).items()}, own or {}) + if preset: + rec.update({k: preset[k] for k in ("name", "base", "key")}, + endpoints={**(rec.get("endpoints") or {}), **preset["endpoints"]}) + ignored = [] + if not preset: + old_name, old_url = (f"MODEL_{e}_NAME" if e else ""), endpoint_vars(e)[0] + for field, old in (("name", old_name), ("base", old_url)): + if old and env.get(old): + if field in (own or {}): # the record sets the field, or removes it with null + ignored.append(old) + else: + rec[field] = env[old] + rec.setdefault("name", e or settings.DEFAULT_ENDPOINT_NAME) + rec["key"] = key_setting(choice) + return {"id": choice if preset or e else "custom", "name": rec.get("name") or "", "base": rec.get("base") or "", + "key": rec["key"], "endpoints": {k: v for k, v in (rec.get("endpoints") or {}).items() if v}, + "preset": bool(preset), "ignored": ignored} + + +# What a base on a preset's host also serves, its endpoints, suggested on Settings' Endpoints rows: a hint, +# never saved unasked. +KNOWN_HOSTS = {urllib.parse.urlsplit(d["base"]).hostname: {p: urllib.parse.urlsplit(join(d["base"], e)).path for p, e in d["endpoints"].items()} + for d in PRESETS.values()} + + +def suggestions(base: str) -> dict: + """{protocol: full URL} a base on a known host suggests (``KNOWN_HOSTS``); {} for another host.""" + parts = urllib.parse.urlsplit(base or "") + host = parts.netloc.rpartition("@")[2] # never a user or password the base may hold + return {p: f"{parts.scheme}://{host}{path}" for p, path in KNOWN_HOSTS.get(parts.hostname or "", {}).items()} + + +def records(env=None) -> List[dict]: + """Every provider: the presets, then the OpenAI-compatible endpoint at MODEL_API_URL when it is set, then + each custom provider by name.""" + env = os.environ if env is None else env + custom = sorted((record(choice("custom", i), env) for i in settings.endpoint_ids(env)), key=lambda r: r["name"].casefold()) + return [record(p, env) for p in PRESETS] + ([record("custom", env)] if env.get("MODEL_API_URL") else []) + custom + + +def url(rec: dict, protocol: str) -> str: + """Where ``rec`` is asked in ``protocol``: its endpoint for it, else its base and the protocol's default path.""" + return join(rec["base"], rec["endpoints"].get(protocol) or PROTOCOLS[protocol]["path"]) + + +def serves(rec: dict, protocol: str) -> Optional[bool]: + """True when ``rec`` lists the protocol, False when a preset does not, None (not known) for a custom + provider that does not say: a hint for pickers to sort and warn by.""" + return True if protocol in rec["endpoints"] else False if rec["preset"] else None + + +def ready(rec: dict, env=None) -> bool: + """Whether a provider can be asked here: a preset whose key is set, a custom provider whose base is a + URL that holds no credential (its key is optional).""" + env = os.environ if env is None else env + return bool(env.get(rec["key"])) if rec["preset"] else bool(rec["base"]) and not settings.endpoint_url_problem(rec["base"]) + + +def protocol_for(choice: str, apis, env=None) -> str: + """The protocol a consumer speaking ``apis`` (in its order of preference) uses on a provider: the first + the provider lists, else the first it may serve, else the first.""" + rec = record(choice, env) + apis = list(apis) or ["chat"] + return next((a for a in apis if serves(rec, a)), next((a for a in apis if serves(rec, a) is not False), apis[0])) + + +def auth(protocol: str, key: str) -> dict: + """The headers that carry ``key`` in ``protocol``: a bearer header, or Anthropic's ``x-api-key``.""" + if PROTOCOLS[protocol]["auth"] == "x-api-key": + from nullius.standard.relay import ANTHROPIC_VERSION + return {"x-api-key": key, "anthropic-version": ANTHROPIC_VERSION} + return {"Authorization": "Bearer " + key} if key else {} + + # The builder's steps (command-builder-core.js, STEP): an error names the step that edits it. STEP = {"benchmark": 0, "runner": 1, "dataset": 2, "setup": 3, "agent": 4, "model": 5, "limits": 6, "where": 7, "review": 8} @@ -41,19 +232,9 @@ GONE = ("The Native runner is no longer offered: the benchmark's own code runs as its Original setup " "on the Nullius runner.") -PROVIDERS = ("openrouter", "openai", "custom") -PROVIDER_LABEL = {"openrouter": "OpenRouter", "openai": "OpenAI", "custom": settings.DEFAULT_ENDPOINT_NAME} -# Where a provider is served: OpenRouter and OpenAI at their own endpoint, -# fixed; an OpenAI-compatible endpoint has no default and must be named -# (Settings: MODEL_API_URL, or an endpoint's own). -PROVIDER_BASE = {"openrouter": "https://openrouter.ai/api/v1", "openai": "https://api.openai.com/v1", "custom": ""} -# The setting a run's key is read from, per provider (web/execute.py, -# credentials_for): a key of its own for OpenRouter and for OpenAI, the -# endpoint's key for an OpenAI-compatible one. -PROVIDER_KEY = {"openrouter": "OPENROUTER_API_KEY", "openai": "OPENAI_API_KEY", "custom": "MODEL_API_KEY"} -# What makes a provider one a run can start on here: its key, for OpenRouter -# and OpenAI; its endpoint, for an OpenAI-compatible one, whose key is optional. -PROVIDER_NEEDS = dict(PROVIDER_KEY, custom="MODEL_API_URL") +# The providers a run is sent to, in the order the builder prefers them. A run asks for what its agent +# speaks, so the builder offers each only where its protocols fit (``offered``). +PROVIDERS = ("openrouter", "openai", "anthropic", "custom") def endpoint_vars(endpoint: str = "") -> tuple: @@ -63,6 +244,14 @@ def endpoint_vars(endpoint: str = "") -> tuple: return ("MODEL_API_URL", "MODEL_API_KEY") if not endpoint else (f"MODEL_{endpoint}_API_URL", f"MODEL_{endpoint}_API_KEY") +def base_of(url_var: str, env=None) -> str: + """The base a run's ``url_var`` names (``execution.Plan``): a custom provider's, from its record when it + has one (MODEL__API_URL is its old setting), else the variable's own value.""" + env = os.environ if env is None else env + i = url_var[6:-8] if url_var.startswith("MODEL_") and url_var.endswith("_API_URL") else "" + return record(choice("custom", i), env)["base"] if i and settings.ENDPOINT_ID.match(i) else env.get(url_var, "") + + def choice(provider: str, endpoint: str = "") -> str: """A provider as the builder and the judge pickers offer it: ``custom:`` for a named endpoint.""" return f"custom:{endpoint}" if provider == "custom" and endpoint else provider @@ -74,38 +263,46 @@ def split(value: str) -> tuple: return (p, e) if p == "custom" and settings.ENDPOINT_ID.match(e) else (str(value or ""), "") -def _with_url(env) -> List[tuple]: - """(id, URL) of each OpenAI-compatible endpoint whose URL is set; "" is MODEL_API_URL's.""" - return [(e, env[endpoint_vars(e)[0]]) for e in ("", *settings.endpoint_ids(env)) if env.get(endpoint_vars(e)[0])] +def _custom(env) -> List[dict]: + """The custom providers whose base is set: the one at MODEL_API_URL first, then the named ones by name.""" + return [r for r in records(env) if not r["preset"] and r["base"]] def endpoints(env=os.environ) -> List[dict]: - """The OpenAI-compatible endpoints a run can be sent to, each whose URL is set and - holds no credential (``broken``): the one at MODEL_API_URL first, then the named - ones by name. Names only, never a value.""" - out = [{"id": e, "name": (env.get(f"MODEL_{e}_NAME") if e else "") or e or settings.DEFAULT_ENDPOINT_NAME} - for e, url in _with_url(env) if not settings.endpoint_url_problem(url)] - return sorted(out, key=lambda x: (x["id"] != "", x["name"].casefold())) + """The custom providers a run can be sent to, each whose base is set and holds no credential + (``broken``): the one at MODEL_API_URL first, then the named ones by name. Names only, never a value.""" + return [{"id": split(r["id"])[1], "name": r["name"]} for r in _custom(env) if ready(r, env)] def broken(env=os.environ) -> List[str]: - """The endpoint choices whose URL is set but is refused (a credential in it, or no + """The custom providers whose base is set but is refused (a credential in it, or no http(s) URL): never offered, and a recipe on one is told to fix it in Settings.""" - return [choice("custom", e) for e, url in _with_url(env) if settings.endpoint_url_problem(url)] + return [r["id"] for r in _custom(env) if not ready(r, env)] def label(value: str, env=os.environ) -> str: - """What a choice is called: the provider's name, or the endpoint's.""" + """What a choice is called: the provider's name.""" p, e = split(value) - if not e: - return PROVIDER_LABEL.get(p, p) - return next((x["name"] for x in endpoints(env) if x["id"] == e), env.get(f"MODEL_{e}_NAME") or e) + return record(value, env)["name"] if p in PRESETS or p == "custom" else p def needs(value: str) -> str: - """The setting that makes a choice one a run can start on.""" + """The setting that makes a choice one a run can start on: a preset's key, a custom provider's base.""" p, e = split(value) - return endpoint_vars(e)[0] if e else PROVIDER_NEEDS.get(p, PROVIDER_NEEDS["custom"]) + return PRESETS[p]["key"] if p in PRESETS else endpoint_vars(e)[0] + + +def offered(apis, env=None) -> List[str]: + """The run providers that may serve a consumer speaking ``apis``: every custom one, a preset only + when it lists one of them. Anthropic is not offered to a ``chat``-only agent.""" + return [p for p in PROVIDERS if any(serves(record(p, env), a) is not False for a in apis)] + + +def run_choices(env=None) -> List[dict]: + """The run providers as the builder reads them (command-builder-core.js): id, name, the setting that + makes one ready, and the protocols it lists (None: a custom provider, which may serve any).""" + return [{"id": p, "label": label(p, os.environ if env is None else env), "needs": needs(p), + "protocols": list(record(p, env)["endpoints"]) if p in PRESETS else None} for p in PROVIDERS] # OpenRouter names a model with its maker in front (openai/gpt-4o); a @@ -129,32 +326,33 @@ def served_as(model: str, provider: str) -> str: def available(env=os.environ) -> List[str]: """The choices whose setting is set (``needs``), in the order the builder - prefers them: OpenRouter, OpenAI, then each OpenAI-compatible endpoint.""" - return [p for p in PROVIDERS[:2] if env.get(PROVIDER_NEEDS[p])] + [choice("custom", e["id"]) for e in endpoints(env)] + prefers them: OpenRouter, OpenAI, Anthropic, then each custom provider.""" + return [p for p in PROVIDERS if p in PRESETS and env.get(PRESETS[p]["key"])] + [choice("custom", e["id"]) for e in endpoints(env)] def embedding_problem(model: str, provider: str) -> str: """Bare OpenAI embedding names work on OpenAI or OpenRouter; qualified names require OpenRouter.""" allowed = ("openrouter",) if "/" in model else ("openai", "openrouter") if provider not in allowed: - names = " or ".join(PROVIDER_LABEL[p] for p in allowed) + names = " or ".join(PRESETS[p]["name"] for p in allowed) return f"{model} needs the {names} provider for this search." return "" def _where(provider: str, endpoint: str, env) -> tuple: - """(base URL, key) a choice is asked at; ValueError, before any request, when it cannot be asked.""" - url_var, key_var = endpoint_vars(endpoint) - base = PROVIDER_BASE[provider] if provider != "custom" else env.get(url_var, "") - key = env.get(key_var if provider == "custom" else PROVIDER_KEY[provider], "") + """(base URL, key, headers) a choice is asked at, the headers its own protocol's (Anthropic's + ``x-api-key``); ValueError, before any request, when it cannot be asked.""" + rec = record(choice(provider, endpoint), env) + base, key = rec["base"], env.get(rec["key"], "") if not base: raise ValueError("Set the provider endpoint in Settings, or enter a model ID.") - if provider == "openai" and not key: + if provider in ("openai", "anthropic") and not key: raise ValueError("Add the provider key in Settings, or enter a model ID.") parsed = urllib.parse.urlsplit(base) if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment: raise ValueError("The provider endpoint must be an HTTP or HTTPS base URL.") - return base.rstrip("/"), key + native = "messages" if serves(rec, "messages") and not serves(rec, "chat") else "chat" + return base.rstrip("/"), key, auth(native, key) class _NoRedirect(urllib.request.HTTPRedirectHandler): @@ -165,25 +363,25 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): _MOST = 8 * 1024 * 1024 -def _get(url: str, headers: dict, opener=None) -> bytes: +def _get(url: str, headers: dict, opener=None, timeout: float = 12) -> bytes: """One GET, never another method: no redirect followed, so a key stays at its endpoint; at most 8 MiB read.""" req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "nullius", **headers}) - with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=12) as response: + with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=timeout) as response: content = response.read(_MOST + 1) if len(content) > _MOST: raise ValueError("response too large") return content -def _models(content: bytes) -> list[dict]: - """The model list of a ``{"data": [{"id": ...}]}`` answer, by name; ValueError when there is none.""" +def _models(content: bytes, keep=lambda row: True) -> list[dict]: + """The model list of a ``{"data": [{"id": ...}]}`` answer, by name, of the rows ``keep`` accepts; ValueError when there is none.""" data = json.loads(content) rows = data.get("data") if isinstance(data, dict) else None if not isinstance(rows, list): raise ValueError("missing model list") models = {} for row in rows: - if not isinstance(row, dict): + if not isinstance(row, dict) or not keep(row): continue model = row.get("id") if not isinstance(model, str) or not model or len(model) > 256 or any(c.isspace() or ord(c) < 32 for c in model): @@ -193,28 +391,61 @@ def _models(content: bytes) -> list[dict]: return sorted(models.values(), key=lambda m: (m["name"].casefold(), m["id"])) -def _bearer(key: str) -> dict: - return {"Authorization": "Bearer " + key} if key else {} - - def list_models(provider: str, env=None, *, opener=None) -> list[dict]: """Read model IDs from the selected provider, or ``custom:``'s endpoint; credentials never leave its endpoint.""" env = os.environ if env is None else env provider, endpoint = split(provider) if provider not in PROVIDERS: raise ValueError("Choose a model provider.") - base, key = _where(provider, endpoint, env) + base, _, headers = _where(provider, endpoint, env) try: - return _models(_get(base + "/models", _bearer(key), opener)) + return _models(_get(base + "/models" + ("?limit=1000" if provider == "anthropic" else ""), headers, opener)) except (OSError, ValueError, TypeError, http.client.HTTPException): # Provider errors can contain keys or private endpoint details; do not echo them. raise ValueError("Model list unavailable. Enter a model ID manually.") from None -# Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is -# public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test. -TESTED = ("openrouter", "openai", "anthropic", "custom") -ANTHROPIC = ("https://api.anthropic.com/v1", "ANTHROPIC_API_KEY") +DECISION_TTL = 6 * 3600 # seconds OpenRouter's list of decision models is kept before it is asked again + + +def _decides(row: dict) -> bool: + arch = row.get("architecture") + out = arch.get("output_modalities") if isinstance(arch, dict) else None + return isinstance(out, list) and "decisions" in out + + +def decision_models(env=None, *, fetch: bool = True, opener=None, clock=time.time) -> list[str]: + """OpenRouter's decision models by ID, from its public list (no key sent, no redirect followed), kept + ``DECISION_TTL`` in the cache. ``fetch`` false, or the list unreadable: the cached IDs, however old; + [] when there are none. Only the filtered list has them: the unfiltered one leaves them out.""" + file = os.path.join(paths.cache_dir(env), "openrouter-decision-models.json") + try: + with open(file, encoding="utf-8") as f: + cached = json.load(f) + at, models = cached["at"], [m for m in cached["models"] if isinstance(m, str)] + except (OSError, ValueError, TypeError, KeyError): + at, models = 0, [] + if not fetch or (models and isinstance(at, (int, float)) and clock() - at < DECISION_TTL): + return models + try: + fresh = [m["id"] for m in _models(_get(PRESETS["openrouter"]["base"] + "/models?output_modalities=decisions", + {}, opener, timeout=5), keep=_decides)] + except (OSError, ValueError, TypeError, http.client.HTTPException): + return models + if not fresh: + return models + try: + os.makedirs(os.path.dirname(file), exist_ok=True) + with open(file + ".tmp", "w", encoding="utf-8") as f: + json.dump({"at": clock(), "models": fresh}, f) + os.replace(file + ".tmp", file) + except OSError: + pass # an unwritable cache asks again next time + return fresh + + +# Settings' Test, for each run provider (TypeSafe has no free request): only requests that cost nothing. A +# model list everywhere; OpenRouter's is public, so its key is asked at its own key-information endpoint. _TESTS_LOCK = threading.Lock() @@ -235,26 +466,22 @@ def test(choice: str, env=None, *, opener=None, clock=time.monotonic) -> dict: ValueError, before any request, when the provider has no test or its settings are incomplete.""" env = os.environ if env is None else env provider, endpoint = split(choice) - if provider not in TESTED: + if provider not in PROVIDERS: raise ValueError("This provider has no test.") - if provider == "anthropic": - from nullius.standard.relay import ANTHROPIC_VERSION - base, key = ANTHROPIC[0], env.get(ANTHROPIC[1], "") - auth, models = {"x-api-key": key, "anthropic-version": ANTHROPIC_VERSION}, base + "/models?limit=1000" - else: - base, key = _where(provider, endpoint, env) - auth, models = _bearer(key), base + "/models" - if not key and provider != "custom": + rec = record(choice, env) + if rec["preset"] and not env.get(rec["key"]): raise ValueError("Add the provider key in Settings.") + base, key, headers = _where(provider, endpoint, env) + models = base + "/models" + ("?limit=1000" if provider == "anthropic" else "") out = {"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "reached": False, "key": "not tested", "models": None, "ms": 0, "error": ""} start = clock() try: # OpenRouter's model list is public: its key goes to its key endpoint only - out["models"] = len(_models(_get(models, {} if provider == "openrouter" else auth, opener))) + out["models"] = len(_models(_get(models, {} if provider == "openrouter" else headers, opener))) out["reached"] = True if provider == "openrouter": - _get(base + "/key", auth, opener) + _get(base + "/key", headers, opener) out["key"] = "accepted" elif provider != "custom": out["key"] = "accepted" @@ -262,7 +489,7 @@ def test(choice: str, env=None, *, opener=None, clock=time.monotonic) -> dict: out["key"] = "not checked" else: try: - _get(models, _bearer("nullius-wrong-" + secrets.token_hex(8)), opener) + _get(models, auth("chat", "nullius-wrong-" + secrets.token_hex(8)), opener) out["key"] = "not checked" except urllib.error.HTTPError as e: e.close() diff --git a/app/nullius/recipes/launch.py b/app/nullius/recipes/launch.py index 3bf99551..0c74ef47 100644 --- a/app/nullius/recipes/launch.py +++ b/app/nullius/recipes/launch.py @@ -67,6 +67,8 @@ def payload(cat, jobs, workspace_name=""): "runners": [standard.RUNNER, providers.HARBOR], # why a saved recipe of a runner no longer offered does not open (a benchmark's own scripts: its Original setup) "runnerGone": providers.GONE, + # the providers a run is sent to, from the registry: the Model step offers each where its protocols fit + "providers": providers.run_choices(), # the standard's benchmarks, agents and capabilities, and why a pair does not fit (standard/recipe.py, choices) "standard": standard.choices(), "parameters": PARAMETERS, "benchmarks": [asdict(b) | {"label": b.label} for b in cat.benchmarks], diff --git a/app/nullius/settings.py b/app/nullius/settings.py index d351ad60..f241a09a 100644 --- a/app/nullius/settings.py +++ b/app/nullius/settings.py @@ -14,6 +14,11 @@ variable the process environment holds is shown read-only, because the file could not override it anyway. +A model provider's record (``nullius.providers``) is one JSON-valued setting of the +``provider`` kind, without its key. There the environment does not replace the file: +its value is a partial JSON object merged over the file's (``merge_patch``), and the +page holds read-only only the fields the environment sets (``state``, ``locked``). + ``example`` renders ``app/.env.example`` from the same table, so the two cannot drift (a test compares them). """ @@ -35,8 +40,14 @@ _LOCAL_KEY_OK = re.compile(r"^[!-~]+$") # the same, any length: a local server's key is often a word ("ollama", "EMPTY") _TEXT_OK = re.compile(r"^[^\x00-\x1f]{1,400}$") _MODEL_OK = re.compile(r"^~?[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,159}$") # an OpenRouter model id: [~]vendor/model[:variant] -_S1_OK = re.compile(r"^(typesafe|openrouter)$") -_LLM_JUDGE_OK = re.compile(r"^(openrouter|openai|local(:[A-Z0-9]+(_[A-Z0-9]+)*)?)$") +# a provider choice each judge can use: a preset that serves its protocols, or custom (local, its old name); +# System One's custom provider is a named one, the only kind whose record lists a decision protocol +_ID = r"[A-Z0-9]+(_[A-Z0-9]+)*" +_S1_OK = re.compile(rf"^(typesafe|openrouter|openai|(custom|local):{_ID})$") +_LLM_JUDGE_OK = re.compile(rf"^(openrouter|openai|anthropic|(custom|local)(:{_ID})?)$") +# the protocols each judge speaks (``judge.client``: LLM_PROTOCOLS, SYSTEM_ONE_PROTOCOLS) +_LLM_PROTOCOL_OK = re.compile(r"^(chat|messages)$") +_S1_PROTOCOL_OK = re.compile(r"^(systemone|decisions|openai-decisions)$") _ENVNAME_OK = re.compile(r"^[A-Z_][A-Z0-9_]*$") _BOOL = {"0", "1", "true", "false", "yes", "no"} @@ -56,11 +67,17 @@ def _paths(v: str) -> Optional[str]: "LD_PRELOAD", "LD_LIBRARY_PATH", "SSH_AUTH_SOCK"} -def reserved_name(n: str) -> bool: +def reserved_name(n: str, records: bool = True) -> bool: """A name that is already something else's: a setting, a start-up option, another spelling of one, the run's own or the shell's. An agent key or a - benchmark's secret may not take it.""" - return n in BY_NAME or n in STARTUP or n in INSTEAD or n in _NOT_AGENT_KEYS or n.startswith("NULLIUS_") or bool(_ENDPOINT_VAR.match(n)) + benchmark's secret may not take it. ``records``: a provider record's name + too (a preset's, ``MODEL_``), which an agent key beside it in the + settings file may not take; a benchmark's licence, kept apart and never in + this environment (``secrets``), may.""" + if bool(_PROVIDER_VAR.match(n)) or getattr(BY_NAME.get(n), "kind", "") == "provider": + return records + return (n in BY_NAME or n in STARTUP or n in INSTEAD or n in _NOT_AGENT_KEYS or n.startswith("NULLIUS_") + or bool(_ENDPOINT_VAR.match(n))) def _agent_key_names(v: str) -> Optional[str]: @@ -132,7 +149,7 @@ class Variable: name: str group: str purpose: str - kind: str # secret | localkey | endpoint | mirror | path | paths | url | repo | name | bool | text | envname | hosts | constraints | model | llmprovider | s1provider | envnames + kind: str # secret | localkey | endpoint | mirror | provider | path | paths | url | repo | name | bool | text | envname | hosts | constraints | model | llmprovider | s1provider | llmprotocol | s1protocol | envnames example: str = "" # shown as a placeholder, written to .env.example as a comment @property @@ -145,6 +162,9 @@ def validate(self, value: str) -> Optional[str]: return "a single line" if value != value.strip(): return "no leading or trailing whitespace" + if self.kind == "provider": + from nullius import providers + return providers.record_problem(self.name, value) return { "secret": _check(_SECRET_OK, "a nonempty single-line value without spaces"), "localkey": _check(_LOCAL_KEY_OK, "a nonempty single-line value without spaces"), @@ -161,8 +181,10 @@ def validate(self, value: str) -> Optional[str]: "hosts": _hosts, "constraints": _constraints, "model": _check(_MODEL_OK, "a model id as OpenRouter lists it, vendor/model"), - "llmprovider": _check(_LLM_JUDGE_OK, "openrouter, openai, local or local:"), - "s1provider": _check(_S1_OK, "typesafe or openrouter"), + "llmprovider": _check(_LLM_JUDGE_OK, "a provider: openrouter, openai, anthropic, custom or custom:"), + "s1provider": _check(_S1_OK, "a provider: typesafe, openrouter, openai or custom:"), + "llmprotocol": _check(_LLM_PROTOCOL_OK, "a protocol: chat or messages"), + "s1protocol": _check(_S1_PROTOCOL_OK, "a protocol: systemone, decisions or openai-decisions"), "envnames": _agent_key_names, }[self.kind](value) @@ -179,21 +201,28 @@ def validate(self, value: str) -> Optional[str]: ENDPOINT_ID = re.compile(r"^(?=.{1,32}$)[A-Z0-9]+(?:_[A-Z0-9]+)*$") _ENDPOINT_VAR = re.compile(r"^MODEL_([A-Z0-9]+(?:_[A-Z0-9]+)*)_(NAME|API_URL|API_KEY)$") ENDPOINT_PARTS = ("NAME", "API_URL", "API_KEY") +# A custom provider's record, MODEL_: the id may not end like one of the old settings above +_PROVIDER_VAR = re.compile(r"^MODEL_(?!(?:.*_)?(?:NAME|API_URL|API_KEY)$)([A-Z0-9]+(?:_[A-Z0-9]+)*)$") # What a person sets. Everything else the application reads is a start-up # option with a command-line form or a sensible default (STARTUP below) and # is not on the page. VARIABLES: List[Variable] = [ Variable("MODEL_API_KEY", GROUPS[0], "The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word " + repr(MODEL_KEY_PLACEHOLDER) + " as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.", "localkey", "sk-…"), - Variable("MODEL_API_URL", GROUPS[0], "The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL__NAME, MODEL__API_URL and MODEL__API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.", "endpoint", "https://…/v1"), - Variable("OPENAI_API_KEY", GROUPS[0], "An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), - Variable("OPENROUTER_API_KEY", GROUPS[0], "An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own endpoint, https://openrouter.ai/api/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), - Variable("ANTHROPIC_API_KEY", GROUPS[0], "An Anthropic key, for a run of the Nullius standard whose provider is Anthropic: its relay sends the calls to Anthropic's own endpoint, https://api.anthropic.com/v1, with this key. Only the relay holds it - never an agent, a recipe or a log.", "secret"), - Variable("TYPESAFE_API_KEY", GROUPS[0], "TypeSafe.ai key for the System One judgement on TypeSafe's own endpoint, POST https://api.typesafe.ai/v1/systemone — sent as a bearer header there and nowhere else.", "secret", "ts-…"), - Variable("SYSTEM_ONE_PROVIDER", GROUPS[0], "Where the System One judgement is asked: typesafe (its own endpoint, with TYPESAFE_API_KEY) or openrouter (https://openrouter.ai/api/v1/systemone, the same contract, with OPENROUTER_API_KEY). Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "s1provider"), - Variable("SYSTEM_ONE_MODEL", GROUPS[0], "The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev.", "model"), - Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local: (the endpoint MODEL__API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.", "llmprovider"), + Variable("MODEL_API_URL", GROUPS[0], "The base URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More custom providers, each with a name, are a record MODEL_ and a key MODEL__API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.", "endpoint", "https://…/v1"), + Variable("OPENAI_API_KEY", GROUPS[0], "An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), + Variable("OPENROUTER_API_KEY", GROUPS[0], "An OpenRouter key, for a recipe whose provider is OpenRouter: that run is sent to OpenRouter's own base with this key, never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.", "secret"), + Variable("ANTHROPIC_API_KEY", GROUPS[0], "An Anthropic key, for a run of the Nullius standard whose agent speaks Anthropic's Messages, and for the LLM judge: the calls go to Anthropic's own base with this key. Only the relay and the judge hold it, never an agent, a recipe or a log.", "secret"), + Variable("TYPESAFE_API_KEY", GROUPS[0], "A TypeSafe key, for the System One judgement on TypeSafe's own base: sent as a bearer header there and nowhere else.", "secret", "ts-…"), + Variable("SYSTEM_ONE_PROVIDER", GROUPS[0], "The provider the System One judgement is asked on: one that serves a decision protocol, such as typesafe or openrouter, with that provider's own key. Unset: TypeSafe when its key is set, else OpenRouter when its key is.", "s1provider"), + Variable("SYSTEM_ONE_PROTOCOL", GROUPS[0], "The decision protocol System One is asked in, on a provider that serves more than one: systemone, decisions or openai-decisions. Unset, or one the provider does not serve: on OpenRouter systemone for TypeSafe's models and decisions for another maker's, else the first the provider serves.", "s1protocol"), + Variable("SYSTEM_ONE_MODEL", GROUPS[0], "The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest or typesafe/jev-1.14. Unset: the provider's default.", "model"), + Variable("LLM_JUDGE_PROVIDER", GROUPS[0], "The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, anthropic, custom (the endpoint at MODEL_API_URL) or custom: (a custom provider), asked with that provider's own key and base from the model providers above. local and local: are read as custom. Unset: the first of them that is set, Anthropic last.", "llmprovider"), + Variable("LLM_JUDGE_PROTOCOL", GROUPS[0], "The protocol the LLM judge is asked in, on a provider that serves both: chat (OpenAI Chat Completions) or messages (Anthropic Messages). Unset, or one the provider does not serve: the first the provider serves, chat before messages.", "llmprotocol"), Variable("LLM_JUDGE_MODEL", GROUPS[0], "The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.", "model"), + *(Variable(name, GROUPS[0], f"Additions to the {title} provider, as a JSON object without the key: endpoints of protocols the preset does not list, " + '{"endpoints": {"": "/path" or "https://full/url"}}. In the environment, a partial object merged over this one.', "provider") + for name, title in (("OPENROUTER", "OpenRouter"), ("OPENAI", "OpenAI"), ("ANTHROPIC", "Anthropic"), ("TYPESAFE", "TypeSafe"))), Variable("AGENT_KEYS", GROUPS[2], "The names of the agent keys this file holds, comma-separated: the keys an agent needs of its own vendor, beside the model's (Cursor's CLI agent a CURSOR_API_KEY, Copilot's a GH_TOKEN). Each is a line of its own in this file. A Harbor run gets every one of them in its environment - over stdin to a host - and a Nullius standard run's never; none is ever written into a recipe or a log.", "envnames"), Variable("HOSTS", GROUPS[1], "Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are.", "hosts", "user@host[:port][:/path/to/key]"), Variable("HF_ENDPOINT", GROUPS[3], "Where Hugging Face downloads go: https://huggingface.co when unset, or a mirror's address. A run's dataset downloads, installs and drivers get it as HF_ENDPOINT before they load huggingface_hub; the environment wins over this file, a run on a host is sent it.", "mirror", "https://hf-mirror.com"), @@ -222,6 +251,9 @@ def validate(self, value: str) -> Optional[str]: } BY_NAME: Dict[str, Variable] = {v.name: v for v in VARIABLES} LOADED_KEYS: set = set() # variables ``load`` set from the file (so: not held by the process environment) +# A provider record both the process environment and the file hold: the environment's own partial value, +# merged over the file's into the process environment (``merge_patch``) +ENV_PART: Dict[str, str] = {} _FROM: Dict[str, str] = {} # which file each of them came from, so a refresh of one file never drops another's # Names other tools give the model settings, and names this application # used before: a file written with one is read by nobody here, and the page @@ -263,6 +295,62 @@ def parse(path: str) -> List[Tuple[str, Optional[str], Optional[str]]]: return out +def json_object(text: Optional[str]) -> Optional[dict]: + """``text`` as a JSON object, or None when it is not one.""" + try: + got = json.loads(text or "") + except ValueError: + return None + return got if isinstance(got, dict) else None + + +def merge_patch(target, patch): + """``patch`` applied to ``target`` as RFC 7396 says: objects merge key by + key, ``null`` removes a key, anything else replaces.""" + if not isinstance(patch, dict): + return patch + out = dict(target) if isinstance(target, dict) else {} + for k, v in patch.items(): + if v is None: + out.pop(k, None) + else: + out[k] = merge_patch(out.get(k), v) + return out + + +def compose(first: dict, then: dict) -> dict: + """One patch that does ``first`` and then ``then``, its nulls kept, so a + preset below them still loses a field either removes.""" + out = dict(first) + for k, v in then.items(): + out[k] = compose(out[k], v) if isinstance(v, dict) and isinstance(out.get(k), dict) else v + return out + + +def is_record(name: str) -> bool: + """Whether ``name`` is a provider record: a preset's additions or a custom provider's ``MODEL_``.""" + var = BY_NAME.get(name) + return var.kind == "provider" if var else bool(_PROVIDER_VAR.match(name)) and bool(ENDPOINT_ID.match(name[6:])) + + +def _merge_env(name: str, file_value: str) -> None: + """The process environment's value of a record: the environment's own + partial object (kept in ``ENV_PART``) merged over the file's. A value + that is not an object wins whole, as any other setting does.""" + part = ENV_PART.setdefault(name, os.environ.get(name, "")) + first, then = json_object(file_value) or {}, json_object(part) + os.environ[name] = json.dumps(compose(first, then)) if then is not None else part + + +def locked(name: str) -> List[str]: + """The fields of a record the process environment sets: ``base``, ``endpoints.chat``. The page holds only these read-only.""" + part = json_object(ENV_PART.get(name, os.environ.get(name) if name not in LOADED_KEYS else None)) + if part is None: + return [] + return sorted(f"{k}.{p}" if isinstance(v, dict) else k + for k, v in part.items() for p in (v if isinstance(v, dict) else [None])) + + def load(path: str) -> set: """Apply the file to the process environment: existing variables win. Returns the names the file supplied.""" @@ -271,6 +359,9 @@ def load(path: str) -> set: if key and key not in os.environ: os.environ[key] = value or "" applied.add(key) + elif key and key not in LOADED_KEYS and is_record(key): + _merge_env(key, value or "") + _FROM[key] = os.path.abspath(path) LOADED_KEYS.update(applied) _FROM.update((k, os.path.abspath(path)) for k in applied) return applied @@ -292,7 +383,14 @@ def refresh(path: str) -> Dict[str, List[str]]: if os.environ.pop(key, None) is not None: done["removed"].append(key) LOADED_KEYS.discard(key) + for key in [k for k in ENV_PART if _FROM.get(k) == here]: + before = os.environ.get(key) + _merge_env(key, values.get(key, "")) + if os.environ.get(key) != before: + done["updated"].append(key) for key, value in values.items(): + if key in ENV_PART: + continue if key in LOADED_KEYS: if key in os.environ and os.environ[key] != value: os.environ[key] = value @@ -302,6 +400,10 @@ def refresh(path: str) -> Dict[str, List[str]]: LOADED_KEYS.add(key) _FROM[key] = here done["loaded"].append(key) + elif is_record(key): + _merge_env(key, value) + _FROM[key] = here + done["updated"].append(key) return {k: sorted(v) for k, v in done.items()} @@ -342,12 +444,20 @@ def in_file(path: str) -> Dict[str, str]: def agent_key_names(env=None) -> List[str]: - """The agent keys' names, as AGENT_KEYS lists them; a list that would - not validate names none.""" - raw = (os.environ if env is None else env).get("AGENT_KEYS", "") - if _agent_key_names(raw): - return [] - return [n.strip() for n in raw.split(",") if n.strip()] + """The agent keys' names, as AGENT_KEYS lists them, each once: a name that + is not one, or is already something else's (``reserved_agent_keys``), is + left out, never the whole list.""" + out: List[str] = [] + for n in (n.strip() for n in (os.environ if env is None else env).get("AGENT_KEYS", "").split(",")): + if _ENVNAME_OK.match(n) and not reserved_name(n) and n not in out: + out.append(n) + return out + + +def reserved_agent_keys(env=None) -> List[str]: + """The names AGENT_KEYS lists that are now something else's (a provider record's): Settings says so.""" + return [n for n in dict.fromkeys(n.strip() for n in (os.environ if env is None else env).get("AGENT_KEYS", "").split(",")) + if _ENVNAME_OK.match(n) and reserved_name(n)] def agent_key_values(env=None) -> Dict[str, str]: @@ -356,24 +466,35 @@ def agent_key_values(env=None) -> Dict[str, str]: return {n: env[n] for n in agent_key_names(env) if env.get(n)} +def _custom_record(name: str, env) -> bool: + """A custom provider's record that ``env`` holds: ``MODEL_`` whose value is a JSON object. Another + value under such a name (a shell's MODEL_PATH) is not one.""" + return is_record(name) and name not in BY_NAME and json_object(env.get(name)) is not None + + def endpoint_ids(env=None) -> List[str]: - """The ids of the named OpenAI-compatible endpoints: every id one of whose settings is set.""" + """The ids of the custom providers (the named OpenAI-compatible endpoints): every id whose record, + ``MODEL_``, or one of whose old settings is set.""" env = os.environ if env is None else env - return sorted({m.group(1) for m in map(_ENDPOINT_VAR.match, env) if m and ENDPOINT_ID.match(m.group(1))}) + return sorted({m.group(1) for m in map(_ENDPOINT_VAR.match, env) if m and ENDPOINT_ID.match(m.group(1))} + | {n[6:] for n in env if _custom_record(n, env)}) def endpoint_settings(i: str) -> List[str]: - """The settings of the endpoint ``i``: its name, URL and key.""" - return [f"MODEL_{i}_{part}" for part in ENDPOINT_PARTS] + """The settings of the custom provider ``i``: its record, then its old name and URL, and its key.""" + return [f"MODEL_{i}"] + [f"MODEL_{i}_{part}" for part in ENDPOINT_PARTS] def endpoint_names(env=None) -> List[str]: - """Every setting of a named endpoint that ``env`` holds: what a run never inherits.""" - return sorted(n for n in (os.environ if env is None else env) if _ENDPOINT_VAR.match(n)) + """Every setting of a custom provider that ``env`` holds: what a run never inherits.""" + env = os.environ if env is None else env + return sorted(n for n in env if _ENDPOINT_VAR.match(n) or _custom_record(n, env)) def endpoint_variable(name: str) -> Optional[Variable]: - """A named endpoint's setting as the page and ``save_many`` treat it, or None for another name.""" + """A custom provider's setting as the page and ``save_many`` treat it, or None for another name.""" + if is_record(name) and name not in BY_NAME: + return Variable(name, GROUPS[0], f"The record of the custom provider {name[6:]}, as a JSON object without the key.", "provider") m = _ENDPOINT_VAR.match(name) if not m or not ENDPOINT_ID.match(m.group(1)): return None @@ -385,7 +506,7 @@ def new_endpoint_id(name: str, taken) -> str: """An id for a new endpoint, from its name: capitals, digits and underscores, not one of ``taken``.""" base = re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")[:28].strip("_") or "ENDPOINT" n, out = 1, base - while out in taken: + while out in taken or not _PROVIDER_VAR.match("MODEL_" + out): # never an id whose record reads as an old setting n += 1 out = f"{base}_{n}" return out @@ -470,11 +591,19 @@ def state(var: Variable, file_values: Dict[str, str]) -> dict: problem = endpoint_url_problem(value) if var.kind == "endpoint" and value else None # constraints set in the environment or the file by hand: said, and every run refused until fixed invalid = _constraints(value) if var.kind == "constraints" and value else None + record = var.kind == "provider" + if record and value: + why = var.validate(value) + problem = why and f"a provider record: {why}" return {"name": var.name, "group": var.group, "purpose": var.purpose, "kind": var.kind, "secret": var.secret, "example": var.example, "source": source, "shown": shown, - "shadowed": source == "environment" and var.name in file_values and file_values[var.name] != value, - "advice": LOCAL_KEY_ADVICE if var.kind == "localkey" and value is None and os.environ.get(var.name[:-3] + "URL") else "", - "problem": f"This URL must be {problem}." if problem else f"{var.name} must be {invalid}." if invalid else ""} + # a record's environment fields are merged over the file's, never a whole value shadowing it + "shadowed": not record and source == "environment" and var.name in file_values and file_values[var.name] != value, + "locked": locked(var.name) if record else [], + "advice": LOCAL_KEY_ADVICE if var.kind == "localkey" and value is None and ( + os.environ.get(var.name[:-3] + "URL") or (json_object(os.environ.get(var.name[:-8])) or {}).get("base")) else "", + "problem": (f"This must be {problem}." if record else f"This URL must be {problem}.") if problem + else f"{var.name} must be {invalid}." if invalid else ""} def states(path: str) -> List[dict]: @@ -528,6 +657,10 @@ def save_many(path: str, changes: Dict[str, Optional[str]]) -> None: lines.pop() _write_lines(path, lines) for name, value in changes.items(): + if is_record(name) and name in os.environ and name not in LOADED_KEYS: + _merge_env(name, value or "") # a record: the environment's fields stay over the file's + _FROM[name] = os.path.abspath(path) + continue if name in os.environ and name not in LOADED_KEYS: continue # the process environment holds it; the file is for the next start if value is None: diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 56be6ddb..7f296740 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -66,7 +66,7 @@ import sys import tempfile -from nullius import constraints as limits, paths, prices as price_list, progress, secrets, settings +from nullius import constraints as limits, paths, prices as price_list, progress, providers, secrets, settings # The legacy module command needs saved path overrides before install.HOME # and other runtime defaults are computed, just like the unified CLI does. if __name__ == "__main__" and "--from-run" not in sys.argv[1:]: @@ -97,12 +97,11 @@ def run_env(provider: str, env=os.environ) -> dict: ``execution``'s launcher hands every run its one key as ``OPENAI_API_KEY`` and its endpoint as ``OPENAI_BASE_URL`` (empty for OpenAI's own), so they are named here as the provider's own settings and nothing else is.""" - p = run.PROVIDERS[provider] out = {k: v for k, v in env.items() if k in run.KEEP} if env.get("OPENAI_API_KEY"): - out[p["key"]] = env["OPENAI_API_KEY"] - if "base_var" in p and env.get("OPENAI_BASE_URL"): - out[p["base_var"]] = env["OPENAI_BASE_URL"] + out[providers.key_setting(provider)] = env["OPENAI_API_KEY"] + if provider == "custom" and env.get("OPENAI_BASE_URL"): + out["MODEL_API_URL"] = env["OPENAI_BASE_URL"] return out @@ -651,7 +650,7 @@ def main(argv=None): + ("; or nop, oracle: a control the benchmark declares; or original, model: one of its setups" if name == "run" else "")) p.add_argument("--model", required=name == "conform", help="the model (a control needs none; original takes one of the paper's, its first by default)" if name == "run" else None) - p.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS)) + p.add_argument("--provider", default="openai", choices=sorted(providers.PROVIDERS)) p.add_argument("--jobs", default="jobs_standard", help="where jobs are written (default jobs_standard/, gitignored)") p.add_argument("--unsandboxed", action="store_true", help="allow an agent, or a setup check without Docker, to execute code on this machine as you") @@ -700,7 +699,7 @@ def main(argv=None): v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") v.add_argument("--limit", type=int, help="the first N units of each dataset (the record says so)") - v.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS), help="a judged benchmark's: the judge's provider") + v.add_argument("--provider", default="openai", choices=sorted(providers.PROVIDERS), help="a judged benchmark's: the judge's provider") v.add_argument("--judge-model", help="a judged benchmark: this judge instead of its official one") v.add_argument("-j", "--concurrency", type=int, default=4) v.add_argument("--jobs", default="jobs_standard", help="where the control jobs are written (default jobs_standard/)") diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 125c2b2f..bf5bcf86 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -924,7 +924,7 @@ def _secrets(w: str, secrets, local_ok: bool, settings) -> List[str]: n = s.get("name") if not isinstance(n, str) or not _ENV_NAME.match(n) or n in seen: bad.append(f"{sw}.name: a variable name, each once") - elif settings.reserved_name(n): + elif settings.reserved_name(n, records=False): bad.append(f"{sw}.name: {n} is already the name of something else") seen.add(n) to, kind = s.get("to"), s.get("kind", "value") @@ -947,7 +947,7 @@ def _secrets(w: str, secrets, local_ok: bool, settings) -> List[str]: bad.append(f"{sw}: path and env are a file's; a {kind} reaches its process as the variable {n}") continue if v is not None: - if not isinstance(v, str) or not _ENV_NAME.match(v) or settings.reserved_name(v) or v in seen: + if not isinstance(v, str) or not _ENV_NAME.match(v) or settings.reserved_name(v, records=False) or v in seen: bad.append(f"{sw}.env: a variable name of its own, which will hold the file's path") seen.add(v) if to == "driver": diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index bc60de26..ca3a4ab7 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() + key_header=r.key_header, fields=True).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 9e397d1c..13f9a7ff 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -42,6 +42,7 @@ from nullius.providers import PROVIDERS, STEP from nullius.standard import hostreq, definitions, ledger, match, ports, prove +CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's. RUNNER = {"key": HARNESS, "label": "Nullius", "builtin": True, "standard": True, @@ -174,7 +175,7 @@ def skills_of(port: str, record: Optional[dict]) -> List[dict]: def _other_page(o: dict) -> dict: return {"key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"], - "featureKind": o.get("kind", ""), "protocol": protocol(o), "version": str((o.get("upstream") or {}).get("version") or ""), + "featureKind": o.get("kind", ""), "protocol": protocol(o), "apis": list((o.get("model") or {}).get("apis") or ["chat"]), "version": str((o.get("upstream") or {}).get("version") or ""), "default": o.get("_version") == o.get("_default"), "offers": list(o.get("offers") or []), # each proof with the kit version it passed and, on an older kit than the current one, a note saying so @@ -413,7 +414,15 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") if provider not in PROVIDERS: - add("model", "Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.") + add("model", CHOOSE_PROVIDER) + # what each part asks of the provider, as the builder reads it (command-builder-core.js, consumersOf): an agent its + # apis, a setup chat; a judged benchmark's judge chat + for pk in picks if provider in PROVIDERS else (): + apis = None if pk["control"] else (pk["o"].get("model") or {}).get("apis") or ["chat"] + if apis and provider not in providers.offered(apis): + add("model", f"{providers.label(provider)} does not serve what {pk['o']['label']} speaks: choose another provider.") + if provider in PROVIDERS and (b.get("score") or {}).get("judge") and provider not in providers.offered(["chat"]): + add("model", f"{providers.label(provider)} does not serve what {b['label']}'s judge speaks: choose another provider.") model = str(recipe.get("model") or "").strip() if not _MODEL.match(model) and (needs_model or model): add("model", "Choose a model or enter its ID.") @@ -682,7 +691,7 @@ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: return {"errors": [f"no {kind} {key!r}"], "script": "", "jobs": [], "label": key} o = dict(o, label=definitions.title(o)) if provider not in PROVIDERS: - errors.append("Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.") + errors.append(CHOOSE_PROVIDER) if not _MODEL.match(model or ""): errors.append("Choose a model or enter its ID.") backend = definitions.names.backend(backend) diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a8c59455..e3f97845 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -437,14 +437,17 @@ def _said(self, e: Exception) -> str: text = text.replace(k, "[REDACTED]") return redact(text)[:200] - def _headers(self, caller) -> Dict[str, str]: + def _headers(self, caller, tag: str = "") -> Dict[str, str]: + """The headers sent on: the key as ``key_header`` says on the agent's own routes; a route pinned + to a model of the benchmark's (a judge, a simulator, embeddings) speaks chat, so a bearer header.""" out = forwarded(caller) has = {k.lower() for k in out} # a name in any case is the same header: never send it twice + x_api_key = self.key_header == "x-api-key" and tag not in self.models and tag not in self.embedding_models for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ - (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"): + (("anthropic-version", ANTHROPIC_VERSION),) * x_api_key: if name.lower() not in has: out[name] = value - if self.key_header == "x-api-key": + if x_api_key: out["x-api-key"] = self.key else: out["Authorization"] = "Bearer " + self.key @@ -499,7 +502,7 @@ def _forward(self, h: BaseHTTPRequestHandler): conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed try: - conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers)) + conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, tag)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: status, kind = 502, "application/json" diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 200ddac0..315c46d8 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -98,12 +98,6 @@ def driver_path() -> str: os.replace(part, link) return root -PROVIDERS = { - "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, - "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"}, - "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, - "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, -} TURN_SECONDS = 900 DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) @@ -118,18 +112,20 @@ class Refused(Exception): def provider_endpoint(provider: str, env=os.environ) -> tuple: - p = PROVIDERS.get(provider) - if not p: - raise Refused(f"provider {provider!r} is not one of {list(PROVIDERS)}") - base = p.get("base") or env.get(p["base_var"]) or "" - key = env.get(p["key"]) or ("local" if provider == "custom" else "") + """(base, key) of a run provider (``providers.PROVIDERS``), from its record: ``custom`` is the one at + MODEL_API_URL, which a run's launcher fills from the run's own provider (``cli.run_env``).""" + if provider not in providers.PROVIDERS: + raise Refused(f"provider {provider!r} is not one of {list(providers.PROVIDERS)}") + rec = providers.record(provider, env) + base = rec["base"] + key = env.get(rec["key"]) or ("local" if provider == "custom" else "") if not base: - raise Refused(f"{p['base_var']} is not set in Settings") + raise Refused("MODEL_API_URL is not set in Settings") if not key: - raise Refused(f"{p['key']} is not set in Settings") + raise Refused(f"{rec['key']} is not set in Settings") if not key.isprintable() or any(ord(c) > 255 for c in key) or key != key.strip(): # http.client refuses such a value, and quotes it whole in its error - raise Refused(f"{p['key']} holds a character a request header cannot carry (a line break, a space or a " + raise Refused(f"{rec['key']} holds a character a request header cannot carry (a line break, a space or a " "byte-order mark around it, or one outside Latin-1): set it again in Settings") return base, key @@ -161,6 +157,16 @@ def units_file(path: str) -> dict: return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} +def key_header(provider: str, other: dict, env=os.environ) -> str: + """How the agent's relay sends the key: by the protocol the agent (or capability) speaks on this + provider. Refused, as ``recipe.compile`` refuses it, on a provider that serves none of them.""" + apis = (other.get("model") or {}).get("apis") or ["chat"] + if provider not in providers.offered(apis, env): + raise Refused(f"{providers.label(provider, env)} does not serve what {other.get('key') or 'the agent'} speaks " + f"({', '.join(apis)}): choose another provider") + return providers.PROTOCOLS[providers.protocol_for(provider, apis, env)]["auth"] + + def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit" @@ -538,6 +544,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) + self.key_header = "authorization" if keyless else key_header(provider, other, env) self.limit, self.concurrency = limit, limits.concurrency(max(1, int(concurrency)), self.constraints) self.units_file = units_file self.repeat = max(1, int(repeat or 1)) @@ -929,8 +936,8 @@ def ready(self) -> None: # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's. if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), - prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False, - key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() + prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), + unit_caps=False).start() # embeddings and rerank: a bearer key def runtimes_needed(self) -> List[str]: """What the agent needs in an own image that the image may not have: @@ -1312,7 +1319,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] return None # a control with no judge: no model is called record = os.path.join(trial, ".work", "relay") side = tuple((self.other.get("model") or {}).get("side_models") or ()) - key_header = PROVIDERS[self.provider].get("key_header", "authorization") + key_header = self.key_header if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header, ledger=self.ledger_file()).start() diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index d20fd5b5..5ed1d5ba 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -82,7 +82,7 @@ function createStandard(benchmark = '', other = '') { const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0]; const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [], - model_choice: 'any', provider: firstAvailable(PROVIDERS, 'openai'), model: bareModel(schema.defaultModel || ''), limit: '', concurrency: '', unsandboxed: false, where: 'local', + model_choice: 'any', provider: firstAvailable(providersOf({}), 'openai'), model: bareModel(schema.defaultModel || ''), limit: '', concurrency: '', unsandboxed: false, where: 'local', backend: 'local', options: {}}; return b ? setStandardBenchmark(r, b.key, other) : r; } @@ -172,7 +172,10 @@ const defaults = () => Object.fromEntries(parameters().map(p => [p.key, p.default])); // The provider is a recipe field and the model a bare name. Each consumer gets // the prefix matching its credential bindings; a typed model prefix takes precedence. - const PROVIDERS = ['openrouter', 'openai', 'custom']; + // The providers a run is sent to, from the backend's registry (providers.run_choices): each with its name, the + // setting that makes it ready, and the protocols it lists (null: a custom provider, which may serve any). + const PROVIDER_INFO = Object.fromEntries((schema.providers || []).map(p => [p.id, p])); + const PROVIDERS = Object.keys(PROVIDER_INFO); const PROVIDER_PREFIX = /^(openrouter|openai)\//; const bareModel = m => String(m || '').replace(PROVIDER_PREFIX, ''); const providerOf = (r, m) => { if(r.modelFormat === 'provider-id' && PROVIDERS.includes(r.provider))return r.provider; const x = String(m || '').match(PROVIDER_PREFIX); return x ? x[1] : (PROVIDERS.includes(r.provider) ? r.provider : 'openrouter'); }; @@ -201,8 +204,8 @@ // being optional - is set in Settings (schema.credentials, runners.available), in the order the Model step prefers // them. With none, every provider stays visible. A schema that does not say (a test's) is // checked by the server alone. - const PROVIDER_LABEL = {openrouter: 'OpenRouter', openai: 'OpenAI', custom: 'OpenAI-compatible'}; - const PROVIDER_NEEDS = {openrouter: 'OPENROUTER_API_KEY', openai: 'OPENAI_API_KEY', custom: 'MODEL_API_URL'}; + const PROVIDER_LABEL = Object.fromEntries(PROVIDERS.map(p => [p, PROVIDER_INFO[p].label])); + const PROVIDER_NEEDS = Object.fromEntries(PROVIDERS.map(p => [p, PROVIDER_INFO[p].needs])); // A named OpenAI-compatible endpoint: the recipe keeps provider custom and the endpoint's id, which names its // settings (MODEL__API_URL); the Model step offers it as the choice custom: (nullius/providers.py). const ENDPOINT_ID = /^(?=.{1,32}$)[A-Z0-9]+(?:_[A-Z0-9]+)*$/; @@ -213,7 +216,17 @@ const needsOf = c => endpointOf(c) ? 'MODEL_' + endpointOf(c) + '_API_URL' : PROVIDER_NEEDS[c] || PROVIDER_NEEDS.custom; const available = () => ((schema.credentials || {}).available || []).filter(p => PROVIDERS.includes(kindOf(p))); const firstAvailable = (offered, otherwise) => available().find(p => offered.includes(p)) || otherwise; - const providersOf = () => PROVIDERS; + // What each part of a run that calls the model speaks: a Harbor run and a benchmark's own scripts or judge + // chat, an agent of the standard its definition's apis. A provider is offered when it may serve every part. + function consumersOf(r) { + if (!isStandard(r)) return [['chat']]; + const b = standardBench(r), all = standard().agents.concat(standard().capabilities); + const out = chosenOf(r).filter(k => !((b && b.controls) || []).some(c => c.key === k)).map(k => (all.find(o => o.key === k) || {}).apis || ['chat']); + if (b && b.judged) out.push(['chat']); + return out.length ? out : [['chat']]; + } + const serving = (p, apis) => !PROVIDER_INFO[p].protocols || apis.some(a => PROVIDER_INFO[p].protocols.includes(a)); + const providersOf = r => PROVIDERS.filter(p => !r || consumersOf(r).every(apis => serving(p, apis))); const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel); const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r); // The Model step's choice into the recipe: its provider, and for a named endpoint its id. @@ -227,6 +240,8 @@ const offered = providersOf(r), have = available().filter(p => offered.includes(kindOf(p))), p = choiceOf(r); const or = xs => xs.slice(0, -1).join(', ') + (xs.length > 1 ? ' or ' : '') + xs[xs.length - 1]; if (have.includes(p)) return ''; + if (PROVIDERS.includes(kindOf(p)) && !offered.includes(kindOf(p))) + return labelOf(p) + ' does not serve what this recipe\'s agent speaks. Choose ' + or(offered.map(labelOf)) + ' on the Model step.'; if (!have.length) return 'Run needs a model key, and none is set in Settings: set ' + or(offered.map(x => PROVIDER_NEEDS[x])) + ' there.'; if (((schema.credentials || {}).broken || []).includes(p)) return 'This recipe runs on ' + labelOf(p) + ', and its URL in Settings holds a credential or is not an http(s) URL. Put the key in the Key field in Settings.'; @@ -291,7 +306,7 @@ function create() { const r = {version: 1, harness: 'harbor', name: 'harbor-experiment', title: 'Harbor experiment', benchmark: '', source: '', dataset: '', path: '', generatorRoot: '', agents: [], harbor: {kind: 'release', version: '', extras: []}, - customAgents: [], model_choice: 'any', provider: firstAvailable(PROVIDERS, providerOf({}, schema.defaultModel || '')), model: '', userModel: '', evalModel: '', keyEnv: (schema.credentialVars && schema.credentialVars.key) || 'API_TOKEN', urlEnv: schema.credentialVars ? (schema.credentialVars.url || '') : 'API_URL', + customAgents: [], model_choice: 'any', provider: firstAvailable(providersOf({}), providerOf({}, schema.defaultModel || '')), model: '', userModel: '', evalModel: '', keyEnv: (schema.credentialVars && schema.credentialVars.key) || 'API_TOKEN', urlEnv: schema.credentialVars ? (schema.credentialVars.url || '') : 'API_URL', tasks: '', concurrency: 1, retries: 3, timeoutMultiplier: 1, agentMinutes: '', attempts: '', setupTimeoutMultiplier: '', verifierTimeoutMultiplier: '', buildTimeoutMultiplier: '', harborDev: '', include: '', output: 'jobs', where: 'local', adapterPaths: {}, params: defaults(), custom: []}; @@ -422,7 +437,7 @@ if (!allowed.includes(r.model)) add(STEP.model, 'Choose one of the paper models on this provider.'); } if (callsModel && !named) add(STEP.model, benchmarkCallsModel(r) ? 'Enter a model for the agent and/or simulated user and evaluator.' : 'Enter a model.'); - if (callsModel && !PROVIDERS.includes(r.provider)) add(STEP.model, 'Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.'); + if (callsModel && !PROVIDERS.includes(r.provider)) add(STEP.model, 'Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider.'); for (const [key,label] of [['keyEnv','API key source variable'],['urlEnv','Endpoint source variable']]) { if (callsModel && (key !== 'urlEnv' || r[key]) && !/^[A-Z_][A-Z0-9_]*$/.test(r[key])) add(STEP.model, label + ' must be an environment variable name.'); } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index f4eb338f..e30501ac 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -600,7 +600,7 @@ if (step === S.model) { const lb = rec.model_choice === 'leaderboard' ? C.leaderboard(rec) : null, paper = rec.model_choice === 'paper' || !!lb; return `

Choose the model.

` + modelChoiceCards(rec) - + providerField(rec, C.PROVIDERS) + + providerField(rec, C.providersOf(rec)) + field('Model', 'model', rec.model, {list:'nb-models', placeholder:paper && C.paperListed(rec) ? 'Choose a paper model' : 'Search or enter a model ID'}) // the benchmark's other models: the user simulator its conversations need, the judge that grades - each its official one // unless named; like the paper, the official one; like the leaderboard, its own and the options it fixes @@ -667,7 +667,7 @@ if(C.needsCredentials(rec)){ const controlOnly=!rec.agents.some(id=>!schema.agents.find(a=>a.id===id)?.control); const tau3=rec.benchmark==='tau3-bench'; - html+=providerField(rec,C.PROVIDERS); + html+=providerField(rec,C.providersOf(rec)); if(tau3&&controlOnly)html+=`
${field('Model · customer','userModel',rec.userModel,{list:'nb-models',placeholder:'vendor/model, as the provider lists it'})}${field('Model · evaluator','evalModel',rec.evalModel,{list:'nb-models',placeholder:'same as the customer'})}
`; else if(tau3)html+=field('Model · agent','model',rec.model,{list:'nb-models',placeholder:'vendor/model, as the provider lists it'})+`
${field('Model · customer','userModel',rec.userModel,{list:'nb-models',placeholder:'same as the agent'})}${field('Model · evaluator','evalModel',rec.evalModel,{list:'nb-models',placeholder:'same as the customer'})}
`; else html+=field('Model','model',rec.model,{list:'nb-models',placeholder:'vendor/model, as the provider lists it'}); diff --git a/app/static/settings/autosave.js b/app/static/settings/autosave.js index 202754f0..72c2809c 100644 --- a/app/static/settings/autosave.js +++ b/app/static/settings/autosave.js @@ -38,6 +38,8 @@ const provider = current.get('value:' + prefix + '_PROVIDER'); if (provider) body.set('value:' + prefix + '_PROVIDER', provider); } + // The provider and the model decide the protocol: redraw the judge's rows from the saved state. + if (['_PROVIDER', '_PROTOCOL', '_MODEL'].some(part => names.has(prefix + part))) ['_PROVIDER', '_PROTOCOL'].forEach(part => names.add(prefix + part)); } // Removing the final host is an explicit removal, not an empty "keep" input. if (body.has('value:HOSTS') && !body.get('value:HOSTS')) body.set('clear:HOSTS', '1'); @@ -52,12 +54,12 @@ return false; } } - // A new endpoint: its name, URL and optional key go together. + // A new custom provider: its name, base and optional key go together. const endpointChanged = changed.some(([key]) => key.startsWith('endpoint_')); if (endpointChanged) { for (const part of ['name', 'url', 'key']) body.set('endpoint_' + part, current.get('endpoint_' + part) || ''); if (!body.get('endpoint_name') || !body.get('endpoint_url')) { - say('Enter the endpoint name and its URL.', true); + say('Enter the provider name and its base URL.', true); return false; } } @@ -93,7 +95,7 @@ for (const [key, value] of changed) saved.set(key, value); for (const name of names) { const field = document.getElementById(name), replacement = fresh.getElementById(name); - if (!field || !replacement || (field.contains(document.activeElement) && document.activeElement.matches('input:not([type=hidden]), textarea, select')) || field.querySelector('input[type=radio]')) continue; + if (!field || !replacement || (field.contains(document.activeElement) && document.activeElement.matches('input:not([type=hidden]), textarea, select, .ui-select-trigger')) || field.querySelector('input[type=radio]')) continue; const keys = [...field.querySelectorAll('[name]')].map(input => input.name); if (keys.some(key => now.get(key) !== current.get(key))) continue; field.replaceWith(replacement); diff --git a/app/static/settings/judges.js b/app/static/settings/judges.js index 656fda22..7f0864a6 100644 --- a/app/static/settings/judges.js +++ b/app/static/settings/judges.js @@ -2,21 +2,39 @@ (() => { const cards = [...document.querySelectorAll('[data-judge-card]')]; const writable = !!document.getElementById('settings-form'); + // The selected attribute as well as the value, so the shared select (controls.js) redraws. + function choose(select, value) { + for (const option of select.options) option.toggleAttribute('selected', option.value === value); + select.value = value; + } + const chosen = card => { const select = card.querySelector('[data-picker]'); return select.value ? select.selectedOptions[0] : null; }; function update(card, states) { - const choices = [...card.querySelectorAll('[data-provider]')]; + const select = card.querySelector('[data-picker]'); + const choices = [...select.options].filter(o => o.value); for (const choice of choices) { - if (states) choice.dataset.available = states.some(s => s.key === choice.dataset.provider && s.ready === 'true') ? 'true' : 'false'; + if (states) choice.dataset.available = states.some(s => s.key === choice.value && s.ready === 'true') ? 'true' : 'false'; choice.hidden = choice.dataset.available !== 'true'; - const input = choice.querySelector('input'); - input.disabled = !writable || choice.hidden; - if (choice.hidden) input.checked = false; } const available = choices.filter(c => !c.hidden); - if (available.length === 1) available[0].querySelector('input').checked = true; - card.querySelector('[data-picker]').classList.toggle('st-judge-single', available.length === 1); + if (select.value && select.selectedOptions[0].hidden) choose(select, ''); + if (available.length === 1) choose(select, available[0].value); + select.disabled = !writable || !available.length; + select.hidden = !available.length; card.querySelector('[data-no-judge-provider]').hidden = available.length > 0; + const on = chosen(card); + // The protocol: a select when the provider lists two this judge speaks, else a quiet line. + const protocol = card.querySelector('[data-protocol-picker]'); + const offered = (on?.dataset.protocols || '').split('|').filter(Boolean); + for (const option of protocol.options) option.hidden = !offered.includes(option.value); + if (on && protocol.value !== on.dataset.protocol) { + choose(protocol, on.dataset.protocol); + protocol.removeAttribute('name'); // shown, not chosen: the saved setting stays as it is + } + protocol.closest('.st-field').hidden = !on || offered.length < 2; + const via = card.querySelector('[data-judge-via]'); + via.hidden = !on || offered.length > 1; + via.textContent = 'via ' + (protocol.selectedOptions[0]?.textContent || ''); const modelField = card.querySelector('.st-field[id$="_MODEL"]'); - const on = card.querySelector('[data-picker] input:checked'); modelField.hidden = !on; const model = modelField.querySelector('input.st-input'); const list = model?.list; @@ -28,25 +46,35 @@ } if (model) model.placeholder = on?.dataset.default || ''; } - // A provider with no list of its own here: its catalogue, asked once when the field is first used. Typing a model still works. + // A provider with no list of its own here: its catalogue, asked once when the field is first used; a live list + // (System One on OpenRouter) replaces the one drawn with the page. Typing a model still works. const catalogues = new Map(); async function offerCatalogue(card, model) { - const on = card.querySelector('[data-picker] input:checked'); - if (!on || on.dataset.models || !model.list) return; + const on = chosen(card); + if (!on || (on.dataset.models && !('live' in on.dataset)) || !model.list) return; const provider = on.value.replace(/^local(?=$|:)/, 'custom'); // local:, a named endpoint, is the builder's custom: - if (!catalogues.has(provider)) { - catalogues.set(provider, nulliusFetch('/new-run/models', {method: 'POST', headers: {'Content-Type': 'application/json', + const url = 'live' in on.dataset ? '/settings/system-one/models' : '/new-run/models'; + if (!catalogues.has(url + provider)) { + catalogues.set(url + provider, nulliusFetch(url, {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': document.querySelector('#settings-form [name=csrf]').value}, body: JSON.stringify({provider})}) - .then(r => { if (!r.ok) throw new Error("Model catalogue unavailable"); return r.json(); }).then(j => { if (j.error) throw new Error(j.error); return j.models || []; }).catch(() => { catalogues.delete(provider); return []; })); + .then(r => { if (!r.ok) throw new Error("Model catalogue unavailable"); return r.json(); }).then(j => { if (j.error) throw new Error(j.error); return j.models || []; }).catch(() => { catalogues.delete(url + provider); return []; })); } - const models = await catalogues.get(provider); - if (card.querySelector('[data-picker] input:checked') !== on) return; + const models = await catalogues.get(url + provider); + if (!models.length) return; + if (chosen(card) !== on) return; model.list.replaceChildren(...models.map(m => { const option = document.createElement('option'); option.value = m.id; if (m.name !== m.id) option.label = m.name; return option; })); } cards.forEach(card => { - card.addEventListener('change', event => { if (event.target.matches('[data-picker] input')) update(card); }); + // Runs before autosave's listener on the form: a protocol picked here is named, so it is saved. + card.addEventListener('change', event => { + if (event.target.matches('[data-protocol-picker]')) { + event.target.name = event.target.dataset.name; + if (chosen(card)) chosen(card).dataset.protocol = event.target.value; + } + if (event.target.matches('[data-picker], [data-protocol-picker]')) update(card); + }); card.addEventListener('focusin', event => { if (event.target.matches('input.st-input')) offerCatalogue(card, event.target); }); }); // Initialize after autosave records the server state, so a sole replacement diff --git a/app/static/settings/settings.css b/app/static/settings/settings.css index 04e71ec0..b101a29e 100644 --- a/app/static/settings/settings.css +++ b/app/static/settings/settings.css @@ -224,17 +224,7 @@ .settings-dialog .dialog-status:empty { display: none; } .settings-dialog [data-cancel] { margin-left: auto; } -/* the judge in use: two cards, one checked once JUDGE is set; the one in use says so either way */ -.st-judges { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 240px), 1fr)); gap: 10px; margin: 0 0 16px; } -.st-judge { display: flex; align-items: flex-start; gap: 10px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 10px; cursor: pointer; } -.st-judge:has(input:checked) { border-color: var(--accent); background: color-mix(in srgb, var(--accent) 8%, transparent); } -.st-judge input { margin-top: 3px; } -.st-judge b { font-weight: 500; } -.st-judge small { display: block; color: var(--text-dim); font-size: 12px; margin-top: 3px; } - [data-when][hidden] { display: none; } -.st-judge small { overflow-wrap: anywhere; } -.st-judge > span { min-width: 0; } /* Grouped settings navigation, shared by the modal and direct page. */ .st-index .st-nav-group { display: flex; flex-direction: column; gap: 2px; } @@ -312,12 +302,12 @@ .st-provider-summary { display: flex; align-items: center; gap: 14px; min-width: 0; } .st-provider-summary > div { flex: 1; min-width: 0; } .st-provider-summary h3 { overflow-wrap: anywhere; } +.st-endpoints { margin-top: 4px; } +.st-endpoint-rows { display: flex; flex-direction: column; gap: 18px; padding-top: 8px; } +.st-endpoints > summary { display: flex; align-items: center; padding: 10px 0; font-size: 13px; } .st-provider-summary .st-about { margin-top: 3px; } .st-provider-done { align-self: flex-end; } .st-provider-ask { position: static; width: 420px; max-width: 100%; box-sizing: border-box; margin: 12px 0 0 auto; } -.st-judge-single { margin-bottom: 0; } -.st-judge-single .st-judge { border: 0; padding: 0; background: transparent; } -.st-judge-single input { display: none; } /* Labels follow the action row, not the combined action and status height. */ .st-guidance .st-row { align-items: start; } diff --git a/app/templates/settings/custom_keys.html b/app/templates/settings/custom_keys.html index 54f51a16..da5e6095 100644 --- a/app/templates/settings/custom_keys.html +++ b/app/templates/settings/custom_keys.html @@ -48,6 +48,7 @@

{{ b.title }}

{% endfor %}

No custom keys yet.

+ {% for name in reserved_agent_keys or [] %}

AGENT_KEYS lists {{ name }}, which is now the name of a Nullius setting. Add this key again with a different name.

{% endfor %}
{% for name in agent_keys %}{% if by.get(name) %} {% set r = by[name] %} diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index a6441266..95f68edb 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -11,22 +11,16 @@ {% set marks = url_for('static', filename='brand/marks.svg') %} {% set hosts_row = by.get('HOSTS') %} {% set chosen = (hosts_row.shown.split(',') | map('trim') | select | list) if hosts_row else [] %} -{% set providers = [('typesafe', 'TypeSafe AI', 'https://api.typesafe.ai/v1/systemone', [('TYPESAFE_API_KEY', 'Key')]), ('openrouter', 'OpenRouter', 'https://openrouter.ai/api/v1', [('OPENROUTER_API_KEY', 'Key')]), - ('openai', 'OpenAI', 'https://api.openai.com/v1', [('OPENAI_API_KEY', 'Key')]), - ('anthropic', 'Anthropic', 'https://api.anthropic.com/v1', [('ANTHROPIC_API_KEY', 'Key')]), - ('local', 'OpenAI-compatible', '', [('MODEL_API_URL', 'Endpoint'), ('MODEL_API_KEY', 'Key')])] %} -{#- each named OpenAI-compatible endpoint is a card of its own, local:, as the LLM judge's picker names it -#} -{% set cards = providers + [] %} -{% for i, name in endpoints %}{% set _ = cards.append(('local:' ~ i, name, '', [('MODEL_' ~ i ~ '_NAME', 'Name'), ('MODEL_' ~ i ~ '_API_URL', 'Endpoint'), ('MODEL_' ~ i ~ '_API_KEY', 'Key')])) %}{% endfor %} - {#- mode, in a provider's editor: 'plain' is edited in place (a name, an endpoint's URL); 'key' is written, never shown, and replaced; 'optional-key' may also be removed. Elsewhere, Replace and Remove. -#} -{% macro field(r, label, writable, options=none, mode='') %} +{% macro field(r, label, writable, options=none, mode='', placeholder='', tip='') %} {% set plain = mode == 'plain' %}
- +
- {% if writable %} + {% if r.locked %} + {{ r.shown or placeholder }}

Set in the process environment.

+ {% elif writable %} {% if r.source != 'unset' and not plain %}
{{ r.shown }} @@ -34,7 +28,7 @@ {% if mode != 'key' %}{% endif %}
{% endif %} - + {% if r.problem %}

{{ r.problem }}

{% endif %} {% if options is not none %}{% for o in options %}{% endfor %}{% endif %} {% if r.source == 'environment' %}

Also set in the process environment: the running process keeps that value until it is restarted; the file applies then.

{% endif %} @@ -52,12 +46,12 @@

Settings

Ready to run - {% set ns = namespace(compatible='MODEL_API_URL' in held) %} - {% for i, _ in endpoints %}{% if ('MODEL_' ~ i ~ '_API_URL') in held %}{% set ns.compatible = true %}{% endif %}{% endfor %} - {% for label, name in [('OpenRouter', 'OPENROUTER_API_KEY'), ('OpenAI', 'OPENAI_API_KEY'), ('Anthropic', 'ANTHROPIC_API_KEY'), ('OpenAI-compatible', 'MODEL_API_URL')] %} - {% set on = ns.compatible if name == 'MODEL_API_URL' else name in held %} - {{ label }}{{ ': set' if on else ': not set' }} + {% set ns = namespace(custom=false) %} + {% for c in provider_cards if not c.preset and c.ready %}{% set ns.custom = true %}{% endfor %} + {% for c in provider_cards if c.preset and c.run %} + {{ c.title }}{{ ': set' if c.ready else ': not set' }} {% endfor %} + Custom{{ ': set' if ns.custom else ': not set' }} {% for jj, name in [(judges[0], 'System One'), (judges[1], 'LLM judge')] %} {{ name }}{{ ': set' if jj else ': not set' }}{% endfor %} {% for item in chosen %} {{ item.split('=', 1)[0] if '=' in item.split('@', 1)[0] else item.split(':/')[0].split(':~')[0] }}{% else %} no run host{% endfor %}
@@ -112,61 +106,71 @@

Settings

Manage the providers available for runs and judges.

- {% macro provider_logo(key) %}{% if key == 'typesafe' %}{% elif key.startswith('local') %}{% else %}{% endif %}{% endmacro %} + {% macro provider_logo(key) %}{% if key == 'typesafe' %}{% elif key.startswith('custom') %}{% else %}{% endif %}{% endmacro %} - {% for key, title, endpoint, names in cards %} - {% set any_set = names | map(attribute=0) | select('in', held) | list %} - {% set need = names[1 if key.startswith('local:') else 0][0] %} - {% set refused = need in held and by[need].problem %} - {% set ready = need in held and not refused %} -
+ {% for c in provider_cards %} +
- {{ provider_logo(key) }} - {% set ns_env = namespace(held=false) %}{% for name, _ in names %}{% if by.get(name) and by[name].source == 'environment' %}{% set ns_env.held = true %}{% endif %}{% endfor %} - {% set removable = writable and any_set %} + {{ provider_logo(c.key) }} + {% set removable = writable and c.configured %} {#- Test asks the saved settings at no cost (providers.test); TypeSafe has no such request -#} - {% set choice = key.replace('local', 'custom', 1) %}{% set last = tested.get(choice) %} - {% set testable = writable and key != 'typesafe' %} -

{{ title }}

{{ ('Endpoint saved' if key.startswith('local') else 'Key saved') if ready else 'Fix the endpoint URL' if refused else ('Endpoint required' if any_set else 'Not configured') }}

+ {% set last = tested.get(c.key) %} + {% set testable = writable and c.testable %} +

{{ c.title }}

{{ c.state }}

{% if testable or last %}

{{ last.said if last }}

{% endif %} - {% if removable and ns_env.held %}

Set in the process environment: remove it there.

{% endif %}
- {% if testable %}{% endif %} - + {% if removable and c.held_by_process %}

Set in the process environment: remove it there.

{% endif %}
+ {% if testable %}{% endif %} + {#- removing asks first, in place, as a benchmark's Delete does -#} - {% if removable %}{% endif %} + {% if removable %}{% endif %}
- {% if removable and not ns_env.held %} - {% endfor %} {% if writable %}{% endif %} {% if writable %} -
-
{{ provider_logo('local') }}

OpenAI-compatible

-
+
+
{{ provider_logo('custom') }}

Custom

+
- {% for part, label, kind in [('name', 'Name', 'text'), ('url', 'Endpoint', 'text'), ('key', 'Key', 'password')] %} + {% for part, label, kind in [('name', 'Name', 'text'), ('url', 'Base', 'text'), ('key', 'Key', 'password')] %}
{% endfor %} @@ -176,6 +180,7 @@

Add a provider

{% endif %} +

Judges

@@ -183,22 +188,41 @@

Judges

{% set s1, llm = judges %} - {% macro picker(name, providers, judge, label) %} -
- {% for key, d in providers.items() %} - {% set available = (d.get('url_var') or d['key']) in held %} - - {% endfor %} -
-

No compatible provider added. Add a provider

+ {#- Provider, then Protocol when the provider lists two this judge speaks (else a quiet line), then Model. -#} + {% macro judge_rows(prefix, choices, judge, label) %} + {% set name, pname = prefix ~ '_PROVIDER', prefix ~ '_PROTOCOL' %} + {% set prow, qrow = by[name], by[pname] %} + {% set on = choices.get(judge.provider) if choices.get(judge.provider, {}).get('ready') else none %} + {% set offered = on.listed if on else [] %} +
+ +
+ +

1 %} hidden{% endif %}>via {{ protocols[judge.protocol].label }}

+

No compatible provider added. Add a provider

+
+
+
+ +
+ {#- named only once set, so a provider change never saves a protocol nobody chose (judges.js) -#} + +
+
{% endmacro %} {% set srow = by.get('SYSTEM_ONE_PROVIDER') %} {% if srow and by.get('SYSTEM_ONE_MODEL') %}
System One -
{{ picker('SYSTEM_ONE_PROVIDER', s1_providers, s1, 'Where System One is asked') }}
+ {{ judge_rows('SYSTEM_ONE', s1_providers, s1, 'Where System One is asked') }} {{ field(by['SYSTEM_ONE_MODEL'], 'Model', writable, s1_providers[s1.provider].models) }}
{% endif %} @@ -206,10 +230,8 @@

Judges

{% if prow and by.get('LLM_JUDGE_MODEL') %}
LLM judge -
{{ picker('LLM_JUDGE_PROVIDER', llm_providers, llm, "The LLM judge's provider") }}
- + {{ judge_rows('LLM_JUDGE', llm_providers, llm, "The LLM judge's provider") }} {{ field(by['LLM_JUDGE_MODEL'], 'Model', writable, []) }} -
{% endif %}
diff --git a/app/templates/traces/_judgement.html b/app/templates/traces/_judgement.html index 0725d7e1..1ad7e4d4 100644 --- a/app/templates/traces/_judgement.html +++ b/app/templates/traces/_judgement.html @@ -12,13 +12,13 @@

{{ 'LLM judge' if llm else 'System One judge' }} experimental

-

{% if llm %}The same questions, asked of {{ j.judge.label if j.keyed else 'the LLM judge Settings names' }}. Its numbers are what the model states, not calibrated: a choice or a score carries no confidence.{% else %}Task outcome, answer quality, and behaviour — as TypeSafe's System One reads the transcript.{% endif %} Experimental: the recorded outcome and reward stand.

+

{% if llm %}The same questions, asked of {{ j.judge.label if j.keyed else 'the LLM judge Settings names' }}. A choice or a score carries no confidence.{% else %}Task outcome, answer quality and behaviour, as {{ j.record.judge.model if j.record else (j.judge.label if j.keyed else 'the System One model Settings names') }} reads the transcript.{% endif %} Experimental: the recorded outcome and reward stand.

{% if j.keyed %}
{% else %}{{ 'LLM judge' if llm else 'System One' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %}
- {% if j.record %}

{{ j.record.judge.model }}{% if j.record.judge.provider not in ('typesafe.ai', 'typesafe') %} via {{ {'openrouter': 'OpenRouter', 'openai': 'OpenAI'}.get(j.record.judge.provider, 'an OpenAI-compatible endpoint' if (j.record.judge.provider or '').startswith('local') else j.record.judge.provider) }}{% endif %} · {{ j.record.questions }}{% if j.earlier %} earlier vocabulary{% endif %} · asked {{ j.record.asked_at[:16]|replace('T', ' ') }}Z{% if j.record.truncated %} · state truncated{% if j.record.states %} ({% for kind, k in j.record.states.items() if k.truncated %}{{ kind }}{{ ', ' if not loop.last }}{% endfor %}){% endif %}{% endif %}{% if j.record.usage and j.record.usage.input_tokens %} · {{ j.record.usage.input_tokens }} input tokens{% endif %}

{% endif %} + {% if j.record %}

{{ j.record.judge.model }}{% if j.record.judge.provider not in ('typesafe.ai', 'typesafe') %} via {{ j.record.judge.provider|provider }}{% endif %}{% if j.record.refusals %} · {{ j.record.refusals|length }} refused{% endif %} · {{ j.record.questions }}{% if j.earlier %} earlier vocabulary{% endif %} · asked {{ j.record.asked_at[:16]|replace('T', ' ') }}Z{% if j.record.truncated %} · state truncated{% if j.record.states %} ({% for kind, k in j.record.states.items() if k.truncated %}{{ kind }}{{ ', ' if not loop.last }}{% endfor %}){% endif %}{% endif %}{% if j.record.usage and j.record.usage.input_tokens %} · {{ j.record.usage.input_tokens }} input tokens{% endif %}

{% endif %} {% if j.hygiene and j.hygiene.kinds %}

{% if j.hygiene.flagged %}⚠ state hygiene {% for kind, raised in j.hygiene.flagged.items() %}`{{ kind }}` state: {% for id, label, p, decided in raised %}{{ label }} {{ '%.0f'|format(p * 100) }}%{% if not decided %} (undecided){% endif %}{{ '; ' if not loop.last }}{% endfor %}{{ ' · ' if not loop.last }}{% endfor %} — the answers from that state are marked ⚠; none is changed {% else %}state hygiene nothing read as addressing the judge or showing a credential on {{ j.hygiene.kinds|join(', ') }}{% endif %}

{% endif %} diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index 9aaa6709..edbbca05 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -84,14 +84,14 @@ def test_a_run_gets_its_own_key_and_no_other_secret(self): self.assertLessEqual({"API_KEY_TYPESAFE", "API_TOKEN"}, set(execution.withheld())) # old spellings of a secret self.assertEqual({"HOST", "PORT"} & set(execution.withheld()), set()) # old names of start-up options are no secret held = {n: "sk-held-" + n.lower() for n in execution.withheld()} - plan = execution.Plan(name="own", script="env", model="m", key_var="OPENROUTER_API_KEY", url_var="", base_url=providers.PROVIDER_BASE["openrouter"]) + plan = execution.Plan(name="own", script="env", model="m", key_var="OPENROUTER_API_KEY", url_var="", base_url=providers.PRESETS["openrouter"]["base"]) with mock.patch.dict(os.environ, held): self.assertEqual([k for k in execution._clean_env() if k in held], []) ex = self.wait(execution.Execution(plan, root=str(self.tmp)).start()) self.assertEqual((ex.state, ex.error), ("done", None), ex.lines) env = dict(l.split("=", 1) for l in (self.tmp / "own" / "run-run.log").read_text().splitlines() if "=" in l) self.assertEqual({k: v for k, v in env.items() if v.startswith("sk-held-")}, {n: held["OPENROUTER_API_KEY"] for n in execution.KEY_NAMES}) - self.assertEqual({n: env.get(n) for n in execution.URL_NAMES}, {n: providers.PROVIDER_BASE["openrouter"] for n in execution.URL_NAMES}) + self.assertEqual({n: env.get(n) for n in execution.URL_NAMES}, {n: providers.PRESETS["openrouter"]["base"] for n in execution.URL_NAMES}) def test_a_run_on_a_named_endpoint_gets_its_url_and_key_alone(self): """Several OpenAI-compatible endpoints in Settings: a run on one is sent diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 0b78317b..85f34ed5 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -276,7 +276,7 @@ def chat(url, body, key): asked = [q for q in ("task_completed",) if f'"{q}"' in body["messages"][1]["content"]] return 200, {"model": "vendor/m", "choices": [{"message": {"content": _json.dumps({"answers": {q: {"noul": 0.3} for q in asked}})}}], "usage": {"prompt_tokens": 10, "completion_tokens": 2}} - llm = client.Judge("openrouter", "sk-or-1", "vendor/m", client.LLM_PROVIDERS["openrouter"]["url"]) + llm = client.Judge("openrouter", "sk-or-1", "vendor/m", "https://openrouter.ai/api/v1/chat/completions") rec = batch.ask_and_store(str(self.tmp), "job", "t", {"trial": (["task_completed"], "the trial state", False)}, llm, post=chat) self.assertEqual(rec["judge"], {"provider": "openrouter", "model": "vendor/m"}) self.assertEqual(store.load(str(self.tmp), "job", "t", lane="llm")["answers"], {"task_completed": {"noul": 0.3}}) diff --git a/app/tests/test_judge_decisions.py b/app/tests/test_judge_decisions.py new file mode 100644 index 00000000..46bd174f --- /dev/null +++ b/app/tests/test_judge_decisions.py @@ -0,0 +1,297 @@ +"""System One in each decision protocol, and the LLM judge on Anthropic's +Messages. Every provider is a fake ``post``: it records what it was sent and +answers with a fixture reply.""" +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +import shutil +import tempfile +import threading +import unittest +from unittest import mock + +from nullius import providers +from nullius.judge import client, questions as Q + +IDS = ["task_completed", "failure_cause", "transcript_quality"] + + +def fake(status, data): + sent = [] + + def post(url, body, key, timeout=120.0, auth=None): + sent.append({"url": url, "body": body, "key": key, "auth": auth}) + return status, data + return post, sent + + +def judge(env): + return client.system_one(env) + + +class Protocols(unittest.TestCase): + def test_each_provider_and_model_gets_its_decision_protocol(self): + got = lambda env: (judge(env).protocol, judge(env).url) + self.assertEqual(got({"TYPESAFE_API_KEY": "k"}), ("systemone", "https://api.typesafe.ai/v1/systemone")) + self.assertEqual(got({"OPENROUTER_API_KEY": "k"}), ("systemone", "https://openrouter.ai/api/v1/systemone")) + self.assertEqual(got({"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_MODEL": "microsoft/microsoft-decision-1"}), + ("decisions", "https://openrouter.ai/api/alpha/decisions")) # another maker's model + self.assertEqual(got({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}), ("openai-decisions", "https://api.openai.com/v1/decisions")) + self.assertEqual(judge({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}).model, "gpt-6-luna") + custom = {"MODEL_GW": '{"base": "https://gw/v1", "endpoints": {"decisions": "/alpha/decisions"}}', "MODEL_GW_API_KEY": "k", + "SYSTEM_ONE_PROVIDER": "custom:GW", "SYSTEM_ONE_MODEL": "m"} + self.assertEqual(got(custom), ("decisions", "https://gw/v1/alpha/decisions")) + self.assertNotIn("custom:LAB", client.system_one_providers({"MODEL_LAB": '{"base": "http://lab/v1"}'})) # lists none: not offered + + def test_a_chosen_protocol_wins_where_the_provider_serves_it_else_the_automatic_one(self): + from nullius import settings + got = lambda env: (judge(env).protocol, judge(env).url) + router = {"OPENROUTER_API_KEY": "k"} + self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="decisions")), ("decisions", "https://openrouter.ai/api/alpha/decisions")) # over TypeSafe's model + self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="systemone", SYSTEM_ONE_MODEL="microsoft/microsoft-decision-1")), + ("systemone", "https://openrouter.ai/api/v1/systemone")) + self.assertEqual(got(dict(router, SYSTEM_ONE_PROTOCOL="openai-decisions")), ("systemone", "https://openrouter.ai/api/v1/systemone")) # not served: automatic + self.assertEqual(client.system_one_providers(dict(router, SYSTEM_ONE_PROTOCOL="decisions"))["openrouter"]["protocol"], "decisions") + self.assertEqual(client.system_one_providers(router)["openrouter"]["listed"], ["systemone", "decisions"]) + both = {"ANTHROPIC_API_KEY": "k", "ANTHROPIC": '{"endpoints": {"chat": "/chat/completions"}}', "LLM_JUDGE_PROVIDER": "anthropic", "LLM_JUDGE_MODEL": "m"} + llm = lambda env: (client.llm_judge(env).protocol, client.llm_judge(env).url) + self.assertEqual(llm(both), ("chat", "https://api.anthropic.com/v1/chat/completions")) # unset: chat before messages + self.assertEqual(llm(dict(both, LLM_JUDGE_PROTOCOL="messages")), ("messages", "https://api.anthropic.com/v1/messages")) + self.assertEqual(client.llm_providers(both)["anthropic"]["listed"], ["chat", "messages"]) + self.assertEqual(llm({"OPENROUTER_API_KEY": "k", "LLM_JUDGE_MODEL": "m", "LLM_JUDGE_PROTOCOL": "messages"}), + ("chat", "https://openrouter.ai/api/v1/chat/completions")) # not served: automatic + for name, spoken in (("LLM_JUDGE_PROTOCOL", client.LLM_PROTOCOLS), ("SYSTEM_ONE_PROTOCOL", client.SYSTEM_ONE_PROTOCOLS)): + var = settings.BY_NAME[name] + self.assertEqual([p for p in client.providers.PROTOCOLS if var.validate(p) is None], list(spoken)) # what that judge speaks, no more + self.assertIsNotNone(var.validate("Chat")) + + +class SystemOne(unittest.TestCase): + def test_systemone_keeps_the_providers_cost_and_records_a_refusal(self): + post, _ = fake(200, {"model": "~typesafe/jev-latest", "answers": { + "task_completed": {"noul": 0.8}, "failure_cause": {"type": "refusal", "refusal": "not enough evidence"}}, + "usage": {"input_tokens": 900, "output_tokens": 0, "cost": 0.00004}}) + rec = client.ask("state", IDS, judge({"OPENROUTER_API_KEY": "sk-or-1"}), post=post) + self.assertEqual(rec["answers"], {"task_completed": {"noul": 0.8}}) + self.assertEqual(rec["refusals"], {"failure_cause": "not enough evidence"}) # said, not dropped + self.assertEqual(rec["usage"]["cost_usd"], 0.00004) + merged = client.merge([rec, dict(rec, refusals={"transcript_quality": "refused"})]) + self.assertEqual(set(merged["refusals"]), {"failure_cause", "transcript_quality"}) + + def test_a_402_says_what_it_means(self): + post, _ = fake(402, {"error": {"code": 402, "message": "Insufficient credits"}}) + with self.assertRaises(client.JudgeError) as e: + client.ask("state", IDS, judge({"OPENROUTER_API_KEY": "sk-or-1"}), post=post, sleep=lambda s: None) + self.assertEqual(e.exception.status, 402) + self.assertIn("credit is spent", str(e.exception)); self.assertIn("Insufficient credits", str(e.exception)) + + def test_decisions_reads_confidence_and_its_error_shape(self): + env = {"OPENROUTER_API_KEY": "sk-or-1", "SYSTEM_ONE_MODEL": "inception/mercury-decide"} + post, sent = fake(200, {"model": "inception/mercury-decide", "answers": {"task_completed": {"noul": 0.3, "confidence": 0.9}}, + "usage": {"input_tokens": 10, "output_tokens": 0, "cost": 0.001}}) + rec = client.ask("state", IDS, judge(env), post=post) + self.assertEqual(sent[0]["url"], "https://openrouter.ai/api/alpha/decisions") + self.assertEqual(set(sent[0]["body"]), {"model", "state", "questions"}) # the same request as systemone + self.assertEqual(rec["answers"]["task_completed"], {"noul": 0.3, "confidence": 0.9}) + self.assertEqual(rec["judge"], {"provider": "openrouter", "model": "inception/mercury-decide"}) + post, _ = fake(400, {"error": {"code": "invalid_model", "message": "Box questions are Jev-only"}}) + with self.assertRaises(client.JudgeError) as e: + client.ask("state", IDS, judge(env), post=post) + self.assertIn("Box questions are Jev-only", str(e.exception)) + + + def test_a_null_refusal_beside_an_answer_is_the_answer(self): + post, _ = fake(200, {"answers": {"task_completed": {"noul": 0.7, "refusal": None}}}) + rec = client.ask("state", IDS, judge({"OPENROUTER_API_KEY": "sk-or-1"}), post=post) + self.assertEqual(rec["answers"], {"task_completed": {"noul": 0.7}}); self.assertNotIn("refusals", rec) + + def test_a_reply_of_another_shape_is_a_judge_error(self): + for env, data in (({"OPENROUTER_API_KEY": "k"}, {"answers": [1, 2]}), + ({"OPENROUTER_API_KEY": "k"}, {"answers": {"failure_cause": {"choice": "a", "probabilities": {"a": "x"}}}}), + ({"OPENROUTER_API_KEY": "k"}, ["not", "an", "object"]), + ({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}, {"answers": [{"name": ["task_completed"]}]}), + ({"OPENAI_API_KEY": "k", "SYSTEM_ONE_PROVIDER": "openai"}, + {"answers": [{"name": "failure_cause", "choice": next(iter(Q.QUESTIONS["failure_cause"]["criteria"])), "probabilities": 5}]})): + post, _ = fake(200, data) + with self.assertRaises(client.JudgeError, msg=data) as e: + client.ask("state", IDS, judge(env), post=post) + self.assertEqual(e.exception.status, 502); self.assertIn("could not be read", str(e.exception)) + + +class Redirects(unittest.TestCase): + def test_a_redirect_is_an_error_and_the_key_goes_no_further(self): + seen = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_POST(self): + seen.append((self.server.server_address[1], self.path, self.headers.get("Authorization"))) + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + if self.path == "/judge": + self.send_response(302); self.send_header("Location", f"http://127.0.0.1:{other.server_address[1]}/stolen") + else: + self.send_response(200) + self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"{}") + + do_GET = do_POST + + first, other = ThreadingHTTPServer(("127.0.0.1", 0), Handler), ThreadingHTTPServer(("127.0.0.1", 0), Handler) + for srv in (first, other): + threading.Thread(target=srv.serve_forever, daemon=True).start() + self.addCleanup(srv.server_close); self.addCleanup(srv.shutdown) + with self.assertRaises(client.JudgeError) as e: + client.http_post(f"http://127.0.0.1:{first.server_address[1]}/judge", {}, "sk-fixture-redirect") + self.assertEqual(e.exception.status, 302); self.assertIn("redirects", str(e.exception)) + self.assertEqual([s[1] for s in seen], ["/judge"]) # never the other host + + +LISTED = {"data": [{"id": "typesafe/jev-1.14", "architecture": {"output_modalities": ["decisions"]}}, + {"id": "perplexity/pplx-decider-v1.3:free", "architecture": {"output_modalities": ["decisions"]}}, + {"id": "~liquid/d1-latest", "architecture": {"output_modalities": ["text", "decisions"]}}, + {"id": "openai/gpt-6", "architecture": {"output_modalities": ["text"]}}, + {"id": "odd/string", "architecture": {"output_modalities": "decisions"}}, + {"id": "odd/none"}]} + + +class DecisionModels(unittest.TestCase): + """OpenRouter's decision models: its public list, filtered, cached, and the short list without either.""" + + def setUp(self): + self.home = tempfile.mkdtemp(prefix="nullius-decisions-"); self.addCleanup(shutil.rmtree, self.home, True) + self.env, self.now, self.sent = {"NULLIUS_HOME": self.home}, 1000.0, [] + + def opener(self, body=LISTED): + def fetch(req, timeout): + self.sent.append((req.full_url, dict(req.header_items()), timeout)) + if isinstance(body, Exception): + raise body + return io.BytesIO(json.dumps(body).encode()) + return fetch + + def models(self, body=LISTED, **kw): + return providers.decision_models(self.env, opener=self.opener(body), clock=lambda: self.now, **kw) + + def test_only_decision_models_are_kept_and_no_key_is_sent(self): + env = dict(self.env, OPENROUTER_API_KEY="sk-or-fixture") + got = providers.decision_models(env, opener=self.opener(), clock=lambda: self.now) + self.assertEqual(sorted(got), ["perplexity/pplx-decider-v1.3:free", "typesafe/jev-1.14", "~liquid/d1-latest"]) + (url, headers, timeout), = self.sent + self.assertEqual(url, "https://openrouter.ai/api/v1/models?output_modalities=decisions") + self.assertNotIn("Authorization", headers); self.assertNotIn("sk-or-fixture", json.dumps(headers)) + self.assertEqual(timeout, 5) + + def test_the_cache_is_used_while_fresh_and_kept_when_the_network_fails(self): + first = self.models() + self.assertEqual(self.models(OSError("unreached")), first); self.assertEqual(len(self.sent), 1) # a hit: not asked + self.now += providers.DECISION_TTL + 1 + self.assertEqual(self.models(OSError("unreached")), first); self.assertEqual(len(self.sent), 2) # stale: asked, kept + self.assertEqual(self.models({"data": [{"id": "new/one", "architecture": {"output_modalities": ["decisions"]}}]}), ["new/one"]) + self.assertEqual(self.models(OSError("unreached")), ["new/one"]) # fresh again + self.now += providers.DECISION_TTL + 1 + self.assertEqual(self.models(fetch=False), ["new/one"]); self.assertEqual(len(self.sent), 3) # never asked without fetch + + def test_no_network_and_no_cache_is_no_list(self): + for body in (OSError("unreached"), {"data": {}}, {"data": []}, ValueError("x")): + self.assertEqual(self.models(body), []) + self.assertEqual(self.models(fetch=False), []) + + def test_a_redirect_is_refused(self): + seen = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_GET(self): + seen.append(self.path) + self.send_response(302); self.send_header("Location", f"http://127.0.0.1:{srv.server_address[1]}/elsewhere") + self.send_header("Content-Length", "0"); self.end_headers() + + srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + self.addCleanup(srv.server_close); self.addCleanup(srv.shutdown) + with mock.patch.dict(providers.PRESETS["openrouter"], base=f"http://127.0.0.1:{srv.server_address[1]}/api/v1"): + self.assertEqual(providers.decision_models(self.env), []) + self.assertEqual(seen, ["/api/v1/models?output_modalities=decisions"]) # never followed + + def test_the_suggestions_are_the_list_with_the_alias_first_or_the_short_list(self): + with mock.patch.object(providers, "decision_models", return_value=["z/one", "typesafe/jev-1.14", "a/two:free", "~typesafe/jev-latest"]) as live: + self.assertEqual(client.system_one_models("openrouter"), ["~typesafe/jev-latest", "typesafe/jev-1.14", "a/two:free", "z/one"]) + live.assert_called_with(fetch=False) + client.system_one_models("openrouter", fetch=True); live.assert_called_with(fetch=True) + self.assertEqual(client.system_one_models("openai"), ["gpt-6-luna"]) + self.assertEqual(client.system_one_providers({"OPENROUTER_API_KEY": "k"})["openrouter"]["model"], "~typesafe/jev-latest") + with mock.patch.object(providers, "decision_models", return_value=["b/x"]): + self.assertEqual(client.system_one_models("openrouter"), ["~typesafe/jev-latest", "b/x"]) + with mock.patch.object(providers, "decision_models", return_value=[]): + self.assertEqual(client.system_one_models("openrouter"), client.SYSTEM_ONE_MODELS["openrouter"]) + + def test_the_protocol_follows_the_model_not_the_list(self): + on = lambda model, **env: client.system_one({"OPENROUTER_API_KEY": "k", "SYSTEM_ONE_MODEL": model, **env}).protocol + with mock.patch.object(providers, "decision_models", return_value=["google/gemini-decision-1", "typesafe/jev-1.14"]): + self.assertEqual(on("typesafe/jev-1.14"), "systemone") + self.assertEqual(on("google/gemini-decision-1"), "decisions") + self.assertEqual(on("someone/typed-by-hand"), "decisions") + self.assertEqual(on("google/gemini-decision-1", SYSTEM_ONE_PROTOCOL="systemone"), "systemone") + + +class OpenAIDecisions(unittest.TestCase): + ENV = {"OPENAI_API_KEY": "sk-oa-1", "SYSTEM_ONE_PROVIDER": "openai"} + + def test_the_request_is_openais_shape(self): + post, sent = fake(200, {"model": "gpt-6-luna", "answers": [], "usage": {}}) + client.ask("the state", IDS, judge(self.ENV), post=post) + body = sent[0]["body"] + self.assertEqual((body["model"], body["input"]), ("gpt-6-luna", "the state")) + self.assertNotIn("state", body) + asked = {q["name"]: q for q in body["questions"]} + self.assertEqual(list(asked), IDS) + yes_no = asked["task_completed"] + self.assertEqual(yes_no["type"], "predicate"); self.assertNotIn("criteria", yes_no) + self.assertIn("True means: " + Q.QUESTIONS["task_completed"]["criteria"]["true"], yes_no["instructions"]) # criteria fold into it + self.assertEqual([c["value"] for c in asked["failure_cause"]["choices"]], list(Q.QUESTIONS["failure_cause"]["criteria"])) + self.assertEqual([lv["label"] for lv in asked["transcript_quality"]["levels"]], ["0", "1", "2"]) + + def test_the_reply_is_read_into_the_record_and_a_refusal_is_said(self): + option = next(iter(Q.QUESTIONS["failure_cause"]["criteria"])) + post, _ = fake(200, {"model": "gpt-6-luna", "usage": {"input_tokens": 50, "output_tokens": 3}, + "answers": [{"name": "task_completed", "type": "predicate", "probability": 0.7}, + {"name": "failure_cause", "type": "choice", "choice": option, + "probabilities": [{"value": option, "probability": 0.6}]}, + {"name": "transcript_quality", "type": "refusal", "refusal": "cannot judge"}, + {"name": "not_asked", "type": "predicate", "probability": 1}]}) + rec = client.ask("s", IDS, judge(self.ENV), post=post) + self.assertEqual(rec["answers"]["task_completed"], {"noul": 0.7}) + self.assertEqual(rec["answers"]["failure_cause"]["probabilities"], {option: 0.6}) + self.assertEqual(rec["refusals"], {"transcript_quality": "cannot judge"}) + self.assertNotIn("not_asked", rec["answers"]); self.assertEqual(rec["usage"]["input_tokens"], 50) + + def test_a_score_out_of_its_levels_is_left_out(self): + post, _ = fake(200, {"answers": [{"name": "transcript_quality", "type": "score", "level": "7"}]}) + self.assertEqual(client.ask("s", IDS, judge(self.ENV), post=post)["answers"], {}) + post, _ = fake(200, {"answers": [{"name": "transcript_quality", "type": "score", "level": "2"}]}) + self.assertEqual(client.ask("s", IDS, judge(self.ENV), post=post)["answers"]["transcript_quality"]["score"], 2.0) + + +class LLMJudgeOnMessages(unittest.TestCase): + def test_anthropic_is_asked_in_its_own_format_with_its_own_headers(self): + env = {"ANTHROPIC_API_KEY": "sk-ant-1", "LLM_JUDGE_PROVIDER": "anthropic", "LLM_JUDGE_MODEL": "claude-x"} + j = client.llm_judge(env) + self.assertEqual((j.provider, j.protocol, j.url), ("anthropic", "messages", "https://api.anthropic.com/v1/messages")) + post, sent = fake(200, {"model": "claude-x", "content": [{"type": "text", "text": '{"answers": {"task_completed": {"noul": 0.9}}}'}], + "usage": {"input_tokens": 120, "output_tokens": 9}}) + rec = client.ask("state", IDS, j, post=post) + self.assertEqual(sent[0]["auth"]["x-api-key"], "sk-ant-1"); self.assertIn("anthropic-version", sent[0]["auth"]) + body = sent[0]["body"] + self.assertEqual([m["role"] for m in body["messages"]], ["user"]); self.assertIn("system", body); self.assertNotIn("response_format", body) + self.assertEqual(rec["answers"], {"task_completed": {"noul": 0.9}}) + self.assertEqual((rec["usage"]["input_tokens"], rec["usage"]["output_tokens"]), (120, 9)) + self.assertNotIn("sk-ant-1", repr(j)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_judge_openrouter.py b/app/tests/test_judge_openrouter.py index d8945e11..11a21031 100644 --- a/app/tests/test_judge_openrouter.py +++ b/app/tests/test_judge_openrouter.py @@ -30,28 +30,29 @@ def test_system_one_and_the_llm_judge_are_set_apart(self): self.assertFalse(client.system_one({})); self.assertTrue(client.system_one({"TYPESAFE_API_KEY": "ts-1"})) self.assertEqual(client.system_one({}).lane, "system-one") j = client.llm_judge({"OPENROUTER_API_KEY": "sk-or-1"}) - self.assertEqual((j.provider, j.lane, j.url), ("openrouter", "llm", client.LLM_PROVIDERS["openrouter"]["url"])) + self.assertEqual((j.provider, j.lane, j.url), ("openrouter", "llm", "https://openrouter.ai/api/v1/chat/completions")) self.assertFalse(j); self.assertEqual(j.missing, "a model") # no model: not ready j = client.llm_judge({"OPENROUTER_API_KEY": "sk-or-1", "OPENAI_API_KEY": "sk-oa-1", "LLM_JUDGE_PROVIDER": "openai", "LLM_JUDGE_MODEL": "gpt-x"}) self.assertTrue(j); self.assertEqual((j.key, j.url), ("sk-oa-1", "https://api.openai.com/v1/chat/completions")) local = client.llm_judge({"MODEL_API_URL": "http://box:8000/v1/", "LLM_JUDGE_MODEL": "qwen"}) # the first provider set - self.assertEqual((local.provider, local.url, local.key), ("local", "http://box:8000/v1/chat/completions", "local")) + self.assertEqual((local.provider, local.url, local.key), ("custom", "http://box:8000/v1/chat/completions", "local")) self.assertTrue(local) self.assertNotIn("sk-or-1", repr(client.llm_judge({"OPENROUTER_API_KEY": "sk-or-1"}))) def test_the_llm_judge_on_a_named_endpoint_uses_its_url_and_its_key(self): env = {"MODEL_API_URL": "http://default/v1", "MODEL_API_KEY": "default-key", "MODEL_GPU_NAME": "GPU box", "MODEL_GPU_API_URL": "http://gpu:8000/v1", "MODEL_GPU_API_KEY": "gpu-key", "LLM_JUDGE_MODEL": "qwen"} - self.assertEqual(client.llm_providers(env)["local:GPU"], {"label": "GPU box", "url_var": "MODEL_GPU_API_URL", "key": "MODEL_GPU_API_KEY"}) - j = client.llm_judge(dict(env, LLM_JUDGE_PROVIDER="local:GPU")) - self.assertEqual((j.provider, j.url, j.key, j.label), ("local:GPU", "http://gpu:8000/v1/chat/completions", "gpu-key", "GPU box · qwen")) + self.assertEqual(client.llm_providers(env)["custom:GPU"], {"label": "GPU box", "url": "http://gpu:8000/v1/chat/completions", "key": "MODEL_GPU_API_KEY", + "ready": True, "protocol": "chat", "fit": ["chat", "messages"], "listed": []}) + j = client.llm_judge(dict(env, LLM_JUDGE_PROVIDER="local:GPU")) # the old name of custom:GPU + self.assertEqual((j.provider, j.url, j.key, j.label), ("custom:GPU", "http://gpu:8000/v1/chat/completions", "gpu-key", "GPU box · qwen")) self.assertTrue(j) keyless = client.llm_judge({"MODEL_GPU_API_URL": "http://gpu:8000/v1", "LLM_JUDGE_PROVIDER": "local:GPU", "LLM_JUDGE_MODEL": "qwen"}) self.assertEqual(keyless.key, "local"); self.assertTrue(keyless) # no key: the placeholder, as a run bad = client.llm_judge({"MODEL_API_URL": "http://u:pw@box/v1", "LLM_JUDGE_PROVIDER": "local", "LLM_JUDGE_MODEL": "qwen"}) self.assertEqual(bad.url, ""); self.assertFalse(bad) # a credential in its URL: not asked gone = client.llm_judge(dict(env, LLM_JUDGE_PROVIDER="local:GONE")) # removed: the first one set - self.assertEqual((gone.provider, gone.key), ("local", "default-key")) + self.assertEqual((gone.provider, gone.key), ("custom", "default-key")) class SystemOneOnOpenRouter(unittest.TestCase): @@ -74,7 +75,7 @@ def post(url, body, key): class OpenRouterJudge(unittest.TestCase): - judge = client.Judge("openrouter", "sk-or-1", "vendor/judge-1", client.LLM_PROVIDERS["openrouter"]["url"]) + judge = client.Judge("openrouter", "sk-or-1", "vendor/judge-1", "https://openrouter.ai/api/v1/chat/completions") def test_one_call_asks_every_question_and_reads_each_answer_by_its_type(self): post, sent = fake(200, completion({ @@ -83,7 +84,7 @@ def test_one_call_asks_every_question_and_reads_each_answer_by_its_type(self): "transcript_quality": {"score": 1}})) rec = client.ask("the state", IDS, self.judge, post=post, kind="trial") url, body, key = sent[0] - self.assertEqual((url, key, body["model"]), (client.LLM_PROVIDERS["openrouter"]["url"], "sk-or-1", "vendor/judge-1")) + self.assertEqual((url, key, body["model"]), ("https://openrouter.ai/api/v1/chat/completions", "sk-or-1", "vendor/judge-1")) self.assertEqual(body["response_format"], {"type": "json_object"}) prompt = body["messages"][1]["content"] for qid in IDS: @@ -116,7 +117,7 @@ def test_refusals_are_said_for_what_they_are(self): with self.subTest(status), self.assertRaisesRegex(client.JudgeError, said): client.ask("s", IDS, self.judge, post=post, sleep=lambda s: None) with self.assertRaisesRegex(client.JudgeError, "needs a model"): - client.ask("s", IDS, client.Judge("openrouter", "sk-or-1", "", client.LLM_PROVIDERS["openrouter"]["url"])) + client.ask("s", IDS, client.Judge("openrouter", "sk-or-1", "", "https://openrouter.ai/api/v1/chat/completions")) def test_a_busy_openrouter_is_asked_again(self): answers = iter([(429, {}), (200, completion({"task_completed": {"noul": 0.4}}))]) diff --git a/app/tests/test_provider_registry.py b/app/tests/test_provider_registry.py new file mode 100644 index 00000000..4203cb20 --- /dev/null +++ b/app/tests/test_provider_registry.py @@ -0,0 +1,217 @@ +"""The provider registry: one record per provider, merged in three layers, +read by every consumer for the protocol it speaks. No request is sent.""" +from contextlib import redirect_stderr, redirect_stdout +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from nullius import migrate, onboarding, paths, providers, settings +from nullius.judge import client +from nullius.recipes.launch import payload +from nullius.catalogue import catalog +from nullius.standard import run + +KEYS = {"OPENROUTER_API_KEY": "sk-or-fixture-0001", "OPENAI_API_KEY": "sk-oa-fixture-0002", + "ANTHROPIC_API_KEY": "sk-ant-fixture-0003", "MODEL_GW_API_KEY": "sk-gw-fixture-0004"} + + +class Records(unittest.TestCase): + def test_a_preset_is_its_fixed_fields_and_the_user_adds_only_other_protocols(self): + env = {"OPENROUTER": json.dumps({"endpoints": {"messages": "/messages"}})} + rec = providers.record("openrouter", env) + self.assertEqual((rec["name"], rec["base"], rec["key"]), ("OpenRouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY")) + self.assertEqual(providers.url(rec, "messages"), "https://openrouter.ai/api/v1/messages") + self.assertEqual(providers.url(rec, "decisions"), "https://openrouter.ai/api/alpha/decisions") # outside its base: a full URL + for fixed in ({"base": "https://proxy/v1"}, {"name": "X"}, {"key": "OTHER_API_KEY"}, {"endpoints": {"chat": "/x"}}): + self.assertIn("fixed", providers.record_problem("OPENROUTER", json.dumps(fixed)), fixed) + self.assertEqual(providers.record("openrouter", {"OPENROUTER": json.dumps(fixed)})["base"], "https://openrouter.ai/api/v1") + + def test_a_preset_adds_a_full_url_only_on_its_own_host(self): + for name, proto, bad in (("TYPESAFE", "decisions", "https://evil.example/d"), ("OPENROUTER", "messages", "https://evil.example/m"), + ("OPENAI", "messages", "https://api.openai.com.evil.example/m")): + why = providers.record_problem(name, json.dumps({"endpoints": {proto: bad}})) + self.assertIn("the preset's key is sent there", why or "", name) + self.assertNotIn("evil", providers.url(providers.record(name.lower(), {name: json.dumps({"endpoints": {proto: bad}})}), proto)) + self.assertIsNone(providers.record_problem("OPENROUTER", json.dumps({"endpoints": {"messages": "https://openrouter.ai/api/v1/messages"}}))) + self.assertIsNone(providers.record_problem("TYPESAFE", json.dumps({"endpoints": {"decisions": "/decisions"}}))) + + def test_a_custom_record_merges_per_protocol_and_null_removes(self): + file = {"name": "Gateway", "base": "https://gw.example/v1", "endpoints": {"chat": "/chat/completions", "decisions": "/alpha/decisions"}} + env = {"endpoints": {"decisions": None, "responses": "https://eu.gw.example/v1/responses"}} + merged = settings.merge_patch(file, env) + self.assertEqual(merged["endpoints"], {"chat": "/chat/completions", "responses": "https://eu.gw.example/v1/responses"}) + rec = providers.record("custom:GW", {"MODEL_GW": json.dumps(settings.compose(file, env))}) + self.assertEqual(rec["endpoints"], merged["endpoints"]); self.assertEqual(rec["key"], "MODEL_GW_API_KEY") + self.assertEqual(providers.url(rec, "decisions"), "https://gw.example/v1/decisions") # removed: the default path + self.assertEqual(settings.merge_patch({"a": {"b": 1}}, {"a": None}), {}) + + def test_a_path_is_joined_with_one_slash_and_a_url_stands_alone(self): + self.assertEqual(providers.join("https://gw.example/v1/", "/chat/completions"), "https://gw.example/v1/chat/completions") + self.assertEqual(providers.join("https://gw.example/v1", "/chat"), "https://gw.example/v1/chat") # /v1 kept: not URL resolution + self.assertEqual(providers.join("https://gw.example/v1", "https://other.example/d"), "https://other.example/d") + good = lambda e: providers.record_problem("MODEL_GW", json.dumps({"endpoints": {"chat": e}})) + self.assertIsNone(good("/v2/chat")); self.assertIsNone(good("https://other.example/chat")) + for bad in ("chat/completions", "//evil.example/x", "/a b", "ftp://x/y", "https://u:p@x/y", "https://x/y?api_key=1", 7, "/v1/../../x", "/.."): + self.assertIsNotNone(good(bad), bad) + self.assertIn("some of chat", providers.record_problem("MODEL_GW", json.dumps({"endpoints": {"carrier-pigeon": "/x"}}))) + self.assertIn("MODEL_GW_API_KEY", providers.record_problem("MODEL_GW", json.dumps({"key": "OPENAI_API_KEY"}))) + self.assertIsNone(providers.record_problem("MODEL_GW", json.dumps({"key": "MODEL_GW_API_KEY", "base": "https://gw/v1"}))) + self.assertIn("JSON object", providers.record_problem("MODEL_GW", "https://gw/v1")) + + def test_a_base_on_a_presets_host_suggests_its_endpoints(self): + self.assertEqual(set(providers.KNOWN_HOSTS), {"openrouter.ai", "api.openai.com", "api.anthropic.com", "api.typesafe.ai"}) + got = providers.suggestions("https://user:pw@openrouter.ai/api/v1") + self.assertEqual(got["decisions"], "https://openrouter.ai/api/alpha/decisions") # never the user or password + self.assertEqual(providers.suggestions("https://api.anthropic.com/v1"), {"messages": "https://api.anthropic.com/v1/messages"}) + self.assertEqual(providers.suggestions("https://gw.example/v1"), {}) + + def test_old_settings_are_read_and_the_record_wins(self): + env = {"MODEL_GPU_NAME": "GPU box", "MODEL_GPU_API_URL": "http://gpu:8000/v1"} + rec = providers.record("custom:GPU", env) + self.assertEqual((rec["name"], rec["base"], rec["ignored"]), ("GPU box", "http://gpu:8000/v1", [])) + rec = providers.record("custom:GPU", dict(env, MODEL_GPU=json.dumps({"base": "http://new:8000/v1"}))) + self.assertEqual((rec["name"], rec["base"], rec["ignored"]), ("GPU box", "http://new:8000/v1", ["MODEL_GPU_API_URL"])) + self.assertEqual(providers.record("custom", {"MODEL_API_URL": "http://box/v1"})["base"], "http://box/v1") + self.assertEqual(settings.endpoint_ids(dict(env, MODEL_GW="{}")), ["GPU", "GW"]) + + def test_a_reserved_name_in_agent_keys_leaves_the_others_in_use(self): + env = {"AGENT_KEYS": "CURSOR_API_KEY, OPENAI,GH_TOKEN,MODEL_GW", "CURSOR_API_KEY": "c", "GH_TOKEN": "g"} + self.assertEqual(settings.agent_key_names(env), ["CURSOR_API_KEY", "GH_TOKEN"]) + self.assertEqual(settings.agent_key_values(env), {"CURSOR_API_KEY": "c", "GH_TOKEN": "g"}) + self.assertEqual(settings.reserved_agent_keys(env), ["OPENAI", "MODEL_GW"]) + + def test_a_shell_variable_named_like_a_record_is_no_provider(self): + env = {"MODEL_NAME": "gpt-4", "MODEL_PATH": "/models", "MODEL_API_URL": "http://box/v1", "MODEL_X_API_KEY": "k"} + self.assertEqual(settings.endpoint_ids({k: v for k, v in env.items() if k != "MODEL_X_API_KEY"}), []) + self.assertFalse(settings.is_record("MODEL_NAME")); self.assertFalse(settings.is_record("MODEL_API_URL")) + self.assertFalse(settings.is_record("MODEL_GPU_API_KEY")); self.assertTrue(settings.is_record("MODEL_GPU")) + self.assertNotEqual(settings.new_endpoint_id("My name", set()), "MY_NAME") # its record would read as an old setting + + def test_no_key_is_ever_in_a_record_the_builder_or_its_payload(self): + env = dict(KEYS, MODEL_GW=json.dumps({"name": "GW", "base": "https://gw.example/v1"})) + with patch.dict(os.environ, env): + shown = json.dumps([providers.records(env), providers.run_choices(env), payload(catalog.load(), [])["providers"]]) + for key in KEYS.values(): + self.assertNotIn(key, shown) + self.assertIn("MODEL_GW_API_KEY", shown) # the setting's name, never its value + + +class Consumers(unittest.TestCase): + def test_a_consumer_gets_the_providers_that_may_serve_its_protocol(self): + self.assertEqual(providers.offered(["chat"], {}), ["openrouter", "openai", "custom"]) # Anthropic: not for a chat-only agent + self.assertEqual(providers.offered(["messages"], {}), ["anthropic", "custom"]) + self.assertEqual(providers.offered(["responses"], {}), ["openrouter", "openai", "custom"]) + env = dict(KEYS, MODEL_GW=json.dumps({"base": "https://gw/v1", "endpoints": {"decisions": "/d"}}), MODEL_LAB=json.dumps({"base": "http://lab/v1"})) + self.assertEqual(list(client._table(["decisions"], env, known=False)), ["openrouter", "custom:GW", "custom:LAB"]) + self.assertEqual(list(client.system_one_providers(env)), ["openrouter", "openai", "typesafe", "custom:GW"]) + px = dict(env, MODEL_PX=json.dumps({"name": "PX", "base": "https://px/v1", "endpoints": {"messages": "/messages"}})) + self.assertEqual(client.llm_providers(px)["custom:PX"]["protocol"], "messages") # the one it lists, before chat + self.assertEqual(providers.protocol_for("anthropic", ["chat", "messages"], env), "messages") + self.assertEqual(run.key_header("anthropic", {"model": {"apis": ["messages"]}}, env), "x-api-key") + self.assertEqual(run.key_header("openrouter", {"model": {"apis": ["chat"]}}, env), "authorization") + with self.assertRaisesRegex(run.Refused, "Anthropic does not serve what reference speaks"): + run.key_header("anthropic", {"key": "reference", "model": {"apis": ["chat"]}}, env) # as recipe.compile refuses it + + def test_each_judge_setting_takes_only_a_provider_that_judge_can_use(self): + s1, llm = settings.BY_NAME["SYSTEM_ONE_PROVIDER"].validate, settings.BY_NAME["LLM_JUDGE_PROVIDER"].validate + for ok in ("typesafe", "openrouter", "openai", "custom:GW", "local:GW"): + self.assertIsNone(s1(ok), ok) + for ok in ("openrouter", "openai", "anthropic", "custom", "custom:GW", "local"): + self.assertIsNone(llm(ok), ok) + self.assertIsNotNone(s1("anthropic")); self.assertIsNotNone(s1("custom")); self.assertIsNotNone(llm("typesafe")) + + def test_the_llm_judge_falls_back_to_anthropic_last(self): + env = {"ANTHROPIC_API_KEY": "k", "MODEL_API_URL": "http://localhost:1/v1"} + self.assertEqual(client.llm_judge(env).provider, "custom") # as before Anthropic was a judge + self.assertEqual(client.llm_judge({"ANTHROPIC_API_KEY": "k"}).provider, "anthropic") + self.assertEqual(client.llm_judge(dict(env, OPENAI_API_KEY="k")).provider, "openai") + + def test_a_run_reads_its_base_and_key_from_the_record(self): + self.assertEqual(run.provider_endpoint("anthropic", KEYS), ("https://api.anthropic.com/v1", KEYS["ANTHROPIC_API_KEY"])) + with self.assertRaises(run.Refused): + run.provider_endpoint("typesafe", KEYS) # no run provider + env = {"MODEL_GW": json.dumps({"base": "https://gw.example/v1"})} + self.assertEqual(providers.base_of("MODEL_GW_API_URL", env), "https://gw.example/v1") + self.assertEqual(providers.available(KEYS), ["openrouter", "openai", "anthropic"]) + + def test_an_anthropic_only_user_is_set_up(self): + self.assertTrue(onboarding.facts({"ANTHROPIC_API_KEY": "k"}, docker=lambda: False)["provider"]) + self.assertTrue(onboarding.facts({"MODEL_GW": json.dumps({"base": "http://gw/v1"})}, docker=lambda: False)["provider"]) + self.assertFalse(onboarding.facts({}, docker=lambda: False)["provider"]) + self.assertFalse(onboarding.facts({"TYPESAFE_API_KEY": "k"}, docker=lambda: False)["provider"]) # the judge's, not a run's + self.assertTrue(onboarding.facts({"MODEL_API_URL": "http://box/v1"}, docker=lambda: False)["provider"]) + + +class Layers(unittest.TestCase): + """settings.load: the environment's partial record is merged over the file's; only its fields lock.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.file = os.path.join(self.tmp.name, "settings.json") + env = patch.dict(os.environ, {"MODEL_GW": json.dumps({"base": "https://gw-eu.example/v1", "endpoints": {"decisions": None}})}) + env.start(); self.addCleanup(env.stop) + for kept in (settings.LOADED_KEYS, settings.ENV_PART, settings._FROM): + saved = kept.copy() + self.addCleanup(lambda k=kept, s=saved: (k.clear(), k.update(s))) + + def test_the_environment_sets_only_its_fields(self): + Path(self.file).write_text(json.dumps({"MODEL_GW": json.dumps({"name": "Gateway", "base": "https://gw.example/v1", + "endpoints": {"chat": "/c", "decisions": "/d"}})})) + settings.load(self.file) + rec = providers.record("custom:GW") + self.assertEqual((rec["name"], rec["base"], rec["endpoints"]), ("Gateway", "https://gw-eu.example/v1", {"chat": "/c"})) + self.assertEqual(settings.locked("MODEL_GW"), ["base", "endpoints.decisions"]) + state = settings.state(settings.endpoint_variable("MODEL_GW"), settings.in_file(self.file)) + self.assertEqual((state["source"], state["locked"], state["shadowed"]), ("environment", ["base", "endpoints.decisions"], False)) + settings.save(self.file, "MODEL_GW", json.dumps({"name": "Renamed", "base": "https://gw.example/v1"})) + self.assertEqual(providers.record("custom:GW")["name"], "Renamed") # the file's field changes now + self.assertEqual(providers.record("custom:GW")["base"], "https://gw-eu.example/v1") # the environment's still wins + Path(self.file).write_text(json.dumps({"MODEL_GW": json.dumps({"name": "By hand"})})) + settings.refresh(self.file) + self.assertEqual(providers.record("custom:GW")["name"], "By hand") + + +class Migration(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + env = patch.dict(os.environ, {"NULLIUS_HOME": self.tmp.name}, clear=True) + env.start(); self.addCleanup(env.stop) + self.target = Path(paths.settings_file()) + self.target.parent.mkdir(parents=True) + + def cli(self, *args): + out = io.StringIO() + with redirect_stdout(out), redirect_stderr(out): + code = migrate.main(["--providers", *args]) + self.assertNotIn("sk-gpu-fixture", out.getvalue()) + return code, out.getvalue() + + def test_an_old_name_as_long_as_it_could_be_moves_whole(self): + self.target.write_text(json.dumps({"MODEL_GPU_NAME": "N" * 400, "MODEL_GPU_API_URL": "http://gpu:8000/v1"})) + self.assertEqual(self.cli("--apply")[0], 0) + self.assertEqual(json.loads(json.loads(self.target.read_text())["MODEL_GPU"])["name"], "N" * 400) # never cut + + def test_old_names_move_into_records_on_apply_only(self): + before = {"MODEL_GPU_NAME": "GPU box", "MODEL_GPU_API_URL": "http://gpu:8000/v1", "MODEL_GPU_API_KEY": "sk-gpu-fixture", + "MODEL_NEW": json.dumps({"base": "http://new/v1"}), "MODEL_NEW_NAME": "ignored name", "MODEL_API_URL": "http://box/v1"} + self.target.write_text(json.dumps(before)) + code, said = self.cli() + self.assertEqual(code, 0); self.assertIn("Preview only", said); self.assertEqual(json.loads(self.target.read_text()), before) + code, said = self.cli("--apply") + self.assertEqual(code, 0) + after = json.loads(self.target.read_text()) + self.assertEqual(json.loads(after["MODEL_GPU"]), {"name": "GPU box", "base": "http://gpu:8000/v1"}) + self.assertEqual(json.loads(after["MODEL_NEW"]), {"base": "http://new/v1", "name": "ignored name"}) + self.assertEqual(after["MODEL_GPU_API_KEY"], "sk-gpu-fixture"); self.assertEqual(after["MODEL_API_URL"], "http://box/v1") # keys keep their names + self.assertFalse({"MODEL_GPU_NAME", "MODEL_GPU_API_URL", "MODEL_NEW_NAME"} & set(after)) + self.assertEqual(self.cli()[1].splitlines()[1], "No old provider settings.") + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_settings.py b/app/tests/test_settings.py index 0279ac0a..b5975def 100644 --- a/app/tests/test_settings.py +++ b/app/tests/test_settings.py @@ -43,14 +43,14 @@ def test_every_variable_the_application_reads_is_in_the_table(self): def test_the_page_offers_what_a_person_sets_and_no_more(self): self.assertEqual([v.name for v in settings.VARIABLES], - ["MODEL_API_KEY", "MODEL_API_URL", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "TYPESAFE_API_KEY", "SYSTEM_ONE_PROVIDER", "SYSTEM_ONE_MODEL", "LLM_JUDGE_PROVIDER", "LLM_JUDGE_MODEL", "AGENT_KEYS", "HOSTS", "HF_ENDPOINT", "HOST_CONSTRAINTS"]) + ["MODEL_API_KEY", "MODEL_API_URL", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "TYPESAFE_API_KEY", "SYSTEM_ONE_PROVIDER", "SYSTEM_ONE_PROTOCOL", "SYSTEM_ONE_MODEL", "LLM_JUDGE_PROVIDER", "LLM_JUDGE_PROTOCOL", "LLM_JUDGE_MODEL", + "OPENROUTER", "OPENAI", "ANTHROPIC", "TYPESAFE", "AGENT_KEYS", "HOSTS", "HF_ENDPOINT", "HOST_CONSTRAINTS"]) self.assertEqual(sorted(v.name for v in settings.VARIABLES if v.secret), ["ANTHROPIC_API_KEY", "MODEL_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "TYPESAFE_API_KEY"]) self.assertEqual(settings.BY_NAME["MODEL_API_URL"].kind, "endpoint") # shown, so it may hold no credential for name in ("OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY"): self.assertEqual(settings.BY_NAME[name].example, "") # no placeholder on a settings field self.assertEqual({settings.BY_NAME[n].group for n in ("MODEL_API_KEY", "MODEL_API_URL", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "TYPESAFE_API_KEY")}, {"Model credentials"}) - from nullius import providers - self.assertIn(providers.PROVIDER_BASE["openrouter"], settings.BY_NAME["OPENROUTER_API_KEY"].purpose) # the endpoint is fixed, and said + self.assertEqual({settings.BY_NAME[n].kind for n in ("OPENROUTER", "OPENAI", "ANTHROPIC", "TYPESAFE")}, {"provider"}) # a preset's additions self.assertIn("OpenAI-compatible", settings.BY_NAME["MODEL_API_KEY"].purpose) def test_the_servers_address_and_port_are_start_up_options(self): diff --git a/app/tests/test_standard_environments.py b/app/tests/test_standard_environments.py index fcf2e526..01624174 100644 --- a/app/tests/test_standard_environments.py +++ b/app/tests/test_standard_environments.py @@ -86,6 +86,9 @@ def test_each_wrong_field_is_named(self): self.assertTrue(any(".to: one of ('agent', 'driver')" in p for p in said), said) said = self.problems({"x": dict(tool, needs_secrets=[{"name": "OPENAI_API_KEY", "to": "agent"}])}) self.assertTrue(any("already the name of something else" in p for p in said), said) + for name in ("OPENAI", "MODEL_LICENSE"): # a provider record's name: a licence, kept apart, may take it + said = self.problems({"x": dict(tool, needs_secrets=[{"name": name, "to": "agent"}])}) + self.assertFalse(any("already the name" in p for p in said), said) said = self.problems({"x": dict(tool, needs_secrets=[{"name": "A", "to": "agent"}, {"name": "A", "to": "driver"}])}) self.assertTrue(any("each once" in p for p in said), said) diff --git a/app/tests/test_standard_mcp_machine.py b/app/tests/test_standard_mcp_machine.py index b58d0253..10ad6229 100644 --- a/app/tests/test_standard_mcp_machine.py +++ b/app/tests/test_standard_mcp_machine.py @@ -336,7 +336,7 @@ def setUp(self): def runner(self): r = run.Run.__new__(run.Run) - r.base, r.key, r.model, r.provider = "https://x/v1", "k", "m", "custom" + r.base, r.key, r.model, r.provider, r.key_header = "https://x/v1", "k", "m", "custom", "authorization" r.judge_model = r.simulator_model = r.setup_driver = None r.backend, r.other, r.bench, r.name, r.image = "container", agent(), bench("/defs/b"), "job", "img" r.up, r.data, r.ds, r.licences = "/up", "/data", {"id": "all"}, {} diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 83197949..1bd8ad9a 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -706,6 +706,16 @@ def test_a_429_reaches_the_caller_as_sent_and_is_not_retried(self): self.assertEqual(len(KeptAlive.seen), 1) self.assertEqual(r.calls("u")[-1]["status"], 429) + def test_a_benchmark_route_beside_an_anthropic_agent_takes_a_bearer_key(self): + r = self.relay(model="claude-x", key_header="x-api-key") + self.post(r, "u", "/messages", {"model": "claude-x", "max_tokens": 10, "messages": []}, {"x-api-key": relay.PLACEHOLDER_KEY}) + self.assertIn("x-api-key", {k.lower() for k in KeptAlive.seen[-1][1]}) + r.url("u-judge", model="judge-x") + self.post(r, "u-judge", "/chat/completions", {"model": "any", "messages": []}) + sent = {k.lower(): v for k, v in KeptAlive.seen[-1][1].items()} + self.assertEqual(sent["authorization"], "Bearer sk-real") + self.assertFalse({"x-api-key", "anthropic-version"} & set(sent)) + def test_anthropic_messages_take_the_key_as_x_api_key_and_their_usage_is_read(self): r = self.relay(model="claude-x", key_header="x-api-key") status, _, body = self.post(r, "u", "/messages", {"model": "claude-x", "max_tokens": 10, "messages": []}, diff --git a/app/tests/test_standard_setups.py b/app/tests/test_standard_setups.py index e4b451ba..a1f42aee 100644 --- a/app/tests/test_standard_setups.py +++ b/app/tests/test_standard_setups.py @@ -296,6 +296,12 @@ def compiled(self, **kw): r.update(**kw) return recipe.compile(r) + def test_a_setup_and_a_judge_need_a_provider_that_serves_chat(self): + said = [e["message"] for e in self.compiled(others=["original"], provider="anthropic", model="claude-x")["errors"]] + self.assertIn("Anthropic does not serve what LoCoMo original speaks: choose another provider.", said) + said = [e["message"] for e in self.compiled(others=["nop"], benchmark="longmemeval", datasets=["longmemeval_s"], provider="anthropic")["errors"]] + self.assertIn("Anthropic does not serve what LongMemEval's judge speaks: choose another provider.", said) + def test_one_original_participant_uses_the_selected_model_scope(self): paper = self.compiled(others=["original"], model_choice="paper", model="gpt-3.5-turbo-16k") custom = self.compiled(others=["original"], model_choice="any", model="vendor/their-model") diff --git a/app/ui_tests/test_command_builder_providers.py b/app/ui_tests/test_command_builder_providers.py index deefbc84..ceae9aa9 100644 --- a/app/ui_tests/test_command_builder_providers.py +++ b/app/ui_tests/test_command_builder_providers.py @@ -202,7 +202,7 @@ def test_an_openrouter_tau3_run_binds_its_key_and_holds_no_other(self): harbor.write_text('#!/usr/bin/env bash\nenv > "$NULLIUS_JOBS"/../harbor.env\n'); harbor.chmod(0o755) # what Harbor, and every container it starts, could read held = {n: 'sk-held-' + n.lower() for n in execution.withheld()} plan = execution.Plan(name='tau3', script=out['script'], model=out['model'], parts=out['parts'], jobs=out['jobs'], - key_var='OPENROUTER_API_KEY', url_var='', base_url=providers.PROVIDER_BASE['openrouter']) + key_var='OPENROUTER_API_KEY', url_var='', base_url=providers.PRESETS['openrouter']['base']) with patch.dict(os.environ, dict(held, PATH=str(tmp / 'bin') + os.pathsep + os.environ['PATH'])), patch.object(execution, 'SYNC_EVERY', 0.05): ex = execution.Execution(plan, root=str(tmp / 'runs')).start() ex.thread.join(60) diff --git a/app/ui_tests/test_harbor_routes.py b/app/ui_tests/test_harbor_routes.py index 1324c9bc..f8fb04d6 100644 --- a/app/ui_tests/test_harbor_routes.py +++ b/app/ui_tests/test_harbor_routes.py @@ -109,6 +109,18 @@ def test_a_fork_is_built_here_and_scanned_from_the_page(self): self.assertEqual((got["agents"], got["version"]), (["claude-code", "nop", "oracle", "terminus-2"], "0.22.0+fork.1")) self.assertTrue((self.tmp / "pins" / ("git-" + up["commit"]) / "wheels").is_dir()) + def test_a_provider_that_serves_no_chat_is_refused_model_or_not(self): + from web import execute + script = '"$NULLIUS_HARBOR" run --agent nop' + with patch.dict(os.environ, {"MODEL_API_URL": "http://box.example/v1", "TYPESAFE_API_KEY": "ts-fixture-0123456789", "ANTHROPIC_API_KEY": "sk-ant-fixture-0123"}): + for provider, said in (("typesafe", "TypeSafe does not serve"), ("anthropic", "Anthropic does not serve"), ("bogus", "bogus does not serve")): + for model in ({"model": "m"}, {}): + r = self.post("/run", {"name": "p-0101", "where": "local", "script": script, **model, + "recipe": {"version": 1, "harness": "harbor", "provider": provider}}) + self.assertEqual(r.status_code, 400, (provider, model)); self.assertIn(said, r.json["error"]) + self.assertEqual(self.started, []) + self.assertEqual(execute.credentials_for("typesafe")["key_var"], "MODEL_API_KEY") # never a judge's key, as provider_problem reads it + def test_the_pin_travels_with_the_run(self): script = '"$NULLIUS_HARBOR" run --agent nop' pin = {"kind": "repo", "repo": "owner/harbor", "revisionKind": "tag", "revision": "v1", "commit": "c" * 40, "extras": ["jw"]} diff --git a/app/ui_tests/test_judge_cards.py b/app/ui_tests/test_judge_cards.py new file mode 100644 index 00000000..fc995bc9 --- /dev/null +++ b/app/ui_tests/test_judge_cards.py @@ -0,0 +1,93 @@ +"""A judge card follows its provider as it changes, through the JavaScript served to the page.""" +import json +from pathlib import Path +import shutil +import subprocess +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "static" / "settings" / "judges.js" + +# A card as settings.html draws it, in a fake DOM of the few calls judges.js makes. +PROGRAM = r''' +const options = list => list.map(([value, data, text]) => ({value, hidden: false, textContent: text, dataset: data || {}, + toggleAttribute() {}})); +const select = (list, attrs = {}) => ({options: list, index: list.findIndex(o => o.value === attrs.value), dataset: {}, hidden: false, disabled: false, + attrs, get value() { return this.index < 0 ? '' : this.options[this.index].value; }, + set value(v) { this.index = this.options.findIndex(o => o.value === v); }, + get selectedOptions() { return this.index < 0 ? [] : [this.options[this.index]]; }, + get name() { return this.attrs.name; }, set name(v) { this.attrs.name = v; }, + removeAttribute(n) { delete this.attrs[n]; }, closest() { return this.field; }, + matches(s) { return s.split(', ').includes(this.kind); }}); +const provider = select(options([['', {}], ['openrouter', {available: 'true', protocol: 'chat', protocols: 'chat'}, 'OpenRouter'], + ['anthropic', {available: 'true', protocol: 'chat', protocols: 'chat|messages'}, 'Anthropic']]), {value: 'anthropic', name: 'value:LLM_JUDGE_PROVIDER'}); +provider.kind = '[data-picker]'; +const protocol = select(options([['chat', {}, 'OpenAI Chat Completions'], ['messages', {}, 'Anthropic Messages']]), {value: 'chat'}); +protocol.kind = '[data-protocol-picker]'; protocol.dataset.name = 'value:LLM_JUDGE_PROTOCOL'; protocol.field = {hidden: false}; +const model = {placeholder: '', list: {replaceChildren() {}}}, modelField = {hidden: false, querySelector: () => model}; +const via = {hidden: false, textContent: ''}, none = {hidden: false}; +const parts = {'[data-picker]': provider, '[data-protocol-picker]': protocol, '[data-judge-via]': via, '[data-no-judge-provider]': none, '.st-field[id$="_MODEL"]': modelField}; +const listeners = {}, card = {querySelector: s => parts[s], addEventListener(t, f) { listeners[t] = f; }}; +const doc = {}; +global.document = {querySelectorAll: () => [card], getElementById: () => ({dispatchEvent() {}}), + addEventListener(t, f) { doc[t] = f; }, createElement: () => ({})}; +require(process.argv[1]); +const view = () => ({provider: provider.value, hidden: provider.hidden, protocol: protocol.value, named: protocol.attrs.name || '', + protocolRow: !protocol.field.hidden, via: via.hidden ? '' : via.textContent, none: !none.hidden, model: !modelField.hidden}); +const pick = (s, v) => { s.value = v; listeners.change({target: s}); return view(); }; +const out = []; +doc.DOMContentLoaded(); out.push(view()); +out.push(pick(provider, 'openrouter')); +out.push(pick(provider, 'anthropic')); +out.push(pick(protocol, 'messages')); +out.push(pick(provider, 'openrouter')); +doc['nullius-settings-saved']({detail: {providers: [{key: 'anthropic', ready: 'true'}]}}); out.push(view()); +doc['nullius-settings-saved']({detail: {providers: []}}); out.push(view()); +process.stdout.write(JSON.stringify(out)); +''' + +# System One on OpenRouter: the list drawn with the page, then the live one asked when the field is first used. +LIVE = PROGRAM.split("require(process.argv[1]);")[0].replace( + "['openrouter', {available: 'true', protocol: 'chat', protocols: 'chat'}, 'OpenRouter']", + "['openrouter', {available: 'true', protocol: 'systemone', protocols: 'systemone|decisions', models: '~typesafe/jev-latest', live: ''}, 'OpenRouter']" +).replace("const model = {placeholder: '', list: {replaceChildren() {}}}", + "const shown = []; const model = {placeholder: '', matches: () => true, list: {replaceChildren(...o) { shown.push(o.map(x => x.value)); }, append() {}}}") + r''' +const asked = []; +global.nulliusFetch = (url, init) => { asked.push([url, JSON.parse(init.body)]); + return Promise.resolve({ok: true, json: () => Promise.resolve({ok: true, models: [{id: '~typesafe/jev-latest', name: '~typesafe/jev-latest'}, {id: 'b/decider:free', name: 'b/decider:free'}]})}); }; +document.querySelector = () => ({value: 'token'}); +require(process.argv[1]); +doc.DOMContentLoaded(); provider.value = 'openrouter'; listeners.change({target: provider}); +listeners.focusin({target: model}); +setTimeout(() => { listeners.focusin({target: model}); setTimeout(() => process.stdout.write(JSON.stringify({asked, shown: shown.slice(-2)}))); }); +''' + + +class JudgeCard(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.node = shutil.which("node") + if not cls.node: + raise unittest.SkipTest("Node is required for the judge cards") + + def test_the_protocol_follows_the_provider_and_is_saved_only_once_chosen(self): + got = json.loads(subprocess.run([self.node, "-e", PROGRAM, str(SCRIPT)], capture_output=True, text=True, check=True).stdout) + row = lambda provider, protocol, named, shown, via: {"provider": provider, "hidden": False, "protocol": protocol, "named": named, + "protocolRow": shown, "via": via, "none": False, "model": True} + self.assertEqual(got[0], row("anthropic", "chat", "", True, "")) # two protocols: a select, unnamed until chosen + self.assertEqual(got[1], row("openrouter", "chat", "", False, "via OpenAI Chat Completions")) # one: a quiet line + self.assertEqual(got[2], row("anthropic", "chat", "", True, "")) + self.assertEqual(got[3], row("anthropic", "messages", "value:LLM_JUDGE_PROTOCOL", True, "")) # chosen: named, so autosave writes it + self.assertEqual(got[4], row("openrouter", "chat", "", False, "via OpenAI Chat Completions")) # not served there: shown, not saved + self.assertEqual(got[5], row("anthropic", "messages", "", True, "")) # the sole provider left is chosen + self.assertEqual(got[6], {"provider": "", "hidden": True, "protocol": "messages", "named": "", "protocolRow": False, + "via": "", "none": True, "model": False}) # none left: Add a provider + + def test_system_one_on_openrouter_asks_the_live_list_once_when_the_field_is_used(self): + got = json.loads(subprocess.run([self.node, "-e", LIVE, str(SCRIPT)], capture_output=True, text=True, check=True).stdout) + self.assertEqual(got["asked"], [["/settings/system-one/models", {"provider": "openrouter"}]]) # once, for both uses + self.assertEqual(got["shown"], [["~typesafe/jev-latest", "b/decider:free"]] * 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_routes.py b/app/ui_tests/test_routes.py index db3381e3..7ff87ecf 100644 --- a/app/ui_tests/test_routes.py +++ b/app/ui_tests/test_routes.py @@ -302,6 +302,12 @@ def test_a_trial_has_two_judgements_in_two_tabs(self): self.assertIn("asked of OpenRouter · vendor/m", html) self.assertNotIn("sk-or-route-test", html) + def test_a_judgement_names_its_provider_from_the_registry(self): + named = web.app.jinja_env.filters["provider"] + with patch.dict(os.environ, {"MODEL_GW": json.dumps({"name": "Gateway", "base": "https://gw.example/v1"})}): + self.assertEqual([named(p) for p in ("openrouter", "anthropic", "custom:GW", "local:GW")], + ["OpenRouter", "Anthropic", "Gateway", "Gateway"]) + def test_trial_html_does_not_inline_large_conversation_or_trace_payloads(self): response = self.get_ok("/trial/active-run/task-001") self.assertLess(len(response.data), 250_000) diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index b8dacaf4..80a0661c 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -118,7 +118,8 @@ def test_every_variable_is_listed_with_its_state_and_no_secret_value(self): settings.load(self.env_file) html = self.page() for v in settings.VARIABLES: - self.assertIn(f'id="{v.name}"', html) + added = v.kind == "provider" and next(p for p in providers.PROTOCOLS if p not in providers.PRESETS[v.name.lower()]["endpoints"]) + self.assertIn(f'id="{v.name}.endpoints.{added}"' if added else f'id="{v.name}"', html) # a preset's record: a row it may add self.assertIn('title="Held by the process', html) # the token, from the process self.assertEqual(html.count('********'), 3) # the token and the OpenRouter and OpenAI keys: eight stars each, self.assertNotIn("characters", html) # never their length @@ -133,19 +134,64 @@ def test_each_judge_card_starts_on_its_provider_and_holds_each_provider_s_key_ro Path(self.env_file).write_text("OPENROUTER_API_KEY=sk-or-fromthefile\n") settings.load(self.env_file) html = self.page() - self.assertIn('name="value:SYSTEM_ONE_PROVIDER" value="openrouter" checked', html) # the one in use: OpenRouter, the only key - self.assertIn('name="value:LLM_JUDGE_PROVIDER" value="openrouter" checked', html) - self.assertNotIn('value="typesafe" checked', html) - self.assertIn('data-provider="typesafe" data-available="false" hidden', html) - self.assertIn('data-provider="openrouter" data-available="true"', html) - self.assertIn('data-provider="openai" data-available="false" hidden', html) judge_section = html.split('id="judge"')[1].split('id="custom-keys"')[0] + for prefix in ("SYSTEM_ONE", "LLM_JUDGE"): # Provider, Protocol, Model, each labelled + labels = [judge_section.index(f'for="value-{prefix}_{part.upper()}">{part}') for part in ("Provider", "Protocol", "Model")] + self.assertEqual(labels, sorted(labels)) + self.assertIn(f'