diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 53f55d38..d88bddcf 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -140,6 +140,20 @@ "original", "model" ], + "names": { + "m3": "M3 Agent", + "prompt": "PromptAgent", + "claude": "Claude computer use", + "gpt54": "GPT-5.4", + "openai_cua": "OpenAI CUA", + "qwen": "Qwen", + "os_symphony": "OS-Symphony", + "evocua": "EvoCUA", + "autoglm": "AutoGLM", + "autoglm_v": "AutoGLM-V", + "mano": "Mano", + "muse_spark": "Muse Spark" + }, "about": { "m3": "MiniMax's M3Agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API, 100 steps", "prompt": "OSWorld's PromptAgent: run_multienv.py, GPT-4o on screenshots over Chat Completions, 15 steps", @@ -205,6 +219,7 @@ }, "vm_provider": { "label": "VM provider", + "step": "where", "summary": "--provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own.", "values": [ "docker" diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 36b36fa8..b4aaafb3 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -9,7 +9,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--attempts K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] - PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] + PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force] PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed @@ -636,11 +636,16 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) + jobs = options(a.job) + for c in jobs: + if c not in definitions.CONTROLS: + sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}") held, spill = licences(b, from_run=a.from_run) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, - env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images) + env=run_env(a.provider) if a.from_run else None, licences=held, + bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images) except run.Refused as e: print("refused:", e) return 2 @@ -765,6 +770,10 @@ def main(argv=None): v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)") v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only") v.add_argument("--launch", help="the launch id the control jobs of every shard record") + v.add_argument("--bench-option", action="append", metavar="NAME=VALUE", + help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs") + v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR", + help="a finished control job over the whole dataset, checked against its bound instead of running the control") v.add_argument("--remove-images", action="store_true", help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)") g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index ea7f6d13..88a0941e 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -725,7 +725,7 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, - max_calls: Optional[int] = None) -> List[str]: + timeout: Optional[int] = None, max_calls: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -764,6 +764,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] + if timeout: + a += ["--timeout", str(timeout)] return a @@ -1025,11 +1027,12 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, - serving: Optional[Dict[str, List[str]]] = None, max_calls: Optional[int] = None): + serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None, max_calls: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.max_calls = max_calls # the run's cap on its agent's model calls, when it has one self.ledger = ledger # the file of the run's spending cap, when it has one self.serving = dict(serving or {}) # by model, the serving providers that may answer it + self.timeout = timeout self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1055,7 +1058,8 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger, serving=self.serving, max_calls=self.max_calls) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, + timeout=self.timeout, max_calls=self.max_calls) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 8bf1516f..932c5fc3 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -470,7 +470,7 @@ def _bench_options(d: dict) -> List[str]: ``embeddings`` model called through the run's relay, ``host`` requirements or ``files`` by name - and ``when`` it applies: another option's values (a retriever's, only with a retrieval). ``limit``: a bound the upstream has of its own (its steps, its errors), offered with the run's limits; ``by``: - its default under each value of another option (an agent's own steps), for a default of ``default``.""" + its default under each value of another option (an agent's own steps), for a default of ``default``. ``names``: a name for some of its values, shown in their place. ``step``: ``where`` for a choice of where the work runs, ``dataset`` for one its datasets need, the builder step that asks it.""" opts = d.get("options") if opts is None: return [] @@ -490,6 +490,8 @@ def _bench_options(d: dict) -> List[str]: bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)") if "limit" in o and (not isinstance(o["limit"], bool) or "pattern" not in o): bad.append(f"{w}.limit: true for a bound the upstream has of its own, a number its pattern allows") + if o.get("step", "dataset") != "dataset" and (o["step"] != "where" or "values" not in o) or "step" in o and o.get("limit"): + bad.append(f"{w}.step: where, a choice of where the work runs among its values, or dataset") by = o.get("by") if by is not None and not (isinstance(by, dict) and len(by) == 1 and all( k != name and k in opts and isinstance(m, dict) and set(m) == set(opts[k].get("values") or [None]) @@ -501,6 +503,10 @@ def _bench_options(d: dict) -> List[str]: about = o.get("about") or {} if not isinstance(about, dict) or any(option_problem(o, k) is not None or not isinstance(v, str) for k, v in about.items()): bad.append(f"{w}.about: a line for some of its values") + names = o.get("names", {}) + if not isinstance(names, dict) or any(k not in (o.get("values") or []) or not isinstance(v, str) or not v.strip() for k, v in names.items()) \ + or len(set(names.values())) != len(names): + bad.append(f"{w}.names: a name for some of its values") needs = o.get("needs") or {} if not isinstance(needs, dict): bad.append(f"{w}.needs: value -> {{embeddings, reranker, host, files}}") @@ -1044,6 +1050,25 @@ def control(bench: dict, which: str) -> Optional[dict]: "_kind": "control", "_ref": which, "_here": bench.get("_here", "")} +def comparable(orig: dict, model: str) -> bool: + """Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do.""" + c = orig.get("comparable", True) + return model in c if isinstance(c, list) else bool(c) + + +def upstream_name(orig: dict, model: str, provider: str) -> str: + """``model`` by the name the original's upstream knows, when given as ``provider`` lists it.""" + pre = (orig.get("providers") or {}).get(provider, "") + names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []} + return model if model in (orig.get("models") or []) else names.get(model, model) + + +def paper_models(orig: dict) -> List[str]: + """The original's models whose runs compare with the paper's.""" + c = orig.get("comparable", True) + return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c] + + def _leaderboard(d: dict) -> List[str]: """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the upstream's leaderboard fixes besides the ranked model, from its own submission @@ -1131,8 +1156,11 @@ def _setups(s) -> List[str]: bad.append(f"{w}.turn_seconds: a whole number of seconds") if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()): bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase") - if "comparable" in x and not isinstance(x["comparable"], bool): - bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only") + c = x.get("comparable", True) + if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c) + or x.get("default") not in c): + bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; " + "or those of its models whose runs compare with the paper's, its default among them") else: allowed |= {"runs_as"} orig = s.get("original") if isinstance(s.get("original"), dict) else None @@ -1224,7 +1252,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple model = model or orig["default"] if model not in orig["models"]: return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): " - f"{model} is not one - choose the model setup for another") + f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else "")) if provider and provider not in orig["providers"]: return None, (f"the paper's models are not served by {provider}: the original runs on " + " or ".join(orig["providers"])) @@ -1240,7 +1268,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple asked, sent = x["runs_as"], model units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"} return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", - "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True), + "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model), "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which, "_here": bench.get("_here", "")}, "" @@ -1322,7 +1350,7 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") @@ -1330,6 +1358,8 @@ def problems(d: dict, kind: str) -> List[str]: if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " "in a sentence - the driver is then sent attempt with setup, turn and answer") + if "gated" in d and not isinstance(d["gated"], bool): + bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept") s = d.get("score") or {} if s.get("kind") not in SCORES or not s.get("says"): bad.append(f"score: {{kind: one of {SCORES}, says}}") diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index ca3a4ab7..35e133a3 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=r.key_header, fields=True).start() + key_header=r.key_header, fields=True, timeout=r.call_seconds).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} @@ -230,7 +230,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k except (run._Ended, jsonl.Failed) as e: r.say(f" {tag} {unit}: {e}") try: - score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600) + score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds) except jsonl.Failed: score = {} if os.path.isdir(os.path.join(base, "agent")): diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 4949c38d..4fe19059 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -57,9 +57,9 @@ def _text(content) -> str: def _usage_total(calls: List[dict]) -> dict: - ok = [c for c in calls if c.get("status") == 200] + ok = [c for c in calls if c.get("status") == 200 or c.get("reply")] u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok) - costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))] + costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float)) and c.get("cost_from") != "failed"] return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"), "cost": sum(costs) if costs else None, "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None), @@ -169,6 +169,9 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili reply = c.get("reply") or {} st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"), "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"} + if c.get("status") == 200 and reply.get("tool_calls"): + st["tool_calls"] = [{"tool_call_id": f"call-{len(steps) + 1}-{k + 1}", "function_name": t.get("name") or "", + "arguments": _arguments(t.get("arguments"))} for k, t in enumerate(reply["tool_calls"])] if reply.get("usage"): st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"], "cached_tokens": reply["usage"]["cached"]} @@ -183,6 +186,17 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}} +def _arguments(said) -> dict: + """A tool call's arguments as a dict; text that is no JSON object goes under ``raw``.""" + try: + got = json.loads(said) if isinstance(said, str) else said + except ValueError: + got = None + if said is None: + return {} + return got if isinstance(got, dict) else {"raw": said if isinstance(said, str) else json.dumps(said)} + + def _control(other: dict) -> dict: """A control's run says so, and who wrote the control: the runs pages read a standard job named ``nop`` or ``oracle`` as a control only by this.""" @@ -317,7 +331,7 @@ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tu if at is not None: turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True, content={"input": st.get("message")})) - elif agent or st.get("tool_calls"): + elif agent or (st.get("tool_calls") and not st.get("model_name")): spans.extend(_step_spans(i + 1, st, source)) if not spans and not turns: return None @@ -512,6 +526,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st # benchmark a score and no pass or fail - the trial reads as Scored rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value} res["verifier_result"] = {"rewards": rewards} + if score.get("stopped"): + res["verifier_result"]["stopped"] = str(score["stopped"]) parts = score.get("parts") if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0: # a score over several parts (a conversation's questions): the job's mean weighs it by them @@ -531,9 +547,15 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) if trajectory is not None: _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1)) - timeline = trace_from_run(trajectory, calls, other, produce) - if timeline is not None: - _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + try: + timeline = trace_from_run(trajectory, calls, other, produce) + if timeline is not None: + _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + except Exception as e: + try: + _write(os.path.join(trial_dir, "agent", "trace.error.txt"), type(e).__name__ + "\n") + except OSError: + pass for name, text in (extra_logs or {}).items(): _write(os.path.join(trial_dir, "agent", name), text) _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1)) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index a2fd8cf7..deffe288 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -19,7 +19,11 @@ comments, line endings and sha256 may differ from shard to shard, and the merged job records the first shard's list. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: -the shards absent, and the units a shard planned and has no trial for. +the shards absent, and the units a shard planned and has no trial for. A +trial whose record names another job, benchmark, version, control or agent +than its shard, or a job itself merged despite differences, is refused +unless ``force``. Shards run with different benchmark options keep them each +in ``merged_from``. Files are hard-linked into the merged job where the file system allows (a copy where it does not), so a merge of a thousand trials costs no second @@ -45,6 +49,11 @@ ("number of shards", ("standard", "shard", "of")), ("retries per attempt", ("standard", "retries"))) +LISTED = 1000 +OWN = (("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), + ("control", ("standard", "control", "name")), ("agent or capability", ("standard", "plugged"))) + + class Refused(Exception): """The jobs cannot be merged as asked; the message says why.""" @@ -170,6 +179,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: # every trial by its unit: two of one unit are never merged, force or not by_unit: Dict[str, List[tuple]] = {} names: Dict[str, List[str]] = {} + foreign = [] for s in shards: s["trials"] = _trials(s["dir"]) s["results"] = [_read(os.path.join(s["dir"], t, "result.json")) for t in s["trials"]] @@ -177,6 +187,22 @@ def check(job_dirs: List[str], force: bool = False) -> dict: unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) + wrong = [label for label, path in OWN if _at(res, path) != _at(s["config"], path)] + if _at(res, ("standard", "job")) != (s["config"].get("job_name") or s["name"]): + wrong.append("job") + if wrong: + foreign.append(f"{s['name']}/{t} names another {', '.join(wrong)} than its job") + inherited = [f"{s['name']}: {w}" for s in shards for w in _at(s["config"], ("standard", "merged_despite")) or []] + if inherited: + if not force: + raise Refused("a job given was itself merged despite differences - " + "; ".join(inherited[:5]) + + " (--force merges it anyway, and the merge says so)") + differ += inherited + if foreign: + shown = f"{len(foreign)} trial(s) not of the job they are in: " + "; ".join(foreign[:5]) + (" ..." if len(foreign) > 5 else "") + if not force: + raise Refused(shown + " (--force merges them anyway, and the merge says so)") + differ.append(shown) twice = {u: w for u, w in by_unit.items() if len(w) > 1} if twice: shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10]) @@ -191,9 +217,10 @@ def check(job_dirs: List[str], force: bool = False) -> dict: if len(of) == 1: n = of.pop() have = sorted(s["shard"].get("index") for s in shards if s["shard"]) - absent = [i for i in range(1, (n or 0) + 1) if i not in have] - if absent: - notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") + if isinstance(n, int) and n <= LISTED: + absent = [i for i in range(1, n + 1) if i not in have] + if absent: + notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") dup = sorted({i for i in have if have.count(i) > 1}) if dup: notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)") @@ -298,8 +325,15 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: cfg["job_name"] = name std = cfg.setdefault("standard", {}) std.pop("shard", None) + own = len({json.dumps(_at(s["config"], ("standard", "benchmark_options")), sort_keys=True) for s in shards}) > 1 + if own: + std.pop("benchmark_options", None) std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]), - **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards] + **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {}), + **({"benchmark_options": _at(s["config"], ("standard", "benchmark_options")) or {}} if own else {}), + **({"merged_from": _at(s["config"], ("standard", "merged_from"))} + if _at(s["config"], ("standard", "merged_from")) else {})} + for s in shards] if got["units_total"] is not None: std["units_total"] = got["units_total"] if got["differ"]: diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 20682da6..cd50ef4e 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -30,6 +30,8 @@ pages pair them as a launch's controls); the units and leak checks talk to the driver directly and leave nothing behind. A control calls no model, so it needs no key - except a judged benchmark's, whose judge still grades. +``--bench-option`` reaches the checks' driver and each control's run; +``--job nop=DIR`` holds a merged launch of a control to its bound. A check that passes is recorded with its date, its counts and its job; one that fails is taken back, as ``conform`` takes back a port - what it reached kept @@ -41,6 +43,8 @@ """ from __future__ import annotations +import json +import math import os import re import shutil @@ -48,8 +52,7 @@ import time from typing import Callable, Dict, List, Optional -from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, retries, run, shard as shards +from nullius.standard import definitions, install, job, jsonl, retries, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -66,7 +69,8 @@ class _Driver: """The benchmark's driver, started as a run starts it, for the checks that talk to it directly (units, leak).""" - def __init__(self, bench: dict, ds: dict, work: str, licences=None): + def __init__(self, bench: dict, ds: dict, work: str, licences=None, options: Optional[dict] = None): + self.options = options if options is not None else options_for(bench, ds) self.up = install.upstream(bench) self.data = install.dataset(bench, ds, self.up) install.driver_packages(bench) # its driver.install, once: start() then finds it @@ -81,10 +85,15 @@ def start(self, n: int) -> jsonl.Proc: env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), - **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults + **run.option_env(self.options)}) return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() +def options_for(bench: dict, ds: dict, given: Optional[dict] = None, which=shutil.which) -> dict: + """The benchmark's options on ``ds`` for a control's run, at ``given`` or their defaults.""" + return run.resolve_bench_options(bench, ds["id"], given, "yours", which)[0] + + def units_check(driver: _Driver, ds: dict) -> dict: """Two drivers, one ``units`` call each: the same ids, unique, plain.""" got = [] @@ -209,44 +218,164 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str, provider: str, model: str, judge_model: Optional[str], env, concurrency: int, say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None, - licences=None, remove_images: bool = False) -> dict: + licences=None, options: Optional[dict] = None, job_dir: Optional[str] = None, + ids: Optional[List[str]] = None, remove_images: bool = False) -> dict: """Run one control over the dataset (or its first ``limit`` units, of - ``shard`` when given) and hold it to its bound: the pass rate, or the mean score.""" + ``shard`` when given) and hold it to its bound: the pass rate, or the mean score. + ``job_dir``: a control run already, held to the bound instead; it must cover ``ids``.""" other = definitions.control(bench, which) - job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, - jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), - env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images, retries=1, retry_if=retries.rate_limited) # as conform's - j = parse.parse_job(job_dir) - n = len(j.trials) - errored = sum(1 for t in j.trials if t.exception_type) + problems = [] + if job_dir: + job_dir = os.path.abspath(job_dir.rstrip("/")) + problems += job_problems(bench, ds, which, job_dir, ids or []) + cfg = _config(job_dir) + else: + job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, + jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), + say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, + licences=licences, bench_options=options, + remove_images=remove_images, retries=1, retry_if=retries.rate_limited) # as conform's + cfg = {} + field = FIELD[bench["score"]["kind"]] + results = [res for _, res in trials(job_dir)] + n = len(results) + errored = sum(1 for res in results if res.get("exception_info")) + values = [verdict(bench, res) for res in results if not res.get("exception_info")] + unscored = sum(1 for v in values if v is None) + values = [v for v in values if v is not None] + top = sum(1 for v in values if v >= 1.0) if bench["score"]["kind"] == "pass": - top = sum(1 for t in j.trials if t.reward == 1.0) value = top / n if n else 0.0 else: - scores = [t.score for t in j.trials if t.score is not None] - top = sum(1 for s in scores if s >= 1.0) - value = sum(scores) / len(scores) if scores else 0.0 + value = sum(values) / len(values) if values else 0.0 (bound, expected), = other["expect"].items() within = value <= expected if bound == "at_most" else value >= expected - problems = [] if not n: problems.append("no unit ran") if errored: problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials") + if unscored: + problems.append(f"{unscored} unit(s) ended with no {field} from 0 to 1 and no error: read the job's trials") if n and not within: problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}") rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected}, "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems} - if judge_model or (bench["score"].get("judge") or {}).get("model"): - rec["judge"] = judge_model or bench["score"]["judge"]["model"] + ran_with = _options_of(cfg) if cfg else (options or {}) + if ran_with: + rec["options"] = ran_with + judge = (cfg.get("standard") or {}).get("judge") if cfg else judge_model or (bench["score"].get("judge") or {}).get("model") + if judge: + rec["judge"] = judge return rec +FIELD = {"pass": "reward", "score": "score"} +UNITS_FAILED = "the units check failed" + + +def trials(job_dir: str) -> List[tuple]: + """(trial, its result.json) for each trial folder of a job; an unreadable record is {}.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if name.startswith(".") or not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + res = json.load(f) + except (OSError, ValueError): + res = {} + out.append((name, res if isinstance(res, dict) else {})) + return out + + +def verdict(bench: dict, res: dict) -> Optional[float]: + """The trial's value in the field its benchmark's score kind names, when it is a finite number from 0 to 1.""" + v = (((res.get("verifier_result") or {}).get("rewards")) or {}).get(FIELD[bench["score"]["kind"]]) + if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0.0 <= v <= 1.0: + return None + return float(v) + + +def _options_of(cfg: dict): + """The benchmark options a job ran with: one set, or each distinct set of its merged shards.""" + std = cfg.get("standard") or {} + if std.get("benchmark_options"): + return std["benchmark_options"] + sets, todo = [], list(std.get("merged_from") or []) + while todo: + m = todo.pop() + if m.get("benchmark_options") or not m.get("merged_from"): + sets.append(m.get("benchmark_options") or {}) + else: + todo += m.get("merged_from") or [] + distinct = {json.dumps(x, sort_keys=True): x for x in sets} + return [distinct[k] for k in sorted(distinct)] if any(sets) else [] + + +def _config(job_dir: str) -> dict: + try: + with open(os.path.join(job_dir, "config.json"), encoding="utf-8") as f: + got = json.load(f) + return got if isinstance(got, dict) else {} + except (OSError, ValueError): + return {} + + +def job_problems(bench: dict, ds: dict, which: str, job_dir: str, ids: List[str]) -> List[str]: + """Why ``job_dir`` is not one run of ``which`` over each unit of ``ids`` once.""" + cfg = _config(job_dir) + std = cfg.get("standard") or {} + if cfg.get("runner") != "standard": + return [f"{job_dir}: not a job of the Nullius standard"] + want = {"benchmark": bench["key"], "benchmark version": job._version(bench).get("benchmark_version"), + "control": which, "dataset": f"{bench['key']}/{ds['id']}"} + have = {"benchmark": std.get("benchmark"), "benchmark version": std.get("benchmark_version"), + "control": (std.get("control") or {}).get("name"), + "dataset": ((cfg.get("datasets") or [{}])[0] or {}).get("name")} + bad = [f"its {k} is {have[k]!r}, not {want[k]!r}" for k in want if have[k] != want[k]] + if std.get("merged_despite"): + bad.append("merged despite differences: " + "; ".join(std["merged_despite"])) + if (std.get("repeat") or 1) > 1: + bad.append(f"each unit ran {std['repeat']} times") + ran: Dict[str, int] = {} + shard_jobs = {m.get("job") for m in std.get("merged_from") or []} + alien: Dict[str, List[str]] = {} + for trial, res in trials(job_dir): + tstd = res.get("standard") if isinstance(res.get("standard"), dict) else {} + own = {"benchmark": tstd.get("benchmark"), "benchmark version": tstd.get("benchmark_version"), + "control": (tstd.get("control") or {}).get("name"), + "who ran it": (tstd.get("plugged"), (res.get("agent_info") or {}).get("name")), + "job": tstd.get("job")} + expect = {"benchmark": want["benchmark"], "benchmark version": want["benchmark version"], "control": which, + "who ran it": (which, which), "job": cfg.get("job_name")} + wrong = [k for k in expect if own[k] != expect[k]] + if shard_jobs and tstd.get("shard_job") not in shard_jobs: + wrong.append("shard") + for k in wrong: + alien.setdefault(k, []).append(trial) + u = str(tstd.get("unit") or trial) + ran[u] = ran.get(u, 0) + 1 + for k, names in alien.items(): + bad.append(f"{len(names)} trial(s) whose own record names another {k} than the job: {', '.join(names[:3])}") + known = set(ids) + missing = [u for u in ids if u not in ran] + extra = sorted(u for u in ran if u not in known) + twice = sorted(u for u, k in ran.items() if k > 1) + if missing: + bad.append(f"{len(missing)} of the dataset's {len(ids)} units have no trial: {', '.join(missing[:5])}" + (" ..." if len(missing) > 5 else "")) + if extra: + bad.append(f"{len(extra)} trial(s) of units the dataset does not have: {', '.join(extra[:5])}") + if twice: + bad.append(f"{len(twice)} unit(s) with more than one trial: {', '.join(twice[:5])}") + return [f"{os.path.basename(job_dir)}: {b}" for b in bad] + + def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs", provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None, env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print, shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None, + bench_options: Optional[dict] = None, jobs: Optional[Dict[str, str]] = None, remove_images: bool = False) -> Dict[str, dict]: """Every check on every dataset (or ``dataset``); with ``write``, what passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the @@ -254,13 +383,26 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = the units check all of them, and each record says which shard. ``remove_images``: each unit's own image is removed after it, so the oracle pulls it again (by default they stay). Returns - {dataset: {check: record}}, each record with ``ok`` and its ``problems``.""" + {dataset: {check: record}}, each record with ``ok`` and its ``problems``. + ``jobs``: {control: job directory}, held to its bound instead of run.""" env = env if env is not None else os.environ stamp = stamp or time.strftime(STAMP) + jobs = dict(jobs or {}) sets = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not sets: raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})] + if jobs: + if dataset is None and len(sets) > 1: + raise run.Refused("a control's job is a run on one dataset: name it (--dataset)") + if limit or (shard and shard[1] > 1): + raise run.Refused("a control's job is held over the whole dataset: no limit, no shard") + for c, d in jobs.items(): + if c not in declared: + raise run.Refused(f"{bench['label']} declares no {c} control") + if not os.path.isdir(d): + raise run.Refused(f"{d}: no such job") + opts = {ds["id"]: options_for(bench, ds, bench_options) for ds in sets} if not declared: say(f"{bench['label']} declares no controls: only its units and nothing else can be checked") got: Dict[str, dict] = {} @@ -270,11 +412,11 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = checks: Dict[str, dict] = {} try: try: - driver = _Driver(bench, ds, work, licences=licences) + driver = _Driver(bench, ds, work, licences=licences, options=opts[ds["id"]]) except install.InstallFailed as e: raise run.Refused(f"could not install what the proof needs: {e}") checks["units"] = units_check(driver, ds) - ids = checks["units"].pop("ids") + every = ids = checks["units"].pop("ids") if shard and shard[1] > 1: ids = [u for _, u in shards.select(ids, *shard)] ids = ids[:limit] if limit else ids @@ -285,10 +427,18 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = finally: shutil.rmtree(work, ignore_errors=True) for c in declared: - checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, - model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, - shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, - remove_images=remove_images) + if not checks["units"]["ok"] and not jobs.get(c): + checks[c] = {"ok": False, "units": 0, "source": definitions.control(bench, c)["source"], + "problems": [UNITS_FAILED + ": the control was not run"]} + else: + checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, + model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, + shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, + options=opts[ds["id"]], job_dir=jobs.get(c), ids=every, + remove_images=remove_images) + if not checks["units"]["ok"]: + checks[c]["problems"].append(UNITS_FAILED + ", so this job cannot be checked against the dataset") + checks[c]["ok"] = False _say(say, c, checks[c]) total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"] for name, r in checks.items(): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 1ee2509c..f9275891 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -101,6 +101,13 @@ def _all() -> Dict[str, List[dict]]: return {k: definitions.load(k, every=True)[0] for k in ("benchmark", "agent", "capability")} +def _shown(o: dict, v: str, values: dict) -> str: + """An option's value as Review shows it: its name, or for ``default`` under ``by`` the value it stands for.""" + by = next(iter((o.get("by") or {}).items()), None) + v = by[1].get(values.get(by[0]), v) if v == "default" and by else v + return (o.get("names") or {}).get(v) or v + + def _bench_page(b: dict) -> dict: """A benchmark as a page shows it. ``controls``: the ones it declares, each with who wrote it and where it comes from - what the builder offers @@ -131,9 +138,10 @@ def _bench_page(b: dict) -> dict: # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], "reference_agent": x.get("reference_agent", ""), "turn_seconds": x.get("turn_seconds"), - "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), - **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), - "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} + "comparable": k == "original" and b["setups"]["original"].get("comparable", True) is not False, "units": (b["setups"]["original"].get("units") or b.get("units") or {}), + **({"models": definitions.paper_models(x), "default": x["default"], "providers": list(x["providers"]), + "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in definitions.paper_models(x)} + for p, pre in x["providers"].items()}} if "models" in x else {"any": True, "providers": list(x["providers"])})} for k, x in (b.get("setups") or {}).items() if k == "original"], # its own options (a search, a mode): each value with its line and what it needs; and the host requirements @@ -142,14 +150,14 @@ def _bench_page(b: dict) -> dict: "leaderboard": b.get("leaderboard") or None, "attempts": (b.get("limits") or {}).get("attempts", ""), # what a unit's time limits and a conversation's length are when the recipe forces none - "timeouts": {"setup": run.DRIVER_SECONDS, "image": run.IMAGE_SECONDS, "score": run.DRIVER_SECONDS, + "timeouts": {"setup": run.DRIVER_SECONDS, "image": run.IMAGE_SECONDS, "score": (b.get("limits") or {}).get("score_seconds") or run.DRIVER_SECONDS, "agent": (b.get("limits") or {}).get("turn_seconds") or run.TURN_SECONDS, "turns": (b.get("limits") or {}).get("turns") or run.MAX_TURNS, - "catalogue": [k for k, f in (("agent", "turn_seconds"), ("turns", "turns")) if (b.get("limits") or {}).get(f)]}, + "catalogue": [k for k, f in (("agent", "turn_seconds"), ("turns", "turns"), ("score", "score_seconds")) if (b.get("limits") or {}).get(f)]}, "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [], "setups": o.get("setups") or ["yours"], - **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, - "limit": bool(o.get("limit")), "by": o.get("by") or {}} + **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "names": o.get("names") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, + "limit": bool(o.get("limit")), "step": o.get("step") or "", "by": o.get("by") or {}} for n, o in (b.get("options") or {}).items()}, "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]} for n, r in hostreq.all_of(b).items()}, @@ -359,6 +367,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "Choose a model.") continue method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" + if method == "original": + picked = definitions.upstream_name(orig, picked, provider_) o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") @@ -435,8 +445,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()): add("model", "No paper models are listed for this benchmark; choose Any of your choice.") elif model_choice == "paper" and needs_model and orig.get("models"): - if recipe.get("model") not in orig["models"]: - add("model", "Choose one of the paper's listed models.") + if recipe.get("model") not in definitions.paper_models(orig): + add("model", "Choose one of the paper's listed models, or Any of your choice for another of the original's.") elif provider_ not in orig.get("providers", {}): add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") @@ -740,9 +750,9 @@ def row(p: dict) -> list: [STEP["runner"], "Runner", "Nullius"], [STEP["dataset"], "Datasets" if many else "Dataset", ", ".join(f"{x.get('label') or x['id']}" + (f" · {x['units']} {nouns}" if x.get("units") else "") for x in dss)], - *[[STEP["dataset"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(), - f"{v}" + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"] - for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items()], + *[[STEP["limits" if (b.get("options") or {})[k].get("limit") else "dataset" if (b.get("options") or {})[k].get("step") else "agent"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(), + _shown((b.get("options") or {})[k], v, per_ds[next(x["id"] for x in dss if per_ds[x["id"]].get(k) == v)]) + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"] + for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() if (b.get("options") or {})[k].get("step") != "where"], *[row(p) for p in picks], *[[STEP["agent"], "Options" + (f" of {p['o']['label']}" if several else ""), ", ".join(f"{k} {v}" + ("" if k in p["given"] else " (its default)") for k, v in p["opts"].items())] @@ -768,7 +778,9 @@ def row(p: dict) -> list: *([[STEP["limits"], f"Model calls per {noun}", str(calls)]] if calls else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" - + (f" · {boat_size} VMs" if backend == "boat" else "")]] + + (f" · {boat_size} VMs" if backend == "boat" else "") + + "".join(f" · {w.get('label') or k.replace('_', ' ').capitalize()} {(w.get('names') or {}).get(v) or v}" for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() + if (w := (b.get("options") or {})[k]).get("step") == "where")]] if agents_local: notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: " "the local backend is no sandbox.") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 82320991..aea7aed5 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -13,10 +13,12 @@ apart from the rest (:func:`reply_facts`), the reply also says, when it does, the serving provider that answered (``served_by``), how it ended (``end``) and an error it holds though its status was a success (``error``); reading -them never changes what the caller receives, nor the rest of the record. The +them never changes what the caller receives, nor the rest of the record but a failed Responses object's cost. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the -user's price list, ``nullius.prices``), marked as an estimate. +user's price list, ``nullius.prices``), marked as an estimate. A Responses object answered 200 that failed (its ``end``) with no usage +costs 0, marked as failed: the provider keeps no generation for it, and a +key's usage compared with the relay's sum would show one billed after all. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -49,6 +51,8 @@ """ from __future__ import annotations +import base64 +import binascii import hashlib import http.client import json @@ -85,6 +89,13 @@ # its next use, before any of the reply is read: worth one new connection. _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError) ANTHROPIC_VERSION = "2023-06-01" +CONTEXT_ERROR = "ContextWindowExceededError" +STOP = "stop" +OVERFLOW = "context window" +COMPLETION = re.compile(r"^/(?:chat/completions|completions|messages|responses)$") +_WHOLE = ((b"\x89PNG\r\n\x1a\n", b"IEND\xaeB`\x82", "image/png"), (b"\xff\xd8\xff", b"\xff\xd9", "image/jpeg"), + (b"GIF87a", b";", "image/gif"), (b"GIF89a", b";", "image/gif")) +_CONTEXT = re.compile(r"context[ _-]?(length|window|limit)|prompt is too long|token count .{0,40}exceeds", re.I) def _cut(text, n=4000): @@ -157,7 +168,7 @@ def _usage(u: Optional[dict]) -> Optional[dict]: inp += cached + written else: written = (det or {}).get("cache_write_tokens") or 0 - got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0} + got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or (rdet or {}).get("thinking_tokens") or 0} if written: got["cache_write"] = written hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0 @@ -191,6 +202,25 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def _hollow(path: str, body: bytes) -> Optional[tuple]: + """A 200 to a model call that holds no reply: the status it stands for, 429 or 402 when an error it held says so, else + 502, and its kind alone, since its words can quote the request.""" + if not COMPLETION.search(path.split("?")[0]): + return None + try: + got = json.loads(body) + except ValueError: + return 502, "no reply" + if not isinstance(got, dict): + return 502, "no reply" + if read_reply(path, body, False): + return None + code = got["error"].get("code") if isinstance(got.get("error"), dict) else None + code = code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code < 600 else None + return ((code if code in (402, 429) else 502), "an error" + (f" {code}" if code else "")) if got.get("error") \ + else (502, "no reply") + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: """What a reply said: its text, its tool calls, its usage.""" if not streamed: @@ -245,7 +275,7 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): t, c = _text_of_responses(ev.get("response") or {}) return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: @@ -485,6 +515,37 @@ def _request_tail(req: dict) -> dict: "reasoning_effort": effort if isinstance(effort, str) else None} +def _image(data: str) -> Optional[str]: + """An inline image's type when its bytes show a whole one: its head, and its end after it.""" + try: + raw = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + return None + if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP": + return "image/webp" if int.from_bytes(raw[4:8], "little") + 8 <= len(raw) else None + if raw[:2] == b"BM": + return "image/bmp" if int.from_bytes(raw[2:6], "little") <= len(raw) else None + return next((t for head, tail, t in _WHOLE if raw.startswith(head) and tail in raw[len(head):]), None) + + +def _images(x) -> List[dict]: + """Each inline image a request carries, as Messages or chat sends one: its declared type, the type its bytes + show whole (None when they show none) and its size, never a byte of it.""" + if isinstance(x, list): + return [i for v in x for i in _images(v)] + if not isinstance(x, dict): + return [] + src, url = x.get("source"), x.get("image_url") + url = url.get("url") if isinstance(url, dict) else url + if x.get("type") == "image" and isinstance(src, dict) and src.get("type") == "base64" and isinstance(src.get("data"), str): + said, data = src.get("media_type"), src["data"] + elif isinstance(url, str) and url.startswith("data:") and ";base64," in url: + said, data = url[5:].split(";", 1)[0], url.split(";base64,", 1)[1] + else: + return _images(list(x.values())) + return [{"type": str(said)[:40] if said else None, "is": _image(data), "bytes": len(data)}] + + def _digest(obj) -> str: text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True) return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16] @@ -541,8 +602,9 @@ class _Pool: KEEP = 16 # idle connections kept per provider; more are closed - def __init__(self, base: str): + def __init__(self, base: str, timeout: float = TIMEOUT): u = urllib.parse.urlsplit(base) + self.timeout = timeout self.https = u.scheme == "https" self.host, self.port = u.hostname, u.port or (443 if self.https else 80) self.prefix = u.path.rstrip("/") @@ -558,7 +620,7 @@ def take(self, fresh: bool = False): kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection with self.lock: self.opened += 1 - return kind(self.host, self.port, timeout=TIMEOUT), False + return kind(self.host, self.port, timeout=self.timeout), False def give(self, conn, r): """Back to the pool when the reply was read whole and the provider @@ -605,7 +667,7 @@ class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, - facts: Optional[bool] = None, max_calls: Optional[int] = None): + facts: Optional[bool] = None, timeout: float = TIMEOUT, max_calls: Optional[int] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -628,7 +690,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s put on every such request in place of the caller's own. ``facts``: a pinned Chat Completions or Responses request asks OpenRouter for its routing facts, where its reply names who served it; by default when - ``base`` is OpenRouter's. + ``base`` is OpenRouter's. ``timeout``: the seconds one call may wait. ``max_calls``: the most model calls the agent's own route of a unit may make (:func:`agent_route`), each answered call counted.""" self.fields = fields @@ -648,7 +710,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s self.models: Dict[str, str] = {} self.embedding_models: Dict[str, str] = {} self.lock = threading.Lock() - self.pool = _Pool(self.base) + self.pool = _Pool(self.base, timeout) self.server: Optional[ThreadingHTTPServer] = None os.makedirs(record_dir, exist_ok=True) @@ -721,8 +783,13 @@ def _headers(self, caller, tag: str = "", facts: bool = False) -> Dict[str, str] out["Authorization"] = "Bearer " + self.key return out + def _asked(self) -> bool: + return asked(self.dir) is not None + def _forward(self, h: BaseHTTPRequestHandler): - """One call of a caller, ``h``: sent on, passed back, recorded.""" + """One call of a caller, ``h``: sent on, passed back, recorded. A caller's own call + refused for its context window also leaves STOP beside the records; one made once + another asked by STOP is refused unsent and unrecorded.""" bits = h.path.split("/", 2) if len(bits) < 3 or not _TAG.match(bits[1]): h.send_error(404, "no route: //v1/...") @@ -754,6 +821,9 @@ def _forward(self, h: BaseHTTPRequestHandler): routing_asked = req.get("provider") req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if not side and not self.models.get(tag) and self._asked(): + h.send_error(400, "the unit was asked to stop") + return if self.max_calls and agent_route(tag) and h.command == "POST" and isinstance(req, dict) and req.get("model") and not _free(rest): with self.lock: made = self.answered.get(tag, 0) @@ -781,6 +851,7 @@ def _forward(self, h: BaseHTTPRequestHandler): t0 = time.time() conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed + cut_off = None try: # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) @@ -801,10 +872,17 @@ def _forward(self, h: BaseHTTPRequestHandler): except Exception as e: # cut off mid-reply status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() + empty = _hollow(rest, body) if status == 200 and not streamed and h.command == "POST" \ + and not (isinstance(req, dict) and req.get("stream")) else None + if empty: + (status, inside), kind = empty, "application/json" + back = [(n, v) for n, v in back if n.lower() in ("retry-after", "retry-after-ms")] + body = json.dumps({"error": {"message": f"the endpoint answered 200 with {inside}"}}).encode() def record(body): rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None, - "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {}, + "model_asked": asked, "status": status, + "request": dict(_request_tail(req), sha256=_digest(req)) if isinstance(req, dict) else {}, "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True @@ -816,8 +894,8 @@ def record(body): if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None - if status == 200: - if self.max_calls and agent_route(tag) and isinstance(req, dict) and req.get("model") and not _free(rest): + if status == 200 or cut_off: + if status == 200 and self.max_calls and agent_route(tag) and isinstance(req, dict) and req.get("model") and not _free(rest): with self.lock: self.answered[tag] = self.answered.get(tag, 0) + 1 rec["reply"] = read_reply(rest, body, streamed) @@ -827,21 +905,30 @@ def record(body): if k not in rec["reply"]}) except Exception: pass - cost, how = self._cost(rec["model"], rec["reply"].get("usage")) + cost, how = (0.0, "failed") if rec["reply"].get("end") == "failed" and not rec["reply"].get("usage") \ + else self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - else: - rec["error"] = _said(status, body) + if status != 200: + rec["error"] = cut_off or _said(status, body) code, wait = _code(body), _retry_after(back) if code: rec["error_code"] = code if wait is not None: rec["retry_after"] = wait # the provider's word on when to call again (pace) + sent = _images(req) if status in (400, 413, 422) else [] + if sent: + rec["images"] = sent + if empty: + rec["answered"] = 200 if booking is not None: rec["booked_usd"] = booked try: - if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) + if status == 200 or cut or empty: # what it cost; unknown (no token count, cut off), the booking is kept whole + whole = not (cut_off or rec.get("cost_from") == "failed") + known = cost if whole or (cost is not None and cost > booked) else None + self.ledger.settle(booking, known, rec["model"], + ((rec.get("reply") or {}).get("usage") or {}).get("output") if whole else None) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -849,6 +936,9 @@ def record(body): with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") + if not side and not self.models.get(tag) and overflowed(rec): + with open(os.path.join(self.dir, STOP), "a") as f: + f.write(OVERFLOW + "\n") h.send_response(status) if not any(n.lower() == "content-type" for n, _ in back): @@ -864,6 +954,8 @@ def record(body): while True: chunk = r.read1(65536) if not chunk: + if r.length: + raise http.client.IncompleteRead(b"", r.length) whole = True break parts.append(chunk) @@ -872,8 +964,8 @@ def record(body): h.wfile.flush() except OSError: pass # the caller went away; the record still completes - except Exception: - pass + except Exception as e: + status, cut, cut_off = 502, True, "the endpoint's reply was cut off: " + self._said(e) record(b"".join(parts)) # before the connection closes: the caller's end of stream else: record(body) # before the caller has the reply, so a reader after it finds the call @@ -1005,16 +1097,30 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True +def asked(record_dir: str) -> Optional[str]: + """What a guard's STOP in ``record_dir`` says, or None: none there, or only the context window's.""" + try: + with open(os.path.join(record_dir, STOP)) as f: + said = next((ln.strip() for ln in f.read().splitlines() or [""] if ln.strip() != OVERFLOW), None) + except OSError: + return None + return None if said is None else said or STOP + + # what a unit whose agent reached the run's cap on model calls carries (``max_calls``): an agent error, counted # against the agent, as a timeout is (runs.classify) CALLS_ERROR = "UnitCallCapError" -def failure(calls: List[dict]) -> Optional[tuple]: +def failure(calls: List[dict], words: bool = True, agent: bool = False) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's). A call + succeeded (a failure the agent recovered from is not the unit's). + ``words`` False: the status alone, never the provider's text, which can quote + a gated task. ``agent``: the agent's last call over the model's context + window is ``ContextWindowExceededError``, its own, whatever side call followed, and a side call's 200 does not + recover the agent's own. Calls are taken in the order they began. A call the run's spending cap refused is the unit's wherever it falls: once one is refused, every later one is.""" cut = next((c for c in calls if c.get("refused") == "cap"), None) @@ -1026,10 +1132,20 @@ def failure(calls: List[dict]) -> Optional[tuple]: many = next((c for c in calls if c.get("refused") == "calls"), None) if many: # its agent made the calls it was allowed: the agent's, as its time running out is return CALLS_ERROR, many.get("error") or "" - if not calls or calls[-1].get("status") == 200: + calls = _begun(calls) + ok = [i for i, c in enumerate(calls) if c.get("status") == 200] + after, side = max((i for i in ok if not (agent and calls[i].get("side"))), default=-1), max(ok, default=-1) + calls = [c for i, c in enumerate(calls) if i > after and c.get("status") != 200 and not (c.get("side") and i < side)] + if not calls: return None + own = next((c for c in reversed(calls) if not c.get("side")), None) if agent else None + if own is not None and overflowed(own): + said = (own.get("error") or "") if words else ("the endpoint answered 400 to the agent's call, longer than the model's " + "context window (provider text not kept: gated benchmark)") + return CONTEXT_ERROR, said last = calls[-1] - s, said = last.get("status"), last.get("error") or "" + s = last.get("status") + said = (last.get("error") or "") if words else f"the endpoint answered {s} (provider text not kept: gated benchmark)" if s == 429: # OpenAI's spent credit comes as a 429 too: a usage limit, no pace mends it return ("ApiUsageLimitError" if last.get("error_code") == "insufficient_quota" else "RateLimitError"), said @@ -1037,13 +1153,23 @@ def failure(calls: List[dict]) -> Optional[tuple]: return "ApiUsageLimitError", said # the provider's credit spent (the relay's own refusals say so above, or are its own) if s in (401, 403): return "AgentAuthenticationError", said - if (s == 404 and "model" in said.lower()) or (s == 400 and _NO_MODEL.search(said)): + if (s == 404 and "model" in (last.get("error") or "").lower()) or (s == 400 and _NO_MODEL.search(last.get("error") or "")): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said return "ApiError", said +def _begun(calls: List[dict]) -> List[dict]: + timed = all(isinstance(c.get("t0"), (int, float)) for c in calls) + return sorted(calls, key=lambda c: c["t0"]) if timed else list(calls) + + +def overflowed(call: dict) -> bool: + """A 400 for a request longer than the model's context window, by the provider's words.""" + return call.get("status") == 400 and bool(_CONTEXT.search(call.get("error") or "")) + + # -- the relay as a process of its own: a container backend's sidecar ---------- def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL @@ -1077,6 +1203,7 @@ def serve(argv=None) -> int: "or --routing given") ap.add_argument("--routing", action="store_true", help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") + ap.add_argument("--timeout", type=float, default=TIMEOUT, help="seconds one call may wait for the provider's reply") a = ap.parse_args(argv) serving, bad = parse_served_by(a.served_by, "") if serving and not bad: @@ -1101,7 +1228,8 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, max_calls=a.max_calls) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, + timeout=a.timeout, max_calls=a.max_calls) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/retries.py b/app/nullius/standard/retries.py index a3d09459..7c19d1ca 100644 --- a/app/nullius/standard/retries.py +++ b/app/nullius/standard/retries.py @@ -33,6 +33,7 @@ "ModelNotFoundError", # this runner's own: a secret no second try provides, the spending caps and the calls cap, a stopped run "EnvironmentSecretMissingError", ledger.ERROR, ledger.UNIT_ERROR, relay.CALLS_ERROR, "CancelledError", + relay.CONTEXT_ERROR, }) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 7518e9f9..2b36233b 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -54,6 +54,7 @@ """ from __future__ import annotations +import fcntl import hashlib import json import os @@ -69,7 +70,7 @@ from collections import Counter from concurrent.futures import FIRST_EXCEPTION, CancelledError, ThreadPoolExecutor, wait from contextlib import contextmanager -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact @@ -115,6 +116,10 @@ def driver_path() -> str: _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) +HELD: dict = {} +LIVE: dict = {} + + class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" @@ -407,7 +412,24 @@ def get(self, name: str) -> Optional[dict]: return {"kind": kind, "value": text} if text else None +def resolve_bench_options(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + which=shutil.which) -> Tuple[dict, dict]: + """The benchmark's options for a run on ``dataset`` and what their values need; refused before anything starts.""" + vals, bad = definitions.bench_options_of(bench, dataset, given, setup) + if bad: + raise Refused("; ".join(bad)) + needs = definitions.option_needs(bench, vals) + lacking = hostreq.problems(bench, needs["host"], which) + if lacking: + raise Refused(f"{', '.join(f'{k}={v}' for k, v in vals.items())} needs on this machine " + + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) + else ". Run it on a machine that has it")) + return vals, needs + + class Run: + call_seconds: float = relay.TIMEOUT timeouts: Dict[str, float] = {} # the run's own time limits by phase (__init__), none by default forced_turns: Optional[int] = None max_calls: Optional[int] = None # the run's cap on a unit's agent's model calls (__init__), none by default @@ -686,6 +708,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas raise Refused("--max-calls caps an agent's model calls: a capability's unit route carries the benchmark's reader too") self.max_calls = max_calls self.max_turns = max_turns or (bench.get("limits") or {}).get("turns") or MAX_TURNS + self.score_seconds = (bench.get("limits") or {}).get("score_seconds") or DRIVER_SECONDS + self.call_seconds = (bench.get("limits") or {}).get("call_seconds") or relay.TIMEOUT self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -762,8 +786,25 @@ def driver(self) -> jsonl.Proc: return d def start(self) -> str: + lck, mine = os.path.join(self.job_dir, ".work", "job.lck"), object() + try: + return self._start(lck, mine) + finally: + if LIVE.get(lck) is mine: + del LIVE[lck] + + def _start(self, lck, mine) -> str: t0 = time.time() os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True) + held = HELD.get(lck) or open(lck, "a") + try: + if LIVE.setdefault(lck, mine) is not mine: + raise BlockingIOError + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused(f"another run under the job name {self.name} is running: " + "start this one once it ends, or under another name") from None + HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) @@ -1033,7 +1074,8 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, if self._stopped.is_set() or self._halt.is_set(): return # each attempt whose last try ended with a try to follow (``unit``: what it held stays held) - todo = [name for name, res in latest.items() if name in self._counted and self.retry_if(res)] + todo = [name for name, res in latest.items() if name in self._counted and self.retry_if(res) + and relay.asked(os.path.join(self.job_dir, name, ".work", "relay")) is None] if not todo: return self.progress.emit("units", "progress", detail=f"trying {len(todo)} again: retry {retry} of {self.retries}") @@ -1175,7 +1217,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), - unit_caps=False, serving=self.serving).start() # embeddings and rerank: a bearer key + unit_caps=False, serving=self.serving, timeout=self.call_seconds).start() # embeddings and rerank: a bearer key def runtimes_needed(self) -> List[str]: """What the agent needs in an own image that the image may not have: @@ -1219,6 +1261,11 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home") + earlier = os.path.join(trial, ".work", f"earlier.{time.time_ns()}") + for part in (os.path.join(".work", "relay"), "result.json", "verifier", "agent"): + if os.path.lexists(os.path.join(trial, part)): + os.makedirs(earlier, exist_ok=True) + os.replace(os.path.join(trial, part), os.path.join(earlier, os.path.basename(part))) os.makedirs(ws, exist_ok=True) # The workspace is a repository of its own: agents look up to the git root # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept @@ -1303,17 +1350,17 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", self.phase_limit("score", DRIVER_SECONDS) + (self.step_limit("verify", step) if step else 0)) + self.phase(name, unit, "verify", self.phase_limit("score", self.score_seconds) + (self.step_limit("verify", step) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.phase_limit("score", DRIVER_SECONDS), + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.phase_limit("score", self.score_seconds), **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e), retryable=not isinstance(e, jsonl.NoReply)) + raise _Ended(_kind(e, "VerifierError"), str(e), retryable=not isinstance(e, jsonl.NoReply)) finished = True except _Ended as e: exc = (e.kind, e.said) @@ -1337,17 +1384,25 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: for ln in getattr(rl, "mcp_listeners", None) or []: ln.stop() calls = rl.calls(tag) if rl is not None else [] - failed = relay.failure(calls) + words = not self.bench.get("gated") + failed = relay.failure(calls, words, agent=True) if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR, relay.CALLS_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is # the endpoint's, and a cap's refusal is the cap's failed = None - if not failed and rl is not None and exc is not None and exc[0] == "VerifierError": + if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") + or self.upstream and exc and ": VerifierError: " in exc[1]): + failed = None + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and (score or {}).get("stopped") not in (None, "", relay.OVERFLOW) \ + and rl is not None and relay.asked(rl.dir) is not None \ + and [(c.get("status"), c.get("error_code") == "insufficient_quota") for c in relay._begun(calls) if not c.get("side")][-1:] == [(429, False)]: + failed = None + if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): # the judge's or the simulated user's model failing (a 429, a refused request) is the # unit's infrastructure, as the agent's own would be - not a miss counted against it - failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) + failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}"), words) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score, retry_ok = None, True # a time run out under refused calls is the provider's: tried again @@ -1639,7 +1694,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), - max_calls=self.max_calls).start() + timeout=self.call_seconds, max_calls=self.max_calls).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1669,6 +1724,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, ledger=self.ledger_file(), serving=getattr(self, "serving", None), max_calls=self.max_calls, + timeout=self.call_seconds if self.call_seconds != relay.TIMEOUT else None, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -2245,8 +2301,8 @@ def send(op, **kw): def drive_upstream(self, unit, ws, tag, transcript, rl): """The setup's unit: the driver runs the upstream's own answerer on it (``answer``), asked as ``asked`` through the unit's relay, which sends - the run's model in its place. A reply with no text - a reasoning model - that spent the upstream's token budget thinking - is said as such.""" + the run's model in its place. A last reply with no text or tool call - a reasoning model + that spent the upstream's token budget thinking - is said as such; a unit a guard stopped is cancelled.""" o = self.other try: # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes @@ -2255,7 +2311,18 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.agent_limit({})) except jsonl.Failed as e: calls = rl.calls(tag) - silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] + stop = relay.asked(rl.dir) + if stop and not _kind(e, None): + raise _Ended(CANCELLED, f"a guard asked the unit to stop ({stop[:200]}); {o['label']}: {e}", + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + own = [c for c in relay._begun(calls) if not c.get("side")][-1:] + failed = [((c["reply"].get("error") or {}).get("code") or "no code") for c in own if (c.get("reply") or {}).get("end") == "failed"] + kind = _kind(e, "ApiError" if failed else None) + if kind: + raise _Ended(kind, f"{o['label']}: {e}" + "".join(f"; {self.model}'s last reply failed ({c})" for c in failed), + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + silent = [c for c in own if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip() + and not (c.get("reply") or {}).get("tool_calls")] if silent: u = (silent[-1].get("reply") or {}).get("usage") or {} thought = u.get("reasoning") or 0 @@ -2270,6 +2337,12 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") +def _kind(e, otherwise): + """A driver's own EnvironmentSetupError, VerifierError, CancelledError or CauseUnknownError keeps its kind.""" + m = re.match(r"\w+: (EnvironmentSetupError|VerifierError|CancelledError|CauseUnknownError): ", str(e)) + return m.group(1) if m else otherwise + + class _Ended(Exception): """A unit's end in an error: ``kind``, Harbor's exception name; ``retryable`` False for one a further try meets again whatever its name (a setup this machine cannot give, a time run out: ``retries``).""" diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 7ad4dca6..26c829e7 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -475,7 +475,7 @@ return '

Where does it run?

' + `
${machine('local', 'This machine', 'Nullius runs it here, in runs/' + rec.name + '.')}${H.map(h => machine(h.entry, h.label, 'Over ssh: the standard\'s code at this commit is sent there, uv put there if missing; its job comes back as it runs.')).join('')}
` + (H.length ? (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '') : `

Hosts are named in ${button('Settings → Hosts', 'data-href="/settings#HOSTS"', 'nb-inlinebutton')}; this machine is the only choice until then.

`) - + '

How it runs there

' + + (b ? benchOptions(rec, b, 'where') : '') + '

How it runs there

' + `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) @@ -549,12 +549,13 @@ function benchOptions(rec, b, scope = 'yours') { const local = ((schema.standardHere || {}).host || {})[b.key] || {}; const chosen = C.chosenOf(rec); - return Object.entries(b.options || {}).filter(([, o]) => !o.limit) // a bound of the upstream's own is with the limits + return Object.entries(b.options || {}).filter(([, o]) => !o.limit && o.step === (['where', 'dataset'].includes(scope) ? scope : '')) // a bound of the upstream's own is with the limits .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) // one that applies only under another's values (a top-k, only with a retrieval) is left out otherwise .filter(([, o]) => Object.entries(o.when || {}).every(([k, vs]) => vs.includes((rec.bench_options || {})[k] ?? (b.options[k] || {}).default))) .filter(([, o]) => { const on = o.setups || ['yours']; + if (o.step) return on.some(s => s === 'yours' ? chosen.some(k => !(b.setups || []).some(x => x.key === k)) : chosen.includes(s)); const shared = on.includes('yours') && on.includes('original'); if (scope === 'shared') return shared; if (shared) return false; @@ -562,21 +563,22 @@ }) .map(([n, o]) => { const cur = (rec.bench_options || {})[n] ?? o.default; + if (scope === 'where' && o.values.length === 1) return `

${esc(o.label)}

${esc((o.names || {})[cur] || cur)}

`; if (o.pattern) return ``; const tileOf = v => { const need = (o.needs || {})[v] || {}; const foot = [need.embeddings ? `${need.embeddings.includes('/') ? 'OpenRouter key' : 'OpenAI or OpenRouter key'}` : '', need.reranker ? `reranker ${esc(need.reranker)}` : '', ...(need.host || []).filter(h => local[h]).map(h => `needs ${esc(h)}`), v === o.default ? 'default' : ''].join(''); - return tile({attr: 'bench-opt', key: n, data: ` data-bench-val="${esc(v)}"`, on: v === cur, title: v, mark: '', tip: (o.about || {})[v] || '', foot}); }; + return tile({attr: 'bench-opt', key: n, data: ` data-bench-val="${esc(v)}"`, on: v === cur, title: (o.names || {})[v] || v, mark: '', tip: (o.about || {})[v] || '', foot}); }; return `

${esc(o.label)}${o.datasets.length ? ' · ' + esc(o.datasets.join(', ')) : ''}

` // by the data: every value with its line (or a need) - cards of one size, the line whole; plain values - one small // card each, all one size, or a select past five of them (a line some have is their tooltip) + (o.values.some(v => ((o.needs || {})[v] || {}).embeddings || ((o.needs || {})[v] || {}).reranker || (((o.needs || {})[v] || {}).host || []).some(h => local[h])) ? `
${o.values.map(tileOf).join('')}
` : o.values.length > 5 - ? `` - : `
${o.values.map(v => ``).join('')}
`); + ? `` + : `
${o.values.map(v => ``).join('')}
`); }).join(''); } // Only credentials needing attention belong in the builder. Without a dataset's default @@ -653,7 +655,7 @@ tip: x.summary || '', meta: x.units ? x.units + ' ' + u.nouns : '', foot: (x.controls || []).map(c => controlChip(c)).join('') + checkChip('stable', (x.proofs || {}).units, (x.failed || {}).units, 'the same ids twice, each unique') + ((b.controls || []).some(c => c.key === 'oracle') ? checkChip('no leak', (x.proofs || {}).leak, (x.failed || {}).leak, 'nothing of the reference shows before the run') : '')})).join('')}
` - + credentialWarnings(b, b.datasets.filter(x => C.datasetsOf(rec).includes(x.id))); + + benchOptions(rec, b, 'dataset') + credentialWarnings(b, b.datasets.filter(x => C.datasetsOf(rec).includes(x.id))); if (step === S.agent) { const agent = b.family === 'agent', all = C.standardOthers(rec), others = all.filter(o => !o.control && !o.setup); const setups = all.filter(o => o.setup), controls = all.filter(o => o.control); @@ -714,7 +716,7 @@ hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) + servedField(rec, 'simulator') : '') + (b.judge ? (paper ? fixed('Model for the judge', lb && lb.judge || b.judge, lb && lb.judge ? 'leaderboard' : 'official') : field('Model for the judge', 'judge_model', rec.judge_model || '', {list: 'nb-models', placeholder: b.judge + ' (official)', hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) + servedField(rec, 'judge') : '') - + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, v, 'leaderboard')).join('') : '') + + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, ((b.options[k] || {}).names || {})[v] || v, 'leaderboard')).join('') : '') + keyLine(rec); } if (step === S.limits) { @@ -778,7 +780,7 @@ // each field shows, greyed, the value used while it is empty; a task that declares its own keeps it const m = s => String(+(s / 60).toFixed(2)); // [placeholder, declared]: the benchmark's or the agent's own reads as a value, Nullius's default stays grey - const def = {setup: [m(d.setup), false], image: [m(d.image), false], agent: agentDefault(rec, b), verify: ['per task', true], score: [m(d.score), false]}; + const def = {setup: [m(d.setup), false], image: [m(d.image), false], agent: agentDefault(rec, b), verify: ['per task', true], score: [m(d.score), (d.catalogue || []).includes('score')]}; const label = (k, l) => k === 'agent' ? 'Agent, whole ' + u.noun : l; return '

Time limits

' + C.STANDARD_PHASES.map(([k, l]) => minutesRow(k, label(k, l), t[k], def[k])).join(''); diff --git a/app/templates/catalogue/benchmark.html b/app/templates/catalogue/benchmark.html index 81ff62d5..f19f0733 100644 --- a/app/templates/catalogue/benchmark.html +++ b/app/templates/catalogue/benchmark.html @@ -115,7 +115,7 @@

Proofs

Options

{% for n, o in std.options.items() %} -

{{ o.label }}{% if o.datasets %} {{ o.datasets|join(', ') }}{% endif %}

+

{{ o.label }}{% if o.datasets %} {{ std.datasets|selectattr('id', 'in', o.datasets)|map(attribute='label')|join(', ') }}{% endif %}

{{ o.summary }}

{% if o.pattern %}
{{ 'Provider default' if n == 'temperature' and o.default == 'default' else o.default }} @@ -124,7 +124,7 @@

{{ o.label }}{% if o.datasets %} {{ o.datasets|jo {% else %}
    {% for v in o['values'] %}{% set need = o.needs.get(v) or {} %} -
  • {{ v }}{% if v == o.default and v != 'default' %} default{% endif %} +
  • {% if o.names.get(v) %}{{ o.names[v] }}{% else %}{{ v }}{% endif %}{% if v == o.default and v != 'default' %} default{% endif %} {% if need.embeddings %}OpenAI key · {{ need.embeddings }}{% endif %} {% for h in need.host or [] %}needs {{ h }}{% endfor %} {% if o.about.get(v) %}{{ o.about[v] }}{% endif %}
  • diff --git a/app/tests/catalogue_docs.py b/app/tests/catalogue_docs.py index 0a55fc44..ddf11510 100644 --- a/app/tests/catalogue_docs.py +++ b/app/tests/catalogue_docs.py @@ -225,7 +225,8 @@ def _load(kind: str, every: bool = True) -> list: def _values(spec: dict) -> str: """An option's values, the default in bold; a pattern when it has no list.""" - values = ", ".join(f"**`{v}`**" if v == spec.get("default") else f"`{v}`" for v in spec.get("values") or []) + named = {v: f"`{v}` ({n})" for v, n in (spec.get("names") or {}).items()} + values = ", ".join(f"**{named.get(v, f'`{v}`')}**" if v == spec.get("default") else named.get(v, f"`{v}`") for v in spec.get("values") or []) return values or f"pattern `{spec.get('pattern')}`, default `{spec.get('default')}`" diff --git a/app/tests/test_dependency_surface.py b/app/tests/test_dependency_surface.py index 12b6b2ee..d6cbd843 100644 --- a/app/tests/test_dependency_surface.py +++ b/app/tests/test_dependency_surface.py @@ -9,6 +9,7 @@ viewer independent of the runtime. """ +import fnmatch import importlib.util import json import os @@ -80,6 +81,26 @@ def test_every_backend_package_is_shipped_once(self): self.assertEqual({p for p in packages if p == "nullius" or p.startswith("nullius.")}, source) self.assertEqual({p.split(".")[0] for p in packages}, {"nullius", "_nullius_app"}) + def test_every_file_a_bundled_definition_runs_is_shipped(self): + """Every file a host's snapshot takes of a bundled entry is package data too.""" + from nullius.standard import payload + shipped = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["setuptools"]["package-data"] + + def covered(package, relative): + parts = relative.split("/") + return any(len(g.split("/")) == len(parts) and all(map(fnmatch.fnmatchcase, parts, g.split("/"))) + for g in shipped[package]) + + base = APP / "nullius" + entries = [("nullius.catalogue", f"{kind}/{e.name}", payload._entry(e)) + for kind in ("agents", "benchmarks", "capabilities") + for e in sorted((base / "catalogue" / kind).iterdir()) if e.is_dir() and e.name != "__pycache__"] + entries += [("nullius.standard", f"kits/{e.name}", payload._entry(e, kit=True)) + for e in sorted((base / "standard" / "kits").iterdir()) if (e / "standard.json").is_file()] + missing = [f"{where}/{name}" for package, where, files in entries for name in files + if not covered(package, f"{where}/{name}")] + self.assertEqual(missing, []) + def test_every_requirement_is_on_the_allowlist(self): for req in requirements(): with self.subTest(req): diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index e0ce5639..6d283043 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -288,6 +288,18 @@ def test_review_asks_for_docker_for_every_setup_and_the_null_control(self): self.assertFalse(prerequisites.ready([{"rows": rows}])) self.assertTrue(prerequisites.ready([{"rows": prerequisites.check_here([], setups, which=lambda p: "/usr/bin/" + p)}])) + def test_a_value_shows_its_name_and_the_recipe_keeps_the_value(self): + out = recipe.compile(self.a_recipe(["original"], bench_options={"agent": "muse_spark"})) + self.assertIn([4, "Agent", "Muse Spark · test_small"], out["review"]) + self.assertIn([6, "Steps per task", "100 (its default) · test_small"], out["review"]) + self.assertIn("agent=muse_spark", out["script"]) + self.assertEqual(out["review"][-1], [7, "Where", "this machine · local backend · VM provider docker"]) + b = copy.deepcopy(definitions.get("benchmark", "osworld1")) + b["options"]["agent"]["names"]["nobody"] = "Nobody" + b["options"]["thinking"]["step"] = "agent" + self.assertTrue(any("agent.names" in p for p in definitions.problems(b, "benchmark"))) + self.assertTrue(any("thinking.step" in p for p in definitions.problems(b, "benchmark"))) + def test_the_null_control_is_refused_as_it_starts_on_a_machine_without_docker(self): b = definitions.get("benchmark", "osworld1") with self.assertRaises(run.Refused) as e: diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py new file mode 100644 index 00000000..b25ff1b2 --- /dev/null +++ b/app/tests/test_standard_context_window.py @@ -0,0 +1,321 @@ +"""An agent's last call over the model's window ends the unit as ContextWindowExceededError, whatever followed.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from nullius.runs import classify, parse +from nullius.standard import definitions, install, job, relay, retries, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +WORDS = "the gated task's own sentence" +OPENROUTER = f"HTTP 400: This endpoint's maximum context length is 1000 tokens. However, you requested 1200: {WORDS}" +ANTHROPIC = f"HTTP 400: prompt is too long: 1200 tokens > 1000 maximum: {WORDS}" +EMPTY = "HTTP 400: messages: at least one message is required" +AGENT_OK = {"status": 200} +OVERFLOW = {"status": 400, "error": OPENROUTER} +SIDE_EMPTY = {"status": 400, "error": EMPTY, "side": True} +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's runner ended on its simulated user's error") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' +STOPPED = ANSWERING.replace('raise RuntimeError("the upstream\'s runner ended on its simulated user\'s error")', + 'return {"answer": "DONE"}').replace('{"reward": 0.0, "detail": {}}', + '{"reward": 0.5, "detail": {}, "stopped": "context window"}') +UNREAD = '''from nullius.standard.jsonl import serve +class VerifierError(RuntimeError): + pass +def score(unit, dir, **_): + raise VerifierError("the evaluation did not read the VM") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": lambda unit, dir, setup, model, base_url, key, **_: {"answer": "DONE"}, "score": score}) +''' +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Failure(unittest.TestCase): + def test_the_agents_overflow_outranks_the_side_call_after_it(self): + calls = [AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, agent=True), (relay.CONTEXT_ERROR, OPENROUTER)) + self.assertFalse(retries.retryable({"exception_info": {"exception_type": relay.CONTEXT_ERROR}})) + exc, said = relay.failure(calls, words=False, agent=True) + self.assertEqual(exc, "ContextWindowExceededError") + self.assertIn("answered 400", said) + self.assertIn("context window", said) + self.assertNotIn(WORDS, said) + self.assertNotIn("maximum context length", said) + + def test_anthropics_words_and_a_later_rate_limit(self): + calls = [AGENT_OK, {"status": 400, "error": ANTHROPIC}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_the_agents_overflow_as_the_last_call(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW], words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_an_overflow_the_agent_recovered_from_is_not_the_units(self): + self.assertIsNone(relay.failure([OVERFLOW, AGENT_OK], agent=True)) + + def test_a_side_call_that_went_through_after_it_does_not_recover_the_agent(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW, {"status": 200, "side": True}], agent=True)[0], relay.CONTEXT_ERROR) + limited = [AGENT_OK, {"status": 429, "error": "slow down"}, {"status": 502, "error": "x"}, + {"status": 429, "error": "slow down"}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(limited, words=False, agent=True)[0], "RateLimitError") + self.assertIsNone(relay.failure(limited[1:3] + [AGENT_OK, {"status": 200, "side": True}], agent=True)) + self.assertIsNone(relay.failure(limited, words=False)) + calls = [AGENT_OK, OVERFLOW, {"status": 200, "side": True}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + self.assertEqual(relay.failure(calls, words=False)[0], "RateLimitError") + calls = [{"status": 429, "error": "x"}, {"status": 502, "error": "x", "side": True}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "RateLimitError") + + def test_the_calls_are_read_in_the_order_they_began(self): + abandoned = {"status": 502, "error": "x", "t0": 1} + self.assertIsNone(relay.failure([dict(AGENT_OK, t0=2), abandoned], agent=True)) + self.assertEqual(relay.failure([{"status": 429, "error": "x", "t0": 2}, dict(AGENT_OK, t0=1)])[0], "RateLimitError") + self.assertEqual(relay.failure([dict(AGENT_OK, t0=1), {"status": 429, "error": "x"}], agent=True)[0], "RateLimitError") + + def test_another_400_of_the_agent_stays_the_infrastructures(self): + calls = [AGENT_OK, {"status": 400, "error": "HTTP 400: invalid image"}, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_side_calls_overflow_is_not_the_agents(self): + calls = [AGENT_OK, dict(OVERFLOW, side=True)] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_judges_or_simulators_route_is_never_the_agents_context(self): + calls = [AGENT_OK, OVERFLOW] + self.assertEqual(relay.failure(calls, words=False)[0], "ApiError") + self.assertEqual(relay.failure(calls)[0], "ApiError") + + def test_only_a_400_says_so(self): + self.assertTrue(relay.overflowed(OVERFLOW)) + self.assertTrue(relay.overflowed({"status": 400, "error": ANTHROPIC})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: input length and `max_tokens` exceed context " + "limit: 196000 + 8192 > 200000"})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: The input token count (1200) exceeds the " + "maximum number of tokens allowed (1000)."})) + self.assertFalse(relay.overflowed(dict(OVERFLOW, status=500))) + self.assertFalse(relay.overflowed({"status": 400, "error": EMPTY})) + self.assertFalse(relay.overflowed({"status": 400})) + + def test_it_is_the_agents_result_not_the_infrastructure(self): + self.assertEqual(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.AGENT_ERROR) + self.assertNotIn(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.INFRA_OUTCOMES) + fed = job.feed_outcome("score", None, (relay.CONTEXT_ERROR, "said")) + self.assertEqual((fed["outcome"], fed["exc"]), ("agent_error", relay.CONTEXT_ERROR)) + + +class Calls: + """The agent's calls ending on an overflow, then a refused side call.""" + def __init__(self, own, dir=os.devnull): + self.own, self.dir = own, dir + + def calls(self, tag): + return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Unit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "paper") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, setups=SETUPS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def ran(self, own, gated=True, record=os.devnull): + b = dict(self.b, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own, record)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f).get("exception_info"), parse.parse_job(job_dir) + + def test_the_unit_is_the_agents_context_said_by_status_alone(self): + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn("answered 400", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + (trial,) = ran.trials + self.assertEqual(trial.outcome, classify.AGENT_ERROR) + self.assertFalse(trial.is_infra) + self.assertEqual(ran.n_infra, 0) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY], gated=False) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn(WORDS, said["exception_message"]) + + def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW]) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + + def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_a_429(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace('"stopped": "context window"', '"stopped": "batch cap"')) + record = os.path.join(self.tmp, "record") + os.makedirs(record) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("batch cap\n") + limited = {"status": 429, "error": "slow down"} + for calls in ([AGENT_OK, limited], [AGENT_OK, limited, {"status": 502, "error": "down", "side": True}]): + said, ran = self.ran(calls, record=record) + self.assertIsNone(said, calls) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + with open(os.path.join(ran.trials[0].dir, "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + said, _ = self.ran([AGENT_OK, limited]) + self.assertEqual(said["exception_type"], "RateLimitError") + said, _ = self.ran([AGENT_OK, dict(limited, error_code="insufficient_quota")], record=record) + self.assertEqual(said["exception_type"], "ApiUsageLimitError") + for own in (AGENT_OK, {"status": 401, "error": "no"}, {"status": 502, "error": "down"}): + said, _ = self.ran([own, dict(limited, side=True)], record=record) + self.assertEqual(said["exception_type"], "RateLimitError", own) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("context window\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + with open(os.path.join(record, relay.STOP), "a") as f: + f.write("batch cap\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace(', "stopped": "context window"', "")) + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD) + said, ran = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "VerifierError") + self.assertIn("the evaluation did not read the VM", said["exception_message"]) + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD.replace("raise VerifierError", "raise RuntimeError")) + said, _ = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + + def test_a_unit_run_again_under_its_jobs_name_starts_with_no_earlier_record_and_keeps_a_new_stop(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + record = os.path.join(jobs, "again", "a__u0000", ".work", "relay") + os.makedirs(record) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "verifier")) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "agent")) + for name, text in (("u0000.jsonl", json.dumps({"status": 429, "error": "slow down"}) + "\n"), (relay.STOP, "batch cap\n"), + ("../../verifier/score.json", "{}"), ("../../result.json", "{}"), ("../../agent/old.txt", "")): + with open(os.path.join(record, name), "w") as f: + f.write(text) + seen = [] + + def phase(r, name, unit, phase, timeout=None): + if phase == "env_setup": + os.makedirs(record, exist_ok=True) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("key cap\n") + + def unit_relay(r, tag, trial, *a, **k): + with open(os.path.join(trial, ".work", "relay", relay.STOP)) as f: + seen.append((sorted(os.listdir(os.path.join(trial, ".work", "relay"))), f.read())) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "phase", phase), patch.object(run.Run, "unit_relay", unit_relay): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + self.assertEqual(seen, [([relay.STOP], "key cap\n")]) + (earlier,) = [d for d in os.listdir(os.path.dirname(record)) if d.startswith("earlier.")] + earlier = os.path.join(os.path.dirname(record), earlier) + with open(os.path.join(earlier, "relay", relay.STOP)) as f: + self.assertEqual(f.read(), "batch cap\n") + for part in (("verifier", "score.json"), ("result.json",), ("agent", "old.txt")): + self.assertTrue(os.path.isfile(os.path.join(earlier, *part))) + self.assertFalse(os.path.exists(os.path.join(jobs, "again", "a__u0000", "verifier", "score.json"))) + + def test_a_run_under_a_job_name_another_run_holds_is_refused_before_it_writes(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + job_dir = os.path.join(jobs, "again") + lck = os.path.join(job_dir, ".work", "job.lck") + tried = subprocess.run([sys.executable, "-c", "import fcntl, sys\nfcntl.flock(open(sys.argv[1], 'a'), fcntl.LOCK_EX | fcntl.LOCK_NB)", lck], + capture_output=True) + self.assertNotEqual(tried.returncode, 0) + run.HELD.pop(lck).close() + before = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time\nf = open(sys.argv[1], 'a')\n" + "fcntl.flock(f, fcntl.LOCK_EX)\nprint(flush=True)\ntime.sleep(30)", + lck], stdout=subprocess.PIPE) + self.addCleanup(holder.stdout.close) + self.addCleanup(holder.kill) + holder.stdout.readline() + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])), \ + self.assertRaisesRegex(run.Refused, "another run under the job name again is running"): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + after = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + self.assertEqual(after, before) + + def test_a_second_run_in_the_same_process_under_a_live_job_name_is_refused(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + go = lambda: run.run(self.b, o, model="paper-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + name="live", say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + tried = [] + + def relay(r, *a, **k): + if not tried: + tried.append(None) + for again in (go, r.start): + try: + tried.append(again()) + except run.Refused as e: + tried.append(e) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "unit_relay", relay): + go() + self.assertEqual(len(tried), 3) + for refused in tried[1:]: + self.assertIsInstance(refused, run.Refused) + self.assertIn("another run under the job name live is running", str(refused)) + go() + + def test_a_side_call_failing_after_the_agents_success_stays_the_infrastructures(self): + said, ran = self.ran([AGENT_OK, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ApiError") + self.assertEqual(ran.trials[0].outcome, classify.API_ERROR) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index e1c6b84e..1b12115f 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -166,6 +166,107 @@ def test_a_standard_job_named_oracle_without_the_record_is_no_control(self): self.assertFalse(parse.parse_job(j.dir).is_control) # a runner's arm called oracle stays an arm +SLOW_SCORE = '''import os, time +from nullius.standard.jsonl import serve +CHECKED = os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "checked")) +def setup(unit, dir): + return dict({"prompt": "hi"}, **({"verify": {"run": "true"}} if CHECKED else {})) +def score(unit, dir, transcript, answer, verified=None): + time.sleep(2) + return {"reward": 0.0, "passed": False, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "score": score}) +''' + + +class Box: + image = "" + + def give_back(self, paths): + pass + + def stop(self): + pass + + +class ScoreLimit(unittest.TestCase): + """A null control whose driver does its work in ``score``, waited for ``limits.score_seconds``.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_SCORE) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def problems(self, limits): + return definitions.problems(dict(self.b, limits=limits), "benchmark") + + def trial(self, limits): + b = dict(self.b, limits=limits) + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", + jobs_root=os.path.join(self.tmp, "jobs"), env={}, say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f) + + def verify_phase(self): + jobs = os.path.join(self.tmp, "jobs") + with open(os.path.join(jobs, os.listdir(jobs)[0], "events.jsonl")) as f: + return next(e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "verify") + + def test_the_limit_is_a_whole_number_of_seconds_from_1(self): + self.assertEqual(self.problems({"score_seconds": 1800}), []) + for v in (0, -1, 1.5, "600", True, 100001): + self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): + r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) + self.assertEqual(r.score_seconds, 600) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}) + self.assertEqual((r.score_seconds, r.phase_limit("score", r.score_seconds)), (1800, 1800)) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}, timeouts={"score": 60}) + self.assertEqual(r.phase_limit("score", r.score_seconds), 60) + page = recipe._bench_page(dict(self.b, limits={"score_seconds": 1800}))["timeouts"] + self.assertEqual((page["score"], "score" in page["catalogue"]), (1800, True)) + self.assertEqual(recipe._bench_page(self.b)["timeouts"]["score"], run.DRIVER_SECONDS) + + def test_a_score_past_the_limit_is_the_verifiers_error_and_one_within_it_scores(self): + late = self.trial({"score_seconds": 1}) + self.assertEqual(late["exception_info"]["exception_type"], "VerifierError") + self.assertIn("score: no reply in 1 s", late["exception_info"]["exception_message"]) + self.assertIsNone(late.get("verifier_result")) + on_time = self.trial({"score_seconds": 10}) + self.assertIsNone(on_time.get("exception_info")) + self.assertEqual(on_time["verifier_result"]["rewards"]["reward"], 0.0) + + def test_the_verify_phase_carries_the_score_limit_and_an_own_images_check(self): + self.trial({"score_seconds": 10}) + self.assertEqual(self.verify_phase()["timeout_sec"], 10) + shutil.rmtree(os.path.join(self.tmp, "jobs")) + open(os.path.join(self.b["_here"], "checked"), "w").close() + checked = [] + with patch.object(run.Run, "unit_environment", lambda r, setup: ("task", {"image": "own"})), \ + patch.object(run.Run, "not_run", lambda r, e, d: None), \ + patch.object(run.Run, "unit_relay", lambda r, *a, **k: None), \ + patch.object(run.Run, "own_box", lambda r, *a: Box()), \ + patch.object(run.Run, "box_changes", lambda r, box: ""), \ + patch.object(run.Run, "release_image", lambda r, image: None), \ + patch.object(run.Run, "verify", lambda r, step, box, trial: checked.append(step) or {"exit": 0}): + got = self.trial({"score_seconds": 10}) + self.assertIsNone(got.get("exception_info")) + self.assertEqual(checked, [{"run": "true"}]) + self.assertEqual(self.verify_phase()["timeout_sec"], 10 + run.TURN_SECONDS) + + class Prove(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() @@ -281,6 +382,254 @@ def test_units_that_may_not_be_redistributed_are_counted_never_named(self): self.assertIn("held-unit-2", b["proven"]["leaky"]["failed"]["leak"]["problems"][0]) +class ProveOptions(unittest.TestCase): + """The benchmark's options reach the checks' driver and each control's run; a merged control job is held as one.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "kits", "agent-task-v1") + shutil.copytree(os.path.join(definitions.KITS, "agent-task-v1"), here) + with open(os.path.join(here, "driver.py")) as f: + src = f.read() + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(src.replace('return {"units": list(WORDS)}', + 'import os\n return {"units": list(WORDS) if os.environ.get("NULLIUS_OPTION_SITES") == "on" else []}')) + with open(os.path.join(here, "standard.json")) as f: + d = json.load(f) + d["options"] = {"sites": {"summary": "The sites.", "values": ["off", "on"], "default": "off"}, + "lane": {"summary": "A lane.", "values": ["a", "b"], "default": "a"}, + "tool": {"summary": "A tool.", "values": ["none", "x"], "default": "none", + "needs": {"x": {"host": ["tool"]}}}} + d["host"] = {"tool": {"summary": "A program this machine lacks.", "check": ["nullius-no-such-program-k5"]}} + with open(os.path.join(here, "standard.json"), "w") as f: + json.dump(d, f) + self.b, bad = definitions._read(os.path.join(here, "standard.json"), "benchmark", "agent-task-v1") + assert not bad, bad + self.jobs = os.path.join(self.tmp, "jobs") + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def prove(self, **kw): + kw.setdefault("stamp", "2026-01-02__03-04-05") + return prove.prove(self.b, jobs_root=self.jobs, env={}, say=lambda s: None, **kw) + + def config(self, job_dir): + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"] + + def nop_shard(self, i, n, lane): + return run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=self.jobs, env={}, + say=lambda s: None, shard=(i, n), launch="L", bench_options={"sites": "on", "lane": lane}) + + def test_the_options_reach_the_units_driver_and_each_controls_run(self): + got = self.prove(bench_options={"sites": "on"}, write=False)["kit"] + self.assertTrue(all(r["ok"] for r in got.values()), got) + self.assertEqual((got["units"]["units"], got["nop"]["passed"], got["oracle"]["passed"]), (3, "0/3", "3/3")) + self.assertEqual(self.config(os.path.join(self.jobs, got["nop"]["job"]))["benchmark_options"], + {"sites": "on", "lane": "a", "tool": "none"}) + self.assertEqual(got["nop"]["options"], {"sites": "on", "lane": "a", "tool": "none"}) + dflt = self.prove(write=False, stamp="2026-01-02__03-04-06")["kit"] + self.assertEqual(dflt["units"]["problems"], ["units: no ids"]) + self.assertEqual(dflt["nop"]["problems"], ["the units check failed: the control was not run"]) + + def test_a_name_or_a_value_the_benchmark_does_not_allow_is_refused_before_anything_runs(self): + was = json.dumps(self.b.get("proven")) + for given, why in (({"sites": "maybe"}, "'maybe' is not one of off, on"), ({"nope": "x"}, "has no option 'nope'")): + with self.assertRaises(run.Refused) as e: + self.prove(bench_options=given) + self.assertIn(why, str(e.exception)) + self.assertFalse(os.path.exists(self.jobs)) + self.assertEqual(json.dumps(self.b.get("proven")), was) + + def test_an_option_whose_value_needs_a_program_this_machine_lacks_is_refused_before_any_install(self): + with patch.object(install, "upstream", side_effect=AssertionError("installed")) as up: + with self.assertRaises(run.Refused) as e: + self.prove(bench_options={"sites": "on", "tool": "x"}) + self.assertIn("nullius-no-such-program-k5 not on PATH", str(e.exception)) + up.assert_not_called() + self.assertFalse(os.path.exists(self.jobs)) + + def test_a_job_is_not_recorded_when_the_units_check_fails(self): + from nullius.standard import merge + whole = merge.merge([self.nop_shard(i, 4, "a") for i in (1, 3)], os.path.join(self.jobs, "nop-merged"))["out"] + self.b["datasets"][0]["units"] = 4 + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertFalse(got["units"]["ok"]) + self.assertFalse(got["nop"]["ok"]) + self.assertIn("the units check failed, so this job cannot be checked against the dataset", got["nop"]["problems"]) + kept = self.b["proven"]["kit"] + self.assertNotIn("nop", kept) + self.assertNotIn("units", kept) + self.assertEqual(kept["failed"]["nop"]["job"], "nop-merged") + self.assertIn("the units check failed, so this job cannot be checked against the dataset", + kept["failed"]["nop"]["problems"]) + + def test_a_control_is_not_run_nor_kept_when_the_units_check_fails(self): + self.assertIn("nop", self.b["proven"]["kit"]) + self.b["datasets"][0]["units"] = 4 + got = self.prove(bench_options={"sites": "on"})["kit"] + self.assertFalse(got["units"]["ok"]) + for c in ("nop", "oracle"): + self.assertEqual(got[c]["problems"], ["the units check failed: the control was not run"]) + self.assertNotIn(c, self.b["proven"]["kit"]) + self.assertIn(c, self.b["proven"]["kit"]["failed"]) + self.assertFalse(os.path.exists(self.jobs) and any("prove-" in x for x in os.listdir(self.jobs))) + + def test_a_trial_of_another_job_cannot_be_merged_into_a_proof(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, "a") for i in (1, 3)] + trial = next(t for t in sorted(os.listdir(shards[0])) if os.path.isfile(os.path.join(shards[0], t, "result.json"))) + path = os.path.join(shards[0], trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + with self.assertRaises(merge.Refused) as e: + merge.merge(shards, os.path.join(self.jobs, "never")) + self.assertIn("names another job than its job", str(e.exception)) + forced = merge.merge(shards, os.path.join(self.jobs, "forced"), force=True)["out"] + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": forced}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_a_merge_of_merges_keeps_what_any_of_them_was_merged_despite(self): + from nullius.standard import merge + good_shard, bad_shard = (self.nop_shard(i, 4, "a") for i in (1, 3)) + trial = next(t for t in sorted(os.listdir(bad_shard)) if os.path.isfile(os.path.join(bad_shard, t, "result.json"))) + path = os.path.join(bad_shard, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + good = merge.merge([good_shard], os.path.join(self.jobs, "good"))["out"] + bad = merge.merge([bad_shard], os.path.join(self.jobs, "bad"), force=True)["out"] + with self.assertRaises(merge.Refused): + merge.merge([good, bad], os.path.join(self.jobs, "never")) + both = merge.merge([good, bad], os.path.join(self.jobs, "both"), force=True)["out"] + std = self.config(both) + self.assertTrue(any(w.startswith("bad: ") and "not of the job they are in" in w for w in std["merged_despite"])) + self.assertEqual([m.get("merged_from", [{}])[0].get("job") for m in std["merged_from"]], [os.path.basename(good_shard), os.path.basename(bad_shard)]) + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": both}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_the_cli_hands_its_options_and_jobs_to_the_proof(self): + from nullius import progress + from nullius.standard import cli + seen = {} + with patch.object(cli, "_bench", return_value=self.b), patch.object(cli, "licences", return_value=(None, None)), \ + patch.object(cli, "_exit_on_term"), patch.object(progress, "ECHO", None), \ + patch.object(prove, "prove", side_effect=lambda b, **kw: seen.update(kw) or {}): + self.assertEqual(cli.main(["prove", "agent-task-v1", "--from-run", "--dataset", "kit", "--bench-option", "sites=on", + "--bench-option", "lane=b", "--job", "nop=/x/merged"]), 0) + self.assertEqual((seen["bench_options"], seen["jobs"]), ({"sites": "on", "lane": "b"}, {"nop": "/x/merged"})) + with self.assertRaises(SystemExit): + cli.main(["prove", "agent-task-v1", "--from-run", "--job", "reference=/x"]) + + def test_a_merged_launch_is_held_to_the_bound_as_one_job_and_must_cover_every_unit(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, lane) for i, lane in ((1, "a"), (3, "b"))] + whole = merge.merge(shards, os.path.join(self.jobs, "nop-merged"))["out"] + before = sorted(os.listdir(self.jobs)) + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertTrue(got["nop"]["ok"], got["nop"]) + self.assertEqual((got["nop"]["job"], got["nop"]["units"], got["nop"]["passed"]), ("nop-merged", 3, "0/3")) + self.assertEqual(self.b["proven"]["kit"]["nop"]["job"], "nop-merged") + self.assertEqual(self.b["proven"]["kit"]["nop"]["options"], + [{"lane": "a", "sites": "on", "tool": "none"}, {"lane": "b", "sites": "on", "tool": "none"}]) + self.assertEqual(sorted(set(os.listdir(self.jobs)) - set(before)), ["2026-01-02__03-04-05__agent-task-v1__kit__prove-oracle"]) + self.assertEqual([m["benchmark_options"]["lane"] for m in self.config(whole)["merged_from"]], ["a", "b"]) + half = merge.merge(shards[:1], os.path.join(self.jobs, "nop-half"))["out"] + part = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": half}, write=False, stamp="2026-01-02__03-04-06")["kit"]["nop"] + self.assertFalse(part["ok"]) + self.assertTrue(any("units have no trial" in p for p in part["problems"]), part["problems"]) + wrong = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"oracle": whole}, write=False, stamp="2026-01-02__03-04-07")["kit"]["oracle"] + self.assertIn("nop-merged: its control is 'nop', not 'oracle'", wrong["problems"]) + trial = next(t for t in sorted(os.listdir(whole)) if os.path.isfile(os.path.join(whole, t, "result.json"))) + with open(os.path.join(whole, trial, "result.json")) as f: + res = json.load(f) + res.pop("verifier_result", None) + res.pop("exception_info", None) + with open(os.path.join(whole, trial, "result.json"), "w") as f: + json.dump(res, f) + blank = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole}, write=False, + stamp="2026-01-02__03-04-08")["kit"]["nop"] + self.assertFalse(blank["ok"]) + self.assertIn("1 unit(s) ended with no reward from 0 to 1 and no error: read the job's trials", blank["problems"]) + for kw in ({"limit": 1}, {"shard": (1, 2)}): + with self.assertRaises(run.Refused): + self.prove(dataset="kit", jobs={"nop": whole}, **kw) + with self.assertRaises(run.Refused): + self.prove(jobs={"nop": os.path.join(self.tmp, "absent")}) + + +NUMERIC = {"key": "numeric", "_ref": "numeric", "label": "Numeric", "family": "agent", "needs": ["agent.task/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "score", "says": "a number from 0 to 1"}, + "controls": {"nop": {"source": "nullius", "from": "Nothing runs."}}, "datasets": [{"id": "all", "units": 3}]} + + +class HeldJob(unittest.TestCase): + """A control's job held to its bound, read from each trial's own record.""" + + def setUp(self): + from nullius.standard import job + self.tmp = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, self.tmp, True) + self.dir = os.path.join(self.tmp, "nop-job") + os.makedirs(self.dir) + nop = definitions.control(NUMERIC, "nop") + self.units = ["u1", "u2", "u3"] + job.write_config(self.dir, job="nop-job", bench=NUMERIC, other=nop, model="", provider="openai", dataset="all", + units=self.units, concurrency=1, granted=False) + results = [job.write_trial(os.path.join(self.dir, f"{u}__u{i:04d}"), job="nop-job", bench=NUMERIC, other=nop, model="", + unit=u, t0=1_760_000_000.0, t1=1_760_000_001.0, produce=(None, None), score={"reward": 0.0}, + exception=None, calls=[], trajectory=None) for i, u in enumerate(self.units)] + job.write_result(self.dir, "nop-job", 1_760_000_000.0, 1_760_000_002.0, results, 3) + + def held(self): + return prove.control_check(NUMERIC, NUMERIC["datasets"][0], "nop", limit=None, jobs_root=self.tmp, stamp="s", + provider="openai", model="", judge_model=None, env={}, concurrency=1, + say=lambda s: None, job_dir=self.dir, ids=self.units) + + def edit(self, unit, change): + trial = next(t for t in os.listdir(self.dir) if t.startswith(unit + "__")) + path = os.path.join(self.dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + change(res) + with open(path, "w") as f: + json.dump(res, f) + + def test_a_clean_job_passes(self): + r = self.held() + self.assertTrue(r["ok"], r["problems"]) + self.assertEqual((r["passed"], r["value"], r["units"]), ("0/3", 0.0, 3)) + + def test_a_reward_without_a_score_a_score_below_0_and_nan_are_no_score(self): + for value in ({"reward": 1.0}, {"score": -100}, {"score": float("nan")}, {"score": True}, {"score": "0"}): + self.edit("u2", lambda res: res["verifier_result"].update(rewards=value)) + r = self.held() + self.assertFalse(r["ok"], value) + self.assertIn("1 unit(s) ended with no score from 0 to 1 and no error: read the job's trials", r["problems"], value) + self.assertEqual(r["value"], 0.0) + + def test_a_trial_another_run_wrote_is_refused(self): + self.edit("u1", lambda res: res["standard"].update(benchmark="other", control={"name": "oracle"}, plugged="oracle")) + self.edit("u3", lambda res: res["standard"].update(job="another-job")) + r = self.held() + self.assertFalse(r["ok"]) + said = " | ".join(r["problems"]) + for k in ("benchmark", "control", "who ran it", "job"): + self.assertIn(f"whose own record names another {k} than the job", said) + + LEAKY = '''from nullius.standard.jsonl import serve import os ANSWER = "the reference answer is forty-two" diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py new file mode 100644 index 00000000..320fe3be --- /dev/null +++ b/app/tests/test_standard_gated.py @@ -0,0 +1,134 @@ +"""A gated benchmark's failing endpoint is said by its status alone, down to the trial's result.json.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius.standard import definitions, install, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +CONTROLS = {"nop": {"source": "nullius", "from": "Nothing runs; the answer is empty."}} +WORDS = "the gated task's own sentence" +FAILING = '''from nullius.standard.jsonl import serve +def score(unit, dir, transcript, answer, **_): + raise RuntimeError("the LLM grader's endpoint answered 500, and the task was scored without it") +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, "score": score}) +''' + + +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's grader said: status 500: " + %r) +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' % WORDS +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Calls: + """The unit's relay as Run.unit reads it: each route's calls, the judge's ending on a refusal with words.""" + def __init__(self, side): + self.side, self.dir = side, os.devnull + + def calls(self, tag): + if tag.endswith("-" + self.side): + return [{"status": 200}, {"status": 500, "error": f"status 500: {WORDS}"}] + return [] + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Gated(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "failing") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(FAILING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def saved(self, side, **bench): + b = dict(self.b, **bench) + jobs = os.path.join(self.tmp, "jobs-" + side + str(bench)) + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", jobs_root=jobs, env={}, + say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_refused_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.saved(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 500", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said = self.saved("judge") + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn(WORDS, said["exception_message"]) + + def answered(self, side, gated): + """drive_upstream's answer failing on its judge's or simulated user's refusal.""" + here = os.path.join(self.tmp, "paper") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + b = dict(self.b, _here=here, setups=SETUPS, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"answer-{side}-{gated}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_answer_failing_on_its_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.answered(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 500", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_an_answer_failing_on_a_benchmark_not_gated_keeps_its_words(self): + said = self.answered("judge", gated=False) + self.assertEqual(said["exception_type"], "NonZeroAgentExitCodeError") + self.assertIn(WORDS, said["exception_message"]) + + def test_the_failure_alone(self): + calls = [{"status": 429, "error": WORDS}] + self.assertEqual(relay.failure(calls), ("RateLimitError", WORDS)) + exc, said = relay.failure(calls, words=False) + self.assertEqual(exc, "RateLimitError") + self.assertNotIn(WORDS, said) + self.assertEqual(relay.failure([{"status": 404, "error": "no such model"}], words=False)[0], "ModelNotFoundError") + + def test_gated_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) + self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", + definitions.problems(dict(self.b, gated="yes"), "benchmark")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_parts.py b/app/tests/test_standard_parts.py index ea875b87..318af120 100644 --- a/app/tests/test_standard_parts.py +++ b/app/tests/test_standard_parts.py @@ -35,6 +35,15 @@ def trial(self, name, f1s): exception=None, calls=[], trajectory=None) return d + def test_a_stopped_score_says_so_in_the_result(self): + for name, score in (("stopped", {"reward": 0.5, "stopped": "batch cap"}), ("whole", {"reward": 0.5})): + job.write_trial(os.path.join(self.job, name), job="job", bench=BENCH, other={"key": "original", "_kind": "upstream"}, + model="m", unit=name, t0=0, t1=1, produce=(0, 1), score=score, exception=None, calls=[], trajectory=None) + with open(os.path.join(self.job, "stopped", "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + with open(os.path.join(self.job, "whole", "result.json")) as f: + self.assertNotIn("stopped", json.load(f)["verifier_result"]) + def test_the_mean_is_over_every_question(self): self.trial("conv-a", [1.0] * 4) # 4 questions, all right self.trial("conv-b", [0.0] * 12) # 12 questions, all wrong diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 3ab1a31e..83895966 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -3,6 +3,7 @@ A driver or capability process answers one JSON line per request.""" from __future__ import annotations +import base64 import http.client import json import os @@ -34,6 +35,22 @@ b'{"type":"message","content":[{"type":"output_text","text":"hi"}]}],' b'"usage":{"input_tokens":20,"output_tokens":5,"input_tokens_details":{"cached_tokens":10}}}}\n\n') +RESP_FAILED = json.dumps({"object": "response", "id": "gen-1", "status": "failed", "output": [], "usage": None, + "error": {"code": "server_error", "message": "the request said x"}}).encode() +RESP_FAILED_SSE = (b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","output":[],' + b'"error":{"code":"server_error","message":"the request said x"}}}\n\n') +RESP_INCOMPLETE = json.dumps({"object": "response", "status": "incomplete", "output": [], + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 30, "output_tokens": 8}}).encode() + +HOLLOW = {"hollow": json.dumps({"id": "gen-1", "provider": "Minimax", "model": "m"}).encode(), + "hollow-error": json.dumps({"id": "gen-1", "error": {"code": 502, "message": "the request said x"}}).encode(), + "hollow-429": json.dumps({"id": "gen-1", "error": {"code": 429, "message": "the request said x"}}).encode(), + "hollow-402": json.dumps({"id": "gen-1", "error": {"code": 402, "message": "the request said x"}}).encode(), + "hollow-404": json.dumps({"id": "gen-1", "error": {"code": 404, "message": "the request said x"}}).encode(), + "count": json.dumps({"input_tokens": 12}).encode(), "no-choice": json.dumps({"choices": []}).encode(), + "not-json": b"the request said x"} + class Upstream(BaseHTTPRequestHandler): seen = [] @@ -51,6 +68,23 @@ def do_POST(self): self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") elif mode == "no-usage": self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") + elif mode in ("400", "413", "422"): + self.reply(int(mode), b'{"error":{"message":"could not process image"}}', "application/json") + elif mode in ("cut", "short"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header(*(("Transfer-Encoding", "chunked") if mode == "cut" else ("Content-Length", str(len(CHAT_SSE) + 9)))) + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(CHAT_SSE), CHAT_SSE) if mode == "cut" else CHAT_SSE) + elif mode in HOLLOW: + self.reply(200, HOLLOW[mode], "application/json", *[("X-Generation-Id", "gen-1")] * (mode == "not-json"), + ("Retry-After", "30")) + elif self.path.endswith("/responses") and mode == "failed": + self.reply(200, RESP_FAILED, "application/json") + elif self.path.endswith("/responses") and mode == "failed-sse": + self.reply(200, RESP_FAILED_SSE, "text/event-stream") + elif self.path.endswith("/responses") and mode == "incomplete": + self.reply(200, RESP_INCOMPLETE, "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -58,10 +92,11 @@ def do_POST(self): else: self.reply(200, json.dumps(CHAT).encode(), "application/json") - def reply(self, status, body, kind): + def reply(self, status, body, kind, *headers): try: self.send_response(status) - self.send_header("Content-Type", kind) + for header in (("Content-Type", kind),) + headers: + self.send_header(*header) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -90,6 +125,7 @@ def post(self, tag, path, body): with urllib.request.urlopen(req) as r: return r.status, r.read() except urllib.error.HTTPError as e: + self.retry_after = e.headers.get("Retry-After") return e.code, e.read() def test_a_units_agent_is_refused_past_its_cap_on_model_calls(self): @@ -151,6 +187,68 @@ def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): self.assertEqual(rec["reply"]["text"], "hello") self.assertEqual(rec["reply"]["usage"]["input"], 7) + def test_a_stream_cut_off_after_its_headers_is_a_failed_call(self): + for mode in ("cut", "short"): + self.post("u" + mode, "/chat/completions", {"model": "m", "stream": True, "mode": mode, "messages": []}) + rec = self.relay.calls("u" + mode)[-1] + self.assertEqual(rec["status"], 502) + self.assertIn("cut off", rec["error"]) + self.assertEqual(rec["reply"]["usage"]["input"], 7) + self.assertEqual(job._usage_total([rec])["cost"], rec["cost_usd"]) + + def test_a_refused_call_keeps_each_inline_image_by_its_type_wholeness_and_size_alone(self): + png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20 + b"IEND\xaeB`\x82\n").decode() + cut = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20).decode() + jpeg = base64.b64encode(b"\xff\xd8\xff" + b"y" * 9 + b"\xff\xd9tail").decode() + webp = base64.b64encode(b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"z" * 4 + b"\n").decode() + image = lambda kind, data: {"type": "image", "source": {"type": "base64", "media_type": kind, "data": data}} + content = [image("image/png", png), image("image/png", cut), image("image/gif", "not base64!"), + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg}"}}, + {"type": "input_image", "image_url": f"data:image/webp;base64,{webp}"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "grade"}] + sent = [{"type": "image/png", "is": "image/png", "bytes": len(png)}, {"type": "image/png", "is": None, "bytes": len(cut)}, + {"type": "image/gif", "is": None, "bytes": 11}, {"type": "image/jpeg", "is": "image/jpeg", "bytes": len(jpeg)}, + {"type": "image/webp", "is": "image/webp", "bytes": len(webp)}] + for mode in ("400", "413", "422", "429", None): + self.post("img", "/messages", {"model": "m", "mode": mode, "messages": [{"role": "user", "content": content}]}) + rec = self.relay.calls("img")[-1] + with self.subTest(mode): + self.assertEqual(rec.get("images"), sent if mode in ("400", "413", "422") else None) + self.assertFalse(any(data in json.dumps(rec) for data in (png, cut, jpeg, webp))) + + def test_a_200_to_a_model_call_with_no_reply_is_a_502_said_by_its_kind_alone(self): + for mode, path, code, said, kind in ( + ("hollow", "/messages", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/chat/completions", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/responses", 502, "no reply", "NetworkConnectionError"), + ("hollow-error", "/messages", 502, "an error 502", "NetworkConnectionError"), + ("hollow-429", "/messages", 429, "an error 429", "RateLimitError"), + ("hollow-404", "/completions", 502, "an error 404", "NetworkConnectionError"), + ("hollow-402", "/messages", 402, "an error 402", "ApiUsageLimitError"), + ("not-json", "/messages", 502, "no reply", "NetworkConnectionError")): + tag = f"uh-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + rec = self.relay.calls(tag)[-1] + self.assertEqual(self.retry_after, "30") + with self.subTest(mode=mode, path=path): + self.assertEqual((status, rec["status"]), (code, code)) + self.assertEqual(json.loads(body)["error"]["message"], f"the endpoint answered 200 with {said}") + self.assertEqual(rec["error"], f"HTTP {code}: the endpoint answered 200 with {said}") + self.assertEqual(rec["answered"], 200) + self.assertFalse({"reply", "cost_usd"} & set(rec)) + self.assertNotIn("the request said", json.dumps(rec)) + self.assertEqual(relay.failure([rec], agent=True)[0], kind) + self.assertEqual(self.post("uk-asked-stream", "/messages", {"model": "m", "mode": "hollow", "stream": True, + "messages": []}), (200, HOLLOW["hollow"])) + for mode, path in (("count", "/messages/count_tokens"), ("no-choice", "/chat/completions"), ("hollow", "/models"), + ("hollow", "/threads/t1/messages")): + tag = f"uk-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + with self.subTest(path): + self.assertEqual((status, body), (200, HOLLOW[mode])) + self.assertEqual(self.relay.calls(tag)[-1]["status"], 200) + self.assertNotIn("answered", self.relay.calls(tag)[-1]) + def test_streamed_responses_read_from_the_completed_event(self): status, body = self.post("u3", "/responses", {"model": "m", "stream": True, "input": "x"}) self.assertEqual(body, RESP_SSE) @@ -159,6 +257,25 @@ def test_streamed_responses_read_from_the_completed_event(self): self.assertEqual(rec["reply"]["tool_calls"][0]["name"], "shell") self.assertEqual(rec["reply"]["usage"]["cached"], 10) + def test_a_failed_response_with_no_usage_costs_nothing(self): + for tag, mode in (("uf1", "failed"), ("uf2", "failed-sse")): + status, _ = self.post(tag, "/responses", {"model": "m", "mode": mode, "input": "x", "stream": mode == "failed-sse"}) + self.assertEqual(status, 200) + rec = self.relay.calls(tag)[-1] + self.assertEqual((rec["reply"]["end"], rec["reply"]["error"]["code"]), ("failed", "server_error")) + self.assertEqual((rec["cost_usd"], rec["cost_from"]), (0.0, "failed")) + failed = {"status": 200, "cost_usd": 0.0, "cost_from": "failed"} + self.assertEqual(job._usage_total([failed, {"status": 200}])["cost"], None) + self.assertEqual({k: job._usage_total([failed, {"status": 200, "cost_usd": 0.5, "cost_from": "provider"}])[k] + for k in ("cost", "cost_from")}, {"cost": 0.5, "cost_from": "provider"}) + + def test_an_incomplete_response_is_priced_from_its_usage(self): + self.post("uf3", "/responses", {"model": "m", "mode": "incomplete", "input": "x"}) + rec = self.relay.calls("uf3")[-1] + self.assertTrue(rec["reply"]["end"].startswith("incomplete")) + self.assertEqual(rec["cost_from"], "estimate") + self.assertGreater(rec["cost_usd"], 0) + def test_calls_are_recorded_by_their_unit(self): self.post("ua", "/chat/completions", {"model": "m", "messages": []}) self.post("ub", "/chat/completions", {"model": "m", "messages": []}) @@ -261,6 +378,54 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_stream_cut_off_or_a_failed_response_with_no_usage_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + for tag, path, body in (("uc", "/chat/completions", {"model": "m", "stream": True, "mode": "cut", "messages": []}), + ("uf", "/responses", {"model": "m", "mode": "failed", "input": "x"})): + req = urllib.request.Request(r.url(tag) + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,), (failed,) = r.calls("uc"), r.calls("uf") + self.assertEqual((cut["status"], failed["cost_from"]), (502, "failed")) + self.assertIsNotNone(cut.get("cost_usd")) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], cut["booked_usd"] + failed["booked_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_stream_cut_off_that_already_cost_more_than_its_booking_counts_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "stream": True, "mode": "cut", "messages": []} + req = urllib.request.Request(r.url("uc") + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with mock.patch.object(ledger, "bound", lambda *a, **k: 1e-9): + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,) = r.calls("uc") + got = led.state() + self.assertEqual((cut["status"], got["unsettled_usd"]), (502, 0.0)) + self.assertGreater(cut["cost_usd"], cut["booked_usd"]) + self.assertAlmostEqual(got["spent_usd"], cut["cost_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_unit_asked_to_stop_books_nothing_and_a_200_with_no_reply_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + with open(os.path.join(r.dir, relay.STOP), "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(r, "u1", {"model": "m", "messages": []})[0], 400) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"], got["unsettled_usd"]), (0.0, 0, 0.0)) + os.unlink(os.path.join(r.dir, relay.STOP)) + self.assertEqual(self.post(r, "u1", {"model": "m", "mode": "hollow", "messages": []})[0], 502) + (call,) = r.calls("u1") + got = led.state() + self.assertEqual((call["answered"], got["spent_usd"], got["held_usd"]), (200, 0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], call["booked_usd"]) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply (2 output tokens here) a call that names no limit is booked those, at the same price.""" @@ -482,6 +647,11 @@ def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): self.assertEqual(relay._request_tail({"input": [], "reasoning": {"effort": "high"}})["reasoning_effort"], "high") self.assertIsNone(relay._request_tail({"messages": []})["reasoning_effort"]) + def test_reasoning_is_read_as_reasoning_or_thinking_tokens(self): + self.assertEqual(relay._usage({"completion_tokens_details": {"reasoning_tokens": 3}})["reasoning"], 3) + self.assertEqual(relay._usage({"output_tokens_details": {"reasoning_tokens": 5}})["reasoning"], 5) + self.assertEqual(relay._usage({"output_tokens": 9, "output_tokens_details": {"thinking_tokens": 7}})["reasoning"], 7) + class ServeTest(unittest.TestCase): """The relay as a process of its own: the key from its environment only, diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index f23ef69e..096b43bc 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -954,6 +954,25 @@ def test_without_a_retry_the_providers_pause_still_holds_the_next_unit_back(self pauses = [float(m[1]) for line in f if (m := re.search(r"no unit starts for ([0-9.]+) s", json.loads(line).get("detail") or ""))] self.assertTrue(pauses and pauses[0] >= 1.5, pauses) # the provider's 2 s, not the 0.5 s pause + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding", retries=1) + real = r.unit + + def unit(*a): + res = real(*a) + if res and res["trial_name"].startswith("model__"): + with open(os.path.join(r.job_dir, res["trial_name"], ".work", "relay", relay.STOP), "w") as f: + f.write("batch cap\n") + return res + r.unit = unit + job_dir = r.start() + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index c895787e..9b26507f 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -191,7 +191,8 @@ def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): """An error that echoes the request's credentials, and a routing object holding a token: the record keeps neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" - Upstream.always = (200, "application/json", json.dumps({"error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + Upstream.always = (200, "application/json", json.dumps({"choices": [{"message": {"content": "ok"}}], + "error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) for serving in (None, {"m-run": ["prov-a"]}): with self.subTest(serving=serving): r = self.relay(model="m-run", key=KEY, serving=serving) @@ -260,15 +261,15 @@ def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) - def test_what_the_record_read_before_is_read_as_before(self): - """The facts are read apart: a failed Responses stream's text, tool calls and usage stay as the relay read - them before (it took nothing from a ``response.failed`` event), its error beside them.""" + def test_a_failed_responses_stream_is_read_beside_its_facts(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage are read from its + ``response.failed`` event, its error beside them.""" body = sse({"type": "response.output_text.delta", "delta": "part"}, {"type": "response.failed", "response": {"object": "response", "status": "failed", "error": {"code": "server_error", "message": "failed"}, "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], "usage": {"input_tokens": 5, "output_tokens": 2}}}) - self.assertEqual(relay.read_reply("/responses", body, True), {"text": "", "tool_calls": [], "usage": None}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "x", "tool_calls": [], "usage": relay._usage({"input_tokens": 5, "output_tokens": 2})}) self.assertEqual(relay.reply_facts("/responses", body, True), {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) @@ -276,9 +277,7 @@ def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own is added, and a unit's failure reads it as before (a 200 is no failure).""" r = self.relay(model="m-run") - errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", - {"code": 502, "message": "Provider returned error"}), - (sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + errors = [(sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", {"code": "server_error", "message": "upstream ended"}), diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index fdd3bdcc..ebe97fc6 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -17,7 +17,7 @@ from nullius import workspace from nullius.runs import launches, parse -from nullius.standard import job, merge, shard +from nullius.standard import job, merge, prove, shard BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} @@ -63,14 +63,16 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", reward=lambda place: float(place % 2)): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", bench_options=None, + reward=lambda place: float(place % 2)): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) os.makedirs(d) planned = list(units) if planned is None else planned job.write_config(d, job=name, bench=bench, other=other, model=model, provider="openai", dataset="all", units=planned, - concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend) + concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend, + bench_options=bench_options) results = [] for k, u in enumerate(units): place = int(u.split("-")[1]) @@ -82,7 +84,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": reward(place)}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) + job.write_result(d, name, t0, t0 + 10 * (min(index, 9) + 1), results, len(planned), budget=budget) return d @@ -237,6 +239,36 @@ def test_units_no_shard_ran_are_said(self): j = parse.parse_job(got["out"]) self.assertEqual((j.n_total_trials, len(j.trials)), (len(self.split[1]) + len(planned), len(self.split[1]) + len(planned) - 1)) + def test_a_launch_of_one_unit_per_shard_merges_and_says_only_the_units_it_lacks(self): + n = 2 ** 64 + units = [f"t-{i}" for i in range(10)] + dirs = [write_shard(self.tmp, f"one-{u}", [u], shard.of(u, n), n) for u in units[:-1]] + got = merge.merge(dirs, os.path.join(self.tmp, "out")) + self.assertEqual((got["trials"], got["notes"]), (9, ["1 of the dataset's 10 units are not in the merge"])) + + def test_shards_run_with_different_benchmark_options_keep_each_its_own(self): + same = {"websites": "none", "gitlab": "none"} + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, bench_options=same), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example")), + write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)] + std = read(os.path.join(merge.merge(dirs, os.path.join(self.tmp, "out"))["out"], "config.json"))["standard"] + self.assertNotIn("benchmark_options", std) + self.assertEqual([m["benchmark_options"]["gitlab"] for m in std["merged_from"]], ["none", "gitlab.example", "none"]) + alike = [write_shard(self.tmp, f"a{i}", self.split[i], i, 3, bench_options=same) for i in (1, 2, 3)] + std = read(os.path.join(merge.merge(alike, os.path.join(self.tmp, "alike"))["out"], "config.json"))["standard"] + self.assertEqual(std["benchmark_options"], same) + self.assertNotIn("benchmark_options", std["merged_from"][0]) + + def test_a_merge_of_merges_keeps_the_options_of_its_inner_shards(self): + same = {"websites": "none", "gitlab": "none"} + inner = merge.merge([write_shard(self.tmp, "s1", self.split[1], 1, 3), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example"))], + os.path.join(self.tmp, "inner"))["out"] + outer = merge.merge([inner, write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)], + os.path.join(self.tmp, "outer"), force=True)["out"] + self.assertEqual(prove._options_of(read(os.path.join(outer, "config.json"))), + [dict(same, gitlab="gitlab.example"), same, {}]) + def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): dirs = self.shards() os.makedirs(os.path.join(self.tmp, "taken", "x")) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index ec8588fc..ee0c3d32 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -7,6 +7,7 @@ import shutil import tempfile import unittest +from unittest import mock from nullius.standard import job from nullius.traces import trace @@ -198,6 +199,31 @@ def test_a_trial_given_no_agent_clock_has_no_setup_phase(self): def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) + def test_a_trace_that_fails_leaves_the_result_and_says_its_error(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + with mock.patch.object(job, "trace_from_run", side_effect=KeyError("the task's own words")): + path = self.write(AGENT, traj) + agent = os.path.dirname(path) + self.assertFalse(os.path.exists(path)) + with open(os.path.join(os.path.dirname(agent), "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["rewards"], {"reward": 1.0}) + with open(os.path.join(agent, "trace.error.txt")) as f: + self.assertEqual(f.read(), "KeyError\n") + self.assertTrue(os.path.exists(os.path.join(agent, "trajectory.json"))) + + def test_a_trace_error_that_cannot_be_written_leaves_the_result(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + write = job._write + + def refusing(path, text): + if os.path.basename(path).startswith("trace"): + raise PermissionError(path) + write(path, text) + + with mock.patch.object(job, "_write", refusing): + path = self.write(AGENT, traj) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(os.path.dirname(path)), "result.json"))) + class Backfill(unittest.TestCase): """A trial written before traces gets one from the files it kept, and nothing else changes.""" diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index 40223bbb..9674ebbf 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -115,7 +115,7 @@ def tree(job_dir): mock.patch.object(install, "upstream", side_effect=install.InstallFailed("git fetch: exit 128")), \ self.assertRaises(run.Refused): preparing.start() - self.assertEqual(tree(preparing.job_dir), [".work", "progress.jsonl"]) + self.assertEqual(tree(preparing.job_dir), [".work", ".work/job.lck", "progress.jsonl"]) self.assertEqual(tree(unknown.job_dir), tree(preparing.job_dir)) self.assertIn("nope", progress.refusal(progress.read(unknown.job_dir))) diff --git a/app/tests/test_standard_versions.py b/app/tests/test_standard_versions.py index 898a0dc4..1c860879 100644 --- a/app/tests/test_standard_versions.py +++ b/app/tests/test_standard_versions.py @@ -217,7 +217,7 @@ def test_new_version_copies_the_default_and_refuses_a_duplicate(self): ''' TOY = dict(BENCH, upstream={"license": "MIT"}, default="1", controls={"nop": {"source": "nullius", "from": "nothing runs"}}, - versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 2}], + versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 3}], "driver": {"command": ["python3", "{here}/driver2.py"]}, "proven": {}}, "1": {"proven": {"all": {"units": {"units": 2, "passed": "2 ids", "on": "2026-10-12"}}}}}) diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 8be2862b..3b8d5797 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -17,7 +17,7 @@ def draw(self, docker, edit=""): schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": {}}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") - + cut(" const consentBox = ", " async function hostReq(") + + cut(" const consentBox = ", " async function hostReq(") + cut(" function benchOptions(", " // Only credentials") + "const out={};for(const b of C.standard().benchmarks)for(const backend of ['local','docker']){" + "const r=C.createStandard(b.key);r.backend=backend;" + edit + "try{out[b.key+' '+backend]=standardWhere(r);}catch(e){out[b.key+' '+backend]=String(e);}}" @@ -61,6 +61,8 @@ def test_a_machine_setup_is_said_to_be_needed_by_the_options_chosen(self): html = self.draw(None)["osworld1 local"] self.assertIn("

    Machine setup

    Needed by the options you chose, on each machine it runs on", html) self.assertIn("docker: Docker on the machine the driver runs on", html) + self.assertIn('VM provider

", self.draw(None, "r.others=[];")["osworld1 local"]) if __name__ == "__main__": diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 66c51bf8..43d1e3a8 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -37,7 +37,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| -| `agent` | **`m3`**, `prompt`, `claude`, `gpt54`, `openai_cua`, `qwen`, `os_symphony`, `evocua`, `autoglm`, `autoglm_v`, `mano`, `muse_spark` | Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit. | `original`, `model` | +| `agent` | **`m3` (M3 Agent)**, `prompt` (PromptAgent), `claude` (Claude computer use), `gpt54` (GPT-5.4), `openai_cua` (OpenAI CUA), `qwen` (Qwen), `os_symphony` (OS-Symphony), `evocua` (EvoCUA), `autoglm` (AutoGLM), `autoglm_v` (AutoGLM-V), `mano` (Mano), `muse_spark` (Muse Spark) | Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit. | `original`, `model` | | `max_steps` | pattern `default\|[1-9][0-9]{0,3}`, default `default` | The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it. | `original`, `model` | | `thinking` | **`adaptive`**, `off` | M3_THINKING_MODE: adaptive, as run_multienv_m3.py's own documented invocation exports it (the model decides whether to think; temperature 1, no top_p), or off, the agent's code default when the variable is unset. | `original`, `model` | | `vm_provider` | **`docker`** | --provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own. | `original`, `model`, `yours` | diff --git a/docs/builder.md b/docs/builder.md index 834f5cf5..517bc0e0 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -24,11 +24,11 @@ Then [read its results](results.md). | --- | --- | | Benchmark | A catalogue entry of either runner, and a version when it has several (`key@version`) | | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | -| Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | +| Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A benchmark option its datasets need (`step: dataset`) is asked here, for the datasets it applies to. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, attempts per unit, retries per attempt, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | -| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | +| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark's option of where the work runs, such as OSWorld's VM provider, is asked here, one value as fixed. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | Original setups run on local processes or in Docker; Docker changes isolation only. diff --git a/docs/cli/read-jobs.md b/docs/cli/read-jobs.md index 86ca991f..c690a671 100644 --- a/docs/cli/read-jobs.md +++ b/docs/cli/read-jobs.md @@ -56,7 +56,7 @@ nullius standard traces

[--dry-run] ## Give past standard runs a trace -A standard run writes `agent/trace.json` as it ends. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. +A standard run writes `agent/trace.json` as it ends; one it cannot build leaves `agent/trace.error.txt`, the error's type, and the trial's result as it is. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. ```bash title="See what it would write, then write it" nullius standard traces ./jobs --dry-run diff --git a/docs/cli/run.md b/docs/cli/run.md index 6ce9f6be..3abf1b8e 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -69,7 +69,7 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | `image` | 3600 s: an own image's pull or build | Environment setup error | | `agent` | The task's `agent_seconds`, else the Original setup's `turn_seconds`, else `limits.turn_seconds`, else 900 s: the agent's whole work on the unit, every turn of a conversation and the simulated user's between them; each add and each recall of a capability (its reader has `limits.turn_seconds`); an Original setup's answerer; the oracle | Agent timeout | | `verify` | The verify step's `seconds`, else the agent's default | Verifier error | -| `score` | 600 s: the driver's `score` | Verifier error | +| `score` | The benchmark's `limits.score_seconds`, else 600 s: the driver's `score` | Verifier error | `--max-turns N` caps a conversation's agent turns over `limits.turns` (default 20); the conversation is scored as it stands. `config.json` records what was forced under `standard.timeouts` and `standard.max_turns`. From the builder, *Limits* takes them in minutes. @@ -112,7 +112,7 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | Provider, network or setup error; agent crash; setup timeout | Yes | | Run or check out of time | Only if the provider was refusing the calls | | Wrong key, spent credit | No; three in a row stop the run | -| A setup this machine lacks, a missing secret, a cap reached, Stop, failed task | No | +| A setup this machine lacks, a missing secret, a cap reached, Stop, a unit a guard asked to stop, a full context window, failed task | No | `conform` and `prove` retry a rate limit once. @@ -152,7 +152,7 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`). A Responses reply names the provider that served it there alone. A Messages request is sent without the header, since its reply names its provider. A model with no pin is sent as before | | Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | | No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing`, `request.routing_asked` when the agent sent its own, `reply.served_by` (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (by choice index when there are several) and `reply.error`, an error held under a 200, its code, type and message alone ([Who served a call](../standard.md#a-run)) | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | | What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name. Where that name is not the slug once lowercased, `held` reads `unknown` though the provider served the call (`Google` for `google-vertex`, `SambaNova` for `sambanova-turbo`) | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | diff --git a/docs/standard.md b/docs/standard.md index 3437a731..2573d2fb 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -141,15 +141,16 @@ record); verifier/reward.txt and score.json | Rule | Detail | | --- | --- | | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | -| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | +| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A 200 with no reply, not streamed, reaches the caller as a 502 (or the 429 or 402 its error held); the record keeps `answered: 200`. A call refused with 400, 413 or 422 records its inline images' types and sizes as `images`, never their bytes | +| Job name | One run at a time per job name: another is refused before it writes anything. A later run under the same name moves each trial's earlier record aside to the trial's `.work/earlier.` | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; a unit asked to stop (a `stop` beside its relay's records) keeps the score its driver gave despite the agent's own last 429; the agent's own last call over the model's context window is `ContextWindowExceededError`, the agent's; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass | | Rate limits | After a `rate_limited` end no unit starts for 60 s, or the provider's `Retry-After`, doubling to 300 s while units keep ending so | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. The key goes as the agent's protocol says: `x-api-key` and `anthropic-version` for `messages`, else a bearer header. Provider `anthropic` serves `messages` only ([protocols](settings/model-providers.md#protocols)) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | | A cap on model calls (`--max-calls`) | Agents only. A unit's relay counts its agent route's answered calls and refuses the one past the cap: the agent's error, as a timeout is ([Run a benchmark](cli/run.md#a-cap-on-model-calls)) | -| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a 200 with no reply | | Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | @@ -361,6 +362,8 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model). With `--units`, `units_total` counts the listed units. The shards' lists hold the same units; a list's file name, comments and sha256 may differ from shard to shard | | `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, serving providers, backend or N are refused unless `--force`. Shards whose lists hold different units, and two trials of one unit, are refused always. `merge` compares the units each list holds, in any order; the sha256 is kept in the record only | | `cli prove --shard I/N` | The controls and the leak check on that shard's units; each record says `shard` | +| Different options per shard | Shards of one launch may differ in `--bench-option` (a unit that needs a service its machine serves); the merged config keeps each shard's in `merged_from` | +| `cli prove --job nop=MERGED` | Checks a merged launch of a control against its bound instead of running it. The job and each of its trials name that benchmark, version, control and job; each unit of the dataset has one trial, scored 0 to 1; the units check passes. `--dataset` named, no `--limit` or `--shard`. The record names the job and its options | ```bash title="Two machines, one merge" nullius standard run humaneval oracle --shard 1/2 --limit 6 # machine 1 @@ -428,16 +431,21 @@ use the source folders and repository checks. | `turn` | The transcript so far; `simulator: {model, base_url, key}` when there is one; `judge` for an upstream that grades as the conversation ends; `attempt` with `limits.attempts` | The next message, `{done: true}`, or `restart` | | `produce` | A capability benchmark's recall, `base_url` and `key` | The reading, through the relay | | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | -| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | +| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by`; `stopped`, the reason of a stop asked while the unit ran, kept as `verifier_result.stopped` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | +An `answer` or `score` error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind. + | Field in `standard.json` | Meaning | | --- | --- | +| `gated` | `true` for a benchmark whose tasks are gated: a unit's failing model endpoint (the agent's, the judge's, the simulated user's) is kept as its status and class, never the provider's words, which can quote the request | | `score.judge` | `{model, says}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another | | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | | `limits.turns` | At most this many agent turns in a conversation (default 20; `cli run --max-turns` forces another) | -| `limits.turn_seconds` | The agent's whole work on a unit, every turn of a conversation included (default 900; a task's `agent_seconds` or `cli run --timeout agent=` before it) | +| `limits.turn_seconds` | The agent's whole work on a unit, every turn of a conversation included (default 900; a task's `agent_seconds`, a setup's own `turn_seconds` or `cli run --timeout agent=` before it) | +| `limits.score_seconds` | How long `score` may take (default 600; `cli run --timeout score=` before it). A driver that runs the unit's environment in `score`, as its null control does, sets it | +| `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -472,9 +480,11 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | `summary`, `values` or `pattern`, `default` | What it changes, its allowed choices or input pattern, and its value when unset | | `datasets` | The datasets it applies to | | `about` | A line about each value | +| `names` | A name for some values, shown in their place; a recipe and the CLI keep the value | | `needs` | What a value needs: an `embeddings` model, a `reranker` model, `host` requirements, or named `files` | | `when` | Another option's values that make it apply | | `limit` | `true`: a bound of the upstream's own (steps, errors). The builder shows it in Limits, under the benchmark's name | +| `step` | `where`: where the work runs, such as the VMs' provider, asked on Where, one value as fixed; `dataset`: what its datasets need, asked on Dataset for the datasets it applies to | | `by` | Its default under each value of another option, e.g. each OSWorld agent's own steps | | `setups` | `["original"]`, `["original", "model"]`: a setting of the paper's pipeline. Without it the option is the port's (`["yours"]`) | @@ -603,9 +613,10 @@ already accepts arbitrary models needs only the `original` method. | Rule | Detail | | --- | --- | | `original` driver method | A model of `models` only, by the name the upstream knows; the relay sends it as `served_as` says, with the prefix `providers` gives. A provider not in `providers` is refused. A model the upstream asks besides the answerer is a `side_models` entry. Without `models`, any model: its own agent runs as the run's model | -| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError` | +| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen. A model that reasons by default can spend that budget thinking: a last reply with neither text nor a tool call is `OutputLengthExceededError`. A last Responses reply that `failed` is an API error. A runner that fails after the unit is asked to stop is `CancelledError` | | `units` | A setup may count its own units; `units` is sent `setup` | | `answer` | Runs the upstream's answerer through the unit's relay; `score` is sent `setup` too. Local processes or the Original setup driver container | +| `comparable` | `false` when no model of `models` is the paper's, so its runs compare with each other only; or the list of those models whose runs compare with the paper's (`paper`), when the setup runs several agents and only some are the paper's: the builder offers those alone as Like the paper, the others as Any of your choice | | Recorded | `standard.setup` in the config and each trial: its name, the model `asked` and `sent`, `comparable_with_paper` | | Attempts | `cli run --attempts K` runs each unit K times; `result.json` carries [`pass_hat_k`](field-guide.md#attempts) | | Retries | `cli run --retries N` runs a failed attempt again: [which failures](cli/run.md#retries) | @@ -649,7 +660,7 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c `cli run KEY nop` and `cli run KEY oracle` need no `--model` and no `--unsandboxed`: no model is called but a judged benchmark's judge. The config and every trial record `standard.control`. -`cli prove KEY [--dataset D] [--limit N]` runs, for each dataset: +`cli prove KEY [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop=JOB_DIR]` runs, for each dataset: | Check | Passes when | | --- | --- | @@ -658,6 +669,8 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c | `units` | Two `units` calls give the same ids: unique, plain enough to name a trial, as many as the dataset says | | `leak` | After `setup`, the oracle's answer and the lines it writes are not already in the prompt or the workspace. A reference under 12 characters is not looked for; the record says how many were skipped | +`--bench-option NAME=VALUE` sets the benchmark's own options for the units check's driver and each control's run, resolved and refused as `cli run` does them, the programs a value needs on this machine included, before anything is installed or started; the control job's config records the values. + What passed is written into `standard.json` per dataset, dated; a failing check is taken back, and a run on fewer units does not replace one on more. ```json title="standard.json · proven"