diff --git a/.claude/skills/import-standard/SKILL.md b/.claude/skills/import-standard/SKILL.md index 8de551fa..bde64b48 100644 --- a/.claude/skills/import-standard/SKILL.md +++ b/.claude/skills/import-standard/SKILL.md @@ -432,8 +432,8 @@ Do not add custom prose-heavy layout or one full-width row per simple value. Check the shared benchmark page at desktop and mobile widths; simple options, detailed prerequisites and single/multiple datasets should remain readable. -For continuous upstream settings such as temperature, use an input `pattern` -rather than an arbitrary shortlist of `values`. Say what an omitted parameter +For continuous upstream settings such as temperature, use an input `pattern`, with an `example` the +pattern allows, rather than an arbitrary shortlist of `values`. Say what an omitted parameter means and retain any setup restrictions; do not imply every model accepts it. Normal upstream scoring rules and expected use of a chosen model belong in diff --git a/AGENTS.md b/AGENTS.md index d3c69efc..deebcb65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,11 @@ and its resources. The source layout stays under `app/`. tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: a benchmark's `standard.json` and `driver` in its folder (τ³-bench's also the MCP server of its domains' tools, `mcp_server`: the upstream's own, run on the machine; OSWorld1's also its null control's - agent, `nop`), `catalogue/agents//` + agent, `nop`; OSWorld2's also `nop`, `api_logs` (each image of its API logs kept once), `websites` (its mock + websites: the checks before a VM start, Overleaf's lock and reset, the stack's command line), `gitlab_server` (its + GitLab: made once on the run's machine with its token, its tasks' lock, the stack's command line), `proxy` (a + tinyproxy of a unit's own for the tasks that ask for a proxy) and, in `startup/`, + `sitecustomize`: its VMs' disk write limit in every process its driver starts), `catalogue/agents//` (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), diff --git a/app/nullius/catalogue/benchmarks/gaia/standard.json b/app/nullius/catalogue/benchmarks/gaia/standard.json index 302aef47..7eb6f314 100644 --- a/app/nullius/catalogue/benchmarks/gaia/standard.json +++ b/app/nullius/catalogue/benchmarks/gaia/standard.json @@ -111,6 +111,7 @@ "label": "Temperature", "summary": "Provider default sends none; a non-negative decimal sets it. The paper does not state it.", "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?", + "example": "0.5", "default": "default", "setups": [ "original" diff --git a/app/nullius/catalogue/benchmarks/gsm8k/standard.json b/app/nullius/catalogue/benchmarks/gsm8k/standard.json index 1525942c..95a34727 100644 --- a/app/nullius/catalogue/benchmarks/gsm8k/standard.json +++ b/app/nullius/catalogue/benchmarks/gsm8k/standard.json @@ -163,7 +163,8 @@ "setups": [ "original" ], - "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?" + "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?", + "example": "0.5" } }, "limits": { diff --git a/app/nullius/catalogue/benchmarks/humaneval/standard.json b/app/nullius/catalogue/benchmarks/humaneval/standard.json index 2837e8f4..9987c4d2 100644 --- a/app/nullius/catalogue/benchmarks/humaneval/standard.json +++ b/app/nullius/catalogue/benchmarks/humaneval/standard.json @@ -67,7 +67,8 @@ "setups": [ "original" ], - "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?" + "pattern": "default|(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?", + "example": "0.5" } }, "limits": { diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index d88bddcf..1c80e187 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -199,6 +199,7 @@ ], "default": "default", "pattern": "default|[1-9][0-9]{0,3}", + "example": "50", "limit": true, "by": { "agent": { diff --git a/app/nullius/catalogue/benchmarks/osworld2/agents.json b/app/nullius/catalogue/benchmarks/osworld2/agents.json new file mode 100644 index 00000000..c558dfbe --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/agents.json @@ -0,0 +1,83 @@ +{ + "about": "The upstream's agents the driver runs, by the agent option: each its own script at the tested commit, as published, on one task. argv is what its scripts/bash wrapper passes (run_multienv_m3.sh, run_multienv_muse_spark.sh), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the image's password as the wrapper passes it, the task's meta file, the unit's results) and less the aws provider's --region and --use_public_ip and the wrapper's one --specific_task_id. M3 also gets the relay's key as --api_key, since its script reads MINIMAX_ANTHROPIC_BASE_URL and MINIMAX_API_KEY before ANTHROPIC_*, and a .env file in the upstream (each script calls load_dotenv) could set them. Muse Spark gets the reasoning effort and output budget of its published run's args.json (max, 262144) in place of its wrapper's (high, 8192), and MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT, since its count of input tokens before each call asks the endpoint for a path OpenRouter does not serve. env points each script's model client at the unit's relay too, and its simulated user's, asked by the name the documented invocation gives it (OSWORLD_USER_SIM_MODEL), which the Original setup sends on as a side model. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1 (the Anthropic client adds /v1/messages), {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples. --max_steps is each wrapper's own, unless the max_steps option sets another. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them.", + "common": [ + "--provider_name", + "docker", + "--path_to_vm", + "{vm}", + "--headless", + "--num_envs", + "1", + "--test_config_base_dir", + "{examples}", + "--test_all_meta_path", + "{meta}", + "--result_dir", + "{results}", + "--client_password", + "osworld-public-evaluation" + ], + "agents": { + "m3": { + "script": "scripts/python/run_multienv_m3.py", + "models": [ + "MiniMax-M3" + ], + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "8192", + "--max_steps", + "100", + "--model", + "{model}", + "--base_url", + "{base}", + "--api_key", + "{key}" + ], + "env": { + "ANTHROPIC_BASE_URL": "{base}", + "ANTHROPIC_API_KEY": "{key}", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + }, + "muse_spark": { + "script": "scripts/python/run_multienv_muse_spark.py", + "models": [ + "muse-spark-1.3" + ], + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "262144", + "--reasoning_effort", + "max", + "--max_steps", + "500", + "--model", + "{model}", + "--base_url", + "{base_url}", + "--save_model_eval_raw_info" + ], + "env": { + "MUSE_SPARK_API_KEY": "{key}", + "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + } + } +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/api_logs.py b/app/nullius/catalogue/benchmarks/osworld2/api_logs.py new file mode 100644 index 00000000..7faca443 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/api_logs.py @@ -0,0 +1,232 @@ +"""OSWorld2's api_logs with each image in them kept once. + +The upstream's M3 agent logs each model call's request with its screenshots as +base64, so one image repeats across files. pack(folder) writes each image once +as images/. and puts that path in its place; restore(folder) gives +each file back byte for byte. Each write is a temporary file renamed over its +name, so a failure leaves every file whole and running again changes nothing. + +python3 api_logs.py pack|restore FOLDER... prints what it did, as JSON.""" +import base64 +import binascii +import contextlib +import fcntl +import hashlib +import json +import os +import re +import stat +import sys +import tempfile + +IMAGES = "images" +TEMPORARY = ".nullius-tmp" +SHORTEST = 256 +VALUE = rb'(?:(?<=: )|(?<=\[)|(?<=, ))"' +BASE64 = re.compile(VALUE + rb'([A-Za-z0-9+/]{%d,}={0,2})"' % SHORTEST) +MARK = b'{"_packed_images": "' + IMAGES.encode() + b'", ' +LINK = re.compile(VALUE + IMAGES.encode() + rb'/([0-9a-f]{64}\.(?:jpg|png|gif|webp))"') + + +def _kind(data): + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "webp" + for magic, ext in ((b"\xff\xd8\xff", "jpg"), (b"\x89PNG\r\n\x1a\n", "png"), (b"GIF87a", "gif"), (b"GIF89a", "gif")): + if data.startswith(magic): + return ext + return None + + +def _image(text): + """The image a base64 value holds, as (its file name, its bytes), or None.""" + if len(text) % 4: + return None + try: + data = base64.b64decode(text, validate=True) + except (binascii.Error, ValueError): + return None + ext = _kind(data) + if ext is None or base64.b64encode(data) != text: + return None + return f"{hashlib.sha256(data).hexdigest()}.{ext}", data + + +def _expand(text, image): + """A packed file's text as it was, with ``image`` giving a file name's bytes.""" + return b"{" + LINK.sub(lambda m: b'"' + base64.b64encode(image(m.group(1).decode())) + b'"', text[len(MARK):]) + + +def _read(path, folder=None): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=folder) + with os.fdopen(fd, "rb") as f: + st = os.fstat(f.fileno()) + if not stat.S_ISREG(st.st_mode): + raise OSError(f"{path} is not a plain file") + return f.read(), st + + +def _write(path, data, like): + fd, tmp = tempfile.mkstemp(prefix="." + os.path.basename(path) + ".", suffix=TEMPORARY, dir=os.path.dirname(path)) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + f.flush() + os.fchmod(f.fileno(), stat.S_IMODE(like.st_mode)) + os.fsync(f.fileno()) + os.utime(tmp, ns=(like.st_atime_ns, like.st_mtime_ns)) + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + + +def _sync(folder): + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _held(path, name, folder=None): + """The bytes of ``path``, a plain file holding the image ``name`` names; None for anything else.""" + try: + data = _read(path, folder)[0] + except OSError: + return None + return data if f"{hashlib.sha256(data).hexdigest()}.{_kind(data)}" == name else None + + +def _inner(fd, name): + return os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + + +def _tidy(fd): + for name in os.listdir(fd): + if name.startswith(".") and name.endswith(TEMPORARY): + with contextlib.suppress(OSError): + os.unlink(name, dir_fd=fd) + + +@contextlib.contextmanager +def _alone(folder): + """The folder locked for this process, cleared of a stopped one's half-written files; None when another holds it.""" + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + yield None + return + _tidy(fd) + with contextlib.suppress(OSError): + images = _inner(fd, IMAGES) + try: + _tidy(images) + finally: + os.close(images) + yield fd + finally: + os.close(fd) + + +def _files(folder): + for name in sorted(os.listdir(folder)): + if name.endswith(".json") and not name.startswith("."): + path = os.path.join(folder, name) + try: + yield path, *_read(path) + except OSError: + continue + + +def _images(folder): + images = os.path.join(folder, IMAGES) + with contextlib.suppress(FileExistsError): + os.mkdir(images) + if not stat.S_ISDIR(os.lstat(images).st_mode): + raise OSError(f"{images} is not a folder") + return images + + +def pack(folder): + done = dict(files=0, packed=0, images=0, written=0, before=0, after=0) + with _alone(folder) as mine: + if mine is None: + return dict(done, busy=True) + images, kept = None, set() + for path, raw, st in _files(folder): + done["files"] += 1 + done["before"] += len(raw) + found = {} + + def link(m): + image = _image(m.group(1)) + if image is None: + return m.group(0) + found[image[0]] = image[1] + return b'"' + f"{IMAGES}/{image[0]}".encode() + b'"' + + packed = raw if not raw.startswith(b'{"') or raw.startswith(MARK) or LINK.search(raw) \ + else MARK + BASE64.sub(link, raw[1:]) + if not found or _expand(packed, found.__getitem__) != raw: + done["after"] += len(raw) + continue + images = images or _images(folder) + new = [name for name in found if name not in kept and _held(os.path.join(images, name), name) is None] + for name in new: + _write(os.path.join(images, name), found[name], st) + if new: + _sync(images) + kept.update(found) + done["written"] += len(new) + _write(path, packed, st) + done["packed"] += 1 + done["after"] += len(packed) + if done["packed"]: + _sync(folder) + done["images"] = len(kept) + return done + + +def restore(folder): + """A file whose image is missing or does not match its name stays packed.""" + done = dict(restored=0, failed=0) + with _alone(folder) as mine, contextlib.ExitStack() as stack: + if mine is None: + return dict(done, busy=True) + try: + images = _inner(mine, IMAGES) + stack.callback(os.close, images) + except OSError: + images = None + cache = {} + + def image(name): + if name not in cache: + cache[name] = None if images is None else _held(name, name, images) + if cache[name] is None: + raise ValueError(f"no image {name}") + return cache[name] + + for path, raw, st in _files(folder): + if not raw.startswith(MARK): + continue + try: + text = _expand(raw, image) + except ValueError: + done["failed"] += 1 + continue + _write(path, text, st) + done["restored"] += 1 + if done["restored"]: + _sync(folder) + return done + + +if __name__ == "__main__": + act = {"pack": pack, "restore": restore}.get(sys.argv[1] if len(sys.argv) > 2 else "") + if act is None: + sys.exit("usage: api_logs.py pack|restore FOLDER...") + print(json.dumps({folder: act(folder) for folder in sys.argv[2:]})) diff --git a/app/nullius/catalogue/benchmarks/osworld2/benchmark.json b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json new file mode 100644 index 00000000..683dbe76 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json @@ -0,0 +1,66 @@ +{ + "key": "osworld2", + "title": "OSWorld2", + "category": "Computer use", + "domain": "Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a VM of its own.", + "scoring": "OSWorld-V2 evaluator", + "harness": [ + "standard" + ], + "origin": "official", + "good_to_know": [ + "Partial credit: each task scores 0 to 1; the score is the mean." + ], + "limitations": [ + "Not the paper's M3 setup: another release, no residential proxy, and 100 steps with thinking off by default; Steps per task 500 and Thinking adaptive restore the budget.", + "Muse Spark's published score came from newer upstream code: scores here do not compare with it.", + "The LLM grader (21 tasks) and the simulated user (6) are Claude Sonnet 4.6: the run's provider must serve it.", + "29 tasks use the public internet: their outcome can change with the live sites.", + "Tasks 103 and 104 are not supported: they stay in the datasets, and an agent's unit ends in an error.", + "A slow or loaded machine, or a run across 16:00 UTC (the VM's midnight), can change scores.", + "The null control scores above 0 on 015, 076, 090 and 092, and on 077 in some runs.", + "Only OSWorld-V2's own M3 and Muse Spark run it; no agent from the Agents page yet.", + "Fetch the gated tasks and prepare the machine first: see OSWorld2 machine setup in the docs." + ], + "todo": [ + "agent.computer/v1's kit, and the driver's agent side: the upstream's DesktopEnv served to any agent.", + "The upstream's other agents, beside M3 and Muse Spark." + ], + "datasets": [ + { + "key": "osworld2", + "title": "OSWorld-V2, release osworld-v2.1", + "tasks": 108, + "official": "https://github.com/xlang-ai/OSWorld-V2", + "source": "Hugging Face (gated)", + "registry": "", + "hub": "", + "verifier": "Each task's own evaluate() on the VM's final state, 0 to 1.", + "access": [ + { + "step": "Ask for access to the gated datasets on Hugging Face", + "note": "xlangai/osworld_v2_tasks and xlangai/osworld_v2_assets_gated" + }, + { + "step": "Download the task classes into the upstream's evaluation_examples/task_class", + "note": "scripts/tools/download_osworld_v2_tasks.py --benchmark-release osworld-v2.1" + }, + { + "step": "Download the assets into the upstream's cache/osworld_v2_assets", + "note": "scripts/tools/download_osworld_v2_assets.py --benchmark-release osworld-v2.1" + }, + { + "step": "Fetch the VM image into the upstream's docker_vm_data", + "note": "osworld-v2-ubuntu-x86.qcow2 from xlangai/v2-image at 6e16459a, about 15 GB unpacked" + }, + { + "step": "Build the proxy image for the 8 tasks that ask for a proxy", + "note": "python3 proxy.py --build, from this benchmark's folder" + } + ], + "notes": "Code Apache-2.0; tasks and assets gated on Hugging Face." + } + ], + "citation": "@misc{yuan26osworld2,\n title={OSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks}, \n author={Mengqi Yuan and Zilong Zhou and Xinzhuang Xiong and Weiming Wu and Jiayang Sun and Jiamin Song and Kaiqian Cui and Bowen Wang and Haoyuan Wu and Yitong Li and Dunjie Lu and Haikong Lu and Qi Zhen and Xinyuan Wang and Jiaqi Deng and Yuhao Yang and Cheng Chen and Boyuan Zheng and Alex Su and Xiao Yu and Hao Zou and Saaket Agashe and Xing Han Lu and Manpreet Kaur and Zhengyang Qi and Vincent Sunn Chen and Frederic Sala and Dayiheng Liu and Junyang Lin and Zhou Yu and Yu Su and Siva Reddy and Xin Eric Wang and Peng Qi and Tianbao Xie and Tao Yu},\n year={2026},\n eprint={2606.29537},\n archivePrefix={arXiv},\n primaryClass={cs.AI},\n url={https://arxiv.org/abs/2606.29537}, \n}", + "citation_source": "https://github.com/xlang-ai/OSWorld-V2" +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/driver.py b/app/nullius/catalogue/benchmarks/osworld2/driver.py new file mode 100644 index 00000000..183d2c8f --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/driver.py @@ -0,0 +1,1387 @@ +"""OSWorld-V2's driver for the Nullius standard (JSON lines on stdin and stdout). + +It runs the Original setup and the null control. OSWorld-V2's own code at the +tested commit ($NULLIUS_UPSTREAM) loads, acts and scores, in processes of its +own; this driver imports none of it. Each unit's script runs in the unit's +folder, so its logs and results stay with the trial. Every process gets the +assets folder (_assets) as OSWORLD_FILE_BASE_URL and the startup folder first on +PYTHONPATH. Mock websites (websites.py), GitLab (gitlab_server.py) and a proxy +(proxy.py) are the unit's services. + +units - every task of the dataset, as domain__id +setup - the task's instruction, for the null control, from nop.py + --instruction: loading a task runs its code, so it runs apart +answer - the Original setup: the agent option's script (agents.json) on the + one task, its model calls through the unit's relay, whose stop file + ends the episode with DONE +score - the upstream's result.txt and result.json; for the null control, + nop.py's. A failed LLM grader or null-control setup, or a read of + the VM or a site that the evaluation could not make, is the + score's error, never a 0 + +Units on one machine start their VMs one after another under START_LOCK, +OSWorld1's own file. A reaper in a session of its own, armed as the script +starts, ends the script's processes once the driver lets it go or dies, +removes the unit's containers and volumes, and packs its api_logs +(api_logs.py).""" +import contextlib +import errno +import fcntl +import glob +import gzip +import importlib.util +import io +import json +import os +import queue +import re +import signal +import socket +import stat +import subprocess +import sys +import tarfile +import threading +import time +import zlib + +if sys.argv[1:2] in (["--reap"], ["--ward"]): + signal.signal(signal.SIGTERM, signal.SIG_IGN) +import gitlab_server +import proxy +import websites + +HERE = os.path.dirname(os.path.abspath(__file__)) +UP = os.environ["NULLIUS_UPSTREAM"] +_ARGS = sys.argv[1:6] if sys.argv[1:2] not in (["--reap"], ["--ward"]) else [] +_FILES = [os.path.join(HERE, n) for n in ("agents.json", "nop.py", os.path.join("startup", "sitecustomize.py"), + "websites.compose.yml", "gitlab.compose.yml")] +AGENTS_FILE, NOP, SITE, websites.OVERRIDE, gitlab_server.OVERRIDE = (_ARGS + _FILES[len(_ARGS):])[:5] +websites.SITE = SITE +with open(AGENTS_FILE, encoding="utf-8") as _f: + AGENTS = json.load(_f) +PASSWORD = AGENTS["common"][AGENTS["common"].index("--client_password") + 1] +EXAMPLES = os.path.join(UP, "evaluation_examples") +VM = os.path.join(UP, "docker_vm_data", "osworld-v2-ubuntu-x86.qcow2") +PROVIDER_IMAGE = "happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9" +ASSETS = os.path.abspath(os.path.join(UP, "cache", "osworld_v2_assets")) +RESULTS = "results" +START_LOCK = "/tmp/nullius-osworld1-vm-start.lck" +STARTED = "Started container with ports" +START_SECONDS = 1000 +SWEEPS = (2, 8, 20) +GRACE = 5 +DOCKER_SECONDS = 60 +READY_SECONDS = 120 +VOLUMES = "volumes" +VOLUME = re.compile(r"[0-9a-f]{64}") +NOTE_SECONDS = 5 +SNAPSHOT_SECONDS = 5 +NOTE_BYTES = 65536 +CLOSE_SECONDS = 60 +EPISODE_SECONDS = 36000 +EVALUATE_SECONDS = 3600 +STOPPED_BY = (("turn limit", "time"), ("context window", "context window"), ("batch cap", "money"), ("key cap", "money"), + ("forecast", "money")) +HELD_SECONDS = 300 +SETUP_FAILED = "setup_failed.json" +COLOUR = re.compile(r"\x1b\[[0-9;]*m") +RECORD = re.compile(r"^(?:\x1b\[[0-9;]*m)?\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d") +CAUSE = re.compile(r"^((?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*(?:Error|Exception|Exit|Interrupt)|ERROR)(?::|$)") +FRAME = re.compile(r'^ File ".*[/\\]desktop_env[/\\]desktop_env\.py", line \d+, in (\w+)$') +INNER = re.compile(r'^ File "(?:.*[/\\])?([^/\\"]*)", line \d+, in (\S+)$') +LISTS = { + "offline36": "001 002 004 010 012 022 029 030 040 042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 " + "076 080 084 085 088 094 103 104 105 106 107 108".split(), + "no_services63": "001 002 003 004 006 009 010 011 012 014 017 022 023 024 025 027 029 030 032 033 034 040 " + "042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 071 074 076 078 080 083 084 " + "085 086 087 088 089 090 092 093 094 095 096 097 099 103 104 105 106 107 108".split(), + "websites35": "005 007 008 013 015 016 018 019 020 021 028 031 035 038 039 043 046 052 057 058 060 065 068 069 " + "070 072 073 077 079 081 082 091 100 101 102".split(), +} +LISTS["websites98"] = sorted(LISTS["no_services63"] + LISTS["websites35"]) +JUDGED = frozenset("003 007 008 019 025 031 035 041 070 074 075 077 078 079 081 082 087 089 090 092 093".split()) +ASKS_USER = frozenset("007 024 026 034 095 098".split()) +RETRIES_SITE_READS = frozenset({"069"}) +GUEST_PORTS = {"082": (3000,)} +MAILBOXES = {"001": ("task_001_thunderbird-profile_fixed.tar.gz",), "006": ("thunderbird-profile.tar.gz",), + "010": ("email_010.tar.gz",)} +SIGN_INS = re.compile(r"(?:^|/)(?:logins[^/]*\.json|signons[^/]*|cookies\.sqlite[^/]*|user\.js)$|/storage/default(?:/|$)") +ONLINE = re.compile(r'^(\s*user_pref\("mail\.server\.\w+\.(?:login_at_startup|check_new_mail|download_on_biff)",\s*)' + r'true(\s*\);)[ \t\r]*$', re.M) +OFFLINE = 'user_pref("offline.autoDetect", false);\nuser_pref("offline.startup_state", 3);\n' +PORT_LOCK = "/tmp/nullius-osworld2-port-{}.lck" +USER_ROUTE = "OSWORLD_USER_SIM_BASE_URL" +GRADER_FAILED = ("Error in compare_images_with_llm: ", "Error in compare_image_edit_with_llm: ", + "Error in _compare_answers_with_llm: ", "Error in compare_text_with_llm: ") +JUDGE_ROUTE = re.compile(r"/([A-Za-z0-9_-]{1,64}-judge)/v1/?$") +IMAGE_TYPES = ("image/png", "image/jpeg", "image/gif", "image/webp") +IMAGE_BYTES = 5 * 1024 * 1024 +GRADER_STATUS = re.compile(r"\b(?:Error code|[Ss]tatus(?: code)?|HTTP)\s*[:=]?\s*([1-5][0-9]{2})\b") +GRADER_NAME = re.compile(r"^((?:[A-Za-z_]\w*\.)*)([A-Za-z_]\w*)(?=:|\s|$)") +_SDK = ("APIError APIStatusError APIConnectionError APITimeoutError APIResponseValidationError BadRequestError " + "AuthenticationError PermissionDeniedError NotFoundError ConflictError UnprocessableEntityError RateLimitError " + "InternalServerError OverloadedError ServiceUnavailableError DeadlineExceededError " + "LengthFinishReasonError ContentFilterFinishReasonError").split() +_HTTPX = ("HTTPError RequestError TransportError TimeoutException ConnectTimeout ReadTimeout WriteTimeout PoolTimeout " + "NetworkError ConnectError ReadError WriteError CloseError ProtocolError LocalProtocolError RemoteProtocolError " + "ProxyError UnsupportedProtocol DecodingError TooManyRedirects HTTPStatusError InvalidURL").split() +_REQUESTS = ("RequestException HTTPError ConnectionError ProxyError SSLError Timeout ConnectTimeout ReadTimeout " + "URLRequired TooManyRedirects MissingSchema InvalidSchema InvalidURL InvalidHeader ChunkedEncodingError " + "ContentDecodingError StreamConsumedError RetryError JSONDecodeError").split() +_BUILTIN = sorted(n for n, v in vars(__import__("builtins")).items() if isinstance(v, type) and issubclass(v, BaseException)) +GRADER_CLASSES = {"anthropic": _SDK, "openai": _SDK, "httpx": _HTTPX, "requests": _REQUESTS, + "requests.exceptions": _REQUESTS, "builtins": _BUILTIN, + "": sorted(set(_SDK + _HTTPX + _REQUESTS + _BUILTIN))} +GRADER_RECORD = re.compile(r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? ERROR " + r"(?:llm_metrics/\d+-[^\]]*|desktopenv\.env)\] (Error in \w+: )(.*)$") +HEAD = r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? \w+ " +SET_UP = re.compile(HEAD + r"[^\]]*\] Environment (?:has been used, reverting to snapshot|is clean, skipping snapshot revert)") +SET_UP_DONE = re.compile(HEAD + r"[^\]]*\] Environment setup complete\.$") +SETUP_ERROR = re.compile(r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? ERROR (?:desktopenv\.)?setup(?:/\d+-[^\]]*)?\] (.*)$") +SETUP_RETRY = re.compile(r"retrying|Upload attempt \d+ failed|Failed to upload file \S", re.I) +STEPPED = re.compile(HEAD + r"(?:lib_run_single|desktopenv\.experiment)(?:/\d+-[^\]]*)?\] " + r"(?:Done:|(?:Phase \d+ )?(?:Step (\d+)|User simulator Q):)") +SIGNALLED = re.compile(HEAD + r"[^\]]*-MainProcess\] Received signal (\d+)\. Gracefully shutting down\.\.\.$") +MAIN_ERROR = re.compile(HEAD + r"[^\]]*-MainProcess\] Unexpected error in main process: ") +WORKER = re.compile(HEAD + r"[^\]]*\] (?:Process (\S+) started\.|(\S+) cleaning up environment\.\.\.)$") +SCREENSHOT = re.compile(HEAD + r"(?:desktopenv\.pycontroller|python)(?:/\d+-[^\]]*)?\] " + r"(Got screenshot successfully|Failed to get screenshot\.)$") +RUNNER = re.compile(HEAD + r"(?:lib_run_single/\d+-[^\]]*|desktopenv\.experiment)\] (?:(?PDone:|(?:Phase (?P\d+) )?(?:Step \d+|User simulator Q):)" + r"|(?PStarting phase \d+ )|(?:Multi-phase r|R)esult: -?\d)") +CONTROLLER = re.compile(HEAD + r"(?:python/\d+-[^\]]*|desktopenv\.pycontroller)\] (.*)$") +THINGS = "file|screenshot|accessibility tree|terminal output|screen size|window size|wallpaper|desktop path|directory tree" +READ = re.compile(rf"An error occurred while trying to get (?:the )?(?P{THINGS}): |Failed to get (?P{THINGS})\. " + rf"Status code: (?P\d+)$|Retrying to get (?:the )?(?P{THINGS})\.$|Failed to get (?P{THINGS})\.$") +STATE = re.compile(HEAD + r"(?:state/\d+-[^\]]*|desktopenv\.getters\.state)\] \[GET_STATE\w*\] (?:Failed to " + r"(fetch state|download file) from \S+: (.*)$)") +SITE_STATUS = re.compile(r"\b(\d{3}) (?:Client|Server) Error\b") +GATEWAY = ("502", "503", "504") +SETUP_DOC = "docs/benchmarks/osworld2-machine.md" +WRITE_OPTIONS = ("NULLIUS_OPTION_WRITE_MBPS", "NULLIUS_OPTION_WRITE_IOPS") +PROBE = "nullius-osworld2-probe-" +PROBE_SECONDS = 600 +PACK_LOG = "api_logs.log" +_STARTUP = [] +_PORTS = [] +_ARMED = {} +_OWN = [] + + +def _unit(unit): + domain, _, eid = unit.partition("__") + return domain, eid + + +def units(dataset=None, setup=None): + if not os.environ.get("NULLIUS_DATASET"): + name = os.environ.get("NULLIUS_DATASET_ID") or dataset or "" + if name not in LISTS: + raise RuntimeError(f"no tasks file for the dataset {name or '(none)'}, and no list of its own") + return {"units": [f"tasks__{e}" for e in LISTS[name]]} + with open(os.environ["NULLIUS_DATASET"], encoding="utf-8") as f: + meta = json.load(f) + return {"units": [f"{d}__{e}" for d, ids in meta.items() for e in ids]} + + +def _task_file(domain, eid): + path = os.path.join(EXAMPLES, "task_class", f"task_{eid}.py") + if not os.path.isfile(path): + raise RuntimeError(f"no task class for {domain}__{eid} at {path}: put the gated task classes in " + f"{os.path.dirname(path)} before a run ({SETUP_DOC})") + return path + + +def _task(dir, domain, eid): + out = os.path.join(dir, "instruction.json") + _run([sys.executable, NOP, "--instruction", EXAMPLES, domain, eid, out], + dir, _env(eid), os.path.join(dir, "setup.log"), "the task's loading") + with open(out, encoding="utf-8") as f: + task = json.load(f) + os.remove(out) + return task + + +def _letting_go(handler): + def let_go(*a, **kw): + try: + return handler(*a, **kw) + except BaseException: + while _OWN: + os.close(_OWN.pop()) + raise + return let_go + + +@_letting_go +def setup(unit, dir, **_): + domain, eid = _unit(unit) + _fresh(dir) + _task_file(domain, eid) + websites.needed(eid) + gitlab_server.needed(eid) + _workspace(dir) + return {"prompt": _task(dir, domain, eid)["instruction"]} + + +LINES = queue.Queue() +CLOSED = threading.Event() + + +class _Waiting: + """What a wait before the runner ends on: the unit's pipe closed, or its turn limit passed.""" + def __init__(self, until=None): + self.until = until + + def wait(self, seconds): + return CLOSED.wait(seconds) or bool(self.until and time.time() > self.until) + + +def _read(): + for line in sys.stdin: + LINES.put(line) + CLOSED.set() + LINES.put(None) + + +def _arm(pgid, dir, vms): + """A reaper that starts once this process closes its pipe or dies; the locks it is given stay held until it ends.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--reap", str(pgid), dir, "vms" if vms else "group"], + stdin=r, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, + pass_fds=websites.held_fds() + tuple(_PORTS) + tuple(_OWN)) + except BaseException: + os.close(w) + raise + finally: + os.close(r) + _ARMED[pgid] = w + + +def _reaper(pgid, dir, vms, seen=()): + """``seen``: the volumes this driver saw.""" + if pgid not in _ARMED: + _arm(pgid, dir, vms) + with contextlib.suppress(OSError), os.fdopen(_ARMED.pop(pgid), "w") as f: + f.write(" ".join(v for v in dict.fromkeys(seen) if VOLUME.fullmatch(v))) + + +def _stop(p, log, what, vms, seen=()): + """The reaper takes the process's group over. + The upstream's own signal handlers are not relied on: they skip the provider's cleanup while a VM boots.""" + _reaper(p.pid, os.path.dirname(log), vms, seen) + raise RuntimeError(f"{what} was stopped: the unit's time ran out") + + +def _alive(pgid): + """A process that waits for its parent to see its exit does not count, so a stopped script ends its group.""" + try: + os.killpg(pgid, 0) + except OSError: + return False + try: + pids = [p for p in os.listdir("/proc") if p.isdigit()] + except OSError: + return True + for pid in pids: + try: + with open(f"/proc/{pid}/stat", encoding="utf-8", errors="replace") as f: + state, _, group = f.read().rpartition(")")[2].split()[:3] + except (OSError, ValueError): + continue + if group == str(pgid) and state not in ("Z", "X"): + return True + return False + + +def _gone(pgid, seconds): + t0 = time.time() + while time.time() - t0 < seconds: + if not _alive(pgid): + return True + time.sleep(0.2) + return False + + +def reap(pgid, dir, vms, seen=()): + """SIGTERM to the group, GRACE seconds, SIGKILL; then the unit's containers removed and its api_logs packed. + Volumes are read before the signals: the upstream removes a container without its volume.""" + seen = [v for v in seen if VOLUME.fullmatch(v)] + if vms: + done = threading.Event() + done.set() + snapshot = threading.Thread(target=_noting, args=(dir, done, seen), daemon=True) + snapshot.start() + snapshot.join(SNAPSHOT_SECONDS) + ended = False + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(pgid, sig) + except ProcessLookupError: + ended = True + break + except OSError: + break + if _gone(pgid, GRACE): + ended = True + break + packing = threading.Thread(target=_pack, args=(dir,)) + if ended: + packing.start() + if vms: + for wait in SWEEPS: + time.sleep(wait) + try: + _remove_vms(dir, seen) + except (RuntimeError, OSError): + pass + try: + _old_probes() + except (RuntimeError, OSError): + pass + if ended: + packing.join() + + +def _pack(dir): + """A failure leaves the folder as it is and the unit's result alone; each outcome is a line in PACK_LOG.""" + try: + folders = glob.glob(os.path.join(glob.escape(dir), RESULTS, "**", "api_logs"), recursive=True) + if folders: + import api_logs + except Exception as e: + _packed(dir, f"api_logs not packed: {type(e).__name__}: {e}") + return + for folder in folders: + try: + done = api_logs.pack(folder) + except Exception as e: + done = {"error": f"{type(e).__name__}: {e}"} + _packed(dir, f"{os.path.relpath(folder, dir)} {json.dumps(done)}") + + +def _packed(dir, line): + try: + fd = os.open(os.path.join(dir, PACK_LOG), os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o644) + with os.fdopen(fd, "a", encoding="utf-8") as f: + f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + line + "\n") + except OSError: + pass + + +class EnvironmentSetupError(RuntimeError): + """The null control's setup failed for good: the score's error, in the driver's own words only.""" + + +class CancelledError(RuntimeError): + pass + + +class CauseUnknownError(RuntimeError): + """Neither the agent's nor the environment's, as far as the log shows.""" + + +class VerifierError(RuntimeError): + """The task's evaluate() raised: the score's error.""" + + +class NoAnswer(RuntimeError): + """A docker command its daemon did not answer in time.""" + + +def _docker(*args, seconds=DOCKER_SECONDS): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds) + except subprocess.TimeoutExpired: + raise NoAnswer(f"docker {args[0]} got no answer in {seconds} s: is its daemon running?") + except OSError: + raise RuntimeError("no docker on this machine's PATH: the upstream's docker provider needs it") + + +def _asked(*args): + """Waits READY_SECONDS, since a failed check before a VM start costs the unit.""" + try: + return _docker(*args, seconds=READY_SECONDS) + except NoAnswer as e: + command = " ".join(a for a in args[:2] if not a.startswith("-")) + raise NoAnswer(f"Docker's daemon did not answer docker {command} in {READY_SECONDS} s: it may be busy; " + "try again or run fewer units at once") from e + + +def _mine(dir): + """The containers that mount the unit's image link, found by the provider image's digest, and their anonymous volumes.""" + mine = os.path.join(os.path.abspath(dir), "vm") + os.sep + every = _docker("ps", "-a", "-q", "--no-trunc", "--filter", f"ancestor={PROVIDER_IMAGE}").stdout.split() + if not every: + return [], [] + out = _docker("inspect", "-f", "{{.Id}}\t{{json .Mounts}}", *every).stdout + ids, volumes = [], [] + for cid, _, mounts in (ln.partition("\t") for ln in out.splitlines() if ln.strip()): + try: + mounts = [m for m in json.loads(mounts) or [] if isinstance(m, dict)] + except ValueError: + continue + if any(str(m.get("Source", "")).startswith(mine) for m in mounts): + ids.append(cid) + volumes += [m["Name"] for m in mounts if m.get("Type") == "volume" and VOLUME.fullmatch(str(m.get("Name")))] + return ids, volumes + + +def _noted(dir, volumes=()): + """Notes each anonymous volume while its container runs: the upstream removes containers without their volumes. + vm/volumes is read and written only as a plain file of this user's with one link.""" + volumes = [v for v in dict.fromkeys(volumes) if VOLUME.fullmatch(v)] + try: + fd = os.open(os.path.join(dir, "vm", VOLUMES), + os.O_RDWR | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + except OSError: + return volumes + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or st.st_nlink != 1: + return volumes + fcntl.flock(fd, fcntl.LOCK_EX) + known = list(dict.fromkeys(v for v in os.pread(fd, NOTE_BYTES, 0).decode("utf-8", "replace").split() + if VOLUME.fullmatch(v))) + new = [v for v in volumes if v not in known] + if new: + os.write(fd, "".join(v + "\n" for v in new).encode()) + return known + new + except OSError: + return volumes + finally: + os.close(fd) + + +def _noting(dir, done, seen): + """Beside the wait for the process, so a slow docker never holds the unit's stop.""" + while True: + try: + found = _mine(dir)[1] + seen.extend(found) + _noted(dir, found) + except (RuntimeError, OSError): + pass + if done.wait(NOTE_SECONDS): + return + + +def _remove_vms(dir, seen=None): + seen = [] if seen is None else seen + ids, volumes = _mine(dir) + seen.extend(volumes) + noted = _noted(dir, volumes) + if ids: + _docker("rm", "-f", "-v", *ids) + gone = [v for v in dict.fromkeys(noted + seen) if VOLUME.fullmatch(v)] + if gone: + _docker("volume", "rm", *gone) + proxy.remove(dir) + + +def _started(p, log, what, until=None): + """Until the VM container runs, the process ends, START_SECONDS pass or ``until``, so the next start does not queue.""" + t0 = time.time() + while p.poll() is None and time.time() - t0 < START_SECONDS and not (until and time.time() > until): + if CLOSED.wait(1): + _stop(p, log, what, True) + try: + with open(log, encoding="utf-8", errors="replace") as f: + if STARTED in f.read(): + return + except OSError: + pass + + +def _setup_failed(failed): + """Types and attempts only, never the message, which could quote the task.""" + try: + with open(failed, encoding="utf-8") as f: + said = json.load(f) + except FileNotFoundError: + return None + except (OSError, ValueError): + said = {} + said = said if isinstance(said, dict) else {} + cause = said.get("cause") + return (f"the environment's setup failed ({said.get('error') or 'no type'}" + + (f" from {cause}" if cause else "") + f", after {said.get('attempts') or '?'} attempt" + + ("" if said.get("attempts") == 1 else "s") + "), and the task was not scored") + + +def _run(argv, cwd, env, log, what, stagger=False, failed=None, until=None, grace=EVALUATE_SECONDS): + """``argv`` in a session of its own, handed to the reaper when it ends or the unit is stopped. + Once ``failed`` exists, the run is an EnvironmentSetupError in the driver's words: the log can quote the task.""" + lock = None + if stagger: + with _set_up(): + lock = os.fdopen(websites.opened(START_LOCK)) + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if _Waiting(until).wait(1): + lock.close() + raise EnvironmentSetupError(f"{what} was not started: the unit's time ran out") + try: + with open(log, "a") as f: + p = subprocess.Popen(argv, cwd=cwd, env=env, stdout=f, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + start_new_session=True) + until = until and min(until, time.time() + EPISODE_SECONDS) + try: + _arm(p.pid, cwd, stagger) + armed = None + except Exception as e: + armed = e + if lock and not armed: + _started(p, log, what, until) + except OSError as e: + raise EnvironmentSetupError(f"{what} did not start: {type(e).__name__}: {e}") from None + finally: + if lock: + lock.close() + if armed: + try: + reap(p.pid, cwd, stagger) + except Exception: + with contextlib.suppress(OSError): + os.killpg(p.pid, signal.SIGKILL) + if stagger: + with contextlib.suppress(RuntimeError, OSError): + _remove_vms(cwd) + raise EnvironmentSetupError(f"{what}'s reaper did not start: {type(armed).__name__}: {armed}") + done, seen, given, hard = threading.Event(), [], None, None + try: + if stagger: + threading.Thread(target=_noting, args=(cwd, done, seen), daemon=True).start() + while p.poll() is None: + if CLOSED.wait(1): + _stop(p, log, what, stagger, seen) + if until and time.time() > until: + hard = hard or time.time() + grace + try: + os.makedirs(os.path.dirname(_stop_file(cwd)), exist_ok=True) + with open(_stop_file(cwd), "a") as f: + f.write("turn limit\n") + until = None + except OSError: + pass + if hard and time.time() > hard: + _reaper(p.pid, cwd, stagger, seen) + raise (VerifierError if _acted(cwd) or failed and _set_up_done(log) else EnvironmentSetupError)( + f"{what} did not end {grace} s after the unit's turn limit, and was stopped") + if failed and given is None and os.path.exists(failed): + given = time.time() + if given is not None and time.time() - given > CLOSE_SECONDS: + _reaper(p.pid, cwd, stagger, seen) + raise EnvironmentSetupError(f"{what} did not end {CLOSE_SECONDS} s after its failure, and was " + f"stopped: {_setup_failed(failed)} (the whole log: {log})") + finally: + done.set() + if p.pid in _ARMED: + _reaper(p.pid, cwd, stagger, seen) + if stagger: + try: + _remove_vms(cwd, seen) + except (RuntimeError, OSError): + pass + said = _setup_failed(failed) if failed else None + if said: + raise EnvironmentSetupError(f"{what} exited {p.returncode}: {said} (the whole log: {log})") + if p.returncode != 0: + kind, said = _tail(log, p.returncode) + raise kind(f"{what} exited {p.returncode}: {said}") + + +def _tail(log, code=0): + """The error class the log's last traceback makes, and that traceback's exception by a known class's name only. + A runner killed, or ended by a signal it did not send itself, is the environment's; one whose worker process never + reached its cleanup, or that died of another signal with no traceback, is of no known cause.""" + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip() for ln in f.read().splitlines()] + except OSError: + return RuntimeError, f"no log at {log}" + start = max((i for i, ln in enumerate(lines) if ln.endswith("Traceback (most recent call last):")), default=len(lines)) + if any(map(RUNNER.match, lines[start:])): + start = len(lines) + signalled = [m.group(1) for m in map(SIGNALLED.match, lines) if m] + if code in (-signal.SIGKILL, -signal.SIGTERM, -signal.SIGHUP) or code >= 0 and signalled and not any(map(MAIN_ERROR.match, lines)): + return EnvironmentSetupError, f"ended by signal {-code if code < 0 else signalled[-1]} (the whole log: {log})" + workers = [m.groups() for m in map(WORKER.match, lines) if m] + if code == 0 and {a for a, _ in workers if a} - {b for _, b in workers} or code < 0 and start == len(lines): + return CauseUnknownError, (f"{f'ended by signal {-code}, with no traceback' if code else 'its worker process ended without its cleanup'}" + f" (the whole log: {log})") + block = lines[start:] + raised = next((ln for ln in block[1:] if ln and not ln[0].isspace()), "") + named = GRADER_NAME.match(raised) + module, name = (named.group(1).rstrip("."), named.group(2)) if named else ("", "") + frames = {m.group(1) for m in map(FRAME.match, block) if m} + said = name if name == "EnvironmentSetupError" or name in GRADER_CLASSES.get(module.replace("._exceptions", ""), ()) \ + else "an exception of the upstream's own" if name else "no exception named" + kind = EnvironmentSetupError if name == "EnvironmentSetupError" or frames & {"reset", "_start_emulator", "__init__"} \ + else VerifierError if frames & {"evaluate", "_call_task_evaluate"} else RuntimeError + shots = [m.group(1) for m in map(SCREENSHOT.match, lines[:start]) if m] + if kind is RuntimeError and name and shots[-1:] == ["Failed to get screenshot."]: + inner = ([m.groups() for m in map(INNER.match, block) if m] or [("", "")])[-1] + step = [m.group(1) for m in map(STEPPED.match, lines[:start]) if m and m.group(1)][-1:] + where = f"in step {step[0]}" if step else "before the first step" + if name == "UnidentifiedImageError" or inner[1] == "_screenshot_bytes" \ + or name == "TypeError" and "NoneType" in raised and inner[0] == "lib_run_single.py": + return EnvironmentSetupError, f"the VM stopped answering {where}: its last screenshot failed (the whole log: {log})" + return CauseUnknownError, f"{said}, after the VM's last screenshot failed {where} (the whole log: {log})" + return kind, f"{said} (the whole log: {log})" + + +def _env(eid, assets=ASSETS, **extra): + """GitLab's token only for a task that needs it.""" + return dict(os.environ, PYTHONPATH=os.pathsep.join([os.path.dirname(os.path.abspath(SITE)), UP]), + PYTHONIOENCODING="utf-8", OSWORLD_FILE_BASE_URL=assets, **websites.env(), **gitlab_server.env(eid), + **extra) + + +def _assets(dir, eid): + """The assets, with the task's mail profiles signed out of their live mailboxes and started offline.""" + if eid not in MAILBOXES: + return ASSETS + own, root = f"task_{eid}", os.path.join(dir, "assets") + names = os.listdir(os.path.join(ASSETS, own)) + missing = sorted(set(MAILBOXES[eid]) - set(names)) + if missing: + raise RuntimeError(f"no {', '.join(missing)} in {os.path.join(ASSETS, own)}") + os.makedirs(os.path.join(root, own), exist_ok=True) + for src, dst in [(os.path.join(ASSETS, n), os.path.join(root, n)) for n in os.listdir(ASSETS) if n != own] + \ + [(os.path.join(ASSETS, own, n), os.path.join(root, own, n)) for n in names if n not in MAILBOXES[eid]]: + if not os.path.lexists(dst): + os.symlink(src, dst) + for name in MAILBOXES[eid]: + _signed_out(os.path.join(ASSETS, own, name), os.path.join(root, own, name)) + return root + + +def _signed_out(src, dst): + with tarfile.open(src) as old, open(dst + ".part", "wb") as raw, \ + gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as gz, tarfile.open(fileobj=gz, mode="w") as new: + for m in old: + name = os.path.normpath(m.name) + if m.islnk() or m.issparse() or os.path.isabs(name) or name.split(os.sep)[0] == "..": + raise RuntimeError(f"{os.path.basename(src)} has {m.name}, which this driver does not rewrite") + if SIGN_INS.search(name): + continue + data = old.extractfile(m) if m.isfile() else None + if data and name.endswith("/prefs.js"): + text = (ONLINE.sub(r"\1false\2", data.read().decode("utf-8")).rstrip("\n") + "\n" + OFFLINE).encode("utf-8") + data, m.size = io.BytesIO(text), len(text) + m.pax_headers.pop("size", None) + new.addfile(m, data) + os.replace(dst + ".part", dst) + + +def _port_free(eid, port): + """Docker's proxy and this bind both set SO_REUSEADDR, so a connection closing there is no refusal. + localhost can be ::1 first, so a listener there would answer the task.""" + for family, host in ((socket.AF_INET, "127.0.0.1"), (socket.AF_INET6, "::1")): + try: + with socket.socket(family, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + except OSError as e: + if e.errno == errno.EADDRINUSE: + raise RuntimeError(f"task {eid} reaches its VM's port {port} at this machine's localhost:{port}, " + f"and another process listens on {host} there: free that port, or run the " + "task on another machine") from None + + +@contextlib.contextmanager +def _published(eid, closed=_Waiting()): + """The task's GUEST_PORTS, for one VM at a time on this machine; the reaper keeps the locks until that VM is gone.""" + fds = [] + try: + for port in GUEST_PORTS.get(eid, ()): + try: + fds.append(websites.opened(PORT_LOCK.format(port))) + except OSError as e: + raise RuntimeError(f"the lock {PORT_LOCK.format(port)} cannot be opened ({type(e).__name__}): " + "remove it if it is not a plain file") from None + while True: + try: + fcntl.flock(fds[-1], fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if closed.wait(1): + raise RuntimeError(f"the unit's time ran out while another unit's VM held port {port}") + _port_free(eid, port) + _PORTS.extend(fds) + yield {_startup().PORTS: " ".join(map(str, GUEST_PORTS[eid]))} if fds else {} + finally: + for fd in fds: + if fd in _PORTS: + _PORTS.remove(fd) + os.close(fd) + + +def _base(url): + return url[:-3] if url.endswith("/v1") else url + + +def _graded(eid, judge): + """The LLM grader's settings from the run's judge route; a judged task with no route is refused before its VM starts.""" + if not judge: + if eid in JUDGED: + raise RuntimeError(f"task {eid} is scored by the LLM grader and this run gives it no judge route: " + "run it with the benchmark's own judge") + return {} + if not isinstance(judge, dict) or not all(isinstance(judge.get(k), str) and judge[k] for k in ("model", "base_url", "key")): + raise RuntimeError("the judge route needs a model, a base_url and a key") + return {"OSWORLD_EVAL_MODEL_PROVIDER": "anthropic", "OSWORLD_EVAL_MODEL_NAME": judge["model"], + "OSWORLD_EVAL_MODEL_BASE_URL": _base(judge["base_url"]), "OSWORLD_EVAL_MODEL_API_KEY": judge["key"]} + + +def _asks_user(eid, name, spec): + """A task with a simulated user and no route for it is refused before its VM starts.""" + if eid in ASKS_USER and not spec.get("env", {}).get(USER_ROUTE): + raise RuntimeError(f"task {eid} has a simulated user and the {name} agent gives it no route: " + f"set {USER_ROUTE} in its agents.json entry") + + +def vm_image(): + """A unit never fetches the image: an interrupted unpacking leaves a broken one.""" + if not os.path.isfile(VM): + raise RuntimeError(f"no VM image at {VM}: put osworld-v2-ubuntu-x86.qcow2 there before a run ({SETUP_DOC})") + return VM + + +def _nat(): + try: + with open("/proc/modules", encoding="utf-8") as f: + if any(ln.split(" ", 1)[0] == "iptable_nat" for ln in f): + return True + except OSError: + pass + return os.path.isdir("/sys/module/iptable_nat") + + +def _provider_image(): + """Pulled inside the provider's port lock, it would hold the next units' starts past their wait.""" + if _asked("image", "inspect", PROVIDER_IMAGE).returncode != 0: + raise RuntimeError(f"no {PROVIDER_IMAGE} on this machine: docker pull {PROVIDER_IMAGE} before a run ({SETUP_DOC})") + + +def _startup(): + """Loaded under another name, so it wraps nothing in this process.""" + if not _STARTUP: + spec = importlib.util.spec_from_file_location("osworld2_startup", SITE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _STARTUP.append(module) + return _STARTUP[0] + + +def _last(said, otherwise): + return ((said or "").strip().splitlines() or [otherwise])[-1] + + +def _old_probes(ready=False): + """This user's probes whose driver is gone or older than PROBE_SECONDS; another user's stay.""" + docker = _asked if ready else _docker + names = docker("ps", "-a", "--filter", f"name={PROBE}{os.getuid()}-", "--format", "{{.Names}}").stdout.split() + old = [] + for name in names: + m = re.fullmatch(re.escape(f"{PROBE}{os.getuid()}-") + r"(\d+)-(\d+)", name) + if not m: + continue + try: + os.kill(int(m.group(1)), 0) + alive = True + except ProcessLookupError: + alive = False + except OSError: + alive = True + if not alive or time.time_ns() - int(m.group(2)) > PROBE_SECONDS * 10 ** 9: + old.append(name) + if old: + docker("rm", "-f", *old) + + +def _probe_name(): + return f"{PROBE}{os.getuid()}-{os.getpid()}-{time.time_ns()}" + + +def _write_limit(): + """Proof before a VM start, by a probe container, that a set write limit holds: a daemon that cannot limit drops it. + An unanswered probe create is tried once more under a new name.""" + if os.environ.get("NULLIUS_OPTION_VM_PROVIDER", "docker") != "docker" \ + or all(os.environ.get(n, "none") == "none" for n in WRITE_OPTIONS): + return + if not os.path.isfile(SITE): + raise RuntimeError(f"the VM's disk write limit cannot be applied: no {SITE}, which sets it in the unit's " + "processes; set Disk write MB/s and Disk write IOPS to none, or restore the file") + site = _startup() + limits = site.wanted(os.environ) + cannot = "the VM's disk write limit cannot be applied: " + if importlib.util.find_spec("docker") is None: + raise site.WriteLimitError(cannot + f"no docker-py in {sys.executable}'s environment, which the upstream's " + "docker provider needs too: install the driver's environment again") + info = _asked("info", "--format", "{{.DockerRootDir}}\t{{.CgroupVersion}}") + root, _, cgroup = (part.strip() for part in info.stdout.partition("\t")) + if info.returncode != 0 or not root: + raise site.WriteLimitError(cannot + "docker info gave no root (" + _last(info.stderr, "no answer") + + "): is Docker's daemon running and usable by this user?") + if cgroup != "2": + raise site.WriteLimitError(cannot + f"Docker runs on cgroup v{cgroup or '?'}, where buffered writes pass a " + f"device limit; {site.ACTION}") + device = site.root_disk(root) + _old_probes(ready=True) + flags = [x for _, rate, _, flag in limits for x in (flag, f"{device}:{rate}")] + + def create(name): + return _asked("create", "--name", name, "--network", "none", "--mount", "type=tmpfs,destination=/storage", + *flags, PROVIDER_IMAGE) + + probe = _probe_name() + try: + try: + made = create(probe) + except NoAnswer as e: + if _asked("rm", "-f", probe).returncode != 0: + raise e + probe = _probe_name() + made = create(probe) + if made.returncode != 0: + raise site.WriteLimitError(cannot + "a probe container was refused: " + _last(made.stderr, "no reason") + + f"; {site.ACTION}") + try: + host = json.loads(_asked("inspect", "-f", "{{json .HostConfig}}", probe).stdout or "{}") or {} + except ValueError: + host = {} + finally: + try: + _asked("rm", "-f", probe) + except RuntimeError: + pass + dropped = [field for _, rate, field, _ in limits if {"Path": device, "Rate": rate} not in (host.get(field) or [])] + if dropped: + said = _last(made.stderr, "") + raise site.WriteLimitError(cannot + f"Docker kept no {', '.join(dropped)} on {device}" + + (f" ({said})" if said else "") + f"; {site.ACTION}") + + +def _ready(domain, eid): + """What a VM start of the task needs, checked before one, so it does not fail minutes later; no /dev/kvm is a warning.""" + image = vm_image() + _task_file(domain, eid) + sites = websites.needed(eid) + lab = gitlab_server.needed(eid) + _startup().network(_asked) + if sites or lab: + websites.ready(websites.suffix(), state=False) + if not os.path.isdir(ASSETS) or not os.listdir(ASSETS): + raise RuntimeError(f"no assets at {ASSETS}: put the gated assets there before a run ({SETUP_DOC})") + _provider_image() + _write_limit() + if not _nat(): + raise RuntimeError("the kernel module iptable_nat is not loaded, and the VM's port forwarding needs it: " + "sudo modprobe ip_tables iptable_nat") + if not os.path.exists("/dev/kvm"): + print("no /dev/kvm: the VM runs without KVM, slowly, and a slow VM can change a task's outcome", + file=sys.stderr, flush=True) + return image + + +@contextlib.contextmanager +def _gitlab(eid, dir, closed=_Waiting()): + """GitLab made once and started from its snapshot for a task of gitlab_server.TASKS, its warden first, however + the unit ends.""" + value = gitlab_server.needed(eid) + with gitlab_server.held(value, closed): + if not value: + yield + return + where = gitlab_server.home() + limits = gitlab_server.unit(where, dir) + with _warden(dir): + gitlab_server.prepare(where, say=lambda said: print(said, file=sys.stderr, flush=True), closed=closed, + limits=limits) + gitlab_server.up(where, limits, closed) + yield + + +@contextlib.contextmanager +def _warden(dir): + """The warden removes GitLab once this process closes its pipe or dies, then lets the locks go.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--ward", str(r), dir], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, + pass_fds=websites.held_fds() + (r,)) + finally: + os.close(r) + try: + yield + finally: + os.close(w) + + +def _disk(): + """The disk the VMs' writes are limited on, for the proxy's own limit; None with no limit.""" + if all(os.environ.get(n, "none") == "none" for n in WRITE_OPTIONS): + return None + site = _startup() + site.wanted(dict(os.environ, **proxy.TAKEN)) + return site.root_disk(_asked("info", "--format", "{{.DockerRootDir}}").stdout.strip()) + + +def _workspace(dir): + """The unit's folder as the upstream's root for the paths its tasks name.""" + link = os.path.join(dir, "evaluation_examples") + if not os.path.lexists(link): + os.symlink(EXAMPLES, link) + os.makedirs(os.path.join(dir, "logs"), exist_ok=True) + + +def _vm(dir, domain, eid): + """The unit's link to the VM image, which the provider mounts by its path: the reaper's mark.""" + image = _ready(domain, eid) + _workspace(dir) + vm = os.path.join(dir, "vm", os.path.basename(VM)) + os.makedirs(os.path.dirname(vm), exist_ok=True) + if not os.path.lexists(vm): + os.symlink(image, vm) + return vm + + +def _agent(): + name = os.environ.get("NULLIUS_OPTION_AGENT", "m3") + if name not in AGENTS["agents"]: + raise RuntimeError(f"no agent {name} in agents.json") + return name, AGENTS["agents"][name] + + +def _paired(model): + """The agent option's agent and its entry, for a model it is published with only.""" + name, spec = _agent() + if model not in spec["models"]: + others = [n for n, s in AGENTS["agents"].items() if model in s["models"]] + raise RuntimeError(f"{model} is not one of the {name} agent's models ({', '.join(spec['models'])}): " + + (f"choose the agent {others[0]} for it, or one of them" if others else "choose one of them")) + return name, spec + + +def _fill(value, slots): + for k, v in slots.items(): + value = value.replace("{" + k + "}", v) + return value + + +def _result(dir, domain, eid): + found = glob.glob(os.path.join(dir, RESULTS, "**", domain, eid, "result.txt"), recursive=True) + return os.path.dirname(found[0]) if found else None + + +def _traj(where, name="traj.jsonl"): + try: + with open(os.path.join(where, name), encoding="utf-8", errors="replace") as f: + lines = f.read().split("\n") + except OSError: + return [] + out = [] + for ln in lines: + with contextlib.suppress(ValueError): + out.append(json.loads(ln)) + return out + + +def _command(dir, model, base_url, key, vm): + """M3's thinking option is M3_THINKING_MODE (empty: the documented default); Muse Spark's reasoning effort is in its argv. + A dotenv file the script loads overrides no variable already set.""" + name, spec = _paired(model) + meta = os.path.join(dir, "test_unit.json") + slots = {"model": model, "base_url": base_url, "key": key, "vm": vm, "up": UP, "meta": meta, + "results": os.path.join(dir, RESULTS), "examples": EXAMPLES, + "base": _base(base_url)} + common, drop = list(AGENTS["common"]), spec.get("drop", []) + for flag in drop: + i = common.index(flag) + del common[i:i + (1 if flag == "--headless" else 2)] + argv = [sys.executable, os.path.join(UP, spec["script"])] + [_fill(a, slots) for a in + common + spec["argv"] + spec.get("model_argv", {}).get(model, [])] + steps = os.environ.get("NULLIUS_OPTION_MAX_STEPS", "default") + if steps != "default" and "--max_steps" in argv: + argv[argv.index("--max_steps") + 1] = steps + elif steps != "default": + argv += ["--max_steps", steps] + extra = {k: _fill(v, slots) for k, v in spec["env"].items()} + if name == "m3": + thinking = os.environ.get("NULLIUS_OPTION_THINKING", "off") + extra.update(M3_THINKING_MODE="adaptive" if thinking == "adaptive" else "", M3_THINKING_BUDGET="") + return spec, meta, argv, extra + + +@contextlib.contextmanager +def _set_up(): + """What fails before the upstream's process starts is the unit's environment's.""" + try: + yield + except (RuntimeError, OSError, LookupError, ValueError, subprocess.SubprocessError, tarfile.TarError, EOFError, + zlib.error) as e: + raise e if isinstance(e, EnvironmentSetupError) else EnvironmentSetupError(f"{type(e).__name__}: {e}") from None + + +def _services(held, dir, domain, eid, until=None): + closed = _Waiting(until) + ports = held.enter_context(_published(eid, closed)) + held.enter_context(websites.held(eid, websites.needed(eid), closed)) + held.enter_context(_gitlab(eid, dir, closed)) + return dict(ports, **held.enter_context(proxy.held(dir, _task(dir, domain, eid)["proxy"], _disk))) + + +@_letting_go +def answer(unit, dir, setup, model, base_url, key, judge=None, **_): + if setup != "original": + raise RuntimeError(f"unknown setup {setup}: OSWorld2 runs original and its null control") + until = time.time() + _turn_seconds() - EVALUATE_SECONDS - CLOSE_SECONDS + domain, eid = _unit(unit) + with contextlib.ExitStack() as held: + with _set_up(): + _fresh(dir) + script = os.path.basename(_agent()[1]["script"]) + log = os.path.join(dir, script.replace(".py", ".log")) + graded = _graded(eid, judge) + _asks_user(eid, *_paired(model)) + vm = _vm(dir, domain, eid) + spec, meta, argv, extra = _command(dir, model, base_url, key, vm) + extra.update(graded, OSWORLD2_STOP=_stop_file(dir)) + with open(meta, "w", encoding="utf-8") as f: + json.dump({domain: [eid]}, f) + services = _services(held, dir, domain, eid, until) + assets = _assets(dir, eid) + _run(argv, dir, _env(eid, assets, **extra, **services), log, script, stagger=True, until=until) + where = _result(dir, domain, eid) + if where is None: + kind, said = _tail(log) + raise kind(f"{script} wrote no result.txt: {said}") + steps = _traj(where) + last = steps[-1].get("action") if steps else "" + return {"answer": last if isinstance(last, str) else json.dumps(last), "model": model} + + +def _fresh(dir): + while _OWN: + os.close(_OWN.pop()) + os.makedirs(dir, exist_ok=True) + _OWN.append(websites.opened(os.path.join(dir, "unit.lck"))) + t0 = time.time() + while True: + try: + fcntl.flock(_OWN[0], fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if CLOSED.wait(1) or time.time() - t0 > HELD_SECONDS: + raise EnvironmentSetupError("an earlier run of this unit, or its reaper, still holds its folder") + ns = time.time_ns() + for name in [RESULTS] + [os.path.basename(p) for p in glob.glob(os.path.join(glob.escape(dir), "*.log"))]: + if os.path.lexists(os.path.join(dir, name)): + os.replace(os.path.join(dir, name), os.path.join(dir, f"{name}.{ns}")) + + +def _turn_seconds(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f)["setups"]["original"]["turn_seconds"] + + +def _score_seconds(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return (json.load(f).get("limits") or {}).get("score_seconds") or 600 + + +def _acted(dir): + for path in glob.glob(os.path.join(glob.escape(os.path.dirname(_stop_file(dir))), "*.jsonl")): + if "-" not in os.path.basename(path): + for call in _traj(os.path.dirname(path), os.path.basename(path)): + if isinstance(call, dict) and not call.get("side"): + return True + return False + + +def _stop_file(dir): + return os.path.join(os.path.dirname(os.path.abspath(dir)), ".work", "relay", "stop") + + +def _said(path): + with open(path, encoding="utf-8") as f: + return [ln.strip() for ln in f.read().splitlines() if ln.strip()] + + +def _stopped(dir): + """The stop that ended the episode's step (sitecustomize's .taken, the stop file if taken before it was written), + its first word and every reason's kind.""" + try: + said = _said(_stop_file(dir) + ".taken") + except OSError: + return {} + with contextlib.suppress(OSError): + said = said or _said(_stop_file(dir)) + return {"stopped": said[0] if said else "stop", "stopped_by": list(dict.fromkeys( + next((why for start, why in STOPPED_BY if ln.startswith(start)), "other") for ln in said))} + + +def _scored(where, how): + """The safety checks' own words are the task's, so they stay in the trial's result.json.""" + try: + with open(os.path.join(where, "result.txt"), encoding="utf-8") as f: + value = float(f.read().strip()) + except (OSError, ValueError): + raise VerifierError("result.txt could not be read as a number") from None + if not -1e-9 <= value <= 1 + 1e-9: + raise VerifierError(f"result.txt holds {value}, not a score from 0 to 1") + steps = _traj(where) + detail = {"result": value, "by": how, "steps": len(steps), + "last_action": steps[-1].get("action") if steps else None} + try: + with open(os.path.join(where, "result.json"), encoding="utf-8") as f: + evaluated = json.load(f) + except (OSError, ValueError): + evaluated = {} + if isinstance(evaluated, dict): + if "score" in evaluated: + detail["score"] = evaluated["score"] + safety = evaluated.get("safety") + if isinstance(safety, dict): + checks = [c for c in safety.get("checks") or [] if isinstance(c, dict)] + detail["safety"] = {"total_penalty": safety.get("total_penalty"), "checks": len(checks), + "failed": sum(1 for c in checks if c.get("result") == "FAIL")} + return {"reward": value, "detail": detail} + + +def _set_up_done(log): + try: + with open(log, encoding="utf-8", errors="replace") as f: + return any(SET_UP_DONE.match(COLOUR.sub("", ln).rstrip("\n")) for ln in f) + except OSError: + return False + + +def _setup_errors(*logs): + """The upstream's setup errors it logged and went on from, from the last attempt to set the task up to the last step, + or to that setup's end with no step: none retried, none of the evaluation's own setup.""" + for log in logs: + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip("\n") for ln in f] + except OSError: + continue + start = max((i for i, ln in enumerate(lines) if SET_UP.match(ln)), default=None) + if start is not None: + end = max((i for i, ln in enumerate(lines) if i > start and STEPPED.match(ln)), default=None) or next( + (i for i, ln in enumerate(lines) if i > start and SET_UP_DONE.match(ln)), len(lines)) + errors = [m for m in map(SETUP_ERROR.match, lines[start:end]) if m and not SETUP_RETRY.search(m.group(1))] + return {"setup_errors": len(errors)} if errors else {} + return {} + + +def _grader_cause(said): + """A grader error record's HTTP status and its class from GRADER_CLASSES, never its words, which can quote the task.""" + said = said.strip() + status, name = GRADER_STATUS.search(said), GRADER_NAME.match(said) + module = name.group(1).rstrip(".") if name else "" + module = module[:-len("._exceptions")] if module.endswith("._exceptions") else module + known = name and name.group(2) in GRADER_CLASSES.get(module, ()) + found = ([f"{module}.{name.group(2)}" if module else name.group(2)] if known else []) \ + + ([f"HTTP {status.group(1)}"] if status else []) + return ", ".join(found) or "no HTTP status or known error class in its record" + + +def _grader_failed(*logs, refused=0): + """The upstream counts a grader error as a failed comparison; only its own ERROR record counts (_grader_cause). + Each of the ``refused`` calls of the agent's files accounts for one 400 or 422 record.""" + for log in logs: + left = refused + try: + with open(log, encoding="utf-8", errors="replace") as f: + for line in f: + m = GRADER_RECORD.match(COLOUR.sub("", line).rstrip("\n")) + status = GRADER_STATUS.search(m.group(2)) if m and left else None + if status and status.group(1) in ("400", "413", "422") and m.group(1) in GRADER_FAILED: + left -= 1 + elif m and m.group(1) in GRADER_FAILED: + raise VerifierError(f"the LLM grader failed in {m.group(1)[len('Error in '):-2]} " + f"({_grader_cause(m.group(2))}), and the upstream counted its " + "comparison as failed; see the judge's calls") + except OSError: + continue + + +def _unread(value, *logs, retried=False): + """A read of the VM or a site that the last episode's evaluation, any of its phases, gave up on unanswered, by what and how only.""" + for log in logs: + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip("\n") for ln in f] + except OSError: + continue + marks = [(i, m) for i, m in enumerate(map(RUNNER.match, lines)) if m] + ends = [k for k, (_, m) in enumerate(marks) if not m["step"] and not m["phase"]] + windows, end = [], marks[ends[-1]][0] if ends else None + for i, m in reversed(marks[ends[-2] + 1 if len(ends) > 1 else 0:ends[-1]] if ends else []): + if m["phase"]: + end = i + elif m["step"] and end is not None: + windows.append((i, end)) + end = None + if m["n"] in (None, "1"): + break + site, unread = None, None + for a, b in windows: + last = None + for ln in lines[a:b]: + state, said = STATE.match(ln), CONTROLLER.match(ln) + read = READ.match(said.group(1)) if said else None + status = SITE_STATUS.search(state.group(2)) if state else None + if state and not site and not retried and (not status or status.group(1) in GATEWAY): + site = (f"a site's {'state' if state.group(1) == 'fetch state' else 'file'}", f"HTTP {status.group(1)}" + if status else "timed out" if "timed out" in state.group(2) else "no answer") + elif read and (read["lost"] or read["said"]): + last = (read["lost"] or read["said"], read["status"]) + elif read and last and read["again"] == last[0]: + continue + elif read and last and read["gave"] == last[0] and not last[1]: + unread = unread or (f"the VM's {last[0]}", "a broken connection") + elif said: + last = None + unread = unread or site + if unread: + raise VerifierError(f"the evaluation could not read {unread[0]} ({unread[1]}), and the upstream scored " + f"the task {value} without it; see the unit's {os.path.basename(log)}") + + +def _refusal(call): + """A judge call refused for the agent's file or its size, by the images its request carried (the relay's ``images``), + never by the provider's words.""" + images = [i for i in call.get("images") or () if isinstance(i, dict)] + if call.get("status") in (400, 413, 422) and any(type(i.get("bytes")) is int and i["bytes"] > IMAGE_BYTES for i in images): + return "agent file size" + if call.get("status") in (400, 422) and any(i.get("is") not in IMAGE_TYPES for i in images): + return "agent file" + return None + + +def _grader_refused(dir, judge): + """A judge-route call not 200 counts unless a later call with the same request reaches 200; its status alone is said. + Some task classes catch the grader's error and score it themselves. The class of each call that refused the agent's + files.""" + route = JUDGE_ROUTE.search(str(judge.get("base_url") or "")) if isinstance(judge, dict) else None + if not route: + return [] + try: + with open(os.path.join(os.path.dirname(os.path.abspath(dir)), ".work", "relay", route.group(1) + ".jsonl"), + encoding="utf-8") as f: + lines = f.read().splitlines() + except OSError: + return [] + calls = [] + for ln in lines: + try: + call = json.loads(ln) + except ValueError: + continue + if isinstance(call, dict): + calls.append(call) + if all(isinstance(c.get("t0"), (int, float)) for c in calls): + calls.sort(key=lambda c: c["t0"]) + refused, failed = [], None + for i, call in enumerate(calls): + if call.get("status") == 200: + continue + asked, recovered = call.get("request"), False + for retry in calls[i + 1:]: + if not isinstance(asked, dict) or not asked.get("sha256") or retry.get("request") != asked: + break + if retry.get("status") == 200: + recovered = True + break + why = None if recovered else _refusal(call) + if not recovered and not why: + failed = failed or call + refused += [why] if why else [] + if failed: + raise VerifierError(f"the LLM grader's endpoint answered {failed.get('status')}, and the task was " + "scored without it; see the judge's calls") + return refused + + +def score(*a, **kw): + try: + return _score(*a, **kw) + finally: + while _OWN: + os.close(_OWN.pop()) + + +def _score(unit, dir, transcript=None, answer=None, setup=None, judge=None, **_): + domain, eid = _unit(unit) + if setup == "original": + where = _result(dir, domain, eid) + if where is None: + raise VerifierError("no result.txt: the upstream's runner did not finish the task") + script = os.path.basename(_agent()[1]["script"]) + stopped = _stopped(dir) + if stopped and not _acted(dir): + raise (EnvironmentSetupError if stopped["stopped"] == "turn limit" else CancelledError)( + f"the unit was stopped ({stopped['stopped']}) before the agent's first step") + refused = _grader_refused(dir, judge) + logs = os.path.join(where, "eval.log"), os.path.join(dir, script.replace(".py", ".log")) + _grader_failed(*logs, refused=len(refused)) + scored = _scored(where, script + ", DesktopEnv.evaluate") + _unread(scored["reward"], *logs, retried=eid in RETRIES_SITE_READS) + return dict(scored, **stopped, **({"grader_refused": sorted(set(refused))} if refused else {}), + **_setup_errors(*logs)) + if setup is not None: + raise RuntimeError(f"unknown setup {setup}: OSWorld2 runs original and its null control") + if answer: + raise RuntimeError("OSWorld2 has no agent side yet: agent.computer/v1 has no kit") + until = time.time() + _score_seconds() - 2 * CLOSE_SECONDS + where = os.path.join(dir, RESULTS, "nop", domain, eid) + failed = os.path.join(where, SETUP_FAILED) + with contextlib.ExitStack() as held: + with _set_up(): + graded = _graded(eid, judge) + vm = _vm(dir, domain, eid) + os.makedirs(where, exist_ok=True) + if os.path.lexists(failed): + os.remove(failed) + services = _services(held, dir, domain, eid, until) + assets = _assets(dir, eid) + _run([sys.executable, NOP, vm, PASSWORD, EXAMPLES, domain, eid, where], + dir, _env(eid, assets, **graded, **services), os.path.join(dir, "nop.log"), "the null control's run", + stagger=True, failed=failed, until=until, grace=CLOSE_SECONDS) + if not os.path.exists(os.path.join(where, "result.txt")): + kind, said = _tail(os.path.join(dir, "nop.log")) + raise kind(f"the null control's run wrote no result.txt: {said}") + refused = _grader_refused(dir, judge) + _grader_failed(os.path.join(dir, "nop.log"), refused=len(refused)) + scored = _scored(where, "nop.py, DesktopEnv.evaluate") + _unread(scored["reward"], os.path.join(dir, "nop.log"), retried=eid in RETRIES_SITE_READS) + return dict(scored, **({"grader_refused": sorted(set(refused))} if refused else {}), **_setup_errors(os.path.join(dir, "nop.log"))) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--reap"]: + reap(int(sys.argv[2]), sys.argv[3], sys.argv[4:5] == ["vms"], sys.stdin.read().split()) + sys.exit(0) + if sys.argv[1:2] == ["--ward"]: + gitlab_server.ward(int(sys.argv[2]), sys.argv[3]) + sys.exit(0) + HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score} + threading.Thread(target=_read, daemon=True).start() + while True: + line = LINES.get() + if line is None: + break + try: + req = json.loads(line) + out = HANDLERS[req.pop("op")](**req) + except Exception as e: + out = {"error": f"{type(e).__name__}: {e}"} + print(json.dumps(out), flush=True) diff --git a/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml b/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml new file mode 100644 index 00000000..0b531ed3 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml @@ -0,0 +1,94 @@ +name: osworld2-gitlab + +x-logging: &logs + driver: json-file + options: + max-size: 10m + max-file: "3" + +services: + gitlab: + image: gitlab/gitlab-ce@sha256:9c17686c3cd1b72f52282543d2dcf7c96add29b4698ec367bfb1455be8f13ce0 + container_name: !reset null + restart: "no" + ports: !reset [] + networks: + web: + aliases: + - nullius-osworld2-gitlab + labels: + caddy: "http://${OSWORLD2_GITLAB_HOST:?as gitlab_server.py sets it}, http://*.${OSWORLD2_GITLAB_HOST:?as gitlab_server.py sets it}" + caddy.reverse_proxy: "nullius-osworld2-gitlab:80" + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + volumes: + - ${OSWORLD2_GITLAB_RB:?as gitlab_server.py sets it}:/etc/gitlab/gitlab.rb:ro + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_GITLAB_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_GITLAB_IOPS:?as gitlab_server.py sets it} + logging: *logs + gitlab-init-token: + profiles: + - upstream-init + volumes: !override [] + entrypoint: !override ["/bin/false"] + command: !override [] + gitlab-runner: + image: gitlab/gitlab-runner@sha256:9d3b2b8be462e18176b82eb40a5f54a5ee6eaa2b4f46dcc362c4801aef90030f + container_name: !reset null + labels: + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + restart: "no" + depends_on: + - dind + networks: + - ci + volumes: !override + - gitlab_runner_config:/etc/gitlab-runner + - dind_certs_client:/certs/client:ro + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_RUNNER_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_RUNNER_IOPS:?as gitlab_server.py sets it} + logging: *logs + dind: + image: docker@sha256:3ef33f2e220b79ed3ef3b99d81746f06f306cd6340e2cb7331d17ae996e74cb6 + labels: + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + restart: "no" + privileged: true + environment: + DOCKER_TLS_CERTDIR: /certs + networks: + ci: + aliases: + - docker + volumes: + - dind_certs_ca:/certs/ca + - dind_certs_client:/certs/client + - dind_data:/var/lib/docker + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_DIND_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_DIND_IOPS:?as gitlab_server.py sets it} + logging: *logs + +networks: + web: + name: ${OSWORLD2_GITLAB_INGRESS:?the mock websites' Caddy network, as gitlab_server.py sets it} + external: true + ci: {} + +volumes: + dind_certs_ca: + dind_certs_client: + dind_data: diff --git a/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py b/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py new file mode 100644 index 00000000..10057fb6 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py @@ -0,0 +1,637 @@ +"""OSWorld2's GitLab: Task-Web/gitlab at COMMIT, for the tasks of TASKS. + +GitLab is that repository's compose project with gitlab.compose.yml over it. +It publishes no port: the mock websites' Caddy serves it as gitlab.. +A unit of TASKS holds LOCK, has prepare() clone it beside the upstream, make +this machine's root token, register the runner and keep the volumes as the +snapshot, once, then starts GitLab from the snapshot; a warden in a session of +its own removes it once the unit lets go or its driver dies. + +python3 gitlab_server.py prepare STACK SUFFIX | up STACK | down STACK +python3 gitlab_server.py compose STACK ARGS...""" +import argparse +import contextlib +import fcntl +import hashlib +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +import websites + +HERE = os.path.dirname(os.path.abspath(__file__)) +OVERRIDE = os.path.join(HERE, "gitlab.compose.yml") +COMMIT = "8655d651722f4254e59e813de9f68a6732ea525c" +REPOSITORY = "https://github.com/Task-Web/gitlab.git" +PROJECT = "osworld2-gitlab" +VM_WRITE = ("NULLIUS_OPTION_WRITE_MBPS", "NULLIUS_OPTION_WRITE_IOPS") +TOKEN = "GITLAB_PRIVATE_TOKEN" +TASKS = frozenset(("026", "041")) +RUNNER_TASKS = frozenset(("041",)) +LOCK = "/tmp/nullius-osworld2-gitlab.lck" +ENV_FILE = ".osworld2.env" +COMPOSE_FILE = ".osworld2.compose.yml" +RB_FILE = ".osworld2.gitlab.rb" +TOKEN_FILE = ".osworld2.token" +RB = """puma['worker_processes'] = 2 +sidekiq['concurrency'] = 5 +prometheus_monitoring['enable'] = false +""" +SHARES = (("GITLAB", 6), ("RUNNER", 1), ("DIND", 1)) +LIMITS = tuple(f"OSWORLD2_GITLAB_{s}_{k}" for s, _ in SHARES for k in ("BPS", "IOPS")) +SETTINGS = ("GITLAB_URL", "GITLAB_PRIVATE_TOKEN", "OSWORLD2_GITLAB_HOST", "OSWORLD2_GITLAB_INGRESS", + "OSWORLD2_GITLAB_DISK", "OSWORLD2_GITLAB_RB", "OSWORLD2_GITLAB_SHARE") + LIMITS +UNUSED = "unused" +SNAPSHOT = "nullius-osworld2-gitlab-snapshot" +STAMP = "made" +SHARE_DEFAULT = (16, 160) +TOKEN_DAYS = 300 +GITLAB_IMAGE = re.compile(r"^\s+image: (gitlab/gitlab-ce@sha256:[0-9a-f]{64})$", re.M) +SHARE_LABEL = "nullius.osworld2.share" +ARCHIVE = "gitlab.tar.gz" +VOLUMES = ("gitlab_config", "gitlab_data", "gitlab_runner_config") +MARK = "gitlab.stack" +WARD_LOG = "gitlab-warden.log" +WARD_SWEEP = 20 +COPY_LABEL = "nullius.osworld2.gitlab.copy" +UNIT_LIMITS = "gitlab.limits" +JOB_IMAGE = "alpine:3.20" +RUNNER_NAME = "osworld-docker-runner" +DOCKER_HOST = "tcp://docker:2376" +MIN_TOKEN = 20 +HTTP_SECONDS = 15 +START_SECONDS = 3600 +RUNNER_SECONDS = 300 +COPY_SECONDS = 3600 +POLL = 10 +PROXIES = {} + + +class GitLabError(RuntimeError): + pass + + +def url(environ=os.environ): + """GitLab's URL, gitlab. before the mock websites' suffix, as their Caddy serves it; None for no sites.""" + value = websites.suffix(environ) + return f"{websites.SCHEME}gitlab.{value}" if value else None + + +def home(environ=os.environ): + """Where this machine's GitLab is cloned: beside the upstream's clone.""" + return os.path.join(os.path.dirname(os.path.abspath(environ["NULLIUS_UPSTREAM"])), f"gitlab-{COMMIT}") + + +def needed(eid, environ=os.environ): + """A task of TASKS without the mock websites, which serve GitLab, is refused, since its class cannot even load.""" + value = url(environ) + if eid not in TASKS: + return None + if not value: + raise GitLabError(f"task {eid} needs GitLab, which the mock websites serve, and this run gives it none: set " + f"Website host suffix to the suffix the stack serves ({websites.SETUP_DOC})") + return value + + +def env(eid, environ=os.environ): + """The variables the upstream reads GitLab by, for a task of TASKS alone: its URL and this machine's token.""" + value = needed(eid, environ) + return {"GITLAB_URL": value, TOKEN: token(home(environ))} if value else {} + + +def share(environ=os.environ): + """GitLab's write share, (MB/s, IOPS): a VM's own limits, SHARE_DEFAULT's where one is none; None with neither.""" + given = [environ.get(n, "none") for n in VM_WRITE] + if given == ["none", "none"]: + return None + found = tuple(d if v == "none" else int(v) for v, d in zip(given, SHARE_DEFAULT)) + allocation(*found) + return found + + +def token(stack): + """This machine's root token for GitLab, made once and kept beside the stack, for this user alone.""" + path = _beside(stack, TOKEN_FILE) + if not os.path.exists(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".nullius-") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(secrets.token_hex(24)) + with contextlib.suppress(FileExistsError): + os.link(tmp, path) + finally: + os.remove(tmp) + with open(path, encoding="utf-8") as f: + value = f.read() + if len(value) < MIN_TOKEN or not value.isalnum(): + raise GitLabError(f"{path} holds no token: remove it, and GitLab is made anew") + return value + + +def _stamp(stack, value): + return hashlib.sha256(f"{stack}\n{value}\n{token(stack)}\n{_override()}\n{RB}".encode()).hexdigest() + + +def _made(): + """The stamp init left in the snapshot, read read-only; None without one, or once the token is TOKEN_DAYS + old, since the upstream's token step makes it for a year.""" + image = GITLAB_IMAGE.search(_override()) + if not _snapshot() or not image or not _found(_docker("image", "inspect", image.group(1)), "no such image", + "looking for GitLab's image"): + return None + said = _docker("run", "--rm", "--network", "none", "--pull", "never", "-v", f"{SNAPSHOT}:/snapshot:ro", + "--entrypoint", "cat", image.group(1), f"/snapshot/{STAMP}") + if said.returncode != 1 or "no such file" not in said.stderr.lower(): + _done(said, f"reading the snapshot {SNAPSHOT}") + found = said.stdout.split() if said.returncode == 0 else [] + fresh = len(found) == 2 and found[1].isdigit() and time.time() - int(found[1]) < TOKEN_DAYS * 86400 + return found[0] if fresh else None + + +def prepare(stack, environ=os.environ, say=None, closed=None, limits=None): + """GitLab made on this machine for the run's sites, once: cloned, configured, pulled, then init; made anew + when the snapshot was made for another stack, host, token or pin, or its token is old.""" + suffix, value = websites.suffix(environ), url(environ) + if not value: + raise GitLabError("GitLab is served by the mock websites: give their suffix") + made = _made() + with contextlib.suppress(GitLabError): + if made == _stamp(stack, value) and _settings(_pinned(stack))["GITLAB_URL"] == value: + return stack + _clone(stack) + configure(stack, suffix, *SHARE_DEFAULT) + _done(compose(stack, ["pull"], seconds=START_SECONDS), "pulling GitLab's images") + down(stack) + if _snapshot(): + _done(_docker("volume", "rm", SNAPSHOT), "removing the old snapshot") + init(stack, say, closed, limits) + return stack + + +def _clone(stack): + with contextlib.suppress(GitLabError): + return _pinned(stack) + shutil.rmtree(stack, ignore_errors=True) + os.makedirs(os.path.dirname(stack), exist_ok=True) + tmp = tempfile.mkdtemp(dir=os.path.dirname(stack), prefix=".nullius-") + try: + for args in (("init", "--quiet"), ("fetch", "--quiet", "--depth", "1", REPOSITORY, COMMIT), + ("checkout", "--quiet", "FETCH_HEAD")): + try: + said = subprocess.run(["git", "-C", tmp, *args], capture_output=True, text=True, + timeout=START_SECONDS, stdin=subprocess.DEVNULL) + except (OSError, subprocess.TimeoutExpired) as e: + raise GitLabError(f"git could not clone {REPOSITORY}: {type(e).__name__}") from None + _done(said, f"cloning {REPOSITORY} at {COMMIT}") + os.replace(tmp, stack) + finally: + shutil.rmtree(tmp, ignore_errors=True) + return _pinned(stack) + + +def _request(path, value, token=None, data=None, method=None): + """``path`` of GitLab at ``value``, as (status, body); no answer is a GitLabError.""" + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + headers = {"User-Agent": "nullius-osworld2", **({"PRIVATE-TOKEN": token} if token else {})} + body = urllib.parse.urlencode(data).encode() if data is not None else None + try: + with opener.open(urllib.request.Request(value + path, data=body, headers=headers, method=method), + timeout=HTTP_SECONDS) as r: + return r.status, r.read(1 << 20) + except urllib.error.HTTPError as e: + e.close() + return e.code, b"" + except (urllib.error.URLError, OSError, ValueError) as e: + reason = getattr(e, "reason", e) + raise GitLabError(f"{value}{path} did not answer ({type(reason).__name__}: {reason})") from None + + +def _json(said, what): + try: + return json.loads(said[1]) + except ValueError: + raise GitLabError(f"{what} answered no JSON") from None + + +def ready(value, token, runner=False): + """GitLab's sign-in page, the root token as an administrator's and, with ``runner``, an online runner.""" + host = urllib.parse.urlsplit(value).hostname + if not websites._resolves(host): + raise GitLabError(f"{host} does not resolve: Website host suffix must be a wildcard DNS name") + if not _signed_in(value): + raise GitLabError(f"{value}/users/sign_in answered no page") + said = _request("/api/v4/user", value, token) + if said[0] != 200 or _json(said, f"{value}/api/v4/user").get("is_admin") is not True: + raise GitLabError(f"{value}/api/v4/user answered {said[0]} for the token {TOKEN}, or not as an " + f"administrator's: docker volume rm {SNAPSHOT}, and GitLab is made anew") + if runner: + said = _request("/api/v4/runners/all?type=instance_type&status=online", value, token) + if said[0] != 200 or not _json(said, f"{value}/api/v4/runners/all"): + raise GitLabError(f"GitLab at {value} has no runner online, and task 041 deploys with one") + return value + + +@contextlib.contextmanager +def held(value, closed): + """With GitLab's URL, GitLab alone, and the mock websites, which serve it, kept from a reset.""" + if not value: + yield + return + with websites.locked(websites.SITES_LOCK, fcntl.LOCK_SH, closed, "the stack was being reset or smoked"), \ + websites.locked(LOCK, fcntl.LOCK_EX, closed, "another unit's GitLab task ran"): + yield + + +@contextlib.contextmanager +def _alone(): + with websites.locked(websites.SITES_LOCK, fcntl.LOCK_SH, None, "the mock websites are being reset or smoked: " + "start or remove GitLab by hand when they are not"), \ + websites.locked(LOCK, fcntl.LOCK_EX, None, "a unit's GitLab task runs: start or remove GitLab by hand " + "when no task that needs it runs"): + yield + + +def unit(where, dir, environ=os.environ): + """The unit's mark for its reaper and, with a write share, the limits its GitLab takes from it.""" + given = share(environ) + limits = os.path.join(dir, UNIT_LIMITS) if given else None + if limits: + lines = {f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) for s, v in allocation(*given).items() + for i, k in enumerate(("BPS", "IOPS"))} + websites._write(limits, "".join(f"{k}={v}\n" for k, v in lines.items()) + "OSWORLD2_GITLAB_SHARE=unit\n") + websites._write(os.path.join(dir, MARK), json.dumps({"stack": where, "limits": limits})) + return limits + + +def reap(dir): + """GitLab removed for a unit that started it, and again WARD_SWEEP seconds later; nothing without its mark.""" + mark = os.path.join(dir, MARK) + try: + with open(mark, encoding="utf-8") as f: + said = json.load(f) + except (OSError, ValueError): + return False + down(said["stack"], said.get("limits")) + time.sleep(WARD_SWEEP) + if _running(): + down(said["stack"], said.get("limits")) + os.remove(mark) + return True + + +def ward(fd, dir): + """The warden: once ``fd`` closes, GitLab is removed, then the locks let go; failures go to WARD_LOG.""" + try: + while os.read(fd, 4096): + pass + except OSError: + pass + try: + reap(dir) + except (GitLabError, websites.WebsiteError, OSError, ValueError, KeyError) as e: + with open(os.path.join(dir, WARD_LOG), "a", encoding="utf-8") as f: + f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {type(e).__name__}: {e}\n") + + +def _running(): + said = _docker("ps", "-aq", "--filter", f"label=com.docker.compose.project={PROJECT}") + return said.returncode == 0 and bool(said.stdout.split()) + + +def _docker(*args, seconds=websites.DOCKER_SECONDS, **kw): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, + env=kw.pop("env", _clean()), stdin=subprocess.DEVNULL, **kw) + except subprocess.TimeoutExpired: + raise GitLabError(f"docker {args[0]} got no answer in {seconds} s") from None + except OSError: + raise GitLabError("no docker on this machine's PATH: GitLab runs on Docker") from None + + +def _clean(): + """Without the project's settings or the token, so compose reads them from its environment files alone.""" + return {k: v for k, v in os.environ.items() if k not in SETTINGS} + + +def _done(said, what): + if said.returncode != 0: + last = ((said.stderr or said.stdout or "").strip().splitlines() or ["no reason"])[-1] + raise GitLabError(f"{what} failed: {last[:300]}") + return said + + +def _pinned(stack): + """``stack`` is Task-Web/gitlab at COMMIT, nothing tracked changed.""" + def git(*args): + try: + return subprocess.run(["git", "-C", stack, *args], capture_output=True, text=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as e: + raise GitLabError(f"git could not read {stack}: {type(e).__name__}") from None + if git("rev-parse", "HEAD").stdout.strip() != COMMIT: + raise GitLabError(f"{stack} is not Task-Web/gitlab at {COMMIT}: give the stack Nullius cloned") + changed = git("status", "--porcelain", "--untracked-files=no") + if changed.returncode != 0 or changed.stdout.strip(): + raise GitLabError(f"{stack} has changes to its files: GitLab runs as published") + return os.path.abspath(stack) + + +def _beside(stack, name): + return os.path.abspath(stack).rstrip(os.sep) + name + + +def _read(path): + with open(path, encoding="utf-8") as f: + return dict(ln.rstrip("\n").partition("=")[::2] for ln in f if "=" in ln) + + +def _settings(stack, limits=None): + path = _beside(stack, ENV_FILE) + try: + return dict(_read(path), **(_read(limits) if limits else {})) + except OSError as e: + raise GitLabError(f"no settings of GitLab at {path} ({type(e).__name__}): run python3 gitlab_server.py " + "prepare STACK SUFFIX first") from None + + +def allocation(mbps, iops): + """Each service's limit by its share, rounded down, so all of them stay within the project's.""" + if mbps < 1 or iops < 1: + raise GitLabError("GitLab's write share is whole numbers above 0") + total = sum(w for _, w in SHARES) + out = {s: (mbps * websites.MB * w // total, iops * w // total) for s, w in SHARES} + if not all(v for pair in out.values() for v in pair): + raise GitLabError(f"a write limit of {iops} IOPS leaves a GitLab service none: give at least {total}") + return out + + +def configure(stack, suffix, mbps, iops): + stack = _pinned(stack) + if not websites.SUFFIX.fullmatch(suffix): + raise GitLabError(f"{suffix!r} is not a host suffix") + shares = allocation(mbps, iops) + websites._network() + rb = _beside(stack, RB_FILE) + websites._write(rb, RB) + settings = {"GITLAB_URL": f"{websites.SCHEME}gitlab.{suffix}", "GITLAB_PRIVATE_TOKEN": UNUSED, + "OSWORLD2_GITLAB_HOST": f"gitlab.{suffix}", "OSWORLD2_GITLAB_INGRESS": f"{websites.PROJECT}_web", + "OSWORLD2_GITLAB_DISK": websites._root_disk()[1], "OSWORLD2_GITLAB_RB": rb, + "OSWORLD2_GITLAB_SHARE": "stack", + **{f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) + for s, v in shares.items() for i, k in enumerate(("BPS", "IOPS"))}} + path = _beside(stack, ENV_FILE) + websites._write(path, "".join(f"{k}={settings[k]}\n" for k in SETTINGS)) + return path, settings + + +def _override(): + try: + with open(OVERRIDE, encoding="utf-8") as f: + return f.read() + except OSError as e: + raise GitLabError(f"no {OVERRIDE} ({type(e).__name__}): this benchmark's folder is incomplete; " + "install Nullius again") from None + + +def _argv(stack, args, limits=None): + copy, text = _beside(stack, COMPOSE_FILE), _override() + try: + with open(copy, encoding="utf-8") as f: + same = f.read() == text + except OSError: + same = False + if not same: + websites._write(copy, text) + return ["compose", "-p", PROJECT, "--project-directory", stack, "-f", os.path.join(stack, "docker-compose.yml"), + "-f", copy, "--env-file", _beside(stack, ENV_FILE), *(["--env-file", limits] if limits else []), *args] + + +def compose(stack, args, capture=True, seconds=websites.RESET_SECONDS, limits=None): + stack = _pinned(stack) + _settings(stack, limits) + websites._compose_version() + if capture: + return _docker(*_argv(stack, args, limits), seconds=seconds) + return subprocess.run(["docker", *_argv(stack, args, limits)], env=_clean(), pass_fds=websites.held_fds()).returncode + + +def _container(stack, service): + said = _done(compose(stack, ["ps", "-a", "-q", service]), f"finding {service}") + found = said.stdout.split() + if len(found) != 1: + raise GitLabError(f"no one container of {service} in {PROJECT}") + return found[0] + + +def _until(seconds, what, check, closed=None): + t0, last = time.time(), "no answer" + while True: + try: + if check(): + return round(time.time() - t0) + except GitLabError as e: + last = str(e)[:200] + if time.time() - t0 >= seconds: + raise GitLabError(f"{what} did not come up in {seconds} s ({last})") + if closed is None: + time.sleep(POLL) + elif closed.wait(POLL): + raise GitLabError(f"{what} was not up when the unit's time ran out") + + +def _signed_in(value): + """Caddy answers a name it serves no site for with an empty page.""" + status, body = _request("/users/sign_in", value) + return status == 200 and bool(body.strip()) + + +def _upstream_script(stack): + """The upstream's gitlab-init-token script, from its own compose file: the override disarms that service.""" + said = _done(_docker("compose", "-p", PROJECT, "--project-directory", stack, "-f", + os.path.join(stack, "docker-compose.yml"), "--env-file", _beside(stack, ENV_FILE), + "config", "--format", "json"), "reading the upstream's project") + return json.loads(said.stdout)["services"]["gitlab-init-token"]["command"][-1] + + +def _make_token(stack): + """The upstream's own token step, run with this machine's docker: no socket is mounted in a container.""" + script, name = _upstream_script(stack), _container(stack, "gitlab") + swaps = ((f'-e {TOKEN}="$${TOKEN}" gitlab ', f"-e {TOKEN} {name} "), + ("docker exec gitlab ", f"docker exec {name} ")) + for old, new in swaps: + if script.count(old) != 1: + raise GitLabError("the upstream's token step is not the one this file knows: check gitlab_server.py " + f"against Task-Web/gitlab at {COMMIT}") + script = script.replace(old, new) + value = token(stack) + done = subprocess.run(["timeout", "--foreground", str(START_SECONDS), "sh", "-c", script.replace("$$", "$")], + capture_output=True, text=True, timeout=START_SECONDS + 60, env=dict(_clean(), **{TOKEN: value}), stdin=subprocess.DEVNULL) + done.stdout, done.stderr = (x.replace(value, TOKEN) for x in (done.stdout, done.stderr)) + _done(done, "making the root token") + + +def _register(stack, value): + """The README's runner registration; a runner the container could not take is removed from GitLab again.""" + status, body = _request("/api/v4/user/runners", value, token(stack), { + "runner_type": "instance_type", "description": RUNNER_NAME, "tag_list": "osworld,docker", + "run_untagged": "true", "locked": "false", "paused": "false"}) + if status != 201: + raise GitLabError(f"GitLab refused a runner ({status})") + made = json.loads(body) + runner = _container(stack, "gitlab-runner") + said = _docker("exec", "-e", "CI_SERVER_TOKEN", runner, "gitlab-runner", "register", "--non-interactive", + "--url", value, "--executor", "docker", "--docker-image", JOB_IMAGE, "--description", RUNNER_NAME, + "--docker-host", DOCKER_HOST, "--docker-tlsverify", "--docker-cert-path", "/certs/client", + env=dict(_clean(), CI_SERVER_TOKEN=made["token"])) + if said.returncode != 0: + _request(f"/api/v4/runners/{made['id']}", value, token(stack), method="DELETE") + _done(said, "registering the runner") + return made["id"] + + +def _copy(stack, settings, tar, keep): + """``tar`` in a container of the project's, under GitLab's write limit; ``keep`` writes the snapshot.""" + image = _done(compose(stack, ["config", "--images"]), "reading the images").stdout.split() + gitlab = next(i for i in image if i.startswith("gitlab/gitlab-ce")) + disk = settings["OSWORLD2_GITLAB_DISK"] + mounts = [x for v in VOLUMES for x in ("-v", f"{PROJECT}_{v}:/v/{v}")] + return _done(_docker("run", "--rm", "--network", "none", "--label", f"com.docker.compose.project={PROJECT}", + "--label", f"{COPY_LABEL}=1", + "--label", f"{SHARE_LABEL}={settings.get('OSWORLD2_GITLAB_SHARE') or 'stack'}", + "--device-write-bps", f"{disk}:{settings['OSWORLD2_GITLAB_GITLAB_BPS']}", + "--device-write-iops", f"{disk}:{settings['OSWORLD2_GITLAB_GITLAB_IOPS']}", + *mounts, "-v", f"{SNAPSHOT}:/snapshot" + ("" if keep else ":ro"), + "--entrypoint", "sh", gitlab, "-c", tar, seconds=COPY_SECONDS), "copying the volumes") + + +def _volumes(): + said = _done(_docker("volume", "ls", "-q", "--filter", f"label=com.docker.compose.project={PROJECT}"), + "listing the volumes") + return said.stdout.split() + + +def _found(said, gone, what): + """Whether Docker found it: False only where its error says ``gone``, any other failure a GitLabError.""" + if said.returncode and gone not in said.stderr.lower(): + _done(said, what) + return said.returncode == 0 + + +def _snapshot(): + return _found(_docker("volume", "inspect", SNAPSHOT), "no such volume", f"looking for the snapshot {SNAPSHOT}") + + +def init(stack, say=None, closed=None, limits=None): + """GitLab made once from nothing: its first start, the root token, the runner; then kept as the snapshot.""" + stack = _pinned(stack) + settings = _settings(stack, limits) + value, said, say = settings["GITLAB_URL"], token(stack), say or (lambda _: None) + if _volumes(): + raise GitLabError(f"{PROJECT} has volumes already: python3 gitlab_server.py down STACK first") + if _snapshot(): + raise GitLabError(f"the snapshot {SNAPSHOT} is there already: remove it (docker volume rm {SNAPSHOT}) to " + "make GitLab anew") + t0 = time.time() + _done(compose(stack, ["up", "-d", "--pull", "never", "gitlab"], limits=limits), "starting GitLab") + say(json.dumps({"started": "gitlab", "up_s": _until(START_SECONDS, "GitLab", lambda: _signed_in(value), closed)})) + _make_token(stack) + ready(value, said) + say(json.dumps({"token": "made", "s": round(time.time() - t0)})) + _done(compose(stack, ["up", "-d", "--pull", "never", "dind", "gitlab-runner"], limits=limits), "starting the runner") + runner = _register(stack, value) + say(json.dumps({"runner": runner, "online_s": _until(RUNNER_SECONDS, "the runner", + lambda: _online(value, said, runner), closed)})) + _done(compose(stack, ["stop"], limits=limits), "stopping GitLab") + _done(_docker("volume", "create", SNAPSHOT), "making the snapshot's volume") + _copy(stack, settings, f"tar -C /v -czf /snapshot/{ARCHIVE} {' '.join(VOLUMES)} && " + f"printf '%s %s' {_stamp(stack, value)} {int(time.time())} > /snapshot/{STAMP}", keep=True) + _done(compose(stack, ["down", "--volumes"], limits=limits), "removing GitLab") + return {"snapshot": SNAPSHOT, "s": round(time.time() - t0)} + + +def _online(value, token, runner): + said = _request(f"/api/v4/runners/{runner}", value, token) + return said[0] == 200 and _json(said, "the runner").get("status") == "online" + + +def up(stack, limits=None, closed=None, say=None): + """GitLab anew from the snapshot, waited on until ready() and reached from the VMs' network.""" + stack = _pinned(stack) + settings = _settings(stack, limits) + value, said = settings["GITLAB_URL"], token(stack) + if not _snapshot(): + raise GitLabError(f"no snapshot {SNAPSHOT}: python3 gitlab_server.py prepare STACK SUFFIX") + t0 = time.time() + _done(compose(stack, ["down", "--volumes"], limits=limits), "removing GitLab") + _done(compose(stack, ["create", "--pull", "never"], limits=limits), "making GitLab's containers") + _copy(stack, settings, f"tar -C /v -xzf /snapshot/{ARCHIVE}", keep=False) + if say: + say(json.dumps({"restored_s": round(time.time() - t0)})) + _done(compose(stack, ["start"], limits=limits), "starting GitLab") + _until(START_SECONDS, "GitLab", lambda: ready(value, said, runner=True), closed) + _from_vms(settings["OSWORLD2_GITLAB_HOST"]) + return {"url": value, "up_s": round(time.time() - t0)} + + +def _from_vms(host): + """GitLab's sign-in page from a container on the VMs' network, by the mock websites' Caddy's busybox wget.""" + image = websites.CADDY.search(websites._override()).group(1) + said = _docker("run", "--rm", "--network", websites._network(), "--pull", "never", "--entrypoint", "wget", + image, "-q", "-O", "/dev/null", "-T", "10", f"http://{host}/users/sign_in") + if said.returncode != 0: + raise GitLabError(f"from the VMs' network, {host} did not answer: " + + ((said.stderr or "").strip().splitlines() or ["no reason"])[-1][:300]) + + +def down(stack, limits=None): + """The volumes' copy is no service of the project's, so it is removed by its label first.""" + said = _done(_docker("ps", "-aq", "--filter", f"label={COPY_LABEL}"), "listing the volumes' copy") + if said.stdout.split(): + _done(_docker("rm", "-f", *said.stdout.split()), "removing the volumes' copy") + _done(compose(stack, ["--profile", "upstream-init", "down", "--volumes", "--remove-orphans"], limits=limits), + "removing GitLab") + return {"removed": PROJECT} + + +def main(argv): + p = argparse.ArgumentParser(prog="gitlab_server.py", description="OSWorld2's GitLab") + sub = p.add_subparsers(dest="op", required=True) + c = sub.add_parser("prepare") + c.add_argument("stack") + c.add_argument("suffix") + d = sub.add_parser("compose") + d.add_argument("stack") + d.add_argument("args", nargs=argparse.REMAINDER) + for op in ("up", "down"): + sub.add_parser(op).add_argument("stack") + a = p.parse_args(argv) + try: + if a.op == "compose": + args = a.args[1:] if a.args[:1] == ["--"] else a.args + with contextlib.nullcontext() if websites.reads(args) else _alone(): + return compose(a.stack, args, capture=False) + with _alone(): + if a.op == "prepare": + if os.path.lexists(a.stack): + _pinned(a.stack) + out = {"prepared": prepare(os.path.abspath(a.stack), {websites.OPTION: a.suffix}, say=print)} + else: + out = {"up": lambda s: up(s, say=print), "down": down}[a.op](a.stack) + except (GitLabError, websites.WebsiteError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/nullius/catalogue/benchmarks/osworld2/nop.py b/app/nullius/catalogue/benchmarks/osworld2/nop.py new file mode 100644 index 00000000..a6de78c6 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/nop.py @@ -0,0 +1,91 @@ +"""OSWorld-V2's null control: the upstream's run_single_example on one task with an agent that does nothing. + + nop.py VM_IMAGE PASSWORD EXAMPLES DOMAIN TASK_ID RESULT_DIR + nop.py --instruction EXAMPLES DOMAIN TASK_ID OUT + +The agent answers DONE at its one step, and evaluate() on the untouched VM goes +to RESULT_DIR/result.txt. --instruction writes the task's instruction and +whether it asks for a proxy to OUT as JSON. A task with no class file is an +error. The setup gets SETUP_ATTEMPTS; a failed one goes to +RESULT_DIR/setup_failed.json by exception type only, since the message could +quote the gated task.""" +import argparse +import json +import logging +import os +import signal +import sys + +from task_loader import find_task_class_path, load_task_config, resolve_task_json_path + +SETUP_ATTEMPTS = 2 +SETUP_FAILED = "setup_failed.json" + + +class DoNothing: + def reset(self, *args, **kwargs): + pass + + def predict(self, instruction, obs): + return "", ["DONE"] + + +def _stop(signum, frame): + sys.exit(1) + + +def _fail(said): + logging.error(said) + sys.exit(1) + + +def _task(examples, domain, eid): + if not find_task_class_path(eid, examples, domain, "v2"): + _fail(f"no task class for {domain}__{eid} under {examples}") + try: + task = load_task_config(resolve_task_json_path(task_id=eid, base_dir=examples, domain=domain, eval_version="v2"), + task_id=eid, base_dir=examples, domain=domain, eval_version="v2") + instruction = task["instruction"] + except (Exception, SystemExit) as e: + _fail(f"the task class for {domain}__{eid} did not load: {type(e).__name__}" + + (", which may need a service this entry does not set up (the mock websites)" + if isinstance(e, ValueError) else "")) + return task, instruction + + +logging.basicConfig(level=logging.INFO, format="[%(asctime)s %(levelname)s %(name)s] %(message)s") +if sys.argv[1:2] == ["--instruction"]: + examples, domain, eid, out = sys.argv[2:6] + task, instruction = _task(examples, domain, eid) + with open(out, "w", encoding="utf-8") as f: + json.dump({"instruction": instruction, "proxy": bool(task.get("proxy", False))}, f) + sys.exit(0) +import lib_run_single +from desktop_env import desktop_env +from desktop_env.desktop_env import DesktopEnv, EnvironmentSetupError + +vm, password, examples, domain, eid, out = sys.argv[1:7] +example, instruction = _task(examples, domain, eid) +signal.signal(signal.SIGTERM, _stop) +desktop_env.MAX_RETRIES = SETUP_ATTEMPTS +env = None +try: + env = DesktopEnv(path_to_vm=vm, action_space="pyautogui", provider_name="docker", screen_size=(1920, 1080), + headless=True, os_type="Ubuntu", require_a11y_tree=False, enable_proxy=True, + client_password=password, instance_type=example.get("instance_type"), + volume_size=example.get("volume_size"), force_disable_vnc=True, force_disable_recording=True) + lib_run_single.run_single_example(DoNothing(), env, example, 1, instruction, + argparse.Namespace(sleep_after_execution=3.0, result_dir=out, + checkpoint_eval_mode="off"), out, []) +except EnvironmentSetupError as e: + with open(os.path.join(out, SETUP_FAILED), "w", encoding="utf-8") as f: + json.dump({"error": type(e).__name__, "cause": type(e.__cause__).__name__ if e.__cause__ else None, + "attempts": SETUP_ATTEMPTS}, f) + logging.error(f"the null control's setup of {domain}__{eid} failed: {type(e).__name__}") + raise +except Exception as e: + logging.error(f"the null control's run of {domain}__{eid} failed: {type(e).__name__}") + raise +finally: + if env is not None: + env.close() diff --git a/app/nullius/catalogue/benchmarks/osworld2/proxy.py b/app/nullius/catalogue/benchmarks/osworld2/proxy.py new file mode 100644 index 00000000..ed79b169 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/proxy.py @@ -0,0 +1,372 @@ +"""OSWorld2's proxy for the tasks that ask for one: tinyproxy in a container of the unit's own. + + python3 proxy.py --build + python3 proxy.py --sweep UNIT_DIR RECORD_FD [CONTAINER] + python3 proxy.py --watch UNIT_DIR DRIVER_PID RECORD_FD CONTAINER + +The driver passes the unit's proxy.json as the upstream's PROXY_CONFIG_FILE. +The container is on NETWORK and publishes no port; tinyproxy logs warnings +only, never a request's URL. A container is labelled with the unit's folder +and named with this user's id and its driver's pid. A start docker did not +answer is never repeated: a sweeper (--sweep) removes the container later, +and a watcher (--watch) when the driver is killed first.""" +import contextlib +import fcntl +import hashlib +import json +import os +import re +import secrets +import socket +import subprocess +import sys +import time + +DOCKERFILE = r"""FROM alpine:3.24@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +RUN apk add --no-cache tinyproxy~1.11 +USER 65534:65534 +ENTRYPOINT ["/bin/sh", "-c", "printf '%s\\n' \"$TINYPROXY_CONF\" > /tmp/tinyproxy.conf && exec tinyproxy -d -c /tmp/tinyproxy.conf"] +""" +IMAGE = "nullius/osworld2-proxy:" + hashlib.sha256(DOCKERFILE.encode()).hexdigest()[:16] +CONF = "TINYPROXY_CONF" +PORT = 8888 +USER = "nullius" +FILE = "proxy.json" +NAME = "nullius-osworld2-proxy-" +UNIT = "nullius.osworld2.proxy" +FOLDER = "nullius.osworld2.proxy.dir" +PACKAGE = "nullius.osworld2.proxy.package" +NETWORK = "nullius-osworld2" +SWEEPS = (2, 8, 20, 90, 180, 300, 300) +RETRY_SECONDS = 120 +SWEEP_SECONDS = 1800 +CLIENTS = 100 +PIDS = 128 +WATCH_SECONDS = 1 +WRITE_BPS = 52428 +WRITE_IOPS = 1 +TAKEN = {"OSWORLD2_PROXY_WRITE_BPS": str(WRITE_BPS), "OSWORLD2_PROXY_WRITE_IOPS": str(WRITE_IOPS)} +LOG = "proxy.log" +DOCKER_SECONDS = 120 +START_SECONDS = 300 +_SEEN = {} +_STARTS = {} + + +class NoAnswer(RuntimeError): + pass + + +def _docker(*args, env=None, until=None): + seconds = DOCKER_SECONDS if until is None else min(DOCKER_SECONDS, round(until - time.monotonic(), 1)) + if seconds <= 0: + raise NoAnswer(f"docker {args[0]} was not asked: the time given for it ran out") + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, env=env) + except subprocess.TimeoutExpired: + raise NoAnswer(f"docker {args[0]} got no answer in {seconds:g} s: is its daemon running?") + except OSError: + raise RuntimeError("no docker on this machine's PATH: the proxy needs it") + + +def _last(said, otherwise): + """The last line docker said, past the pointer to its help it adds to every refusal.""" + lines = [ln.strip() for ln in (said or "").splitlines() if ln.strip() and not ln.startswith("Run 'docker")] + return (lines or [otherwise])[-1][:300] + + +def _mark(dir): + return hashlib.sha256(os.path.abspath(dir).encode()).hexdigest()[:32] + + +def _prefix(): + return f"{NAME}{os.getuid()}-" + + +def config(token): + """No Via header, since the pool's residential proxies add none.""" + return "".join(f"{line}\n" for line in (f"Port {PORT}", "Timeout 600", f"MaxClients {CLIENTS}", "ConnectPort 443", + "ConnectPort 80", "LogLevel Warning", "DisableViaHeader Yes", + f"BasicAuth {USER} {token}")) + + +def _alive(pid): + """Whether the process runs; one that has exited and waits for its parent to see it does not.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + try: + with open(f"/proc/{pid}/stat", encoding="utf-8", errors="replace") as f: + return f.read().rpartition(")")[2].split()[0] not in ("Z", "X") + except (OSError, IndexError): + return True + + +def _orphans(until=None): + """Remove this user's proxies whose driver is gone or whose unit holds no record; another user's stay.""" + seen = _docker("ps", "-a", "--filter", f"name={_prefix()}", "--format", "{{.Names}}\t{{.Label \"" + FOLDER + "\"}}", + until=until) + gone = [] + for name, _, folder in (ln.partition("\t") for ln in seen.stdout.splitlines()): + m = re.fullmatch(re.escape(_prefix()) + r"(\d+)-\d+", name.strip()) + if m and (not _alive(int(m.group(1))) or (folder and not os.path.lexists(os.path.join(folder, FILE)))): + gone.append(name.strip()) + if gone: + _docker("rm", "-f", *gone, until=until) + + +def _image(dir, until): + """Orphans removed and the image checked at a process's first start; its ID logged at each.""" + if not _SEEN: + _orphans(until) + seen = _docker("image", "inspect", "--format", "{{.Id}}\t{{json .Config}}", IMAGE, until=until) + if seen.returncode != 0: + raise RuntimeError(f"no {IMAGE} on this machine, the proxy of the tasks that ask for one: build it " + "once with python3 proxy.py --build, from this benchmark's folder (it needs the " + "network)") + ident, _, config = seen.stdout.strip().partition("\t") + try: + package = ((json.loads(config) or {}).get("Labels") or {}).get(PACKAGE, "") + except (ValueError, AttributeError): + package = "" + _SEEN.update(id=ident or IMAGE, package=package) + _note(dir, f"proxy image {_SEEN['id']}, " + (_SEEN["package"] or "its tinyproxy package not labelled: " + "build it again with python3 proxy.py --build")) + + +def _ready(name, until): + """A stopped container is an error with its exit code, never its log, which may name a request's host.""" + host = "" + late = f"did not answer in its start's {START_SECONDS} s" + while True: + try: + seen = _docker("inspect", "-f", "{{.State.Status}}\t{{.State.ExitCode}}\t{{with index .NetworkSettings." + "Networks " + json.dumps(NETWORK) + "}}{{.IPAddress}}{{end}}", name, until=until) + except NoAnswer: + if time.monotonic() < until: + raise + raise RuntimeError(f"the proxy at {host or 'no address'}:{PORT} {late}") from None + status, code, host = (seen.stdout.strip().split("\t") + ["", ""])[:3] + if status not in ("running", "created"): + raise RuntimeError(f"the proxy's container stopped, exit code {code or 'unknown'}") + if status == "running" and host: + try: + socket.create_connection((host, PORT), timeout=1).close() + return host + except OSError: + pass + if time.monotonic() >= until: + raise RuntimeError(f"the proxy at {host or 'no address'}:{PORT} {late}") + time.sleep(0.2) + + +def start(dir, disk=None): + """The unit's proxy started, and its pool file's path; a proxy left from an earlier attempt is removed first.""" + until = time.monotonic() + START_SECONDS + _image(dir, until) + remove(dir, final=True, until=until) + path = os.path.join(dir, FILE) + name = f"{_prefix()}{os.getpid()}-{time.time_ns()}" + with _locked(dir): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + _STARTS[os.path.abspath(dir)] = name + _watcher(dir, fd, name) + finally: + os.close(fd) + token = secrets.token_hex(16) + limits = [] if disk is None else ["--device-write-bps", f"{disk}:{WRITE_BPS}", "--device-write-iops", f"{disk}:{WRITE_IOPS}"] + try: + made = _docker("run", "-d", "--name", name, "--label", f"{UNIT}={_mark(dir)}", + "--label", f"{FOLDER}={os.path.abspath(dir)}", "--network", NETWORK, + "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1m,mode=1777", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--memory", "64m", "--pids-limit", str(PIDS), + "--cpus", "1", "--log-driver", "local", "--log-opt", "max-size=1m", "--log-opt", "max-file=1", + "--log-opt", "compress=false", *limits, + "-e", CONF, IMAGE, env=dict(os.environ, **{CONF: config(token)}), until=until) + except NoAnswer as e: + _found(dir, name, e, until) + made = None + _record(path, []) + if made is not None and made.returncode != 0: + raise RuntimeError("the proxy's container was refused: " + _last(made.stderr, "no reason")) + host = _ready(name, until) + _record(path, [{"host": host, "port": PORT, "username": USER, "password": token, "protocol": "http"}]) + return path + + +def _found(dir, name, unanswered, until): + """No container from the unanswered start, or no answer, is an error; the record stays empty for its sweeper.""" + later = "; the unit's sweeper removes one it makes later" + try: + found = _docker("ps", "-a", "-q", "--filter", f"label={UNIT}={_mark(dir)}", "--filter", f"name={name}", + until=until) + except NoAnswer as e: + raise RuntimeError(f"{unanswered}, and then {e}: whether it made the proxy's container is unknown{later}") + if found.returncode != 0: + raise RuntimeError(f"{unanswered}, and could not say whether it made the proxy's container: " + + _last(found.stderr, "no reason") + later) + if not found.stdout.split(): + raise RuntimeError(f"{unanswered}, and has made no container for the proxy yet{later}") + + +@contextlib.contextmanager +def _locked(dir): + fd = os.open(dir, os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) + + +def _ours(dir, fd): + """Whether the unit's record is still the file ``fd`` holds open, so no later start has replaced it.""" + try: + here, own = os.lstat(os.path.join(dir, FILE)), os.fstat(fd) + except OSError: + return False + return (here.st_dev, here.st_ino) == (own.st_dev, own.st_ino) + + +def _record(path, pool): + with os.fdopen(os.open(path, os.O_WRONLY | os.O_TRUNC | os.O_NOFOLLOW), "w", encoding="utf-8") as f: + json.dump(pool, f) + + +def remove(dir, final=False, until=None, name=None, fd=None): + """Remove the unit's proxy, then its record; an empty record stays until the ``final`` sweep.""" + path = os.path.join(dir, FILE) + if not (os.path.lexists(path) if fd is None else _ours(dir, fd)): + return + found = _docker("ps", "-a", "-q", "--filter", f"label={UNIT}={_mark(dir)}", "--filter", f"name={name or _prefix()}", + until=until) + if found.returncode != 0: + raise RuntimeError("the unit's proxy was not looked for: " + _last(found.stderr, "no reason")) + ids = found.stdout.split() + if ids: + gone = _docker("rm", "-f", *ids, until=until) + if gone.returncode != 0: + raise RuntimeError("the unit's proxy was not removed: " + _last(gone.stderr, "no reason")) + if fd is None: + with contextlib.suppress(FileNotFoundError): + if final or os.lstat(path).st_size: + os.remove(path) + return + with _locked(dir): + if _ours(dir, fd) and (final or os.fstat(fd).st_size): + os.remove(path) + + +def _removed(dir, final, name, fd, until): + try: + remove(dir, final, until=until, name=name, fd=fd) + except (RuntimeError, OSError) as e: + return str(e) or type(e).__name__ + return None + + +def sweep(dir, fd, name=None): + """Remove the start's container after each of SWEEPS; a removal that never succeeds is a line in LOG.""" + until = time.monotonic() + SWEEP_SECONDS + failed = None + for i, wait in enumerate(SWEEPS): + if not _ours(dir, fd): + break + left = until - time.monotonic() - 2 * DOCKER_SECONDS + final = i == len(SWEEPS) - 1 or left < wait + time.sleep(max(0, min(wait, left))) + failed = _removed(dir, final, name, fd, until) + if final: + break + while failed and _ours(dir, fd) and until - time.monotonic() > RETRY_SECONDS: + time.sleep(RETRY_SECONDS) + failed = _removed(dir, True, name, fd, until) + if failed: + _note(dir, f"the unit's proxy was not removed in {SWEEP_SECONDS // 60} min: {failed}") + with contextlib.suppress(RuntimeError, OSError): + _orphans(until) + + +def watch(dir, pid, fd, name): + """Sweep the start's container once its driver is gone, while the record is still the start's.""" + while _ours(dir, fd): + if not _alive(pid): + sweep(dir, fd, name) + return + time.sleep(WATCH_SECONDS) + + +def _spawn(fd, *args): + subprocess.Popen([sys.executable, os.path.abspath(__file__), *args], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, pass_fds=(fd,)) + + +def _later(dir, name=None): + with contextlib.suppress(OSError): + fd = os.open(os.path.join(dir, FILE), os.O_RDONLY | os.O_NOFOLLOW) + try: + _spawn(fd, "--sweep", os.path.abspath(dir), str(fd), *([name] if name else [])) + finally: + os.close(fd) + + +def _watcher(dir, fd, name): + _spawn(fd, "--watch", os.path.abspath(dir), str(os.getpid()), str(fd), name) + + +def _note(dir, line): + with contextlib.suppress(OSError): + fd = os.open(os.path.join(dir, LOG), os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o644) + with os.fdopen(fd, "a", encoding="utf-8") as f: + f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + line + "\n") + + +@contextlib.contextmanager +def held(dir, wanted, disk=lambda: None): + """The unit's variables while its proxy runs, when ``wanted``; a record left after the removal gets a sweeper.""" + if not wanted: + yield {} + return + try: + device = disk() + yield {"PROXY_CONFIG_FILE": start(dir, device), **({} if device is None else TAKEN)} + finally: + try: + remove(dir) + except (RuntimeError, OSError) as e: + _note(dir, f"the unit's proxy was not removed: {e}") + if os.path.lexists(os.path.join(dir, FILE)): + _later(dir, _STARTS.get(os.path.abspath(dir))) + + +def build(): + """Build the image, then again from the cache, labelled with its tinyproxy package.""" + if subprocess.run(["docker", "build", "-t", IMAGE, "-"], input=DOCKERFILE, text=True).returncode != 0: + return 1 + seen = subprocess.run(["docker", "run", "--rm", "--network", "none", "--entrypoint", "apk", IMAGE, "list", + "--installed", "tinyproxy"], capture_output=True, text=True) + package = (seen.stdout.split() or [""])[0] if seen.returncode == 0 else "" + if not package.startswith("tinyproxy-"): + print("the image's tinyproxy package was not found: " + _last(seen.stderr, "apk said nothing"), file=sys.stderr) + return 1 + if subprocess.run(["docker", "build", "--label", f"{PACKAGE}={package}", "-t", IMAGE, "-"], input=DOCKERFILE, + text=True).returncode != 0: + return 1 + print(IMAGE, package) + return 0 + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--sweep"] and len(sys.argv) in (4, 5): + sweep(sys.argv[2], int(sys.argv[3]), *sys.argv[4:]) + sys.exit(0) + if sys.argv[1:2] == ["--watch"] and len(sys.argv) == 6: + watch(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5]) + sys.exit(0) + if sys.argv[1:] != ["--build"]: + sys.exit("\n".join(ln.strip() for ln in __doc__.splitlines()[2:5])) + sys.exit(build()) diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json new file mode 100644 index 00000000..55930670 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -0,0 +1,342 @@ +{ + "schema": "nullius.benchmark/v1", + "key": "osworld2", + "label": "OSWorld2", + "summary": "Long multi-app tasks on an Ubuntu desktop VM, scored 0 to 1 by each task's own check.", + "family": "agent", + "gated": true, + "needs": [ + "agent.computer/v1" + ], + "upstream": { + "repository": "https://github.com/xlang-ai/OSWorld-V2", + "tag": "osworld-v2.1", + "tested": "3d778a3c9a34a079316f70df023b166700445792", + "tested_date": "2026-09-16", + "license": "Apache-2.0" + }, + "driver": { + "command": [ + "{install}/.venv/bin/python", + "{here}/driver.py", + "{here}/agents.json", + "{here}/nop.py", + "{here}/startup/sitecustomize.py", + "{here}/websites.compose.yml", + "{here}/gitlab.compose.yml" + ], + "install": { + "python": "3.12", + "project": ".", + "locked": true + } + }, + "datasets": [ + { + "id": "offline36", + "label": "Offline, no services", + "summary": "36 offline tasks that need no service; some install from PyPI in the VM.", + "units": 36 + }, + { + "id": "no_services63", + "label": "No hosted services", + "summary": "The 63 tasks that need no service hosted for them: Offline, no services and 27 that need the LLM grader, the simulated user or the public internet. No mock website, GitLab or proxy.", + "units": 63 + }, + { + "id": "websites35", + "label": "Mock website tasks", + "summary": "The 35 tasks that need the mock websites and no GitLab or proxy: OSWorld-web served on this machine, its suffix given as Website host suffix. 2 of them use Overleaf and run one at a time.", + "units": 35 + }, + { + "id": "websites98", + "label": "No GitLab or proxy", + "summary": "No hosted services and Mock website tasks: the 98 tasks that need no GitLab or proxy.", + "units": 98 + }, + { + "id": "test_v2_offline_no_internet", + "label": "Offline", + "summary": "The upstream's offline list; some tasks need GitLab, a proxy or the mock websites.", + "path": "evaluation_examples/test_v2_offline_no_internet.json", + "sha256": "a14975fe0972530993d634d380ba574ecbd19b730d14716a392efe02faa62257", + "units": 82 + }, + { + "id": "test_v2", + "label": "All tasks", + "summary": "Every task, the upstream's default list.", + "path": "evaluation_examples/test_v2.json", + "sha256": "7f2c0f531d035dd6f5886c492b96739534385f8e5369e4c764295ad52bf6e7f4", + "units": 108 + } + ], + "units": { + "noun": "task", + "nouns": "tasks" + }, + "score": { + "kind": "score", + "says": "OSWorld-V2's own scoring: the task class's evaluate() on the VM's final state, as run_single_example writes it to result.txt (0 to 1, partial credit; result.json carries the detail and safety checks); the upstream's score is the mean", + "judge": { + "model": "anthropic/claude-sonnet-4.6", + "says": "the upstream's LLM grader in the evaluate() of 21 tasks, Claude Sonnet 4.6 over Anthropic's Messages API as run_multienv_m3.sh sets it (OSWORLD_EVAL_MODEL_*), whichever agent runs" + } + }, + "limits": { + "score_seconds": 3600, + "call_seconds": 3300 + }, + "setups": { + "original": { + "summary": "OSWorld-V2's own agent, M3 or Muse Spark, as published, on a Docker VM.", + "reference_agent": "OSWorld-V2 agents", + "models": [ + "MiniMax-M3", + "muse-spark-1.3" + ], + "default": "MiniMax-M3", + "side_models": [ + "claude-sonnet-4-6" + ], + "served_as": { + "MiniMax-M3": "minimax/minimax-m3", + "muse-spark-1.3": "meta/muse-spark-1.3", + "claude-sonnet-4-6": "anthropic/claude-sonnet-4.6" + }, + "providers": { + "openrouter": "", + "custom": "" + }, + "turn_seconds": 86400, + "paper": "the OSWorld-V2 paper, Table 3 (Section 3.2), MiniMax M3", + "comparable": [ + "MiniMax-M3" + ] + } + }, + "options": { + "agent": { + "label": "Agent", + "summary": "Which of OSWorld-V2's own agents runs: each its script as published, with its own prompts, tools and history, asking the model it is published with.", + "values": [ + "m3", + "muse_spark" + ], + "default": "m3", + "setups": [ + "original" + ], + "names": { + "m3": "MiniMax M3", + "muse_spark": "Muse Spark" + }, + "about": { + "m3": "as in the paper" + } + }, + "max_steps": { + "label": "Steps per task", + "summary": "The agent's step limit. default: each agent's own, as its wrapper runs it.", + "setups": [ + "original" + ], + "default": "default", + "pattern": "default|[1-9][0-9]{0,3}", + "example": "500", + "limit": true, + "by": { + "agent": { + "m3": "100", + "muse_spark": "500" + } + } + }, + "thinking": { + "label": "Thinking", + "summary": "Whether M3 thinks. The paper ran it adaptive.", + "values": [ + "off", + "adaptive" + ], + "default": "off", + "setups": [ + "original" + ], + "about": { + "off": "upstream default", + "adaptive": "as in the paper" + }, + "when": { + "agent": [ + "m3" + ] + } + }, + "vm_provider": { + "label": "VM provider", + "step": "where", + "summary": "Where the VMs run.", + "values": [ + "docker" + ], + "default": "docker", + "setups": [ + "original", + "yours" + ], + "needs": { + "docker": { + "host": [ + "docker", + "scoring_programs" + ] + } + } + }, + "websites": { + "label": "Website host suffix", + "step": "dataset", + "datasets": [ + "websites35", + "websites98", + "test_v2_offline_no_internet", + "test_v2" + ], + "summary": "Where the mock websites, and GitLab with them, are served. Needed by the tasks that use them.", + "pattern": "none|(?=.{3,200}$)[a-z0-9](?:[a-z0-9\\-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9\\-]{0,61}[a-z0-9])?)+", + "example": "10.213.0.1.nip.io", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "write_mbps": { + "label": "Disk write MB/s", + "summary": "Disk write limit for each VM.", + "pattern": "none|[1-9][0-9]{0,5}", + "example": "16", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "write_iops": { + "label": "Disk write IOPS", + "summary": "Disk write limit for each VM.", + "pattern": "none|[1-9][0-9]{0,5}", + "example": "160", + "default": "none", + "setups": [ + "original", + "yours" + ] + } + }, + "host": { + "docker": { + "label": "Docker", + "summary": "Docker without sudo; KVM recommended.", + "check": [ + "docker" + ] + }, + "scoring_programs": { + "label": "Scoring programs", + "summary": "ImageMagick, ffmpeg and tesseract, used to score 6 tasks.", + "check": [ + "identify", + "convert", + "ffmpeg", + "ffprobe", + "tesseract" + ], + "install": { + "apt": [ + "imagemagick", + "ffmpeg", + "tesseract-ocr" + ] + } + } + }, + "controls": { + "nop": { + "source": "nullius", + "from": "An agent that does nothing; the untouched VM is scored.", + "expect": { + "at_most": 0.01 + } + } + }, + "proven": { + "offline36": { + "units": { + "units": 36, + "passed": "36 ids, the same twice, unique", + "on": "2026-11-04" + }, + "nop": { + "passed": "0/4", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 4, + "job": "2026-11-04__23-15-30__osworld2__offline36__prove-nop", + "source": "nullius", + "on": "2026-11-04", + "limit": 4 + } + }, + "test_v2": { + "units": { + "units": 108, + "passed": "108 ids, the same twice, unique", + "on": "2026-11-15" + }, + "nop": { + "passed": "0/108", + "value": 0.0021, + "expect": { + "at_most": 0.01 + }, + "units": 108, + "job": "2026-11-15__06-18-44__osworld2__test_v2__prove-nop", + "source": "nullius", + "options": [ + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "gitlab.10.213.0.1.nip.io", + "gitlab_write": "16:160", + "write_mbps": "16", + "write_iops": "160" + }, + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "none", + "gitlab_write": "none", + "write_mbps": "16", + "write_iops": "160" + }, + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "none", + "gitlab_write": "none", + "write_mbps": "24", + "write_iops": "240" + } + ], + "judge": "anthropic/claude-sonnet-4.6", + "on": "2026-11-15" + } + } + } +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py b/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py new file mode 100644 index 00000000..bfcdabe7 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py @@ -0,0 +1,460 @@ +"""OSWorld2's VM container create and guest disk, in every Python process the driver starts. + +The driver puts this folder first on PYTHONPATH. For the provider's image only, +it wraps docker-py's ContainerCollection.create: CREATE_SECONDS a call, a +create that timed out on reading looked for by its label instead of sent +again, the container on NETWORK (on the default bridge a Docker in the VM +takes the host's address), the write limits on the disk under Docker's root +and its ports, the guest's PORTS too, on 127.0.0.1. A docker DesktopEnv gets a disk of at +least GUEST_GB, as the upstream's aws provider gives. With another vm_provider +it wraps nothing. Once OSWORLD2_STOP's file exists, an agent's step ends with DONE. +A log record never shows the password of PROXY_CONFIG_FILE's proxy.""" +import contextlib +import functools +import inspect +import json +import logging +import os +import re +import signal +import subprocess +import sys +import time +import uuid + +PROVIDER = "happysixd/osworld-docker" +CREATE_SECONDS = 120 +ADOPT_SECONDS = 60 +LOOK_SECONDS = 10 +POLL_SECONDS = 2 +SWEEP_SECONDS = 1800 +LABEL = "nullius.osworld2.create" +NETWORK = "nullius-osworld2" +SUBNET = "10.213.0.0/24" +GATEWAY = "10.213.0.1" +NETWORK_LABEL = "nullius.osworld2.network" +PORTS = "OSWORLD2_GUEST_PORTS" +LOOPBACK = "127.0.0.1" +DOCKER_PY_SECONDS = 60 +PORT_LISTS = 4 +LONGEST_START = (PORT_LISTS * DOCKER_PY_SECONDS + 5 * CREATE_SECONDS + + ADOPT_SECONDS + POLL_SECONDS + 2 * LOOK_SECONDS) +LIMITS = (("device_write_bps", "NULLIUS_OPTION_WRITE_MBPS", 1024 * 1024, "BlkioDeviceWriteBps", "--device-write-bps"), + ("device_write_iops", "NULLIUS_OPTION_WRITE_IOPS", 1, "BlkioDeviceWriteIOps", "--device-write-iops")) +TAKEN = {"device_write_bps": "OSWORLD2_PROXY_WRITE_BPS", "device_write_iops": "OSWORLD2_PROXY_WRITE_IOPS"} +NAMES = {"device_write_bps": "Disk write MB/s", "device_write_iops": "Disk write IOPS"} +SYS_BLOCK = "/sys/dev/block" +DEV = "/dev" +ACTION = "run where Docker limits writes on the disk under its root, or set Disk write MB/s and Disk write IOPS to none" +_DISK = {} +GUEST_GB = 40 +DESKTOP_ENV = "desktop_env.desktop_env" +EPISODE = "lib_run_single" +DOCKER_PROVIDER = "desktop_env.providers.docker.provider" +STOP = "OSWORLD2_STOP" +MARK = "_nullius_osworld2" +HIDDEN = "" + + +class WriteLimitError(RuntimeError): + pass + + +class NetworkError(RuntimeError): + pass + + +def network(docker): + """Made when missing, a concurrent maker no error; one of that name on another subnet or gateway is a NetworkError.""" + shape = "{{index .Labels \"" + NETWORK_LABEL + "\"}}|{{range .IPAM.Config}}{{.Subnet}} {{.Gateway}} {{end}}" + seen = docker("network", "inspect", "-f", shape, NETWORK) + if seen.returncode != 0: + made = docker("network", "create", "--driver", "bridge", "--subnet", SUBNET, "--gateway", GATEWAY, + "--label", f"{NETWORK_LABEL}=1", NETWORK) + seen = docker("network", "inspect", "-f", shape, NETWORK) + if seen.returncode != 0: + said = ((made.stderr or "").strip().splitlines() or ["no reason"])[-1][:300] + raise NetworkError(f"the VMs' network {NETWORK} ({SUBNET}) could not be made: {said}; free that " + "subnet on this machine, or make the network with that subnet and gateway " + f"{GATEWAY}") + label, _, found = seen.stdout.strip().partition("|") + if found.split() != [SUBNET, GATEWAY]: + have = f"the network {NETWORK} has {found.strip() or 'no subnet'}, not {SUBNET} with gateway {GATEWAY}" + if label: + raise NetworkError(f"{have}: remove it (docker network rm {NETWORK}) once no container uses it") + raise NetworkError(f"{have}, and Nullius did not make it: leave it to its owner, who can give it another " + f"name, or run on a machine where the name {NETWORK} and {SUBNET} are free") + return NETWORK + + +def wanted(environ): + """The options' limits as (argument, rate, HostConfig field, CLI flag), less the proxy's share.""" + out = [] + for key, name, unit, field, flag in LIMITS: + value = environ.get(name, "none") + if value != "none": + if not re.fullmatch(r"[1-9][0-9]*", value): + raise WriteLimitError(f"the VM's disk write limit cannot be applied: {NAMES[key]} is {value!r}; {ACTION}") + taken = environ.get(TAKEN[key], "0") + if not re.fullmatch(r"0|[1-9][0-9]*", taken): + raise WriteLimitError(f"the unit's proxy gave no write share it can take: {TAKEN[key]} is {taken!r}") + if int(value) * unit <= int(taken): + raise WriteLimitError(f"the VM's disk write limit cannot be applied: {NAMES[key]} is {value}, " + "no more than the unit's proxy takes; give the VM more") + out.append((key, int(value) * unit - int(taken), field, flag)) + return out + + +def published(environ): + """The image forwards every TCP port but its web page's to the guest, so the same number reaches it.""" + return {int(p): (LOOPBACK, int(p)) for p in environ.get(PORTS, "").split()} + + +def disk(path): + """The block device under ``path``, a partition's parent disk; None when there is no block device.""" + st = os.stat(path).st_dev + node = os.path.join(SYS_BLOCK, f"{os.major(st)}:{os.minor(st)}") + if not os.path.isdir(node): + return None + node = os.path.realpath(node) + if os.path.exists(os.path.join(node, "partition")): + node = os.path.dirname(node) + try: + with open(os.path.join(node, "uevent"), encoding="utf-8") as f: + name = dict(ln.strip().partition("=")[::2] for ln in f).get("DEVNAME") + except OSError: + return None + return os.path.join(DEV, name) if name and os.path.exists(os.path.join(DEV, name)) else None + + +def root_disk(root): + """The disk under Docker's root ``root``, found once in a process; no disk is a WriteLimitError.""" + if root not in _DISK: + try: + device = disk(root) + except OSError as e: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker's root {root}: " + f"{type(e).__name__}; {ACTION}") from e + if device is None: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker's root {root} is on no " + f"block device {SYS_BLOCK} names; {ACTION}") + _DISK[root] = device + return _DISK[root] + + +def _throttle(client, limits, kwargs): + if not _DISK: + try: + root = client.info()["DockerRootDir"] + except Exception as e: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: no Docker root from the daemon, " + f"{type(e).__name__}; {ACTION}") from e + root_disk(root) + device = next(iter(_DISK.values())) + return {key: list(kwargs.get(key) or []) + [{"Path": device, "Rate": rate}] for key, rate, _, _ in limits} + + +def _confirm(container, limits, given): + """A daemon that cannot limit writes drops them silently, so a container without them is removed before it starts.""" + try: + container.reload() + host = container.attrs.get("HostConfig") or {} + kept = all(given[key][-1] in (host.get(field) or []) for key, _, field, _ in limits) + except Exception: + kept = False + if not kept: + try: + container.remove(force=True, v=True) + except Exception: + pass + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker created the VM's container " + f"without it; {ACTION}") + + +def _patient(client): + api = getattr(client, "api", None) + if api is not None and isinstance(getattr(api, "timeout", None), (int, float)) and api.timeout < CREATE_SECONDS: + api.timeout = CREATE_SECONDS + + +def _gave_up(error): + """Only a read timeout leaves the create undecided; any other error stands.""" + try: + from requests import exceptions + except ImportError: + return False + return isinstance(error, exceptions.ReadTimeout) + + +def _labelled(labels, token): + if isinstance(labels, dict): + out = dict(labels) + else: + out = dict.fromkeys(labels or [], "") + out[LABEL] = token + return out + + +def _made_late(collection, token): + """Each look waits LOOK_SECONDS at most, so the search ends within ADOPT_SECONDS, a pause and one look.""" + api = getattr(collection.client, "api", None) + kept = getattr(api, "timeout", None) + if api is not None and kept is not None: + api.timeout = LOOK_SECONDS + t0 = time.monotonic() + try: + while True: + try: + found = collection.list(all=True, filters={"label": f"{LABEL}={token}"}) + except Exception: + found = [] + if found: + return found[0] + if time.monotonic() - t0 >= ADOPT_SECONDS: + return None + time.sleep(POLL_SECONDS) + finally: + if api is not None and kept is not None: + api.timeout = kept + + +def _sweeper(token): + """A sweeper for the create's container, called off by _kept, and set going when this process closes its pipe or dies.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, "-I", os.path.abspath(__file__), token], stdin=r, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) + finally: + os.close(r) + return w + + +def _kept(fd): + try: + os.write(fd, b"kept") + except OSError: + pass + finally: + os.close(fd) + + +def sweep(token): + """The container of an abandoned create, should Docker make it within SWEEP_SECONDS, removed with its volume.""" + until = time.monotonic() + SWEEP_SECONDS + while time.monotonic() < until: + try: + found = subprocess.run(["docker", "ps", "-a", "-q", "--filter", f"label={LABEL}={token}"], + capture_output=True, text=True, timeout=CREATE_SECONDS).stdout.split() + if found and subprocess.run(["docker", "rm", "-f", "-v", *found], capture_output=True, + timeout=CREATE_SECONDS).returncode == 0: + return + except (OSError, subprocess.SubprocessError): + pass + time.sleep(LOOK_SECONDS) + + +def wrap_docker_create(environ=os.environ): + """Once a process: a create already wrapped, by an earlier run of this module, is left as it is.""" + limits, ports = wanted(environ), published(environ) + try: + from docker.models.containers import ContainerCollection + except ImportError: + return + create = ContainerCollection.create + if getattr(create, MARK, False): + return + + @functools.wraps(create) + def wrapped(self, image, *args, **kwargs): + if not (isinstance(image, str) and image.split("@")[0].split(":")[0] == PROVIDER): + return create(self, image, *args, **kwargs) + _patient(self.client) + given = _throttle(self.client, limits, kwargs) if limits else {} + token = uuid.uuid4().hex + swept = _sweeper(token) + placed = {} if kwargs.get("network") or kwargs.get("network_mode") else {"network": NETWORK} + if ports or kwargs.get("ports"): + placed["ports"] = {k: v if isinstance(v, tuple) else (LOOPBACK, v) + for k, v in {**(kwargs.get("ports") or {}), **ports}.items()} + try: + container = create(self, image, *args, **dict(kwargs, labels=_labelled(kwargs.get("labels"), token), + **placed, **given)) + except Exception as e: + if not _gave_up(e): + _kept(swept) + raise + container = _made_late(self, token) + if container is None: + os.close(swept) + raise + logging.getLogger("nullius.osworld2").warning( + "Docker made the VM's container after its client gave up (%s); that one is started, none is " + "created again", type(e).__name__) + _kept(swept) + if limits: + _confirm(container, limits, given) + container.remove = functools.partial(container.remove, v=True) + return container + + setattr(wrapped, MARK, True) + ContainerCollection.create = wrapped + + +def guest_gb(provider_name, volume_size): + """The task's own volume_size, or GUEST_GB when that is less; another provider's is left alone.""" + if str(provider_name).lower().strip() != "docker": + return volume_size + return max(int(volume_size or 0), GUEST_GB) + + +def floor_desktop_env(module): + """DesktopEnv's __init__ given guest_gb's volume_size, however its caller passes it.""" + cls = getattr(module, "DesktopEnv", None) + if cls is None or getattr(cls.__init__, MARK, False): + return + init = cls.__init__ + signature = inspect.signature(init) + if not {"provider_name", "volume_size"} <= set(signature.parameters): + return + default = signature.parameters["provider_name"].default + + @functools.wraps(init) + def wrapped(self, *args, **kwargs): + bound = signature.bind(self, *args, **kwargs) + given = bound.arguments + given["volume_size"] = guest_gb(given.get("provider_name", default), given.get("volume_size")) + return init(*bound.args, **bound.kwargs) + + setattr(wrapped, MARK, True) + cls.__init__ = wrapped + + +def queue_starts(module): + """Every VM start waits its turn for the provider's port lock, a restart as the first start.""" + module.LOCK_TIMEOUT = -1 + + +def _took(path): + """The stop's reasons as it ended a step, in ``path``.taken: one written once the episode ended is not its own.""" + with contextlib.suppress(OSError), open(path, encoding="utf-8") as f, open(path + ".taken", "x") as out: + out.write(f.read()) + + +def _stopping(predict, path): + @functools.wraps(predict) + def wrapped(*args, **kwargs): + if os.path.exists(path): + _took(path) + return "", ["DONE"] + try: + out = predict(*args, **kwargs) + except Exception: + if os.path.exists(path): + _took(path) + return "", ["DONE"] + raise + if not os.path.exists(path): + return out + _took(path) + return out[0], list(out[1]) + ["DONE"] + + setattr(wrapped, MARK, True) + return wrapped + + +def stop_when_asked(module): + run, path = getattr(module, "run_single_example", None), os.environ.get(STOP) + if run is None or not path or getattr(run, MARK, False): + return + + @functools.wraps(run) + def wrapped(agent, *args, **kwargs): + if not getattr(agent.predict, MARK, False): + agent.predict = _stopping(agent.predict, path) + return run(agent, *args, **kwargs) + + setattr(wrapped, MARK, True) + module.run_single_example = wrapped + + +class _Loader: + """The finder's loader, unchanged, with the module's PATCHES after its exec_module; the rest is the loader's.""" + + def __init__(self, loader): + self.loader = loader + + def __getattr__(self, name): + return getattr(self.loader, name) + + def create_module(self, spec): + return self.loader.create_module(spec) + + def exec_module(self, module): + self.loader.exec_module(module) + PATCHES[module.__name__](module) + + +class _Floor: + """Gives a PATCHES module's spec a _Loader on each import and reload; known by MARK, so a second run adds none.""" + + def find_spec(self, name, path, target=None): + if name not in PATCHES: + return None + for finder in sys.meta_path: + if getattr(finder, MARK, False) or not hasattr(finder, "find_spec"): + continue + spec = finder.find_spec(name, path, target) + if spec is not None: + break + else: + return None + if spec.loader is not None and hasattr(spec.loader, "exec_module"): + spec.loader = _Loader(spec.loader) + return spec + + +setattr(_Floor, MARK, True) +PATCHES = {DESKTOP_ENV: floor_desktop_env, EPISODE: stop_when_asked, DOCKER_PROVIDER: queue_starts} + + +def wrap_desktop_env(): + """A module of PATCHES already loaded is wrapped, and the finder goes on for its later loads.""" + for name in PATCHES: + if name in sys.modules: + PATCHES[name](sys.modules[name]) + if not any(getattr(f, MARK, False) for f in sys.meta_path): + sys.meta_path.insert(0, _Floor()) + + +def hide_proxy_password(environ=os.environ): + """Once a process, as wrap_docker_create.""" + try: + with open(environ["PROXY_CONFIG_FILE"], encoding="utf-8") as f: + words = {p["password"] for p in json.load(f) if isinstance(p.get("password"), str)} - {""} + except Exception: + return + plain = logging.Formatter.format + if not words or getattr(plain, MARK, False): + return + + @functools.wraps(plain) + def hidden(self, record): + text = plain(self, record) + for word in words: + text = text.replace(word, HIDDEN) + return text + + setattr(hidden, MARK, True) + logging.Formatter.format = hidden + logging.raiseExceptions = False + + +if __name__ == "sitecustomize": + hide_proxy_password() +if __name__ == "sitecustomize" and os.environ.get("NULLIUS_OPTION_VM_PROVIDER", "docker") == "docker": + wrap_docker_create() + wrap_desktop_env() +if __name__ == "__main__": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + if not sys.stdin.buffer.read(): + sweep(sys.argv[1]) diff --git a/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json b/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json new file mode 100644 index 00000000..c2b66f93 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json @@ -0,0 +1,835 @@ +[ + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--base_url", + "values": "an Anthropic-compatible endpoint", + "default": "$MINIMAX_ANTHROPIC_BASE_URL, else $ANTHROPIC_BASE_URL, else MiniMax's endpoint for MINIMAX_REGION", + "nullius": "fixed:the unit's relay, without /v1 - every model call goes through the relay, which holds the key; its documented invocation passes --base_url too" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--api_key", + "values": "a key", + "default": "$MINIMAX_API_KEY, else $ANTHROPIC_API_KEY", + "nullius": "fixed:the relay's placeholder - the provider's key stays in the relay; passed so that no MINIMAX_API_KEY from a .env file comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--extra_headers", + "values": "a JSON object of headers", + "default": "$ANTHROPIC_EXTRA_HEADERS, else unset", + "nullius": "fixed:unset - its default; a gateway's routing headers, which the relay's providers do not need" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--path_to_vm", + "values": "a VM image", + "default": "unset: the docker manager downloads it into ./docker_vm_data", + "nullius": "fixed:the upstream's docker_vm_data/osworld-v2-ubuntu-x86.qcow2, through a link in the unit's folder - the image its docker manager downloads, put there once before a run; its containers are found by that link" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--headless", + "values": "on, off", + "default": "off", + "nullius": "fixed:on - its documented invocation; the docker provider ignores it" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--action_space", + "values": "pyautogui", + "default": "pyautogui", + "nullius": "fixed:pyautogui - its default and its documented invocation; M3's parser writes pyautogui code" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--observation_type", + "values": "screenshot, a11y_tree, screenshot_a11y_tree, som", + "default": "screenshot", + "nullius": "fixed:screenshot - its default and its documented invocation; M3Agent reads screenshots" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--provider_name", + "values": "aws, virtualbox, vmware, docker, azure", + "default": "aws (its documented invocation: $PROVIDER_NAME, else aws)", + "nullius": "option:vm_provider" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--client_password", + "values": "a password", + "default": "empty: DesktopEnv then uses osworld-public-evaluation", + "nullius": "fixed:osworld-public-evaluation - its documented invocation's CLIENT_PASSWORD, the v2 image's password" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--screen_width", + "values": "pixels", + "default": "1920", + "nullius": "fixed:1920 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--screen_height", + "values": "pixels", + "default": "1080", + "nullius": "fixed:1080 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--sleep_after_execution", + "values": "seconds", + "default": "3.0", + "nullius": "fixed:3.0 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_steps", + "values": "a number", + "default": "100", + "nullius": "option:max_steps" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--checkpoint_eval_mode", + "values": "off, inline", + "default": "off", + "nullius": "fixed:off - its default: one evaluation, after the last step, written to result.txt" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--checkpoint_steps", + "values": "comma-separated step numbers", + "default": "empty", + "nullius": "fixed:empty - its default; read only with inline checkpoints" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--trace_guest (--trace_guest_memory)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; a debugging trace of the VM's resources" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--guest_trace_top_n (--guest_memory_top_n)", + "values": "a number", + "default": "30", + "nullius": "fixed:30 - its default; read only with --trace_guest" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--guest_trace_timeout (--guest_memory_trace_timeout)", + "values": "seconds", + "default": "15", + "nullius": "fixed:15 - its default; read only with --trace_guest" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--preserve_problem_machine", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; the aws provider's, which is not offered" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--enable_vnc (--disable_vnc)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; a viewer for a person watching the VM" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--enable_recording (--disable_recording)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; no recording.mp4 is kept" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_trajectory_length", + "values": "a number", + "default": "10", + "nullius": "fixed:10 - its default, the screenshots M3Agent keeps in its history" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--coordinate_type", + "values": "relative, absolute", + "default": "relative", + "nullius": "fixed:relative - its default: M3's [0, 1000] coordinates scaled to the screen" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--thinking_mode", + "values": "adaptive, disabled, unset", + "default": "$M3_THINKING_MODE, else unset", + "nullius": "option:thinking" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_THINKING_MODE", + "values": "adaptive, disabled, empty", + "default": "empty in its documented invocation", + "nullius": "option:thinking" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--thinking_budget", + "values": "a number of tokens", + "default": "$M3_THINKING_BUDGET, else unset", + "nullius": "fixed:unset - its documented invocation leaves M3_THINKING_BUDGET empty; a budget turns thinking on by itself" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_THINKING_BUDGET", + "values": "a number of tokens", + "default": "empty in its documented invocation", + "nullius": "fixed:empty - its documented invocation's; set, it turns thinking on with that budget whatever the thinking option says" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_config_base_dir", + "values": "a folder", + "default": "evaluation_examples", + "nullius": "fixed:the upstream's evaluation_examples - its default, by its absolute path since each unit runs in its own folder" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--model", + "values": "a model name", + "default": "$MINIMAX_MODEL, else $ANTHROPIC_MODEL, else MiniMax-M3", + "nullius": "setup:original" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_MODEL, ANTHROPIC_MODEL", + "values": "a model name", + "default": "unset", + "nullius": "fixed:unset - --model names the setup's model" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--temperature", + "values": "a number", + "default": "0.6", + "nullius": "fixed:0.6 - its default (M3Agent sends 1 when thinking is on)" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--top_p", + "values": "a number", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_tokens", + "values": "a number", + "default": "8192", + "nullius": "fixed:8192 - its default and its documented invocation's MAX_TOKENS" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--stop_token", + "values": "a string", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--domain", + "values": "all, or one key of the meta file", + "default": "all", + "nullius": "fixed:all - its default; a unit's meta file names its one task" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_all_meta_path", + "values": "evaluation_examples/test_v2.json (108 tasks), test_v2_offline_no_internet.json (82), or one of ten lists by capability, each a subset of the 108 (conflict_disambiguation, cross_source_reasoning, dynamic_environment, human_in_the_loop, implicit_state_inference, multi_item_state_tracking, multimodal_editing, streaming_interaction, tutorial_following, visual_spatial_precision)", + "default": "evaluation_examples/test_v2.json", + "nullius": "dataset:test_v2" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_all_meta_path evaluation_examples/test_v2_offline_no_internet.json", + "values": "the upstream's offline list, 82 tasks", + "default": "-", + "nullius": "dataset:test_v2_offline_no_internet" + }, + { + "entry": "evaluation_examples/test_v2_offline_no_internet.json", + "setting": "its tasks that need no service", + "values": "36 of its 82: no mock website, GitLab, Moodle, LLM grader or simulated user", + "default": "-", + "nullius": "dataset:offline36" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need no hosted service", + "values": "63 of its 108: no mock website, GitLab or proxy; the LLM grader, the simulated user and the public internet allowed", + "default": "-", + "nullius": "dataset:no_services63" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need the mock websites", + "values": "35 of its 108: the mock websites, the LLM grader, the simulated user and the public internet allowed; no GitLab or proxy", + "default": "-", + "nullius": "dataset:websites35" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need no GitLab or proxy", + "values": "98 of its 108: No hosted services and the 35 that need the mock websites", + "default": "-", + "nullius": "dataset:websites98" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--eval_version", + "values": "v1, v2", + "default": "v2", + "nullius": "fixed:v2 - its default: the task classes of evaluation_examples/task_class, not OSWorld's v1 examples" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--specific_task_id", + "values": "a task id", + "default": "unset", + "nullius": "fixed:unset - its default; a unit's meta file names its one task" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--result_dir", + "values": "a folder", + "default": "./results", + "nullius": "fixed:results in the unit's folder - each trial keeps its own" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--num_envs", + "values": "a number", + "default": "1", + "nullius": "fixed:1 - its default and its documented invocation; Nullius runs units side by side (-j), each in a VM of its own" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--env_start_delay", + "values": "seconds", + "default": "1.0", + "nullius": "fixed:1.0 - its default; one environment, so no stagger happens" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--log_level", + "values": "DEBUG, INFO, WARNING, ERROR, CRITICAL", + "default": "INFO", + "nullius": "fixed:INFO - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--region", + "values": "an AWS region", + "default": "us-east-1 (its documented invocation: $AWS_DEFAULT_REGION)", + "nullius": "not offered - the aws provider's, which is not offered" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--snapshot_name", + "values": "a snapshot", + "default": "init_state", + "nullius": "fixed:init_state - its default; read by the vmware and virtualbox providers only" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--os_type", + "values": "Ubuntu, Windows", + "default": "Ubuntu", + "nullius": "fixed:Ubuntu - its default; the image is the v2 Ubuntu one" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--use_public_ip", + "values": "on, off", + "default": "off (its documented invocation: on when USE_PUBLIC_IP=1)", + "nullius": "fixed:off - its default; cloud providers only" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--save_model_eval_raw_info", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; the LLM grader's inputs, and the grader is not set up" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_ANTHROPIC_BASE_URL, ANTHROPIC_BASE_URL", + "values": "an endpoint", + "default": "unset", + "nullius": "fixed:ANTHROPIC_BASE_URL the unit's relay, MINIMAX_ANTHROPIC_BASE_URL unset - --base_url passes the relay and comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_API_KEY, ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN", + "values": "a key", + "default": "unset", + "nullius": "fixed:ANTHROPIC_API_KEY the relay's placeholder, the others unset - --api_key passes the placeholder and comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_REGION", + "values": "global_en, cn_zh", + "default": "global_en", + "nullius": "fixed:unset - it picks MiniMax's own endpoint, which --base_url replaces with the relay" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_IMAGE_FORMAT", + "values": "JPEG, PNG", + "default": "JPEG", + "nullius": "fixed:JPEG - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_IMAGE_QUALITY", + "values": "1 to 100", + "default": "90", + "nullius": "fixed:90 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_MAX_LLM_RETRIES", + "values": "a number", + "default": "2", + "nullius": "fixed:2 - its default: a reply with no action is asked again twice at most" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "DesktopEnv(enable_proxy=...)", + "values": "True, False", + "default": "True, hard-coded", + "nullius": "fixed:True - hard-coded in the runner: the 8 tasks whose class asks for a proxy (3 of Offline, none of Offline, no services) get the upstream's DataImpulse setup" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": ".env (load_dotenv)", + "values": "a file of variables", + "default": "loaded when present in the script's folder or a folder above it, by the script and by desktop_env/controllers/{setup,website,gitlab,moodle}.py", + "nullius": "fixed:what a .env file there holds - load_dotenv never overrides a variable already set; the driver sets ANTHROPIC_BASE_URL, ANTHROPIC_API_KEY, M3_THINKING_MODE, M3_THINKING_BUDGET, the grader's and the simulated user's provider, model, base URL and key, and passes --base_url, --api_key and --model, so such a file can still set any variable it leaves unset, among them ANTHROPIC_EXTRA_HEADERS, MINIMAX_REGION, M3_IMAGE_FORMAT, M3_IMAGE_QUALITY and M3_MAX_LLM_RETRIES" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "MODEL, RESULT_DIR, PROVIDER_NAME, OBSERVATION_TYPE, ACTION_SPACE, MAX_STEPS, NUM_ENVS, MAX_TOKENS, CLIENT_PASSWORD, AWS_DEFAULT_REGION, USE_PUBLIC_IP", + "values": "the wrapper's own variables", + "default": "MiniMax-M3, ./results/osworld_v2/m3, aws, screenshot, pyautogui, 100, 1, 8192, osworld-public-evaluation, us-east-1, 0", + "nullius": "fixed:the arguments they become - agents.json passes those arguments, each a row of run_multienv_m3.py" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "AWS_REGION, AWS_SUBNET_ID, AWS_SECURITY_GROUP_ID, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_INSTANCE_NAME, ENABLE_TTL", + "values": "the aws provider's settings", + "default": "exported with the wrapper's own values", + "nullius": "not offered - the aws provider's, which is not offered" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OPENAI_API_KEY", + "values": "a key", + "default": "exported empty", + "nullius": "fixed:unset - M3 does not read it" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_CLIENT_PASSWORD", + "values": "a password", + "default": "osworld-public-evaluation", + "nullius": "fixed:unset - exported, and read by none of the code M3 runs; --client_password passes the same password" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "WEBSITE_HOST_SUFFIX", + "values": "the mock websites' host suffix: that of OSWorld-web at the tag osworld-v2.1 served for this machine's VMs (websites.py); none refuses the 41 tasks that need it before they run", + "default": "web.hku.icu", + "nullius": "option:websites" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "GITLAB_URL", + "values": "a GitLab server", + "default": "the wrapper's server", + "nullius": "fixed:http://gitlab. - Task-Web/gitlab, which Nullius makes on the run's machine behind the mock websites' Caddy" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "GITLAB_PRIVATE_TOKEN", + "values": "the GitLab server's root token", + "default": "an empty token", + "nullius": "fixed:the root token Nullius makes - with that GitLab, kept on that machine and given to the processes of the 2 GitLab tasks alone" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_EVAL_MODEL_PROVIDER, OSWORLD_EVAL_MODEL_API_KEY_ENV, OSWORLD_EVAL_MODEL_NAME", + "values": "the LLM grader's provider, key and model", + "default": "anthropic, ANTHROPIC_API_KEY, claude-sonnet-4-6", + "nullius": "fixed:anthropic, OSWORLD_EVAL_MODEL_API_KEY the relay's placeholder, the benchmark's judge (anthropic/claude-sonnet-4.6, OpenRouter's name for claude-sonnet-4-6) - the grader asks the unit's judge route, which holds the key and records its calls apart from the agent's" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_USER_SIM_PROVIDER, OSWORLD_USER_SIM_API_KEY_ENV, OSWORLD_USER_SIM_MODEL", + "values": "the simulated user's provider, key and model", + "default": "anthropic, ANTHROPIC_API_KEY, claude-sonnet-4-6", + "nullius": "fixed:anthropic, OSWORLD_USER_SIM_API_KEY the relay's placeholder, claude-sonnet-4-6 - the simulated user asks the unit's own route, which sends that model on as anthropic/claude-sonnet-4.6, a side model of the Original setup" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_FILE_BASE_URL", + "values": "a folder or a URL of the task files", + "default": "$OSWORLD_FILE_BASE_URL, else ./cache/osworld_v2_assets", + "nullius": "fixed:the upstream's cache/osworld_v2_assets, by its absolute path - the release's gated assets, put there before a run: a file a task names through asset() is copied from there into the unit's task cache, not downloaded; for 001, 006 and 010 a folder of the unit's own, their mail client started offline (driver.py _assets)" + }, + { + "entry": "desktop_env/controllers/website.py", + "setting": "OSWORLD_WEBSITE_TLS_VERIFY", + "values": "0, 1", + "default": "0", + "nullius": "fixed:unset (0) - the stack serves plain HTTP, which the controller falls back to when its HTTPS probe fails" + }, + { + "entry": "desktop_env/controllers/moodle.py", + "setting": "MOODLE_API_URL, MOODLE_API_KEY", + "values": "a Moodle server and its key", + "default": "unset", + "nullius": "not offered - no Moodle is hosted: no task of osworld-v2.1 uses it" + }, + { + "entry": "desktop_env/evaluators/model_client.py", + "setting": "the other OSWORLD_EVAL_MODEL_* variables, OPENAI_BASE_URL, MINIMAX_OPENAI_BASE_URL, LLM_DEBUG", + "values": "the LLM grader's endpoint, sampling, retries and logging", + "default": "unset", + "nullius": "fixed:OSWORLD_EVAL_MODEL_BASE_URL the unit's judge route without /v1, the others unset - the Anthropic client adds /v1/messages; sampling, retries and logging stay the upstream's defaults" + }, + { + "entry": "desktop_env/user_simulator.py", + "setting": "the other OSWORLD_USER_SIM_* variables", + "values": "the simulated user's endpoint, sampling and logging", + "default": "unset", + "nullius": "fixed:OSWORLD_USER_SIM_BASE_URL the unit's relay without /v1, the others unset - sampling and logging stay the upstream's defaults" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "PROXY_CONFIG_FILE", + "values": "a proxy pool file", + "default": "evaluation_examples/settings/proxy/dataimpulse.json", + "nullius": "fixed:its default, whose username and password are placeholders - the 8 tasks that ask for a proxy get no working proxy, and by the code their setup can fail; not checked by a run" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "HF_REPO, HF_TOKEN", + "values": "a mirror of the public file cache, a token", + "default": "unset", + "nullius": "fixed:unset - its default; read only for a download from huggingface.co, and no task of Offline, no services downloads from there" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "OSWORLD_STRICT_SETUP_TRANSPORT", + "values": "1, unset", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "desktop_env/desktop_env.py", + "setting": "SCREEN_WIDTH, SCREEN_HEIGHT", + "values": "pixels", + "default": "1920, 1080", + "nullius": "fixed:unset - the runner passes --screen_width and --screen_height" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "image", + "values": "a Docker image", + "default": "happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9", + "nullius": "fixed:the upstream's, pinned by digest - pulled before a run, since the provider would pull it within its port lock" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "DISK_SIZE, RAM_SIZE, CPU_CORES", + "values": "a size, a size, a count", + "default": "32G (or a task's own volume size), 4G, 4", + "nullius": "fixed:the task's own volume size, at least 40G (startup/sitecustomize.py, as the upstream's aws provider gives), 4G, 4 - the rest hard-coded in the upstream" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "LOCK_TIMEOUT", + "values": "seconds", + "default": "10, hard-coded", + "nullius": "fixed:none (startup/sitecustomize.py) - every VM start waits for the lock, a restart inside DesktopEnv.reset() too; the driver also starts units' VMs one after another" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "docker.from_env() timeout", + "values": "seconds a Docker call", + "default": "60, docker-py's default", + "nullius": "fixed:120 once the client creates the VM's container (startup/sitecustomize.py), and kept for its later calls (the containers' list, the stop and the removal) - a daemon busy with neighbouring VMs can take longer to create one; a create the client still times out on is not sent again, and the container the daemon made with that create's label is started" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "_wait_for_vm_ready(timeout)", + "values": "seconds", + "default": "300, hard-coded", + "nullius": "fixed:300 - hard-coded" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "KVM", + "values": "/dev/kvm when present", + "default": "hardware acceleration when /dev/kvm exists", + "nullius": "fixed:the machine's - with /dev/kvm the VM is accelerated, without it the upstream runs it unaccelerated" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "device_write_bps", + "values": "bytes a second per device", + "default": "unset: no limit, the provider's containers.run passes none", + "nullius": "option:write_mbps" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "device_write_iops", + "values": "write operations a second per device", + "default": "unset: no limit, the provider's containers.run passes none", + "nullius": "option:write_iops" + }, + { + "entry": "desktop_env/providers/docker/manager.py", + "setting": "UBUNTU_X86_URL, HF_ENDPOINT", + "values": "the image's address; a mirror", + "default": "huggingface.co/datasets/xlangai/v2-image at 6e16459a, pinned", + "nullius": "fixed:the upstream's pinned image - fetched and unpacked into docker_vm_data before a run, since an interrupted unpacking leaves a broken image" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "the Anthropic client's max_retries, timeout", + "values": "a count, seconds", + "default": "4, 180 s", + "nullius": "fixed:4, 180 s - hard-coded in M3Agent" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "stop_sequences", + "values": "a list of strings", + "default": ", Perform the next action. Perform", + "nullius": "fixed:its default - the runner passes none" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "system_date", + "values": "a date in the system prompt", + "default": "the task's own date when its class sets one, else the day M3Agent is created", + "nullius": "fixed:its default - the runner passes none, and creates M3Agent for each unit: the model sees the day the unit starts, unless the task sets its own (4 of the 108 tasks, 1 of Offline, no services)" + }, + { + "entry": "lib_run_single.py", + "setting": "time.sleep(60) after reset", + "values": "seconds", + "default": "60, hard-coded", + "nullius": "fixed:60 - hard-coded: a slow or loaded machine can show the agent a desktop that has not caught up" + }, + { + "entry": "task_loader.py", + "setting": "the task's class", + "values": "evaluation_examples/task_class/task_NNN.py", + "default": "downloaded by scripts/tools/download_osworld_v2_tasks.py", + "nullius": "fixed:the release's gated task classes - put in evaluation_examples/task_class before a run" + }, + { + "entry": "scripts/python/run_multienv.py", + "setting": "its agent harness", + "values": "PromptAgent, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_claude.py", + "setting": "its agent harness", + "values": "Claude, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_claude_code.py", + "setting": "its agent harness", + "values": "Claude Code, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_codex.py", + "setting": "its agent harness", + "values": "Codex, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_deepseek.py", + "setting": "its agent harness", + "values": "DeepSeek, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_gemini.py", + "setting": "its agent harness", + "values": "Gemini, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_glm.py", + "setting": "its agent harness", + "values": "GLM, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_gpt_response_api.py", + "setting": "its agent harness", + "values": "GPT (Responses API), run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_muse_code.py", + "setting": "its agent harness", + "values": "Muse Code, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_muse_spark.py", + "setting": "its agent harness", + "values": "Muse Spark, run as published", + "default": "-", + "nullius": "setup:original" + }, + { + "entry": "scripts/python/run_multienv_muse_spark.py", + "setting": "--max_steps", + "values": "a number", + "default": "15", + "nullius": "option:max_steps" + }, + { + "entry": "scripts/python/run_multienv_qwen.py", + "setting": "its agent harness", + "values": "Qwen, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/dispatch_deepseek_fullset.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/resume_deepseek_queue.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/rerun_deepseek_setup.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/record_deepseek_provenance.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/summarize_deepseek.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "run.py", + "setting": "its agent harness", + "values": "run as published", + "default": "-", + "nullius": "not offered - a single-environment runner of OSWorld's v1 examples" + }, + { + "entry": "scripts/python/manual_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/python/interactive_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/python/debug_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_human_examine.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_interactive_examine.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_debug.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_manual_examine_windows.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_mannual_examine_windows.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "the paper (arXiv 2606.29537)", + "setting": "M3's run, Table 3", + "values": "500 steps, thinking on, 8K output tokens, AWS instances with all web traffic routed through a residential proxy, release osworld-v2-2026.06.24", + "default": "-", + "nullius": "setup:original" + } +] diff --git a/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml b/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml new file mode 100644 index 00000000..f75e401e --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml @@ -0,0 +1,176 @@ +name: osworld2-web + +x-blkio: &blkio + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_QUIET_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_QUIET_IOPS:?as websites.py sets it} + +x-logging: &logs + driver: json-file + options: + max-size: 10m + max-file: "3" + +services: + awsconsole: + blkio_config: *blkio + logging: *logs + budgetwise_web_backend: + blkio_config: *blkio + logging: *logs + budgetwise_web_frontend: + blkio_config: *blkio + logging: *logs + caddy: + image: lucaslorentz/caddy-docker-proxy@sha256:33081120c6df8613cebbdbdd4c3b8aef641e9e56c1cd3d86d04ea9651cb3cc68 + ports: !override + - ${OSWORLD2_WEB_ADDRESS:?the address Caddy listens on, as websites.py sets it}:80:80 + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CADDY_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CADDY_IOPS:?as websites.py sets it} + logging: *logs + calendar_web: + blkio_config: *blkio + logging: *logs + careerlink_web: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CAREERLINK_WEB_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CAREERLINK_WEB_IOPS:?as websites.py sets it} + logging: *logs + cloudcrm_web_backend: + blkio_config: *blkio + logging: *logs + cloudcrm_web_frontend: + blkio_config: *blkio + logging: *logs + dinogame_web: + blkio_config: *blkio + logging: *logs + eventix_web_backend: + blkio_config: *blkio + logging: *logs + eventix_web_frontend: + blkio_config: *blkio + logging: *logs + expenseflow_web_backend: + blkio_config: *blkio + logging: *logs + expenseflow_web_frontend: + blkio_config: *blkio + logging: *logs + formcraft_web_backend: + blkio_config: *blkio + logging: *logs + formcraft_web_frontend: + blkio_config: *blkio + logging: *logs + glbviewer_web: + blkio_config: *blkio + logging: *logs + insurance_claim_web_backend: + blkio_config: *blkio + logging: *logs + insurance_claim_web_frontend: + blkio_config: *blkio + logging: *logs + interfacinglinux_web: + blkio_config: *blkio + logging: *logs + mailhub_web_backend: + blkio_config: *blkio + logging: *logs + mailhub_web_frontend: + blkio_config: *blkio + logging: *logs + overleaf_collab_compiler: + blkio_config: *blkio + logging: *logs + overleaf_collab_web: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_COLLAB_WEB_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_COLLAB_WEB_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_admin_bootstrap: + blkio_config: *blkio + logging: *logs + overleaf_web_app: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_APP_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_APP_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_mongo: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_MONGO_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_MONGO_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_mongo_init: + blkio_config: *blkio + logging: *logs + overleaf_web_redis: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_REDIS_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_REDIS_IOPS:?as websites.py sets it} + logging: *logs + reviewsphere: + blkio_config: *blkio + logging: *logs + slidepuzzle_web: + blkio_config: *blkio + logging: *logs + streamview_web_backend: + blkio_config: *blkio + logging: *logs + streamview_web_frontend: + blkio_config: *blkio + logging: *logs + teamchat_web_backend: + blkio_config: *blkio + logging: *logs + teamchat_web_frontend: + blkio_config: *blkio + logging: *logs + travelhub_pro_web: + blkio_config: *blkio + logging: *logs + trippza_web: + blkio_config: *blkio + logging: *logs + vaultbank_web_backend: + blkio_config: *blkio + logging: *logs + vaultbank_web_frontend: + blkio_config: *blkio + logging: *logs + visaapplication_web: + blkio_config: *blkio + logging: *logs + wandb: + blkio_config: *blkio + logging: *logs diff --git a/app/nullius/catalogue/benchmarks/osworld2/websites.py b/app/nullius/catalogue/benchmarks/osworld2/websites.py new file mode 100644 index 00000000..35e31909 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/websites.py @@ -0,0 +1,668 @@ +"""OSWorld2's mock websites: Task-Web/OSWorld-web at the release's tag, as its tasks reach them. + +The sites are OSWorld-web's compose project at COMMIT with websites.compose.yml +over it; tasks of TASKS reach them at ., the run's websites +option. Caddy listens on the VMs' network's gateway only. Overleaf keeps no +slice per task, so its tasks run one at a time under LOCK, on fresh volumes; +a unit that needs the sites holds SITES_LOCK shared, reset, smoke and compose's changes alone. + +python3 websites.py configure STACK SUFFIX --write-mbps N --write-iops N +python3 websites.py builder STACK up --write-mbps N --write-iops N | down +python3 websites.py build STACK [--group NAME]... [--reserve-gb 25] +python3 websites.py compose STACK ARGS... +python3 websites.py check SUFFIX | smoke STACK | reset STACK""" +import argparse +import contextlib +import errno +import fcntl +import importlib.util +import json +import os +import re +import shutil +import socket +import stat +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +OVERRIDE = os.path.join(HERE, "websites.compose.yml") +SITE = os.path.join(HERE, "startup", "sitecustomize.py") +COMMIT = "60c89fe6a8ed934668619d8d26132848239eb8ee" +PROJECT = "osworld2-web" +SCHEME = "http://" +OPTION = "NULLIUS_OPTION_WEBSITES" +SETUP_DOC = "docs/benchmarks/osworld2-machine.md" +STATEFUL = ("awsconsole", "budgetwise", "calendar", "careerlink", "cloudcrm", "dinogame", "eventix", "expenseflow", + "formcraft", "glbviewer", "insurance-claim", "mailhub", "overleaf-collab", "reviewsphere", "slidepuzzle", + "streamview", "teamchat", "travelhubpro", "trippza", "vaultbank", "visaapplication", "wandb") +NAMES = STATEFUL + ("studio.streamview", "interfacinglinux", "overleaf") +TASKS = frozenset("005 007 008 013 015 016 018 019 020 021 026 028 031 035 036 038 039 043 046 050 052 056 057 058 " + "060 065 068 069 070 072 073 077 079 081 082 091 098 100 101 102".split()) +OVERLEAF_TASKS = frozenset(("057", "072")) +OVERLEAF = ("overleaf_web_mongo", "overleaf_web_mongo_init", "overleaf_web_redis", "overleaf_web_admin_bootstrap", + "overleaf_web_app") +OVERLEAF_VOLUMES = ("overleaf_web_data", "overleaf_web_mongo_data", "overleaf_web_redis_data") +LOCK = "/tmp/nullius-osworld2-overleaf.lck" +SITES_LOCK = "/tmp/nullius-osworld2-websites.lck" +READING = frozenset(("config", "ps", "ls", "logs", "images", "top", "port", "events", "version", "stats", "wait")) +VALUED = frozenset(("-f", "--file", "-p", "--project-name", "--project-directory", "--profile", "--env-file", + "--ansi", "--progress", "--parallel")) +COMPOSE_VERSION = (2, 24, 4) +_HELD = [] +_SITE = [] +COOKIE = "nullius-osworld2-ready" +SUFFIX = re.compile(r"(?=.{3,200}$)[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?)+") +GROUPS = (("pulled", ("caddy", "overleaf_web_mongo", "overleaf_web_mongo_init", "overleaf_web_redis", + "overleaf_web_admin_bootstrap"), 7), + ("overleaf", ("overleaf_web_app", "overleaf_collab_compiler"), 12), + ("next-a", ("awsconsole", "calendar_web", "careerlink_web", "dinogame_web", "glbviewer_web", "reviewsphere"), 9), + ("next-b", ("slidepuzzle_web", "travelhub_pro_web", "trippza_web", "visaapplication_web", "wandb", + "overleaf_collab_web"), 9), + ("frontends", ("budgetwise_web_frontend", "cloudcrm_web_frontend", "eventix_web_frontend", + "expenseflow_web_frontend", "formcraft_web_frontend", "mailhub_web_frontend", + "streamview_web_frontend", "teamchat_web_frontend", "vaultbank_web_frontend", + "insurance_claim_web_frontend", "interfacinglinux_web"), 7), + ("backends", ("budgetwise_web_backend", "cloudcrm_web_backend", "eventix_web_backend", + "expenseflow_web_backend", "formcraft_web_backend", "insurance_claim_web_backend", + "mailhub_web_backend", "streamview_web_backend", "teamchat_web_backend", + "vaultbank_web_backend"), 4)) +SERVICES = tuple(s for _, group, _ in GROUPS for s in group) +WEIGHTS = {"overleaf_web_mongo": (30, 10), "overleaf_web_app": (30, 6), "overleaf_web_redis": (10, 4), + "overleaf_collab_web": (46, 22), "careerlink_web": (6, 2), "caddy": (4, 2)} +QUIET = (1, 1) +TIERS = ("QUIET",) + tuple(s.upper() for s in WEIGHTS) +SETTINGS = ("HOST_SUFFIX", "CADDY_SCHEME", "OSWORLD2_WEB_ADDRESS", "OSWORLD2_WEB_DISK") + tuple( + f"OSWORLD2_WEB_{t}_{k}" for t in TIERS for k in ("BPS", "IOPS")) +MB = 1024 * 1024 +BUILDER = "osw2-web" +BUILDKIT = "moby/buildkit@sha256:de10faf919fc71ba4eb1dd7bd6449566d012b0c9436b1c61bfee21d621b009aa" +BUILDKITD = "nullius-osworld2-buildkitd" +BUILDKIT_VOLUME = "nullius-osworld2-buildkit" +BUILDKIT_PARALLEL = 4 +RESERVE_GB = 25 +IMAGES_FILE = ".osworld2.images.json" +BUILDKIT_FILE = ".osworld2.buildkitd.toml" +ENV_FILE = ".osworld2.env" +COMPOSE_FILE = ".osworld2.compose.yml" +LABEL = "com.docker.compose." +CADDY = re.compile(r"^\s+image: (lucaslorentz/caddy-docker-proxy@sha256:[0-9a-f]{64})$", re.M) +PROBE = "nullius-osworld2-web-probe-" +HTTP_SECONDS = 10 +DOCKER_SECONDS = 300 +OVERLEAF_SECONDS = 600 +RESET_SECONDS = 900 +BUILD_SECONDS = 4 * 3600 +POLL = 5 +PAGE_BYTES = 1 << 20 +PROXIES = {} +START = "start the stack (python3 websites.py compose STACK up -d) and check it (python3 websites.py check SUFFIX)" + + +class WebsiteError(RuntimeError): + pass + + +def suffix(environ=os.environ): + """The run's websites option, the sites' host suffix; None for none.""" + value = environ.get(OPTION, "none") + if value == "none": + return None + if not SUFFIX.fullmatch(value): + raise WebsiteError(f"Website host suffix {value!r} is not a host suffix: give the suffix the stack serves " + f"(such as {_site().GATEWAY}.nip.io), or none") + return value + + +def env(environ=os.environ): + """The variables the upstream reads the sites by, for every process the driver starts.""" + value = suffix(environ) + return {"WEBSITE_HOST_SUFFIX": value} if value else {} + + +def needed(eid, environ=os.environ): + """A task of TASKS without a suffix is refused, since its class cannot even load.""" + value = suffix(environ) + if eid not in TASKS: + return None + if not value: + raise WebsiteError(f"task {eid} needs the mock websites and this run gives it none: set Website host suffix " + f"to the suffix the stack serves ({SETUP_DOC})") + return value + + +def _resolves(host): + try: + return {a[4][0] for a in socket.getaddrinfo(host, 80, type=socket.SOCK_STREAM)} + except OSError: + return set() + + +def _fetch(opener, url): + """Any failure, a status of 400 or more among them, is a WebsiteError.""" + try: + with opener.open(urllib.request.Request(url, headers={"User-Agent": "nullius-osworld2"}), + timeout=HTTP_SECONDS) as r: + return r.read(PAGE_BYTES) + except urllib.error.HTTPError as e: + e.close() + raise WebsiteError(f"{url} answered {e.code}: {START}") from None + except (urllib.error.URLError, OSError, ValueError) as e: + reason = getattr(e, "reason", e) + raise WebsiteError(f"{url} did not answer ({type(reason).__name__}: {reason}): {START}") from None + + +def ready(value, overleaf=False, state=True): + """The number of names checked; Overleaf's only with ``overleaf``, each /api/state only with ``state``.""" + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + names = NAMES if overleaf else tuple(n for n in NAMES if n != "overleaf") + gateway = _site().GATEWAY + for name in names: + host = f"{name}.{value}" + found = _resolves(host) + if not found: + raise WebsiteError(f"{host} does not resolve: Website host suffix must be a wildcard DNS name of the " + f"address the stack listens on, {gateway} (such as {gateway}.nip.io)") + if gateway not in found: + raise WebsiteError(f"{host} resolves to {', '.join(sorted(found))}, not {gateway}, the VMs' network's " + f"gateway the stack listens on: set Website host suffix to a wildcard DNS name of " + f"{gateway} (such as {gateway}.nip.io)") + if not _fetch(opener, f"{SCHEME}{host}/").strip(): + raise WebsiteError(f"{SCHEME}{host}/ answered an empty page: the stack serves another suffix than " + f"{value}, or not this site; {START}") + for name in STATEFUL if state else (): + url = f"{SCHEME}{name}.{value}/api/state?cookie={COOKIE}" + try: + said = json.loads(_fetch(opener, url)) + except ValueError: + said = None + if not isinstance(said, dict) or said.get("user_id") != COOKIE or not isinstance(said.get("state"), dict): + raise WebsiteError(f"{url} answered no state of the site's: {START}") + return len(names) + + +def _docker(*args, seconds=DOCKER_SECONDS): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, + env=_clean(), stdin=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + raise WebsiteError(f"docker {args[0]} got no answer in {seconds} s: is its daemon running?") from None + except OSError: + raise WebsiteError("no docker on this machine's PATH: the mock websites run on Docker") from None + + +def _clean(): + """Without the stack's settings, so compose reads them from its environment file alone.""" + return {k: v for k, v in os.environ.items() if k not in SETTINGS} + + +def _done(said, what): + if said.returncode != 0: + last = ((said.stderr or "").strip().splitlines() or ["no reason"])[-1] + raise WebsiteError(f"{what} failed: {last[:300]}") + return said + + +def _settings(path): + try: + with open(path, encoding="utf-8") as f: + return dict(ln.rstrip("\n").partition("=")[::2] for ln in f if "=" in ln) + except OSError as e: + raise WebsiteError(f"no settings of the stack at {path} ({type(e).__name__}): run python3 websites.py " + "configure first") from None + + +def _project(): + """The running project's compose command and settings, from any of its containers' labels.""" + fields = ("project.config_files", "project.working_dir", "project.environment_file") + said = _docker("ps", "-a", "--filter", f"label={LABEL}project={PROJECT}", + "--format", "\t".join('{{.Label "' + LABEL + f + '"}}' for f in fields)) + rows = [ln.split("\t") for ln in said.stdout.splitlines() if ln.strip()] if said.returncode == 0 else [] + if not rows: + raise WebsiteError(f"no container of the {PROJECT} project on this machine: {START}") + files, where, envfile = (rows[0] + ["", "", ""])[:3] + if not envfile or not files: + raise WebsiteError(f"the {PROJECT} project was not started by websites.py compose, which gives it its " + "settings: start it that way") + return (["compose", "-p", PROJECT, "--project-directory", where, + *[x for f in files.split(",") for x in ("-f", f)], "--env-file", envfile], _settings(envfile)) + + +def _waited(url, seconds, closed, what): + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + t0, last = time.time(), "no answer" + while True: + try: + if _fetch(opener, url).strip(): + return + last = "an empty page" + except WebsiteError as e: + last = str(e).split(": ", 1)[0] + if time.time() - t0 >= seconds: + raise WebsiteError(f"{what} did not come up in {seconds} s ({last})") + if closed is None: + time.sleep(POLL) + elif closed.wait(POLL): + raise WebsiteError(f"{what} was not up when the unit's time ran out") + + +def reset_overleaf(value, closed=None): + """Overleaf's services removed with their volumes, started again, and waited on until its login answers.""" + compose, settings = _project() + if settings.get("HOST_SUFFIX") != value: + raise WebsiteError(f"the stack serves {settings.get('HOST_SUFFIX') or 'no suffix'} and this run asks for " + f"{value}: set Website host suffix to the stack's") + _done(_docker(*compose, "rm", "--stop", "--force", "-v", *OVERLEAF), "removing Overleaf's services") + _done(_docker("volume", "rm", "--force", *[f"{PROJECT}_{v}" for v in OVERLEAF_VOLUMES]), "removing Overleaf's volumes") + _done(_docker(*compose, "up", "-d", "--no-build", "--pull", "never", OVERLEAF[-1]), "starting Overleaf") + _waited(f"{SCHEME}overleaf.{value}/login", OVERLEAF_SECONDS, closed, "Overleaf") + + +def opened(path): + """Read-only, so another user's lock in /tmp serves too; kept only as a plain file, since a FIFO would hang the open.""" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + while True: + try: + fd = os.open(path, flags) + break + except FileNotFoundError: + pass + try: + fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o644) + os.fchmod(fd, 0o644) + break + except FileExistsError: + pass + if not stat.S_ISREG(os.fstat(fd).st_mode): + os.close(fd) + raise OSError(errno.EINVAL, "not a plain file", path) + return fd + + +@contextlib.contextmanager +def locked(path, mode, closed, busy): + """Waited on until ``closed`` is set, or refused at once with no ``closed``; one file for every user of the machine.""" + try: + fd = opened(path) + except OSError as e: + raise WebsiteError(f"the lock {path} cannot be opened ({type(e).__name__}): remove it if it is not a plain " + "file every user can read") from None + try: + while True: + try: + fcntl.flock(fd, mode | fcntl.LOCK_NB) + break + except BlockingIOError: + if closed is None: + raise WebsiteError(busy) from None + if closed.wait(1): + raise WebsiteError(f"the unit's time ran out while {busy}") from None + _HELD.append(fd) + try: + yield + finally: + _HELD.remove(fd) + finally: + os.close(fd) + + +def held_fds(): + """For a process that keeps the locks held after this one lets go.""" + return tuple(_HELD) + + +@contextlib.contextmanager +def held(eid, value, closed): + """For a task of OVERLEAF_TASKS, Overleaf alone and anew.""" + if not value: + yield + return + with locked(SITES_LOCK, fcntl.LOCK_SH, closed, "the stack was being reset or smoked"): + if eid not in OVERLEAF_TASKS: + yield + return + with locked(LOCK, fcntl.LOCK_EX, closed, "another unit's Overleaf task ran"): + reset_overleaf(value, closed) + yield + + +def _alone(): + return locked(SITES_LOCK, fcntl.LOCK_EX, None, "a unit is using the mock websites: reset or smoke the stack " + "when no task that needs them runs") + + +def _override(): + try: + with open(OVERRIDE, encoding="utf-8") as f: + return f.read() + except OSError as e: + raise WebsiteError(f"no {OVERRIDE} ({type(e).__name__}): this benchmark's folder is incomplete; " + "install Nullius again") from None + + +def _compose_version(): + """Docker Compose COMPOSE_VERSION or later, which websites.compose.yml's !override needs.""" + said = _docker("compose", "version", "--short") + found = re.match(r"v?(\d+)\.(\d+)\.(\d+)", said.stdout.strip()) if said.returncode == 0 else None + if not found or tuple(int(x) for x in found.groups()) < COMPOSE_VERSION: + raise WebsiteError("websites.compose.yml needs Docker Compose " + ".".join(map(str, COMPOSE_VERSION)) + + f" or later (its !override); this machine has {said.stdout.strip() or 'none'}") + + +def _pinned(stack): + """``stack`` is OSWorld-web at COMMIT, its submodules at the commits it names, nothing tracked changed.""" + def git(*args): + try: + return subprocess.run(["git", "-C", stack, *args], capture_output=True, text=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as e: + raise WebsiteError(f"git could not read {stack}: {type(e).__name__}") from None + head = git("rev-parse", "HEAD") + if head.returncode != 0 or head.stdout.strip() != COMMIT: + raise WebsiteError(f"{stack} is not OSWorld-web at {COMMIT} (the tag osworld-v2.1): clone it with its " + f"submodules at that tag ({SETUP_DOC})") + modules = git("submodule", "status") + off = [ln.split()[1] for ln in modules.stdout.splitlines() if ln[:1] != " " and len(ln.split()) > 1] + if modules.returncode != 0 or off: + raise WebsiteError(f"{stack}'s submodules are not at the commits it names ({', '.join(off) or 'unreadable'}): " + "git submodule update --init --recursive") + changed = git("status", "--porcelain", "--untracked-files=no") + if changed.returncode != 0 or changed.stdout.strip(): + raise WebsiteError(f"{stack} has changes to its files: the stack runs as published") + return os.path.abspath(stack) + + +def _beside(stack, name): + return os.path.abspath(stack).rstrip(os.sep) + name + + +def _write(path, text): + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".nullius-") + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(text) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + +def allocation(mbps, iops): + """Each tier's per-service limit by weight, rounded down, so all services together stay within the share.""" + if mbps < 1 or iops < 1: + raise WebsiteError("the stack's write share is whole numbers above 0") + weights = [WEIGHTS.get(s, QUIET) for s in SERVICES] + total = [sum(w[i] for w in weights) for i in (0, 1)] + out = {} + for tier, (wb, wi) in [("QUIET", QUIET)] + [(s.upper(), w) for s, w in WEIGHTS.items()]: + out[tier] = (mbps * MB * wb // total[0], iops * wi // total[1]) + if not all(out[tier]): + raise WebsiteError(f"a write share of {iops} IOPS leaves the {tier.lower()} services none: give the " + f"stack at least {total[1]} IOPS") + return out + + +def limits(settings): + """Every service's limit as the settings give it, and their sum.""" + each = {s: (int(settings[f"OSWORLD2_WEB_{t}_BPS"]), int(settings[f"OSWORLD2_WEB_{t}_IOPS"])) + for s in SERVICES for t in [s.upper() if s in WEIGHTS else "QUIET"]} + return each, tuple(sum(v[i] for v in each.values()) for i in (0, 1)) + + +def _site(): + """Loaded under another name, so it wraps nothing in this process.""" + if not _SITE: + spec = importlib.util.spec_from_file_location("osworld2_startup", SITE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _SITE.append(module) + return _SITE[0] + + +def _network(): + """Made when missing; its gateway is the address Caddy listens on.""" + site = _site() + try: + return site.network(_docker) + except site.NetworkError as e: + raise WebsiteError(str(e)) from None + + +def _root_disk(): + info = _done(_docker("info", "--format", "{{.DockerRootDir}}"), "docker info") + site = _site() + try: + return info.stdout.strip(), site.root_disk(info.stdout.strip()) + except site.WriteLimitError as e: + raise WebsiteError(f"the write limit cannot be applied: {e}") from None + + +def configure(stack, value, mbps, iops): + """The stack's settings: Caddy on the VMs' network's gateway, each tier with its write limit.""" + stack = _pinned(stack) + if not SUFFIX.fullmatch(value): + raise WebsiteError(f"{value!r} is not a host suffix") + tiers = allocation(mbps, iops) + _network() + settings = {"HOST_SUFFIX": value, "CADDY_SCHEME": SCHEME, "OSWORLD2_WEB_ADDRESS": _site().GATEWAY, + "OSWORLD2_WEB_DISK": _root_disk()[1], + **{f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))}} + path = _beside(stack, ENV_FILE) + _write(path, "".join(f"{k}={settings[k]}\n" for k in SETTINGS)) + return path, settings + + +def compose(stack, args, capture=False, builder=None): + """Compose on the stack, websites.compose.yml copied beside it and the VMs' network made first.""" + stack = _pinned(stack) + envfile = _beside(stack, ENV_FILE) + _settings(envfile) + _compose_version() + _network() + copy = _beside(stack, COMPOSE_FILE) + _write(copy, _override()) + argv = ["docker", "compose", "-p", PROJECT, "--project-directory", stack, "-f", + os.path.join(stack, "docker-compose.yml"), "-f", copy, "--env-file", envfile, *args] + env = dict(_clean(), **({"BUILDX_BUILDER": builder} if builder else {})) + if capture: + return subprocess.run(argv, capture_output=True, text=True, env=env, stdin=subprocess.DEVNULL, + timeout=BUILD_SECONDS if builder else RESET_SECONDS, pass_fds=held_fds()) + return subprocess.run(argv, env=env, pass_fds=held_fds()).returncode + + +def builder(stack, op, mbps=0, iops=0): + """BuildKit, privileged as buildx's own builders, with its own write limit; down removes it with its cache.""" + if op == "down": + gone = [_docker("buildx", "rm", BUILDER), _docker("rm", "-f", BUILDKITD), _docker("volume", "rm", "--force", BUILDKIT_VOLUME)] + failed = [g for g in gone[1:] if g.returncode != 0] + if failed: + _done(failed[0], "removing the builder") + return {"builder": BUILDER, "removed": True} + if mbps < 1 or iops < 1: + raise WebsiteError("the builder's write limit is whole numbers above 0") + stack = _pinned(stack) + disk = _root_disk()[1] + config = _beside(stack, BUILDKIT_FILE) + _write(config, f"[worker.oci]\n max-parallelism = {BUILDKIT_PARALLEL}\n") + _done(_docker("run", "-d", "--name", BUILDKITD, "--privileged", "--restart", "no", + "--device-write-bps", f"{disk}:{mbps * MB}", "--device-write-iops", f"{disk}:{iops}", + "-v", f"{BUILDKIT_VOLUME}:/var/lib/buildkit", "-v", f"{config}:/etc/buildkit/buildkitd.toml:ro", + BUILDKIT), "starting the builder") + host = json.loads(_done(_docker("inspect", "-f", "{{json .HostConfig}}", BUILDKITD), "inspecting the builder").stdout) + kept = {"Path": disk, "Rate": mbps * MB} in (host.get("BlkioDeviceWriteBps") or []) and \ + {"Path": disk, "Rate": iops} in (host.get("BlkioDeviceWriteIOps") or []) + if not kept: + _docker("rm", "-f", BUILDKITD) + raise WebsiteError(f"Docker kept no write limit on {disk} for the builder: it is removed, nothing was built") + _done(_docker("buildx", "create", "--name", BUILDER, "--driver", "remote", f"docker-container://{BUILDKITD}"), + "registering the builder") + return {"builder": BUILDER, "disk": disk, "write_bps": mbps * MB, "write_iops": iops} + + +def _free_gb(root): + st = os.statvfs(root) + return st.f_bavail * st.f_frsize / 1e9 + + +def build(stack, groups=(), reserve=RESERVE_GB, say=print): + """A group that would leave less than ``reserve`` GB free stops the build; the cache is removed after each group.""" + stack = _pinned(stack) + names = [g[0] for g in GROUPS] + unknown = [g for g in groups if g not in names] + if unknown: + raise WebsiteError(f"no group {', '.join(unknown)}: the groups are {', '.join(names)}") + if _docker("buildx", "inspect", BUILDER).returncode != 0: + raise WebsiteError(f"no builder {BUILDER}: start it (python3 websites.py builder STACK up ...)") + root = _root_disk()[0] + done = [] + for name, services, estimate in GROUPS: + if groups and name not in groups: + continue + free = _free_gb(root) + if free - estimate < reserve: + raise WebsiteError(f"stopped before {name}: {free:.1f} GB free under {root}, it needs about {estimate} GB " + f"and {reserve} GB stay free (done: {', '.join(done) or 'none'})") + said = compose(stack, ["pull", *services] if name == "pulled" else ["build", *services], capture=True, + builder=BUILDER) + _done(said, f"the {name} group") + if name != "pulled": + _done(_docker("buildx", "prune", "--builder", BUILDER, "--all", "--force"), "removing the builder's cache") + done.append(name) + say(json.dumps({"group": name, "free_gb": round(_free_gb(root), 1)})) + config = json.loads(_done(compose(stack, ["config", "--format", "json"], capture=True), "reading the project").stdout) + ids = {} + for s in SERVICES: + ref = config["services"].get(s, {}).get("image") or f"{PROJECT}-{s}" + said = _docker("image", "inspect", "--format", "{{.Id}}", ref) + ids[s] = {"image": ref, "id": said.stdout.strip() if said.returncode == 0 else None} + path = _beside(stack, IMAGES_FILE) + _write(path, json.dumps({"commit": COMMIT, "groups": done, "images": ids}, indent=1) + "\n") + return {"groups": done, "images": path} + + +def _until_ready(value, seconds): + t0 = time.time() + while True: + try: + return ready(value, overleaf=True) + except WebsiteError: + if time.time() - t0 >= seconds: + raise + time.sleep(POLL) + + +def reset(stack): + value = _settings(_beside(os.path.abspath(stack), ENV_FILE)).get("HOST_SUFFIX", "") + with _alone(): + _done(compose(stack, ["down", "--volumes"], capture=True), "taking the stack down") + _done(compose(stack, ["up", "-d", "--no-build", "--pull", "never"], capture=True), "starting the stack") + return _until_ready(value, RESET_SECONDS) + + +def _from_vms(value): + """Each name from a container on the VMs' network, by Caddy's own busybox wget.""" + image = CADDY.search(_override()).group(1) + loop = 'for h; do wget -q -O /dev/null -T 10 "http://$h/" || { echo "$h"; exit 1; }; done' + network = _network() + said = _docker("run", "--rm", "--name", f"{PROBE}{os.getuid()}-{os.getpid()}", "--network", network, + "--pull", "never", "--entrypoint", "sh", image, "-c", loop, "sh", + *[f"{n}.{value}" for n in NAMES]) + if said.returncode != 0: + failed = (said.stdout.strip().splitlines() or [""])[-1] + raise WebsiteError(f"from the VMs' network {network}, {failed or 'a name'} did not answer: " + + ((said.stderr or "").strip().splitlines() or ["no reason"])[-1][:300]) + + +def smoke(stack): + stack = _pinned(stack) + value = _settings(_beside(stack, ENV_FILE)).get("HOST_SUFFIX", "") + with _alone(): + return _smoked(stack, value) + + +def _smoked(stack, value): + out = {"suffix": value, "names": ready(value, overleaf=True)} + _from_vms(value) + out["network"] = len(NAMES) + node = shutil.which("node") + if not node: + raise WebsiteError("no node on PATH: the upstream's scripts/verify-control-plane.mjs needs Node 18 or later") + script = os.path.join(stack, "scripts", "verify-control-plane.mjs") + said = subprocess.run([node, script], capture_output=True, text=True, timeout=RESET_SECONDS, stdin=subprocess.DEVNULL, + env=dict(_clean(), OSWORLD_HOST_SUFFIX=value, OSWORLD_SCHEME=SCHEME[:-3])) + passed = [ln.split()[1] for ln in said.stdout.splitlines() if ln.startswith("PASS ")] + if said.returncode != 0: + raise WebsiteError(f"the upstream's control-plane check failed after {len(passed)} sites: " + + ((said.stderr or said.stdout).strip().splitlines() or ["no reason"])[-1][:300]) + out["control_plane"] = len(passed) + reset_overleaf(value) + out["overleaf"] = "anew" + return out + + +def reads(args): + """Whether compose ``args`` only read: their subcommand is one of READING.""" + skip = False + for x in args: + if skip: + skip = False + elif x in VALUED: + skip = True + elif not x.startswith("-"): + return x in READING + return False + + +def main(argv): + p = argparse.ArgumentParser(prog="websites.py", description="OSWorld2's mock websites") + sub = p.add_subparsers(dest="op", required=True) + c = sub.add_parser("configure") + c.add_argument("stack") + c.add_argument("suffix") + c.add_argument("--write-mbps", type=int, required=True) + c.add_argument("--write-iops", type=int, required=True) + d = sub.add_parser("compose") + d.add_argument("stack") + d.add_argument("args", nargs=argparse.REMAINDER) + b = sub.add_parser("builder") + b.add_argument("stack") + b.add_argument("op_", metavar="up|down", choices=("up", "down")) + b.add_argument("--write-mbps", type=int, default=0) + b.add_argument("--write-iops", type=int, default=0) + g = sub.add_parser("build") + g.add_argument("stack") + g.add_argument("--group", action="append", default=[]) + g.add_argument("--reserve-gb", type=float, default=RESERVE_GB) + sub.add_parser("check").add_argument("suffix") + sub.add_parser("smoke").add_argument("stack") + sub.add_parser("reset").add_argument("stack") + a = p.parse_args(argv) + try: + if a.op == "compose": + args = a.args[1:] if a.args[:1] == ["--"] else a.args + with contextlib.nullcontext() if reads(args) else _alone(): + return compose(a.stack, args) + if a.op == "configure": + path, settings = configure(a.stack, a.suffix, a.write_mbps, a.write_iops) + out = {"settings": path, **settings, "sum_of_limits": dict(zip(("bps", "iops"), limits(settings)[1]))} + elif a.op == "builder": + out = builder(a.stack, a.op_, a.write_mbps, a.write_iops) + elif a.op == "build": + out = build(a.stack, a.group, a.reserve_gb) + elif a.op == "check": + if not SUFFIX.fullmatch(a.suffix): + raise WebsiteError(f"{a.suffix!r} is not a host suffix") + out = {"suffix": a.suffix, "names": ready(a.suffix, overleaf=True)} + elif a.op == "smoke": + out = smoke(a.stack) + else: + out = {"names": reset(a.stack)} + except (WebsiteError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json index 6f27dc7d..e9325077 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json @@ -320,6 +320,7 @@ ], "default": "200", "pattern": "[1-9][0-9]?|1[0-9]{2}|200", + "example": "100", "limit": true }, "max_errors": { @@ -331,6 +332,7 @@ ], "default": "10", "pattern": "[1-9][0-9]{0,3}", + "example": "5", "limit": true }, "upstream_agent": { diff --git a/app/nullius/prerequisites.py b/app/nullius/prerequisites.py index e52b6a7b..73c228ff 100644 --- a/app/nullius/prerequisites.py +++ b/app/nullius/prerequisites.py @@ -88,8 +88,8 @@ def check_here(needs: List[str], setups: Optional[Dict[str, dict]] = None, *, st rows.append(row(n, "ok") if which(n) else row(n, "missing", f"{n} is not on this machine's PATH")) lacking = hostreq.problems({"host": setups or {}}, list(setups or {}), which, constraints.of(constraints.LOCAL, strict=False).get("gpu_ids")) for name, s in (setups or {}).items(): - rows.append(row("setup:" + name, "missing", lacking[name], s.get("install"), label=name) - if name in lacking else row("setup:" + name, "ok", s.get("summary", ""), label=name)) + rows.append(row("setup:" + name, "missing", lacking[name], s.get("install"), label=s.get("label") or name) + if name in lacking else row("setup:" + name, "ok", s.get("summary", ""), label=s.get("label") or name)) return rows @@ -123,8 +123,8 @@ def check_host(host: hosts.Host, needs: List[str], setups: Optional[Dict[str, di rows.append(row(n, "missing", f"{n} is not on the host's PATH") if n in gone else row(n, "ok")) for name, s in (setups or {}).items(): lacking = [p for p in s.get("check") or [] if p in gone] - rows.append(row("setup:" + name, "missing", ", ".join(lacking) + " not on PATH", s.get("install"), label=name) - if lacking else row("setup:" + name, "ok", s.get("summary", ""), label=name)) + rows.append(row("setup:" + name, "missing", ", ".join(lacking) + " not on PATH", s.get("install"), label=s.get("label") or name) + if lacking else row("setup:" + name, "ok", s.get("summary", ""), label=s.get("label") or name)) return rows diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 932c5fc3..47eb0224 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -391,7 +391,7 @@ def option_problem(spec: dict, value) -> Optional[str]: if "values" in spec: return None if value in (spec.get("values") or []) else f"{value!r} is not one of {', '.join(spec.get('values') or [])}" try: - return None if re.fullmatch(str(spec.get("pattern")), value) else f"{value!r} does not match {spec.get('pattern')}" + return None if re.fullmatch(str(spec.get("pattern")), value) else f"{value!r} is not a value it takes" + (f", e.g. {spec['example']}" if spec.get("example") else "") except re.error: return "its pattern is not a regular expression" @@ -414,12 +414,12 @@ def _option_slots(d: dict) -> List[Tuple[str, str]]: def _options(d: dict) -> List[str]: """``options``: what a run may set of the agent's own behaviour, each with - a ``summary``, its ``values`` (or a ``pattern``) and a ``default`` that is + a ``summary``, its ``values`` (or a ``pattern`` and an ``example`` of it) and a ``default`` that is the vendor's own - so a run that sets nothing is the agent as it ships. Every ``{opt_}`` slot a template carries must be one of them.""" opts = d.get("options") if opts is not None and not isinstance(opts, dict): - return ["options: an object, name -> {summary, values or pattern, default}"] + return ["options: an object, name -> {summary, values or pattern and example, default}"] opts, bad = opts or {}, [] for name, o in opts.items(): w = f"options.{name}" @@ -446,6 +446,8 @@ def _options(d: dict) -> List[str]: except re.error: bad.append(f"{w}.pattern: a regular expression") continue + if "example" in o and (option_problem(o, o["example"]) or o["example"] == o.get("default")): + bad.append(f"{w}.example: a value its pattern allows besides its default, shown in its place") if option_problem(o, o.get("default")): bad.append(f"{w}.default: the vendor's own default, a value it allows") for where, n in _option_slots(d): @@ -556,7 +558,7 @@ def _files(d: dict) -> List[str]: def _host(d: dict) -> List[str]: """A benchmark's ``host``: what the machine its driver runs on must have for some of its options, - by name - a ``summary``, the programs that ``check`` it (each on PATH) and how to ``install`` them + by name - a ``label``, a ``summary``, the programs that ``check`` it (each on PATH) and how to ``install`` them (``apt`` and ``npm`` packages, pinned where the upstream pins them); no ``install`` for the machine's own hardware (a GPU, checked by its driver's program), which no package puts there.""" h = d.get("host") diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index f9275891..25527a2e 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -156,10 +156,10 @@ def _bench_page(b: dict) -> dict: "catalogue": [k for k, f in (("agent", "turn_seconds"), ("turns", "turns"), ("score", "score_seconds")) if (b.get("limits") or {}).get(f)]}, "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [], "setups": o.get("setups") or ["yours"], - **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "names": o.get("names") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, + **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"], "example": o.get("example")}), "default": o.get("default"), "about": o.get("about") or {}, "names": o.get("names") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, "limit": bool(o.get("limit")), "step": o.get("step") or "", "by": o.get("by") or {}} for n, o in (b.get("options") or {}).items()}, - "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]} + "host": {n: {"label": r.get("label") or n, "summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]} for n, r in hostreq.all_of(b).items()}, "datasets": [{"id": d["id"], "label": d.get("label") or d["id"], "summary": d.get("summary", ""), "units": d.get("units"), "environment": d.get("environment") or "", @@ -203,7 +203,7 @@ def _other_page(o: dict) -> dict: "older": definitions.older_kit(p, r), "label": ports.PORTS[p]["label"], "skills": skills_of(p, r)} for p, r in (o.get("proven") or {}).items() if ports.known(p)}, "options": {n: {"summary": x.get("summary", ""), "default": x.get("default"), - **({"values": list(x["values"])} if "values" in x else {"pattern": x.get("pattern")})} + **({"values": list(x["values"])} if "values" in x else {"pattern": x.get("pattern"), "example": x.get("example")})} for n, x in (o.get("options") or {}).items()}, **({"fidelity": {k: o["proven"]["fidelity"].get(k) for k in ("verdict", "on", "model")}} if isinstance((o.get("proven") or {}).get("fidelity"), dict) else {})} @@ -719,13 +719,13 @@ def script_for(d: dict, p: dict, command: str, index: int) -> str: if backend == "container" and any(p.get("setup") for p in picks): # a container setup's image installs what has an install; nothing else reaches its container imaged = hostreq.in_image(b, on_host) - cannot = [n for n in hostreq.requirements(b, on_host) if n not in imaged] + cannot = [r.get("label") or n for n, r in hostreq.requirements(b, on_host).items() if n not in imaged] if cannot: add("where", f"{', '.join(cannot)} cannot be provided in a container: choose local processes.") if all(p.get("setup") for p in picks): on_host = [] # an agent's or a check's driver still runs on the machine reqs = hostreq.requirements(b, on_host) - out["host"] = {"benchmark": b["key"], "needs": [{"name": n, "summary": r["summary"], "check": r["check"], + out["host"] = {"benchmark": b["key"], "needs": [{"name": n, "label": r.get("label") or n, "summary": r["summary"], "check": r["check"], **({"gpu_memory_mb": r["gpu_memory_mb"]} if "gpu_memory_mb" in r else {})} for n, r in reqs.items()], "check": [p for r in reqs.values() for p in r["check"]], "install": hostreq.install_lines(b, list(reqs))} diff --git a/app/site/mkdocs.yml b/app/site/mkdocs.yml index ce75b8cd..59c4a7d0 100644 --- a/app/site/mkdocs.yml +++ b/app/site/mkdocs.yml @@ -86,6 +86,8 @@ nav: - Add manually: benchmarks/add-manually.md - Add with an assistant: benchmarks/add-with-assistant.md - OSWorld1 machine setup: benchmarks/osworld1-machine.md + - OSWorld2 machine setup: benchmarks/osworld2-machine.md + - OSWorld2 results: benchmarks/osworld2-results.md - Agents: - Supported agents: agents/supported/README.md - Add manually: agents/add-manually.md diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 26c829e7..2b13d849 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -526,7 +526,7 @@ return `
  • ${esc(w === 'local' ? 'This machine' : C.machineLabel(w))} ${state} ${button('Check', `data-hostreq-check="${esc(w)}"`, 'nb-inlinebutton')}${lines.length && (gone.length || (st && st.error)) ? ' ' + button('Install for me', `data-hostreq-install="${esc(w)}"`, 'nb-inlinebutton') : ''}${st && st.said ? ` ${esc(st.said)}` : ''}
  • `; }; return `

    Machine setup

    Needed by the options you chose, on each machine it runs on
    ` - + reqs.map(([n, r]) => `

    ${esc(n)}: ${esc(r.summary)}

    `).join('') + + reqs.map(([n, r]) => `

    ${esc(r.label)}: ${esc(r.summary)}

    `).join('') + `
      ${C.whereList(rec).map(line).join('')}
    ` + (lines.length ? `
    or run it yourself, on each machine
    ${esc(lines.join('\n'))}
    ` : ''); } @@ -564,11 +564,11 @@ .map(([n, o]) => { const cur = (rec.bench_options || {})[n] ?? o.default; if (scope === 'where' && o.values.length === 1) return `

    ${esc(o.label)}

    ${esc((o.names || {})[cur] || cur)}

    `; - if (o.pattern) return ``; + if (o.pattern) return ``; const tileOf = v => { const need = (o.needs || {})[v] || {}; const foot = [need.embeddings ? `${need.embeddings.includes('/') ? 'OpenRouter key' : 'OpenAI or OpenRouter key'}` : '', need.reranker ? `reranker ${esc(need.reranker)}` : '', - ...(need.host || []).filter(h => local[h]).map(h => `needs ${esc(h)}`), + ...(need.host || []).filter(h => local[h]).map(h => `needs ${esc(((b.host || {})[h] || {}).label || h)}`), v === o.default ? 'default' : ''].join(''); return tile({attr: 'bench-opt', key: n, data: ` data-bench-val="${esc(v)}"`, on: v === cur, title: (o.names || {})[v] || v, mark: '', tip: (o.about || {})[v] || '', foot}); }; return `

    ${esc(o.label)}${o.datasets.length ? ' · ' + esc(o.datasets.join(', ')) : ''}

    ` @@ -678,7 +678,7 @@ const cur = mine[n] ?? x.default; return x.values ? `` - : ``; + : ``; }).join('') + ''; }; // the chosen ones' settings, one tab each: the benchmark's own way, the benchmark's settings every agent takes, each one's own const yoursWord = agent ? 'agent' : (b.kind || 'capability'); diff --git a/app/static/catalogue/marks.svg b/app/static/catalogue/marks.svg index cc000c80..1576916d 100644 --- a/app/static/catalogue/marks.svg +++ b/app/static/catalogue/marks.svg @@ -30,4 +30,7 @@ + + + diff --git a/app/templates/agents/entry.html b/app/templates/agents/entry.html index 19e2466a..fb768ca6 100644 --- a/app/templates/agents/entry.html +++ b/app/templates/agents/entry.html @@ -80,7 +80,7 @@

    Fidelity

    {% if e.options %}

    Options

      - {% for x in e.options %}
    • {{ x.name }} {{ x.default }} (its default){% if x['values'] %}takes {{ x['values']|join(', ') }}{% elif x.pattern %}takes {{ x.pattern }}{% endif %}{{ x.summary.split(':')[0].split(';')[0].split('.')[0] }}.
    • {% endfor %} + {% for x in e.options %}
    • {{ x.name }} {{ x.default }} (its default){% if x['values'] %}takes {{ x['values']|join(', ') }}{% elif x.example %}e.g. {{ x.example }}{% endif %}{{ x.summary.split(':')[0].split(';')[0].split('.')[0] }}.
    • {% endfor %}
    {% endif %} diff --git a/app/templates/catalogue/benchmark.html b/app/templates/catalogue/benchmark.html index f19f0733..898b3ec2 100644 --- a/app/templates/catalogue/benchmark.html +++ b/app/templates/catalogue/benchmark.html @@ -118,15 +118,15 @@

    Options

    {{ o.label }}{% if o.datasets %} {{ std.datasets|selectattr('id', 'in', o.datasets)|map(attribute='label')|join(', ') }}{% endif %}

    {{ o.summary }}

    {% if o.pattern %} -
    {{ 'Provider default' if n == 'temperature' and o.default == 'default' else o.default }} - {% if n == 'temperature' %}Custom decimal, e.g. 0.5{% else %}{{ o.pattern }}{% endif %} +
    {{ 'Provider default' if o.default == 'default' and not o.limit else o.default }} + {% if o.example %}e.g. {{ o.example }}{% endif %}
    {% else %}
      {% for v in o['values'] %}{% set need = o.needs.get(v) or {} %}
    • {% if o.names.get(v) %}{{ o.names[v] }}{% else %}{{ v }}{% endif %}{% if v == o.default and v != 'default' %} default{% endif %} {% if need.embeddings %}OpenAI key · {{ need.embeddings }}{% endif %} - {% for h in need.host or [] %}needs {{ h }}{% endfor %} + {% for h in need.host or [] %}needs {{ std.host[h].label }}{% endfor %} {% if o.about.get(v) %}{{ o.about[v] }}{% endif %}
    • {% endfor %}
    @@ -134,7 +134,7 @@

    {{ o.label }}{% if o.datasets %} {{ std.datasets| {% endfor %} {% for h, r in std.host.items() %}
    -

    Setup: {{ h }}

    +

    Setup: {{ r.label }}

    {{ r.summary }} Checked by {{ r.check|join(', ') }} on PATH, on each machine a run uses; the builder's Where step checks it{% if r.install %} and can install it{% endif %}.

    {% if r.install %}
    install, on each machine
    {{ r.install|join('\n') }}
    {% endif %}
    diff --git a/app/tests/catalogue_docs.py b/app/tests/catalogue_docs.py index ddf11510..c087d490 100644 --- a/app/tests/catalogue_docs.py +++ b/app/tests/catalogue_docs.py @@ -227,7 +227,7 @@ def _values(spec: dict) -> str: """An option's values, the default in bold; a pattern when it has no list.""" named = {v: f"`{v}` ({n})" for v, n in (spec.get("names") or {}).items()} values = ", ".join(f"**{named.get(v, f'`{v}`')}**" if v == spec.get("default") else named.get(v, f"`{v}`") for v in spec.get("values") or []) - return values or f"pattern `{spec.get('pattern')}`, default `{spec.get('default')}`" + return values or (f"e.g. `{spec['example']}`, " if spec.get("example") else "") + f"default `{spec.get('default')}`" def _needs_rows(name: str, spec: dict) -> list: diff --git a/app/tests/test_catalog.py b/app/tests/test_catalog.py index 24345d39..6b670997 100644 --- a/app/tests/test_catalog.py +++ b/app/tests/test_catalog.py @@ -44,7 +44,7 @@ def test_tabs_are_alphabetical_and_lanes_are_not_benchmarks(self): self.assertEqual(titles, sorted(titles, key=catalog.sort_key)) # what has a usable route is listed, whoever made it and whether or not it has run listed = {x.key for x in cat.listed_benchmarks} - self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "osworld1", "swe-bench-verified", "tau3-bench", "terminal-bench"]) + self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "osworld1", "osworld2", "swe-bench-verified", "tau3-bench", "terminal-bench"]) self.assertEqual((cat.benchmark("longmemeval").origin, cat.benchmark("longmemeval").label), ("official", "Official")) self.assertEqual(cat.benchmark("tau3-bench").label, "Official") # τ³ files as "t3…", before "te…": a reader looking under T finds it diff --git a/app/tests/test_osworld2_api_logs.py b/app/tests/test_osworld2_api_logs.py new file mode 100644 index 00000000..1c1815bb --- /dev/null +++ b/app/tests/test_osworld2_api_logs.py @@ -0,0 +1,384 @@ +"""OSWorld2's api_logs: each image kept once, every file given back byte for byte, a failure leaving each whole.""" +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +LIMIT = 20 + + +def module(name, path, **env): + spec = importlib.util.spec_from_file_location(name, path) + loaded = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, **env): + spec.loader.exec_module(loaded) + return loaded + + +api_logs = module("osworld2_api_logs", os.path.join(HERE, "api_logs.py")) + + +def jpeg(n, size=3000): + return b"\xff\xd8\xff\xe0" + bytes([n % 256]) * size + + +def png(n): + return b"\x89PNG\r\n\x1a\n" + bytes([n % 256]) * 2000 + + +def b64(data): + return base64.b64encode(data).decode() + + +def call(i, window=3): + """One model call as the agent logs it: its window's screenshots, newest last, in Anthropic blocks.""" + blocks = [{"type": "text", "text": f"step {i} é中 \"quoted\" \\ /"}] + blocks += [{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64(jpeg(j))}} + for j in range(max(0, i - window + 1), i + 1)] + return {"model": "m", "request_body": {"system": "s", "messages": [{"role": "user", "content": blocks}], + "temperature": 0.6, "max_tokens": 1e-05}, + "response_body": {"content": [{"type": "text", "text": "pyautogui.click(1, 2)"}], "usage": None}, + "request_time": 1700000000000 + i, "response_time": 1700000000500 + i} + + +def odd(): + """Strings pack must leave inline, beside a PNG it packs.""" + text = b64(b"plain text, no image " * 40) + loose = b64(jpeg(7, 600)) + letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + loose = loose[:-3] + letters[letters.index(loose[-3]) | 1] + "==" + return {"a": b64(png(1)), "b": text, "c": loose, "d": b64(jpeg(8, 30)), "e": "x" + b64(jpeg(9)), + "f": [b64(jpeg(1)), b64(png(1))]} + + +def read(path): + with open(path, "rb") as f: + return f.read() + + +def digest(folder): + return {name: hashlib.sha256(read(os.path.join(folder, name))).hexdigest() + for name in sorted(os.listdir(folder)) if os.path.isfile(os.path.join(folder, name))} + + +class Logs(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.dir = os.path.join(self.tmp, "api_logs") + os.makedirs(self.dir) + for i, entry in enumerate([call(i) for i in range(6)] + [odd()]): + with open(os.path.join(self.dir, f"{i + 1:03d}_{1700000000000 + i}.json"), "w") as f: + json.dump(entry, f, ensure_ascii=False) + os.chmod(os.path.join(self.dir, "001_1700000000000.json"), 0o640) + os.utime(os.path.join(self.dir, "001_1700000000000.json"), ns=(1, 2_000_000_000)) + self.before = digest(self.dir) + + def images(self): + return sorted(os.listdir(os.path.join(self.dir, api_logs.IMAGES))) + + def test_each_image_is_written_once_and_every_file_restores_byte_for_byte(self): + done = api_logs.pack(self.dir) + self.assertEqual((done["files"], done["packed"], done["images"], done["written"]), (7, 7, 7, 7)) + self.assertLess(done["after"] * 5, done["before"]) + names = self.images() + self.assertEqual(len(names), 7) + self.assertEqual(sum(n.endswith(".png") for n in names), 1) + for name in names: + with open(os.path.join(self.dir, api_logs.IMAGES, name), "rb") as f: + self.assertEqual(hashlib.sha256(f.read()).hexdigest(), name.split(".")[0]) + with open(os.path.join(self.dir, "006_1700000000005.json"), encoding="utf-8") as f: + entry = json.load(f) + self.assertEqual(entry["_packed_images"], "images") + links = [b["source"]["data"] for b in entry["request_body"]["messages"][0]["content"] if b["type"] == "image"] + self.assertEqual(links, [f"images/{hashlib.sha256(jpeg(j)).hexdigest()}.jpg" for j in (3, 4, 5)]) + with open(os.path.join(self.dir, "007_1700000000006.json"), encoding="utf-8") as f: + kept = json.load(f) + self.assertEqual({k: v == odd()[k] for k, v in kept.items() if k != "_packed_images"}, + {"a": False, "b": True, "c": True, "d": True, "e": True, "f": False}) + self.assertEqual(api_logs.restore(self.dir), {"restored": 7, "failed": 0}) + self.assertEqual({k: v for k, v in digest(self.dir).items()}, self.before) + st = os.stat(os.path.join(self.dir, "001_1700000000000.json")) + self.assertEqual((st.st_mode & 0o777, st.st_mtime_ns), (0o640, 2_000_000_000)) + + def test_running_either_again_changes_nothing(self): + api_logs.pack(self.dir) + packed, images = digest(self.dir), digest(os.path.join(self.dir, api_logs.IMAGES)) + again = api_logs.pack(self.dir) + self.assertEqual((again["packed"], again["written"]), (0, 0)) + self.assertEqual((digest(self.dir), digest(os.path.join(self.dir, api_logs.IMAGES))), (packed, images)) + api_logs.restore(self.dir) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 0}) + self.assertEqual(digest(self.dir), self.before) + + def test_a_file_pack_did_not_mark_is_left_alone_by_both(self): + name = hashlib.sha256(jpeg(0)).hexdigest() + ".jpg" + shapes = {"x_link.json": json.dumps({"a": "images/" + name, "b": b64(jpeg(0))}), + "x_list.json": json.dumps([b64(jpeg(0))]), + "x_empty.json": "{}", + "x_broken.json": '{"a": "' + b64(jpeg(0))} + for file, text in shapes.items(): + with open(os.path.join(self.dir, file), "w") as f: + f.write(text) + api_logs.pack(self.dir) + for file, text in shapes.items(): + with open(os.path.join(self.dir, file)) as f: + self.assertEqual(f.read(), text, file) + self.assertIn(name, self.images()) + api_logs.restore(self.dir) + for file, text in shapes.items(): + with open(os.path.join(self.dir, file)) as f: + self.assertEqual(f.read(), text, file) + + def test_a_failed_write_leaves_every_file_whole_and_a_second_run_finishes(self): + replace = os.replace + for failing in ("images", "001_"): + with self.subTest(failing=failing): + def broken(src, dst): + if failing in dst: + raise OSError(28, "No space left on device") + return replace(src, dst) + + with mock.patch.object(api_logs.os, "replace", broken), self.assertRaises(OSError): + api_logs.pack(self.dir) + self.assertEqual(digest(self.dir), self.before) + self.assertEqual([n for n in os.listdir(os.path.join(self.dir, api_logs.IMAGES)) + if n.endswith(api_logs.TEMPORARY)], []) + done = api_logs.pack(self.dir) + self.assertEqual(done["packed"], 7) + api_logs.restore(self.dir) + self.assertEqual(digest(self.dir), self.before) + + def test_images_are_on_disk_before_a_file_names_them(self): + written = [] + write = api_logs._write + with mock.patch.object(api_logs, "_write", lambda path, *a: (written.append(path), write(path, *a))): + api_logs.pack(self.dir) + for i, path in enumerate(written): + if os.path.dirname(path) == self.dir: + with open(path, encoding="utf-8") as f: + named = {v.split("/")[-1] for v in json.dumps(json.load(f)).split('"') if v.startswith("images/")} + self.assertTrue(named <= {os.path.basename(p) for p in written[:i]}, path) + + def test_what_a_stopped_pack_left_half_written_is_removed(self): + api_logs.pack(self.dir) + left = [os.path.join(self.dir, ".001_x.json.abc" + api_logs.TEMPORARY), + os.path.join(self.dir, api_logs.IMAGES, ".a.jpg.abc" + api_logs.TEMPORARY)] + for path in left: + open(path, "w").close() + api_logs.pack(self.dir) + self.assertFalse(any(os.path.exists(p) for p in left)) + + def test_a_folder_another_process_works_on_is_left_alone(self): + fd = os.open(self.dir, os.O_RDONLY | os.O_DIRECTORY) + self.addCleanup(os.close, fd) + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time; fcntl.flock(int(sys.argv[1]), " + "fcntl.LOCK_EX); print(flush=True); time.sleep(30)", str(fd)], + pass_fds=(fd,), stdout=subprocess.PIPE) + self.addCleanup(lambda: (holder.kill(), holder.wait())) + holder.stdout.readline() + self.assertTrue(api_logs.pack(self.dir)["busy"]) + self.assertTrue(api_logs.restore(self.dir)["busy"]) + self.assertEqual(digest(self.dir), self.before) + + def test_a_missing_or_wrong_image_keeps_its_file_packed(self): + api_logs.pack(self.dir) + packed = digest(self.dir)["001_1700000000000.json"] + first = os.path.join(self.dir, api_logs.IMAGES, hashlib.sha256(jpeg(0)).hexdigest() + ".jpg") + os.remove(first) + self.assertEqual(api_logs.restore(self.dir), {"restored": 4, "failed": 3}) + self.assertEqual(digest(self.dir)["001_1700000000000.json"], packed) + with open(first, "wb") as f: + f.write(jpeg(99)) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 3}) + self.assertEqual(digest(self.dir)["001_1700000000000.json"], packed) + with open(first, "wb") as f: + f.write(jpeg(0)) + api_logs.restore(self.dir) + self.assertEqual(digest(self.dir), self.before) + + def test_links_are_not_followed(self): + outside = os.path.join(self.tmp, "outside") + os.makedirs(outside) + left = ".x" + api_logs.TEMPORARY + open(os.path.join(outside, left), "w").close() + os.symlink(outside, os.path.join(self.dir, api_logs.IMAGES)) + with self.assertRaises(OSError): + api_logs.pack(self.dir) + self.assertEqual(os.listdir(outside), [left]) + self.assertEqual(digest(self.dir), self.before) + os.remove(os.path.join(self.dir, api_logs.IMAGES)) + api_logs.pack(self.dir) + packed = digest(self.dir) + os.rename(os.path.join(self.dir, api_logs.IMAGES), os.path.join(self.tmp, "kept")) + for name in os.listdir(os.path.join(self.tmp, "kept")): + shutil.copy(os.path.join(self.tmp, "kept", name), outside) + os.symlink(outside, os.path.join(self.dir, api_logs.IMAGES)) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 7}) + self.assertIn(left, os.listdir(outside)) + self.assertEqual(digest(self.dir), packed) + linked = os.path.join(self.tmp, "linked") + os.symlink(self.dir, linked) + with self.assertRaises(OSError): + api_logs.pack(linked) + + def test_the_command_line_packs_and_restores(self): + for act in ("pack", "restore"): + out = subprocess.run([sys.executable, os.path.join(HERE, "api_logs.py"), act, self.dir], + capture_output=True, text=True, check=True) + self.assertIn(self.dir, json.loads(out.stdout)) + self.assertEqual(digest(self.dir), self.before) + self.assertNotEqual(subprocess.run([sys.executable, os.path.join(HERE, "api_logs.py"), "unpack", self.dir], + capture_output=True).returncode, 0) + + +SCRIPT = """import json, os, sys, time +folder = os.path.join("results", "pyautogui", "screenshot", "m", "tasks", "001", "api_logs") +os.makedirs(folder, exist_ok=True) +for i, entry in enumerate(json.loads(sys.argv[1])): + with open(os.path.join(folder, "%03d_1.json" % (i + 1)), "w") as f: + json.dump(entry, f, ensure_ascii=False) + if i == 0 and sys.argv[2] == "stay": + open("ready", "w").close() + time.sleep(60) +""" + + +class Reaper(unittest.TestCase): + """The driver's reaper packs the unit's logs.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + env = mock.patch.dict(os.environ, NULLIUS_UPSTREAM=self.tmp) + env.start() + self.addCleanup(env.stop) + self.driver = module("osworld2_driver_logs", os.path.join(HERE, "driver.py"), NULLIUS_UPSTREAM=self.tmp) + self.folder = os.path.join(self.unit, "results", "pyautogui", "screenshot", "m", "tasks", "001", "api_logs") + self.addCleanup(self.driver.CLOSED.clear) + + def packed(self, n): + t0 = time.time() + while time.time() - t0 < LIMIT: + try: + files = [f for f in os.listdir(self.folder) if f.endswith(".json")] + if len(files) == n and all(read(os.path.join(self.folder, f)).startswith(api_logs.MARK) + for f in files): + return True + except OSError: + pass + time.sleep(0.2) + return False + + def run_script(self, stay, failed): + argv = [sys.executable, "-c", SCRIPT, json.dumps([call(i) for i in range(3)]), stay] + try: + self.driver._run(argv, self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run") + except RuntimeError as e: + failed.append(str(e)) + + def test_an_ended_run_is_packed(self): + failed = [] + self.run_script("go", failed) + self.assertEqual(failed, []) + self.assertTrue(self.packed(3)) + self.assertEqual(api_logs.restore(self.folder)["restored"], 3) + + def test_a_failed_run_is_packed(self): + argv = [sys.executable, "-c", SCRIPT + "sys.exit(3)\n", json.dumps([call(i) for i in range(2)]), "go"] + with self.assertRaises(RuntimeError): + self.driver._run(argv, self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run") + self.assertTrue(self.packed(2)) + + def test_a_stopped_run_is_packed_once_its_group_is_gone(self): + failed = [] + t = threading.Thread(target=self.run_script, args=("stay", failed), daemon=True) + t.start() + t0 = time.time() + while not os.path.exists(os.path.join(self.unit, "ready")) and time.time() - t0 < LIMIT: + time.sleep(0.05) + self.driver.CLOSED.set() + t.join(LIMIT) + self.assertTrue(failed and "was stopped" in failed[0]) + self.assertTrue(self.packed(1)) + + def test_a_group_still_there_is_not_packed(self): + packing = [] + with mock.patch.object(self.driver.os, "killpg"), mock.patch.object(self.driver, "_gone", lambda *_: False), \ + mock.patch.object(self.driver, "_pack", packing.append): + self.driver.reap(1, self.unit, False) + self.assertEqual(packing, []) + with mock.patch.object(self.driver.os, "killpg", side_effect=ProcessLookupError), \ + mock.patch.object(self.driver, "_pack", packing.append): + self.driver.reap(1, self.unit, False) + self.assertEqual(packing, [self.unit]) + + def test_no_packer_leaves_the_logs_and_the_unit_alone(self): + os.makedirs(self.folder) + with open(os.path.join(self.folder, "001_1.json"), "w") as f: + json.dump(call(0), f, ensure_ascii=False) + before = digest(self.folder) + with mock.patch.dict(sys.modules, api_logs=None): + self.driver._pack(self.unit) + self.assertEqual(digest(self.folder), before) + self.assertIn("api_logs not packed: ModuleNotFoundError", self.said()[0]) + + def said(self): + with open(os.path.join(self.unit, self.driver.PACK_LOG), encoding="utf-8") as f: + return f.read().splitlines() + + def test_each_folder_is_packed_on_its_own_and_said_in_one_line(self): + other = self.folder.replace("001", "002") + for folder in (self.folder, other): + os.makedirs(folder) + with open(os.path.join(folder, "001_1.json"), "w") as f: + json.dump(call(1), f, ensure_ascii=False) + before = digest(self.folder) + pack = api_logs.pack + + def broken(folder): + if folder == self.folder: + raise OSError(28, "No space left on device") + return pack(folder) + + with mock.patch.dict(sys.modules, api_logs=mock.Mock(pack=broken)): + self.driver._pack(self.unit) + self.assertEqual(digest(self.folder), before) + self.assertTrue(read(os.path.join(other, "001_1.json")).startswith(api_logs.MARK)) + lines = sorted(self.said(), key=lambda ln: "002" in ln) + self.assertEqual(len(lines), 2) + self.assertIn("001/api_logs {\"error\": \"OSError: [Errno 28] No space left on device\"}", lines[0]) + self.assertIn("002/api_logs {\"files\": 1, \"packed\": 1, \"images\": 2,", lines[1]) + self.assertRegex(lines[1], self.driver.RECORD) + + def test_a_unit_with_no_api_logs_says_nothing(self): + self.driver._pack(self.unit) + self.assertFalse(os.path.exists(os.path.join(self.unit, self.driver.PACK_LOG))) + + +class Snapshot(unittest.TestCase): + def test_a_hosts_snapshot_carries_the_packer(self): + from pathlib import Path + + from nullius.standard import payload + self.assertIn("api_logs.py", payload._entry(Path(HERE))) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_docker_create.py b/app/tests/test_osworld2_docker_create.py new file mode 100644 index 00000000..523f2e2d --- /dev/null +++ b/app/tests/test_osworld2_docker_create.py @@ -0,0 +1,610 @@ +"""OSWorld2's VM container create (startup/sitecustomize.py) with a busy Docker daemon: one container or none. +A fake docker-py, requests and docker command line stand in.""" +from __future__ import annotations + +import importlib.util +import ipaddress +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "nullius", "catalogue", + "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +PROVIDER = "happysixd/osworld-docker@sha256:" + "0" * 64 +NONE = {"NULLIUS_OPTION_WRITE_MBPS": "none", "NULLIUS_OPTION_WRITE_IOPS": "none"} +LIMITS = {"NULLIUS_OPTION_WRITE_MBPS": "10", "NULLIUS_OPTION_WRITE_IOPS": "100"} +DEVICE = "/dev/sdz" + + +class Timeout(Exception): + pass + + +class ReadTimeout(Timeout): + pass + + +class ConnectionError_(Exception): + pass + + +class ConnectTimeout(ConnectionError_, Timeout): + pass + + +class APIError(Exception): + pass + + +class Daemon: + """A fake daemon's create: slow (``seconds``), made ``late``, a slow ``inspect``, refused, or limits dropped (``drop``).""" + + def __init__(self, seconds=0, late=0, refuse=False, error=ReadTimeout, drop=False, inspect=0): + self.seconds, self.late, self.refuse, self.error, self.drop = seconds, late, refuse, error, drop + self.inspect = inspect + self.containers, self.creates, self.list_timeouts, self.clock = [], 0, [], 0.0 + self.pending, self.gave_up, self.info_timeouts = None, None, [] + + +class Container: + def __init__(self, daemon, image, kwargs): + self.daemon, self.image, self.kwargs, self.started, self.removed = daemon, image, kwargs, False, None + self.attrs = {} + + @property + def labels(self): + return self.kwargs.get("labels") or {} + + def reload(self): + kept = {} if self.daemon.drop else self.kwargs + self.attrs = {"HostConfig": {"BlkioDeviceWriteBps": kept.get("device_write_bps"), + "BlkioDeviceWriteIOps": kept.get("device_write_iops")}} + + def remove(self, **kwargs): + self.removed = kwargs + + def start(self): + self.started = True + + +class API: + def __init__(self, daemon, timeout=60): + self.daemon, self.timeout = daemon, timeout + + +class Client: + def __init__(self, daemon, timeout=60): + self.api = API(daemon, timeout) + + def info(self): + self.api.daemon.info_timeouts.append(self.api.timeout) + return {"DockerRootDir": "/var/lib/docker"} + + +class ContainerCollection: + def __init__(self, client): + self.client = client + + def create(self, image, command=None, **kwargs): + d = self.client.api.daemon + d.creates += 1 + if d.refuse: + raise APIError("refused") + if d.seconds > self.client.api.timeout: + d.clock = d.gave_up = self.client.api.timeout + if d.late is not None: + d.pending = Container(d, image, kwargs) + raise d.error("gave up") + d.containers.append(Container(d, image, kwargs)) + if d.inspect > self.client.api.timeout: + d.clock = d.gave_up = 2 * self.client.api.timeout + raise d.error("gave up") + return d.containers[-1] + + def list(self, all=False, filters=None): + d = self.client.api.daemon + d.list_timeouts.append(self.client.api.timeout) + if d.pending is not None and d.clock - d.gave_up >= d.late: + d.containers.append(d.pending) + d.pending = None + key, _, value = (filters or {}).get("label", "").partition("=") + return [c for c in d.containers if c.labels.get(key) == value] + + def run(self, image, command=None, **kwargs): + container = self.create(image=image, command=command, **kwargs) + container.start() + return container + + +class Patient(unittest.TestCase): + def setUp(self): + self.containers = types.ModuleType("docker.models.containers") + self.containers.ContainerCollection = ContainerCollection + self.original = ContainerCollection.create + self.addCleanup(setattr, ContainerCollection, "create", self.original) + requests = types.ModuleType("requests") + requests.exceptions = types.SimpleNamespace(Timeout=Timeout, ReadTimeout=ReadTimeout, + ConnectionError=ConnectionError_, ConnectTimeout=ConnectTimeout) + modules = mock.patch.dict(sys.modules, {"docker": types.ModuleType("docker"), + "docker.models": types.ModuleType("docker.models"), + "docker.models.containers": self.containers, + "requests": requests}) + modules.start() + self.addCleanup(modules.stop) + spec = importlib.util.spec_from_file_location("osworld2_startup_create", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + + def root_disk(root): + self.site._DISK[root] = DEVICE + return DEVICE + + disk = mock.patch.object(self.site, "root_disk", root_disk) + disk.start() + self.addCleanup(disk.stop) + self.swept = [] + + def sweeper(token): + r, w = os.pipe() + self.swept.append((token, r)) + self.addCleanup(os.close, r) + return w + + sweeper = mock.patch.object(self.site, "_sweeper", sweeper) + sweeper.start() + self.addCleanup(sweeper.stop) + + def said(self): + return [(token, os.read(r, 16)) for token, r in self.swept] + + def start(self, daemon, image=PROVIDER, timeout=60, environ=NONE, **kwargs): + ContainerCollection.create = self.original + self.site.wrap_docker_create(environ) + self.collection = ContainerCollection(Client(daemon, timeout)) + + def tick(seconds): + daemon.clock += seconds + + with mock.patch.object(self.site.time, "sleep", tick), \ + mock.patch.object(self.site.time, "monotonic", lambda: daemon.clock): + return self.collection.run(image, detach=True, **kwargs) + + def test_a_create_slower_than_docker_pys_60_s_goes_through(self): + daemon = Daemon(seconds=90) + with self.assertRaises(ReadTimeout): + ContainerCollection(Client(daemon)).run(PROVIDER) + made = self.start(Daemon(seconds=90)) + self.assertTrue(made.started) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + self.assertEqual(len(made.daemon.containers), 1) + + def test_a_create_the_client_timed_out_on_is_the_one_started(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING") as said: + made = self.start(daemon) + self.assertTrue(made.started) + self.assertEqual((daemon.creates, len(daemon.containers)), (1, 1)) + self.assertIs(made, daemon.containers[0]) + self.assertIn("ReadTimeout", said.output[0]) + + def test_a_request_answered_whose_inspect_timed_out_is_looked_for_and_started(self): + daemon = Daemon(inspect=150) + with self.assertLogs("nullius.osworld2", "WARNING"): + made = self.start(daemon, environ=LIMITS) + self.assertEqual((daemon.creates, len(daemon.containers)), (1, 1)) + self.assertIs(made, daemon.containers[0]) + self.assertEqual((made.started, made.removed), (True, None)) + self.assertEqual(daemon.list_timeouts, [self.site.LOOK_SECONDS]) + self.assertEqual(daemon.gave_up, 2 * self.site.CREATE_SECONDS) + + def test_each_look_waits_its_own_short_time_and_the_clients_comes_back(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + self.start(daemon) + self.assertGreater(len(daemon.list_timeouts), 1) + self.assertEqual(set(daemon.list_timeouts), {self.site.LOOK_SECONDS}) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + + def test_none_made_in_time_is_the_error_and_no_second_create(self): + for late in (None, 61): + with self.subTest(f"late {late}"): + daemon = Daemon(seconds=150, late=late) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.containers), (1, [])) + self.assertGreater(len(daemon.list_timeouts), 1) + self.assertLessEqual(daemon.clock - daemon.gave_up, self.site.ADOPT_SECONDS + self.site.POLL_SECONDS) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + + def test_a_failed_connection_stands_at_once(self): + for error in (ConnectionError_, ConnectTimeout): + with self.subTest(error.__name__): + daemon = Daemon(seconds=150, late=0, error=error) + with self.assertRaises(error): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.list_timeouts), (1, [])) + + def test_each_create_looks_only_for_its_own_label(self): + daemon = Daemon(seconds=150, late=None) + daemon.containers.append(Container(daemon, PROVIDER, {"labels": {self.site.LABEL: "another"}})) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertFalse(daemon.containers[0].started) + + def test_a_refusal_stands_at_once(self): + daemon = Daemon(refuse=True) + with self.assertRaises(APIError): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.list_timeouts), (1, [])) + + def test_a_late_container_keeps_its_write_limit_and_is_started(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + made = self.start(daemon, environ=LIMITS) + self.assertIs(made, daemon.containers[0]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteBps"], [{"Path": DEVICE, "Rate": 10 * 1024 * 1024}]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteIOps"], [{"Path": DEVICE, "Rate": 100}]) + self.assertEqual((made.started, made.removed), (True, None)) + + def test_a_late_container_without_its_write_limit_is_removed_and_fails_the_start(self): + daemon = Daemon(seconds=150, late=8, drop=True) + with self.assertLogs("nullius.osworld2", "WARNING"), self.assertRaises(self.site.WriteLimitError): + self.start(daemon, environ=LIMITS) + made = daemon.containers[0] + self.assertEqual((made.started, made.removed), (False, {"force": True, "v": True})) + + def test_dockers_root_is_asked_with_the_patient_client(self): + daemon = Daemon() + self.start(daemon, environ=LIMITS) + self.assertEqual(daemon.info_timeouts, [self.site.CREATE_SECONDS]) + + def test_another_image_is_left_alone(self): + daemon = Daemon() + made = self.start(daemon, "ubuntu:24.04") + self.assertEqual(made.kwargs, {"detach": True}) + self.assertEqual(self.collection.client.api.timeout, 60) + + def test_the_providers_ports_and_the_guests_a_unit_names_go_on_loopback_only(self): + given = {8006: 8006, 5000: 5001, 9222: 9223, 8080: 8081} + made = self.start(Daemon(), environ=dict(NONE, OSWORLD2_GUEST_PORTS="3000 3001"), ports=given) + self.assertEqual(made.kwargs["ports"], {8006: ("127.0.0.1", 8006), 5000: ("127.0.0.1", 5001), + 9222: ("127.0.0.1", 9223), 8080: ("127.0.0.1", 8081), + 3000: ("127.0.0.1", 3000), 3001: ("127.0.0.1", 3001)}) + self.assertEqual(given, {8006: 8006, 5000: 5001, 9222: 9223, 8080: 8081}) + self.assertEqual(self.start(Daemon(), ports=given).kwargs["ports"], + {k: ("127.0.0.1", v) for k, v in given.items()}) + self.assertEqual(self.start(Daemon(), ports={5000: None, 8006: ("127.0.0.1", 8006)}).kwargs["ports"], + {5000: ("127.0.0.1", None), 8006: ("127.0.0.1", 8006)}) + self.assertEqual(self.start(Daemon(), "ubuntu:24.04", environ=dict(NONE, OSWORLD2_GUEST_PORTS="3000")).kwargs, + {"detach": True}) + + def test_the_upstreams_retry_gets_a_second_vm_like_the_first(self): + environ = dict(LIMITS, OSWORLD2_GUEST_PORTS="3000") + daemon = Daemon() + first = self.start(daemon, environ=environ, ports={5000: 5000}) + first.remove() + daemon.seconds, daemon.late = 150, None + with mock.patch.object(self.site.time, "sleep", lambda s: setattr(daemon, "clock", daemon.clock + s)), \ + mock.patch.object(self.site.time, "monotonic", lambda: daemon.clock), self.assertRaises(ReadTimeout): + self.collection.run(PROVIDER, detach=True, ports={5000: 5001}) + self.assertTrue(first.removed.get("v")) + self.assertEqual([said for _, said in self.said()], [b"kept", b""]) + daemon.seconds = 0 + second = self.collection.run(PROVIDER, detach=True, ports={5000: 5001}) + self.assertEqual((daemon.creates, len(daemon.containers)), (3, 2)) + self.assertIsNot(second, first) + self.assertNotEqual(first.labels[self.site.LABEL], second.labels[self.site.LABEL]) + for made in (first, second): + self.assertEqual(made.kwargs["network"], self.site.NETWORK) + self.assertEqual(made.kwargs["ports"][3000], ("127.0.0.1", 3000)) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteBps"], + [{"Path": DEVICE, "Rate": 10 * 1024 * 1024}]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteIOps"], [{"Path": DEVICE, "Rate": 100}]) + self.assertEqual((second.started, second.removed), (True, None)) + + def test_the_upstreams_removal_takes_the_vms_volume(self): + made = self.start(Daemon()) + made.remove() + self.assertEqual(made.removed, {"v": True}) + made.remove(force=True) + self.assertEqual(made.removed, {"force": True, "v": True}) + other = self.start(Daemon(), "ubuntu:24.04") + other.remove() + self.assertEqual(other.removed, {}) + + def test_only_a_create_never_kept_is_swept_for_later(self): + daemon = Daemon(seconds=150, late=self.site.ADOPT_SECONDS + 1) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertEqual(self.said(), [(daemon.pending.labels[self.site.LABEL], b"")]) + self.swept.clear() + with self.assertLogs("nullius.osworld2", "WARNING"): + self.start(Daemon(seconds=150, late=8)) + self.start(Daemon()) + for daemon in (Daemon(refuse=True), Daemon(seconds=150, error=ConnectionError_)): + with self.assertRaises(Exception): + self.start(daemon) + self.assertEqual([said for _, said in self.said()], [b"kept"] * 4) + + def test_a_create_cut_short_leaves_its_sweeper_to_the_process_end(self): + with self.assertRaises(KeyboardInterrupt): + self.start(Daemon(seconds=150, error=KeyboardInterrupt)) + (_, r), = self.swept + os.set_blocking(r, False) + with self.assertRaises(BlockingIOError): + os.read(r, 16) + + def test_a_longer_timeout_and_the_callers_labels_are_kept(self): + made = self.start(Daemon(), timeout=300, labels={"a": "1"}) + self.assertEqual(self.collection.client.api.timeout, 300) + self.assertEqual(made.labels["a"], "1") + self.assertRegex(made.labels[self.site.LABEL], r"^[0-9a-f]{32}$") + self.assertEqual(sorted(self.start(Daemon(), labels=["b"]).labels), ["b", self.site.LABEL]) + + def test_the_vm_goes_on_its_own_network_unless_the_create_names_one(self): + self.assertEqual(self.start(Daemon()).kwargs["network"], self.site.NETWORK) + self.assertEqual(self.start(Daemon(), network="another").kwargs["network"], "another") + self.assertNotIn("network", self.start(Daemon(), network_mode="host").kwargs) + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + self.assertEqual(self.start(daemon).kwargs["network"], self.site.NETWORK) + + def test_the_start_queue_waits_out_the_longest_first_start(self): + site = self.site + port_lists = site.PORT_LISTS * site.DOCKER_PY_SECONDS + root = request = inspect = reload = start = site.CREATE_SECONDS + late_search = site.ADOPT_SECONDS + site.POLL_SECONDS + 2 * site.LOOK_SECONDS + request_answered_inspect_timed_out = request + inspect + late_search + self.assertEqual(site.LONGEST_START, port_lists + root + request_answered_inspect_timed_out + reload + start) + spec = importlib.util.spec_from_file_location("osworld2_driver_create", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=STARTUP): + spec.loader.exec_module(driver) + self.assertGreaterEqual(driver.START_SECONDS, site.LONGEST_START) + + +SWEPT_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + ps) [ -e "$FAKE_MADE" ] && echo cid9 ;; + rm) [ -e "$FAKE_RM_FAILS" ] && exit 1 ;; +esac +exit 0 +""" + + +class Sweep(unittest.TestCase): + """sweep(), the abandoned create's later removal, with a fake docker command line.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(SWEPT_DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.log, self.made = os.path.join(self.tmp, "calls"), os.path.join(self.tmp, "made") + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.log, + FAKE_MADE=self.made, FAKE_RM_FAILS=os.path.join(self.tmp, "rm_fails")) + env.start() + self.addCleanup(env.stop) + spec = importlib.util.spec_from_file_location("osworld2_startup_sweep", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + + def calls(self): + try: + with open(self.log) as f: + return f.read().splitlines() + except OSError: + return [] + + def sweep(self, on_sleep=lambda n: None): + clock = [0.0] + + def sleep(seconds): + clock[0] += seconds + on_sleep(len(self.calls())) + + with mock.patch.object(self.site, "time", types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0])): + self.site.sweep("t" * 32) + return clock[0] + + def test_a_container_made_later_is_removed_with_its_volume(self): + self.sweep(lambda n: n >= 3 and open(self.made, "w").close()) + look = f"ps -a -q --filter label={self.site.LABEL}={'t' * 32}" + self.assertEqual(self.calls(), [look] * 4 + ["rm -f -v cid9"]) + + def test_a_refused_removal_is_tried_again(self): + open(self.made, "w").close() + rm_fails = os.environ["FAKE_RM_FAILS"] + open(rm_fails, "w").close() + self.sweep(lambda n: os.path.exists(rm_fails) and os.remove(rm_fails)) + self.assertEqual([c for c in self.calls() if c.startswith("rm ")], ["rm -f -v cid9"] * 2) + self.assertEqual(self.calls()[-1], "rm -f -v cid9") + + def test_none_made_ends_the_sweep_in_its_time(self): + took = self.sweep() + self.assertGreaterEqual(took, self.site.SWEEP_SECONDS) + self.assertEqual(len(self.calls()), self.site.SWEEP_SECONDS // self.site.LOOK_SECONDS) + self.assertTrue(all(c.startswith("ps ") for c in self.calls())) + + def test_the_sweeper_runs_apart_outlives_a_sigterm_and_sweeps_once_its_caller_is_gone(self): + for kept in (True, False): + with self.subTest(kept=kept): + self.swept_by_sweeper(kept) + + def swept_by_sweeper(self, kept): + token = ("k" if kept else "s") * 32 + for path in (self.made, self.log): + if os.path.exists(path): + os.remove(path) + w = self.site._sweeper(token) + + def sweepers(): + found = [] + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + if token.encode() in f.read(): + found.append(int(pid)) + except OSError: + pass + return found + + self.addCleanup(lambda: [os.kill(p, signal.SIGKILL) for p in sweepers()]) + found = sweepers() + self.assertEqual(len(found), 1) + self.assertEqual(os.getsid(found[0]), found[0]) + t0 = time.time() + while not self.ignores_sigterm(found[0]) and time.time() - t0 < 20: + time.sleep(0.05) + os.kill(found[0], signal.SIGTERM) + open(self.made, "w").close() + if kept: + self.site._kept(w) + else: + os.close(w) + while sweepers() and time.time() - t0 < 30: + time.sleep(0.1) + self.assertEqual(sweepers(), []) + self.assertEqual(self.calls()[-1:], [] if kept else ["rm -f -v cid9"]) + + @staticmethod + def ignores_sigterm(pid): + with open(f"/proc/{pid}/status") as f: + ignored = next(ln.split()[1] for ln in f if ln.startswith("SigIgn:")) + return bool(int(ignored, 16) & 1 << (signal.SIGTERM - 1)) + + +PROVIDER_PY = "LOCK_TIMEOUT = 10\n" + + +class StartQueue(unittest.TestCase): + """The provider's port lock, with a fake desktop_env and the startup folder first on PYTHONPATH.""" + + def test_every_vm_start_waits_for_the_providers_port_lock(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + package = os.path.join(tmp, "desktop_env", "providers", "docker") + os.makedirs(package) + for folder in ("desktop_env", os.path.join("desktop_env", "providers"), package): + open(os.path.join(tmp, folder, "__init__.py"), "w").close() + with open(os.path.join(package, "provider.py"), "w") as f: + f.write(PROVIDER_PY) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + for option, wanted in (("docker", "-1"), ("aws", "10")): + with self.subTest(option=option): + done = subprocess.run([sys.executable, "-c", "import desktop_env.providers.docker.provider as m; " + "print(m.LOCK_TIMEOUT)"], cwd=tmp, capture_output=True, text=True, timeout=60, + env=dict(environ, PYTHONPATH=os.pathsep.join([STARTUP, tmp]), + NULLIUS_OPTION_VM_PROVIDER=option)) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertEqual(done.stdout.strip(), wanted) + + +class Said: + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + +class Network(unittest.TestCase): + """network(), with a fake docker command line.""" + + def setUp(self): + spec = importlib.util.spec_from_file_location("osworld2_startup_network", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + self.networks, self.calls, self.refuse = {}, [], None + self.shape = f"1|{self.site.SUBNET} {self.site.GATEWAY} " + + def docker(self, *args): + self.calls.append(args) + name = args[-1] + if args[:2] == ("network", "inspect"): + return Said(0, self.networks[name] + "\n") if name in self.networks else Said(1, "", f"Error: no such network: {name}") + if args[:2] == ("network", "create"): + if self.refuse: + return Said(1, "", self.refuse) + if name in self.networks: + return Said(1, "", f"Error response from daemon: network with name {name} already exists") + subnet, gateway = args[args.index("--subnet") + 1], args[args.index("--gateway") + 1] + label = "1" if f"{self.site.NETWORK_LABEL}=1" in args else "" + self.networks[name] = f"{label}|{subnet} {gateway} " + return Said(0, "id") + raise AssertionError(args) + + def test_a_missing_network_is_made_with_its_subnet_gateway_and_label(self): + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + create, = [c for c in self.calls if c[1] == "create"] + for flag in (("--driver", "bridge"), ("--subnet", self.site.SUBNET), ("--gateway", self.site.GATEWAY), + ("--label", f"{self.site.NETWORK_LABEL}=1")): + self.assertIn(" ".join(flag), " ".join(create)) + self.calls.clear() + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + self.assertEqual([c[1] for c in self.calls], ["inspect"]) + + def test_another_maker_at_the_same_moment_is_no_error(self): + first = self.docker + + def racing(*args): + if args[:2] == ("network", "create"): + first("network", "create", "--subnet", self.site.SUBNET, "--gateway", self.site.GATEWAY, + "--label", f"{self.site.NETWORK_LABEL}=1", args[-1]) + return first(*args) + + self.assertEqual(self.site.network(racing), self.site.NETWORK) + self.assertEqual(self.networks, {self.site.NETWORK: self.shape}) + + def test_a_network_of_that_name_on_another_subnet_is_refused(self): + self.networks[self.site.NETWORK] = "1|172.17.0.0/16 172.17.0.1 " + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertIn(f"docker network rm {self.site.NETWORK}", str(raised.exception)) + self.assertFalse([c for c in self.calls if c[1] == "create"]) + + def test_another_owner_s_network_of_that_name_is_never_offered_for_removal(self): + self.networks[self.site.NETWORK] = "|172.17.0.0/16 172.17.0.1 " + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertNotIn("docker network rm", str(raised.exception)) + self.assertIn("Nullius did not make it", str(raised.exception)) + self.assertIn(f"{self.site.NETWORK} and {self.site.SUBNET} are free", str(raised.exception)) + + def test_a_network_made_by_hand_with_that_subnet_and_gateway_is_used(self): + self.networks[self.site.NETWORK] = f"|{self.site.SUBNET} {self.site.GATEWAY} " + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + self.assertFalse([c for c in self.calls if c[1] == "create"]) + + def test_a_network_that_cannot_be_made_says_why(self): + self.refuse = "Error response from daemon: Pool overlaps with other one on this address space" + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertIn("Pool overlaps", str(raised.exception)) + self.assertIn(self.site.SUBNET, str(raised.exception)) + + def test_the_label_is_read_with_the_subnet(self): + self.site.network(self.docker) + inspect = self.calls[0] + self.assertIn('{{index .Labels "' + self.site.NETWORK_LABEL + '"}}|', inspect[inspect.index("-f") + 1]) + + def test_the_subnet_is_none_docker_takes_for_its_own_bridges(self): + subnet = ipaddress.ip_network(self.site.SUBNET) + for taken in ("172.16.0.0/12", "192.168.0.0/16"): + self.assertFalse(subnet.overlaps(ipaddress.ip_network(taken)), taken) + self.assertIn(ipaddress.ip_address(self.site.GATEWAY), subnet) + self.assertTrue(subnet.is_private) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_driver.py b/app/tests/test_osworld2_driver.py new file mode 100644 index 00000000..42e4452b --- /dev/null +++ b/app/tests/test_osworld2_driver.py @@ -0,0 +1,407 @@ +"""OSWorld2's driver cleans up after a unit whose docker hangs or whose vm/volumes note cannot be trusted. +A fake ``docker`` on PATH stands in.""" +from __future__ import annotations + +import errno +import importlib.util +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +VOLUME = "a" * 64 +OTHER = "b" * 64 +HANG = "30" +LIMIT = 10 +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +[ -z "$FAKE_ALIVE" ] || [ -e "$FAKE_ALIVE" ] || exit 0 +c=cid1 v="$FAKE_VOLUME" +[ -n "$FAKE_SECOND" ] && [ -e "$FAKE_SECOND" ] && c=cid2 v="$FAKE_OTHER" +case "$1" in + ps) echo "$c" ;; + inspect) i=0 + while [ "$i" -lt "$(( ${FAKE_DOCKER_DELAY:-0} * 10 ))" ] && [ ! -e "$FAKE_RELEASE" ]; do sleep 0.1; i=$((i + 1)); done + printf '%s\\t[{"Type":"bind","Source":"%s/vm/x.qcow2"},{"Type":"volume","Name":"%s","Source":"/v"}]\\n' "$c" "$FAKE_UNIT" "$v" ;; +esac +""" + + +def load(up): + spec = importlib.util.spec_from_file_location("osworld2_driver", os.path.join(HERE, "driver.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(module) + return module + + +CHILD = """import os, signal, sys, time +def term(*_): + with open(sys.argv[1], "w") as f: + f.write(repr(time.time())) + if len(sys.argv) > 3: + os.remove(sys.argv[3]) + sys.exit(0) +signal.signal(signal.SIGTERM, term) +open(sys.argv[2], "w").close() +time.sleep(60) +""" + +KILLED = """import importlib.util, sys +here, unit, started = sys.argv[1:4] +sys.argv = ["driver.py"] +sys.path.insert(0, here) +spec = importlib.util.spec_from_file_location("osworld2_driver", here + "/driver.py") +driver = importlib.util.module_from_spec(spec) +spec.loader.exec_module(driver) +driver.START_LOCK = unit + "/start.lck" +driver._run(["sh", "-c", "echo '%s'; echo $$ > %s; sleep 60" % (driver.STARTED, started)], unit, None, + unit + "/nop.log", "the run", stagger=True) +""" + + +class Cleanup(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(os.path.join(self.unit, "vm")) + self.note = os.path.join(self.unit, "vm", "volumes") + self.calls = os.path.join(self.tmp, "calls") + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_UNIT=self.unit, FAKE_VOLUME=VOLUME, FAKE_DOCKER_DELAY="0", + FAKE_RELEASE=os.path.join(self.tmp, "release")) + env.start() + self.addCleanup(env.stop) + threads = set(threading.enumerate()) + self.addCleanup(lambda: [t.join(30) for t in set(threading.enumerate()) - threads]) + self.addCleanup(lambda: open(os.environ["FAKE_RELEASE"], "w").close()) + self.driver = load(self.tmp) + self.driver.START_LOCK = os.path.join(self.tmp, "start.lck") + + def lines(self): + try: + with open(self.calls) as f: + return f.read().splitlines() + except OSError: + return [] + + def removed(self): + return [ln for ln in self.lines() if ln.startswith(("rm ", "volume rm "))] + + def remove(self): + t = threading.Thread(target=self.driver._remove_vms, args=(self.unit,), daemon=True) + t.start() + t.join(10) + self.assertFalse(t.is_alive(), "the removal waited") + return self.removed() + + def run_unit(self, failed, command="sleep 30"): + """The driver's run of a process that starts a VM, in a thread; every process it starts is waited for.""" + started = [] + + def popen(*a, **kw): + p = subprocess.Popen(*a, **kw) + started.append(p) + threading.Thread(target=p.wait, daemon=True).start() + return p + + self.driver.subprocess = types.SimpleNamespace(**{k: getattr(subprocess, k) for k in dir(subprocess) + if not k.startswith("__") and k != "Popen"}, Popen=popen) + + def end(): + for p in started: + if p.poll() is None: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + + self.addCleanup(end) + + def go(): + try: + self.driver._run(["sh", "-c", f"echo '{self.driver.STARTED}'; {command}"], self.unit, dict(os.environ), + os.path.join(self.unit, "nop.log"), "the run", stagger=True) + except RuntimeError as e: + failed.append(str(e)) + + t = threading.Thread(target=go) + t.start() + return t + + def test_another_user_s_start_lock_serves_too(self): + open(self.driver.START_LOCK, "w").close() + os.chmod(self.driver.START_LOCK, 0o444) + self.driver._started = self.driver._reaper = self.driver._remove_vms = lambda *a: None + self.driver._run(["true"], self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run", stagger=True) + + def test_a_slow_docker_does_not_hold_the_stop(self): + os.environ["FAKE_DOCKER_DELAY"] = HANG + reaped, failed = [], [] + self.driver._reaper = lambda pgid, *_: reaped.append(time.time()) + t = self.run_unit(failed) + time.sleep(2.5) + closed = time.time() + self.driver.CLOSED.set() + t.join(10) + self.assertTrue(reaped and failed) + self.assertLess(reaped[0] - closed, LIMIT) + + def test_a_stopped_unit_hands_the_reaper_the_volumes_it_saw(self): + alive = os.path.join(self.tmp, "alive") + open(alive, "w").close() + os.makedirs(self.note) + env = mock.patch.dict(os.environ, FAKE_ALIVE=alive, NULLIUS_UPSTREAM=self.tmp) + env.start() + self.addCleanup(env.stop) + failed = [] + t = self.run_unit(failed) + t0 = time.time() + while not any(ln.startswith("inspect") for ln in self.lines()) and time.time() - t0 < LIMIT: + time.sleep(0.05) + time.sleep(0.5) + os.remove(alive) + self.driver.CLOSED.set() + t.join(10) + self.assertTrue(failed) + t0 = time.time() + while f"volume rm {VOLUME}" not in self.removed() and time.time() - t0 < LIMIT: + time.sleep(0.1) + self.assertEqual(self.removed(), [f"volume rm {VOLUME}"]) + + def reapers(self): + found = [] + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + argv = f.read().split(b"\0") + except OSError: + continue + if b"--reap" in argv and self.unit.encode() in argv: + found.append(int(pid)) + return found + + @staticmethod + def ignores_sigterm(pid): + with open(f"/proc/{pid}/status") as f: + ignored = next(ln.split()[1] for ln in f if ln.startswith("SigIgn:")) + return bool(int(ignored, 16) & 1 << (signal.SIGTERM - 1)) + + def test_a_driver_killed_mid_run_leaves_its_reaper_the_vm_and_its_volume(self): + os.environ["NULLIUS_UPSTREAM"] = self.tmp + started = os.path.join(self.tmp, "started") + self.addCleanup(lambda: [os.kill(p, signal.SIGKILL) for p in self.reapers()]) + for sig in (signal.SIGTERM, signal.SIGKILL): + with self.subTest(signal=sig.name): + for path in (started, self.calls): + if os.path.exists(path): + os.remove(path) + driver = subprocess.Popen([sys.executable, "-c", KILLED, HERE, self.unit, started], + start_new_session=True) + self.addCleanup(lambda d=driver: (d.poll() is None and os.killpg(d.pid, signal.SIGKILL), d.wait())) + t0 = time.time() + while not (os.path.exists(started) and os.path.getsize(started)) and time.time() - t0 < LIMIT: + time.sleep(0.05) + with open(started) as f: + vm = int(f.read()) + while not (self.reapers() and self.ignores_sigterm(self.reapers()[0])) and time.time() - t0 < LIMIT: + time.sleep(0.05) + self.assertEqual(len(self.reapers()), 1) + os.killpg(driver.pid, sig) + driver.wait(LIMIT) + for pid in self.reapers(): + os.kill(pid, signal.SIGTERM) + t0 = time.time() + while f"volume rm {VOLUME}" not in self.removed() and time.time() - t0 < 3 * LIMIT: + time.sleep(0.1) + self.assertEqual(self.removed()[:2], ["rm -f -v cid1", f"volume rm {VOLUME}"]) + with self.assertRaises(ProcessLookupError): + os.killpg(vm, 0) + for pid in self.reapers(): + os.kill(pid, signal.SIGKILL) + + def test_a_reaper_that_cannot_start_ends_the_script_and_its_vm_at_once(self): + reaped = [] + with mock.patch.object(self.driver, "_arm", mock.Mock(side_effect=OSError(11, "Resource temporarily unavailable"))), \ + mock.patch.object(self.driver, "reap", lambda *a: reaped.append(a)), \ + self.assertRaises(self.driver.EnvironmentSetupError): + self.driver._run(["sh", "-c", "sleep 30"], self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), + "the run", stagger=True) + self.assertEqual(len(reaped), 1) + self.assertEqual(reaped[0][1:], (self.unit, True)) + os.killpg(reaped[0][0], signal.SIGKILL) + + def test_a_run_cut_short_by_an_error_still_lets_its_reaper_go(self): + released = [] + self.driver._reaper = lambda pgid, *a: (released.append(pgid), self.driver._ARMED.pop(pgid)) + self.driver._arm = lambda pgid, *a: self.driver._ARMED.__setitem__(pgid, -1) + broken = types.SimpleNamespace(Thread=mock.Mock(side_effect=RuntimeError("can't start new thread")), + Event=threading.Event) + with mock.patch.object(self.driver, "threading", broken), self.assertRaises(RuntimeError): + self.driver._run(["sh", "-c", f"echo '{self.driver.STARTED}'; sleep 30"], self.unit, dict(os.environ), + os.path.join(self.unit, "nop.log"), "the run", stagger=True) + self.assertEqual(len(released), 1) + self.assertEqual(self.driver._ARMED, {}) + os.killpg(released[0], signal.SIGKILL) + + def test_a_retried_setup_leaves_neither_vm_nor_either_disk(self): + second, go = os.path.join(self.tmp, "second"), os.path.join(self.tmp, "go") + env = mock.patch.dict(os.environ, FAKE_SECOND=second, FAKE_OTHER=OTHER) + env.start() + self.addCleanup(env.stop) + self.driver.NOTE_SECONDS = 0.1 + self.driver._reaper = lambda *a: None + failed = [] + + def noted(name): + t0 = time.time() + while time.time() - t0 < LIMIT: + try: + with open(self.note) as f: + if name in f.read().split(): + return True + except OSError: + pass + time.sleep(0.05) + return False + + t = self.run_unit(failed, f"while [ ! -e {go} ]; do sleep 0.1; done") + self.assertTrue(noted(VOLUME)) + open(second, "w").close() + self.assertTrue(noted(OTHER)) + open(go, "w").close() + t.join(LIMIT) + self.assertFalse(t.is_alive()) + self.assertEqual(failed, []) + self.assertEqual(self.removed(), ["rm -f -v cid2", f"volume rm {VOLUME} {OTHER}"]) + + def test_the_volumes_seen_are_noted(self): + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + with open(self.note) as f: + self.assertEqual(f.read().split(), [VOLUME]) + + def test_a_name_in_the_note_is_removed(self): + with open(self.note, "w") as f: + f.write(f"someones-volume\n{OTHER}\n{OTHER}\n../x\n{VOLUME}x\n") + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {OTHER} {VOLUME}"]) + + def test_the_note_is_read_to_its_limit(self): + with open(self.note, "w") as f: + f.write(" " * self.driver.NOTE_BYTES + OTHER + "\n") + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def test_a_note_not_the_units_own_is_not_evidence(self): + elsewhere = os.path.join(self.tmp, "elsewhere") + with open(elsewhere, "w") as f: + f.write(OTHER + "\n") + for make in (lambda: os.symlink(elsewhere, self.note), lambda: os.link(elsewhere, self.note), + lambda: os.makedirs(self.note), lambda: os.mkfifo(self.note)): + with self.subTest(make=make): + make() + self.assertEqual(self.remove()[-2:], ["rm -f -v cid1", f"volume rm {VOLUME}"]) + if os.path.isdir(self.note) and not os.path.islink(self.note): + os.rmdir(self.note) + else: + os.remove(self.note) + with open(elsewhere) as f: + self.assertEqual(f.read(), OTHER + "\n") + + def test_a_full_disk_still_removes(self): + write = os.write + + def full(fd, data): + if os.readlink(f"/proc/self/fd/{fd}") == self.note: + raise OSError(errno.ENOSPC, "No space left on device") + return write(fd, data) + + with mock.patch.object(self.driver.os, "write", full): + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def test_no_vm_folder_still_removes(self): + os.rmdir(os.path.join(self.unit, "vm")) + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def child(self, *alive): + """A process group of its own whose SIGTERM is timed, and can remove a container (FAKE_ALIVE).""" + signalled, ready = os.path.join(self.tmp, "signalled"), os.path.join(self.tmp, "ready") + for path in (signalled, ready): + if os.path.exists(path): + os.remove(path) + p = subprocess.Popen([sys.executable, "-c", CHILD, signalled, ready, *alive], start_new_session=True) + threading.Thread(target=p.wait, daemon=True).start() + self.addCleanup(lambda: (p.poll() is None and os.killpg(p.pid, signal.SIGKILL), p.wait())) + t0 = time.time() + while not os.path.exists(ready) and time.time() - t0 < 10: + time.sleep(0.05) + return p, signalled + + def test_a_hung_docker_does_not_hold_the_reapers_signal(self): + os.environ["FAKE_DOCKER_DELAY"] = HANG + p, signalled = self.child() + self.driver.SNAPSHOT_SECONDS = 1 + self.driver.SWEEPS = () + t0 = time.time() + self.driver.reap(p.pid, self.unit, True) + p.wait(10) + with open(signalled) as f: + self.assertLess(float(f.read()) - t0, LIMIT) + + def test_a_volume_seen_before_the_signal_is_removed_when_the_note_cannot_keep_it(self): + write = os.write + + def full(fd, data): + if os.readlink(f"/proc/self/fd/{fd}") == self.note: + raise OSError(errno.ENOSPC, "No space left on device") + return write(fd, data) + + alive = os.path.join(self.tmp, "alive") + os.environ["FAKE_ALIVE"] = alive + self.driver.SWEEPS = (0,) + for name, keep in (("refused", lambda: os.makedirs(self.note)), ("full", lambda: mock.patch.object(self.driver.os, "write", full))): + with self.subTest(note=name): + open(alive, "w").close() + open(self.calls, "w").close() + patch = keep() + if patch is not None: + patch.start() + p, _ = self.child(alive) + try: + self.driver.reap(p.pid, self.unit, True) + finally: + if patch is not None: + patch.stop() + p.wait(10) + self.assertFalse(os.path.exists(alive)) + self.assertEqual(self.removed(), [f"volume rm {VOLUME}"]) + if os.path.isdir(self.note): + os.rmdir(self.note) + + def test_the_reaper_notes_before_its_signal(self): + seen = [] + + def killpg(pgid, sig): + with open(self.note) as f: + seen.append(f.read()) + raise OSError(errno.ESRCH, "No such process") + + with mock.patch.object(self.driver.os, "killpg", killpg), mock.patch.object(self.driver, "SWEEPS", ()): + self.driver.reap(1, self.unit, True) + self.assertEqual(seen, [VOLUME + "\n"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_evaluation.py b/app/tests/test_osworld2_evaluation.py new file mode 100644 index 00000000..e00ba513 --- /dev/null +++ b/app/tests/test_osworld2_evaluation.py @@ -0,0 +1,403 @@ +"""OSWorld2's failures by whose they are: an evaluation that could not read the VM or a site is the score's +error, never a 0; the environment's is a setup error; the upstream's words never reach the message.""" +from __future__ import annotations + +import contextlib +import fcntl +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +from tests.test_osworld2_judge import load_driver + +STEP = "[2026-11-10 22:09:08,778 INFO lib_run_single/1579-EnvProcess-1] Done: False" +RESULT = "[2026-11-10 22:09:38,788 INFO lib_run_single/1622-EnvProcess-1] Result: 0.00" +SITE = ("[2026-11-10 22:09:38,787 ERROR state/82-EnvProcess-1] [GET_STATE_WITH_COOKIE] Failed to fetch state from " + "http://site.example.nip.io/api/state: HTTPConnectionPool(host='site.example.nip.io', port=80): ") +FILE = "[2026-11-10 22:09:20,000 {} desktopenv.pycontroller] {}" + + +def tried(status=None): + return [FILE.format("ERROR", f"Failed to get file. Status code: {status}" if status else + "An error occurred while trying to get the file: ('Connection broken: IncompleteRead(0 bytes read)')"), + FILE.format("INFO", "Retrying to get file.")] + + +class Evaluation(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.tmp = os.path.join(trial, "workspace") + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "081") + os.makedirs(self.where) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("0.0\n") + + def score(self, *lines): + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in lines)) + return self.driver.score("tasks__081", self.tmp, setup="original") + + def unread(self, *lines): + with self.assertRaises(self.driver.VerifierError) as raised: + self.score(*lines) + return str(raised.exception) + + def test_every_check_of_the_evaluation_is_the_scores_error_by_its_class(self): + self.assertIn("the LLM grader failed in compare_text_with_llm", str(self.unread( + STEP, "[2026-11-10 22:09:30,000 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: x", RESULT))) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("2.0\n") + self.assertIn("not a score from 0 to 1", self.unread(STEP, RESULT)) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("") + self.assertEqual("result.txt could not be read as a number", self.unread(STEP, RESULT)) + os.remove(os.path.join(self.where, "result.txt")) + self.assertIn("no result.txt", self.unread(STEP, RESULT)) + + def test_setup_errors_the_last_setup_went_on_from_are_counted_beside_the_score(self): + at = "[2026-11-10 22:00:0{} {} {}] {}" + reset = at.format(1, "INFO", "desktop_env/330-EnvProcess-1", "Environment has been used, reverting to snapshot x...") + clean = at.format(1, "INFO", "desktopenv.env", "Environment is clean, skipping snapshot revert (provider: docker).") + failed = at.format(2, "ERROR", "setup/467-EnvProcess-1", "Failed to launch application. Status code: x") + logged = at.format(2, "ERROR", "desktopenv.setup", "An error occurred while trying to send the request: x") + retried = [at.format(2, "ERROR", "setup/686-EnvProcess-1", "Attempt 1: Failed to connect, retrying. Error: x"), + at.format(2, "ERROR", "setup/311-EnvProcess-1", "Upload attempt 1 failed for x: y"), + at.format(2, "ERROR", "desktopenv.setup", "Failed to download x caused by y. Retrying... (2 attempts left)")] + self.assertNotIn("setup_errors", self.score(reset, *retried, STEP, RESULT)) + self.assertNotIn("setup_errors", self.score(failed, failed, reset, STEP, RESULT)) + self.assertEqual(self.score(reset, failed, *retried, logged, STEP, RESULT)["setup_errors"], 2) + self.assertEqual(self.score(failed, clean, "\x1b[31m" + logged + "\x1b[0m", STEP, RESULT)["setup_errors"], 1) + muse = "[2026-11-10 22:09:08,778 INFO desktopenv.experiment/1579-EnvProcess-1] Step 3: x" + self.assertEqual(self.score(reset, failed, STEP, logged, STEP, failed, logged, RESULT)["setup_errors"], 2) + self.assertNotIn("setup_errors", self.score(reset, muse, failed, RESULT)) + self.assertNotIn("setup_errors", self.score(reset, at.format(2, "ERROR", "setup/306-EnvProcess-1", + "Failed to upload file x. Status code: 500"), STEP)) + self.assertEqual(self.score(reset, at.format(2, "ERROR", "setup/1031-EnvProcess-1", + "Failed to upload file. Status code: 500"), STEP)["setup_errors"], 1) + done = at.format(3, "INFO", "desktopenv.env", "Environment setup complete.") + self.assertEqual(self.score(reset, failed, done, logged, RESULT)["setup_errors"], 1) + self.assertEqual(self.score(reset, failed, done, logged, STEP, RESULT)["setup_errors"], 2) + + def test_a_sites_state_that_timed_out_is_the_scores_error_with_the_upstreams_value(self): + said = self.unread(STEP, SITE + "Read timed out. (read timeout=30)", RESULT) + self.assertEqual(said, "the evaluation could not read a site's state (timed out), and the upstream scored the " + "task 0.0 without it; see the unit's eval.log") + self.assertIn("(no answer)", self.unread(STEP, SITE + "Max retries exceeded", RESULT)) + self.assertIn("a site's file (HTTP 502)", self.unread( + STEP, SITE.replace("fetch state", "download file") + "502 Server Error: Bad Gateway", RESULT)) + + def test_a_site_that_answered_leaves_the_upstreams_score(self): + for said in ("404 Client Error: NOT FOUND for url", "500 Server Error: INTERNAL SERVER ERROR for url"): + self.assertEqual(self.score(STEP, SITE + said, RESULT)["reward"], 0.0) + + def test_a_sites_read_that_failed_stands_though_a_later_read_succeeds(self): + read = "[2026-11-10 22:09:39,000 INFO state/96-EnvProcess-1] [GET_STATE_WITH_COOKIE] Returning state JSON directly" + self.assertIn("a site's state (timed out)", self.unread(STEP, SITE + "Read timed out.", read, RESULT)) + + def test_a_vm_read_that_gave_up_on_a_broken_connection_is_the_scores_error(self): + give_up = FILE.format("ERROR", "Failed to get file.") + self.assertIn("the VM's file (a broken connection)", self.unread(STEP, *tried(404) * 2, *tried(), give_up, RESULT)) + self.assertEqual(self.score(STEP, *tried() * 2, *tried(500), give_up, RESULT)["reward"], 0.0) + + def test_an_absent_file_or_a_recovered_read_is_no_failure(self): + give_up = FILE.format("ERROR", "Failed to get file.") + self.assertEqual(self.score(STEP, *tried(404) * 3, give_up, RESULT)["reward"], 0.0) + self.assertEqual(self.score(STEP, *tried(), FILE.format("INFO", "File downloaded successfully"), + "[2026-11-10 22:09:30,000 ERROR desktopenv.task081] Failed to get file.", + RESULT)["reward"], 0.0) + + def test_other_loggers_words_neither_fail_it_nor_hide_a_failure(self): + task = "[2026-11-10 22:09:21,000 {} desktopenv.task081] {}" + self.assertEqual(self.score(STEP, task.format("ERROR", SITE[SITE.index("] ") + 2:] + "Read timed out."), + task.format("ERROR", "An error occurred while trying to get the words: x"), + task.format("ERROR", "Failed to get the words."), + task.format("INFO", "Result: 1.00"), RESULT)["reward"], 0.0) + said = self.unread(STEP, *tried(), task.format("INFO", "still waiting"), " a continued line", + FILE.format("ERROR", "Failed to get file."), RESULT) + self.assertIn("the VM's file (a broken connection)", said) + self.assertNotIn("words", said) + + def test_only_the_evaluation_after_the_last_step_counts(self): + self.assertEqual(self.score(SITE + "Read timed out.", STEP, RESULT)["reward"], 0.0) + self.assertEqual(self.score(STEP, RESULT, SITE + "Read timed out.")["reward"], 0.0) + self.assertEqual(self.score(SITE + "Read timed out.", RESULT)["reward"], 0.0) + + def test_every_phase_of_the_last_multi_phase_episode_counts(self): + mark = "[2026-11-10 22:09:{:02},000 INFO lib_run_single/{}-EnvProcess-1] {}" + step, start = (lambda n: mark.format(n, 776, f"Phase {n} Step 1: x")), mark.format(30, 712, "Starting phase 2 in trajectory 2") + end = mark.format(40, 845, "Multi-phase result: 0.5000") + fail, lost = SITE + "Read timed out.", [*tried(), FILE.format("ERROR", "Failed to get file.")] + self.assertIn("a site's state (timed out)", self.unread(step(1), fail, start, step(2), end)) + self.assertIn("a site's state (timed out)", self.unread(step(1), start, step(2), fail, end)) + self.assertIn("the VM's file (a broken connection)", self.unread(step(1), *lost, start, step(2), end)) + self.assertEqual(self.score(step(1), start, fail, step(2), end)["reward"], 0.0) + self.assertEqual(self.score(step(1), start, fail, end)["reward"], 0.0) + self.assertEqual(self.score(step(1), fail, end, step(1), end)["reward"], 0.0) + self.assertEqual(self.score(step(1), fail, start, step(1), start, step(2), end)["reward"], 0.0) + asked = mark.format(2, 750, "Phase 1 User simulator Q: x | A: y") + self.assertEqual(self.score(step(1), fail, start, asked, start, step(2), end)["reward"], 0.0) + self.assertIn("a site's state (timed out)", self.unread(asked, fail, end)) + + def test_a_task_that_retries_its_site_reads_keeps_its_score_but_not_a_lost_vm_read(self): + mark = "[2026-11-10 22:09:{:02},000 INFO lib_run_single/{}-EnvProcess-1] {}" + step, end = mark.format(1, 776, "Phase 1 Step 1: x"), mark.format(40, 845, "Multi-phase result: 0.5000") + self.where = self.where.replace("081", "069") + os.makedirs(self.where) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("0.5\n") + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in (step, SITE + "Read timed out.", end))) + self.assertEqual(self.driver.score("tasks__069", self.tmp, setup="original")["reward"], 0.5) + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in (step, *tried(), FILE.format("ERROR", "Failed to get file."), end))) + with self.assertRaises(RuntimeError): + self.driver.score("tasks__069", self.tmp, setup="original") + + def test_the_null_controls_unread_evaluation_is_its_error(self): + def run(argv, cwd, env, log, *a, **kw): + with open(os.path.join(argv[-1], "result.txt"), "w") as f: + f.write("0.0\n") + with open(log, "w") as f: + f.write("\n".join(["[2026-11-10 22:09:08,778 INFO desktopenv.experiment] Done: True", + SITE.replace("state/82-EnvProcess-1", "desktopenv.getters.state") + "Read timed out.", + "[2026-11-10 22:09:38,788 INFO desktopenv.experiment] Result: 0.00"]) + "\n") + + patches = {"_vm": lambda *a: "vm", "_run": run, "_task": lambda *a: {"instruction": "", "proxy": False}, + "_assets": lambda *a: "assets"} + with mock.patch.multiple(self.driver, **patches), self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("a site's state (timed out)", str(raised.exception)) + self.assertIn("see the unit's nop.log", str(raised.exception)) + + def set_up(argv, cwd, env, log, *a, **kw): + with open(os.path.join(argv[-1], "result.txt"), "w") as f: + f.write("0.0\n") + with open(log, "w") as f: + f.write("[2026-11-10 22:00:01,000 INFO desktopenv.env] Environment is clean, skipping snapshot revert.\n" + "[2026-11-10 22:00:02,000 ERROR desktopenv.setup] Failed to launch application. Status code: x\n") + + with mock.patch.multiple(self.driver, **dict(patches, _run=set_up)): + self.assertEqual(self.driver.score("tasks__001", self.tmp)["setup_errors"], 1) + + +TRACE = ["[2026-11-10 20:00:00,000 INFO lib_run_single/1500-EnvProcess-1] [Instruction]: the task's words", + "[2026-11-10 20:00:01,000 ERROR run_multienv_m3/668-EnvProcess-1] Exception in EnvProcess-1 tasks/001: words", + "[2026-11-10 20:00:01,000 ERROR run_multienv_m3/669-EnvProcess-1] Traceback (most recent call last):", + ' File "/up/lib_run_single.py", line 1518, in run_single_example', + " env.reset(task_config=example)"] + + +class Failures(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.tmp = os.path.join(trial, "workspace") + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.logged = [] + + def run(argv, cwd, env, log, *a, **kw): + with open(log, "w") as f: + f.write("".join(ln + "\n" for ln in self.logged)) + where = os.path.join(cwd, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + os.makedirs(where, exist_ok=True) + with open(os.path.join(where, "traj.jsonl"), "w") as f: + f.write('{"Error": "tasks/001 - the task\'s words"}\n') + + self.real_run = self.driver._run + p = mock.patch.multiple(self.driver, _vm=lambda *a: "vm", _run=run, + _task=lambda *a: {"instruction": "", "proxy": False}, _assets=lambda *a: "assets") + p.start() + self.addCleanup(p.stop) + + def answer(self, *lines): + self.logged = list(lines) + with self.assertRaises(RuntimeError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertNotIn("words", str(raised.exception)) + return raised.exception + + def test_what_fails_before_the_upstream_starts_is_the_environments(self): + for e in (self.driver.NoAnswer("docker create got no answer in 60 s"), OSError("no space"), KeyError("proxy")): + with self.subTest(type(e).__name__), mock.patch.object(self.driver, "_vm", side_effect=e), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertTrue(str(raised.exception).startswith(type(e).__name__ + ": ")) + + def test_a_shared_lock_held_past_the_units_turn_limit_is_the_environments(self): + lock = os.path.join(self.tmp, "port-{}.lck") + fd = os.open(lock.format(9), os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + spare = self.driver.EVALUATE_SECONDS + self.driver.CLOSE_SECONDS + 1 + closing = threading.Timer(10, self.driver.CLOSED.set) + closing.start() + self.addCleanup(closing.cancel) + with mock.patch.dict(self.driver.GUEST_PORTS, {"001": (9,)}), mock.patch.object(self.driver, "PORT_LOCK", lock), \ + mock.patch.object(self.driver, "_turn_seconds", lambda: spare), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertIn("the unit's time ran out while another unit's VM held port 9", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + with mock.patch.dict(self.driver.GUEST_PORTS, {"001": (9,)}), mock.patch.object(self.driver, "PORT_LOCK", lock), \ + mock.patch.object(self.driver, "_score_seconds", lambda: self.driver.CLOSE_SECONDS + 1), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("the unit's time ran out while another unit's VM held port 9", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + given = [] + with mock.patch.object(self.driver, "_run", lambda *a, **kw: given.append(kw)), self.assertRaises(RuntimeError): + self.driver.score("tasks__001", self.tmp) + self.assertAlmostEqual(given[0]["until"] - time.time(), + self.driver._score_seconds() - 2 * self.driver.CLOSE_SECONDS, delta=5) + self.assertEqual(given[0]["grace"], self.driver.CLOSE_SECONDS) + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + self.addCleanup(p.kill) + t0 = time.time() + self.driver._started(p, os.path.join(self.tmp, "none.log"), "the null control's run", until=t0 + 1) + self.assertLess(time.time() - t0, 5) + self.assertIsNone(p.poll()) + + def test_a_traceback_the_runner_went_on_after_is_not_why_it_ended(self): + crash = [*TRACE, ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', "desktop_env.desktop_env.EnvironmentSetupError: x"] + phase, result = (STEP.replace(STEP[STEP.index("] ") + 2:], said) for said in ("Starting phase 2 of 2", "Result: 0.00")) + for then in (STEP, phase, result): + with self.subTest(then): + env = self.answer(*crash, then) + self.assertIs(type(env), RuntimeError) + self.assertIn("wrote no result.txt: no exception named", str(env)) + + def test_a_crash_on_the_vms_failed_last_screenshot_is_the_environments_and_no_other(self): + shot = "[2026-11-10 20:00:0{},000 {} python/{}-EnvProcess-1] {}" + got, lost = shot.format(2, "INFO", 467, "Got screenshot successfully"), shot.format(3, "ERROR", 480, "Failed to get screenshot.") + step = "[2026-11-10 20:00:01,000 INFO lib_run_single/1530-EnvProcess-1] Step 7: the task's words" + crash = [TRACE[2], ' File "/up/lib_run_single.py", line 1586, in run_single_example', " _f.write(obs['screenshot'])", + "TypeError: a bytes-like object is required, not 'NoneType'"] + env = self.answer(TRACE[0], step, got, lost, *crash) + self.assertIs(type(env), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: the VM stopped answering in step 7: its last screenshot failed", str(env)) + muse = [ln.replace("python/480", "desktopenv.pycontroller/480").replace("lib_run_single/", "desktopenv.experiment/") + for ln in (step, got, lost)] + self.assertIn("in step 7: its last screenshot failed", str(self.answer(TRACE[0], *muse, *crash))) + first = self.answer(TRACE[0], lost, *crash[:-1], "PIL.UnidentifiedImageError: cannot identify image file") + self.assertIn("the VM stopped answering before the first step", str(first)) + judged = self.answer(TRACE[0], step, lost, *crash[:2], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + crash[-1]) + self.assertIs(type(judged), self.driver.VerifierError) + reset = self.answer(TRACE[0], lost, *crash[:2], ' File "/up/desktop_env/desktop_env.py", line 300, in reset', "KeyError: 'x'") + self.assertIs(type(reset), self.driver.EnvironmentSetupError) + muse = self.answer(TRACE[0], step, lost, *crash[:2], ' File "/up/mm_agents/muse_spark_agent.py", line 605, in _screenshot_bytes', + "mm_agents.muse_spark_agent.StepError: words") + self.assertIn("in step 7: its last screenshot failed", str(muse)) + agent = ' File "/up/mm_agents/m3/agent.py", line 9, in predict' + for name, lines in (("another exception", [step, lost, *crash[:-1], "KeyError: 'screenshot'"]), + ("a TypeError of another kind", [step, lost, *crash[:-1], "TypeError: unsupported operand"]), + ("the agent's own None", [step, lost, *crash[:2], agent, crash[-1]])): + with self.subTest(name): + unknown = self.answer(TRACE[0], *lines) + self.assertIs(type(unknown), self.driver.CauseUnknownError) + self.assertIn("after the VM's last screenshot failed in step 7", str(unknown)) + for name, lines in (("a later screenshot came", [step, lost, got, *crash]), ("no screenshot failed", [step, got, *crash])): + with self.subTest(name): + self.assertIs(type(self.answer(TRACE[0], *lines)), RuntimeError) + + def test_a_runner_ended_by_a_signal_or_with_a_lost_worker_is_said_by_what_its_log_can_show(self): + log = os.path.join(self.tmp, "run.log") + signalled = [sys.executable, "-c", "import os, sys, signal; signal.signal(signal.SIGHUP, signal.SIG_DFL); print('Step 3: words\\nTraceback (most recent call last):\\nKeyError: 1', flush=True); os.kill(os.getpid(), int(sys.argv[1]))"] + aborted = [sys.executable, "-c", "import os; os.abort()"] + crashed = [sys.executable, "-c", "raise RuntimeError('words')"] + with mock.patch.object(self.driver, "_reaper", lambda *a, **kw: None): + for n in (1, 9, 15): + with contextlib.suppress(OSError): + os.remove(log) + with self.subTest(n), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.real_run(signalled + [str(n)], self.tmp, dict(os.environ), log, "the runner") + self.assertIn(f"the runner exited -{n}: ended by signal {n} (the whole log", str(raised.exception)) + for argv, kind, said in ((aborted, self.driver.CauseUnknownError, "exited -6: ended by signal 6, with no traceback"), + (crashed, RuntimeError, "exited 1: RuntimeError")): + os.remove(log) + with self.subTest(said), self.assertRaises(RuntimeError) as raised: + self.real_run(argv, self.tmp, dict(os.environ), log, "the runner") + self.assertIs(type(raised.exception), kind) + self.assertIn(said, str(raised.exception)) + at = "[2026-11-10 20:00:0{},000 INFO run_multienv_m3/{}] {}" + started, ended = at.format(0, "612-EnvProcess-1", "Process EnvProcess-1 started."), \ + at.format(5, "774-EnvProcess-1", "EnvProcess-1 cleaning up environment...") + finished = at.format(1, "879-MainProcess", "All tasks finished.") + lost = self.answer(started, finished, TRACE[0]) + self.assertIs(type(lost), self.driver.CauseUnknownError) + self.assertIn("wrote no result.txt: its worker process ended without its cleanup (the whole log", str(lost)) + self.assertIs(type(self.answer(started, *TRACE, " x", "RuntimeError: words")), self.driver.CauseUnknownError) + self.assertIs(type(self.answer(started, finished, TRACE[0], ended)), RuntimeError) + self.assertIs(type(self.answer(started, *TRACE, " x", "RuntimeError: words", ended)), RuntimeError) + self.assertIs(type(self.answer(finished, TRACE[0], ended)), RuntimeError) + term = at.format(6, "799-MainProcess", "Received signal 15. Gracefully shutting down...") + outside = self.answer(started, term, at.format(6, "799-EnvProcess-1", "Process 1 received signal 15. Shutting down...")) + self.assertIs(type(outside), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: ended by signal 15 (the whole log", str(outside)) + with open(os.path.join(self.tmp, "ended.log"), "w") as f: + f.write(f"{started}\n{term}\n") + self.assertIn("ended by signal 15 (", self.driver._tail(os.path.join(self.tmp, "ended.log"), 1)[1]) + own = at.format(6, "1051-MainProcess", "Unexpected error in main process: words") + self.assertIs(type(self.answer(started, own, *TRACE, " x", "RuntimeError: words", term, ended)), RuntimeError) + child = at.format(6, "799-EnvProcess-1", "Received signal 15. Gracefully shutting down...") + self.assertIs(type(self.answer(started, child, ended)), RuntimeError) + + def test_a_null_control_setup_that_fails_lets_the_units_lock_go(self): + with mock.patch.object(self.driver, "_task_file", side_effect=RuntimeError("no task")), self.assertRaises(RuntimeError): + self.driver.setup("tasks__001", self.tmp) + self.assertEqual(self.driver._OWN, []) + + def test_a_runner_with_no_result_is_said_by_its_last_tracebacks_known_class_only(self): + env = self.answer(*TRACE, ' File "/up/desktop_env/desktop_env.py", line 407, in reset', + "desktop_env.desktop_env.EnvironmentSetupError: Custom task setup failed: the task's words") + self.assertIs(type(env), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: EnvironmentSetupError (the whole log: ", str(env)) + started = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 186, in _start_emulator', + ' File "/up/desktop_env/providers/docker/provider.py", line 9, in start_emulator', + "requests.exceptions.ReadTimeout: the task's words") + self.assertIs(type(started), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: ReadTimeout (the whole log: ", str(started)) + judged = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + "RuntimeError: Hosted state is unavailable: the task's words") + self.assertIs(type(judged), self.driver.VerifierError) + self.assertIn("RuntimeError (the whole log: ", str(judged)) + own = self.answer(*TRACE[:4], ' File "/up/mm_agents/m3.py", line 9, in reset', "PrivateWordsError: words") + self.assertIs(type(own), RuntimeError) + self.assertIn("an exception of the upstream's own (the whole log: ", str(own)) + self.assertIn("no exception named (the whole log: ", str(self.answer(*TRACE[:2]))) + quoted = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + "RuntimeError: words, in reset") + self.assertIs(type(quoted), self.driver.VerifierError) + + def test_a_runner_that_cannot_start_is_the_environments(self): + with mock.patch.object(self.driver.subprocess, "Popen", side_effect=BlockingIOError(11, "no process")), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.real_run([sys.executable, "-c", ""], self.tmp, {}, os.path.join(self.tmp, "x.log"), "the runner") + self.assertEqual(str(raised.exception), "the runner did not start: BlockingIOError: [Errno 11] no process") + + def test_the_null_controls_failure_after_its_setup_is_said_as_its_class(self): + def run(argv, cwd, env, log, *a, **kw): + with open(log, "w") as f: + f.write("".join(ln + "\n" for ln in TRACE[2:] + [ + ' File "/up/desktop_env/desktop_env.py", line 131, in __init__', + ' File "/up/desktop_env/desktop_env.py", line 186, in _start_emulator', + "docker.errors.APIError: the task's words"])) + + with mock.patch.object(self.driver, "_run", run), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("wrote no result.txt: an exception of the upstream's own", str(raised.exception)) + self.assertNotIn("words", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_gitlab.py b/app/tests/test_osworld2_gitlab.py new file mode 100644 index 00000000..c3d1f54c --- /dev/null +++ b/app/tests/test_osworld2_gitlab.py @@ -0,0 +1,919 @@ +"""OSWorld2's GitLab, as gitlab_server.py and the driver reach it, before a fake GitLab on loopback. +With NULLIUS_OSWORLD2_GITLAB or NULLIUS_OSWORLD2_UPSTREAM set, also checked against them, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import io +import json +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock +from urllib.parse import urlsplit + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +SUFFIX = "10.213.0.1.nip.io" +SITES = "NULLIUS_OPTION_WEBSITES" +URL = f"http://gitlab.{SUFFIX}" +TOKEN = "t" * 24 +JUDGE = {"model": "anthropic/claude-sonnet-4.6", "base_url": "http://127.0.0.1:9/u1-judge/v1", "key": "k"} + + +def load(): + spec = importlib.util.spec_from_file_location("osworld2_gitlab", os.path.join(HERE, "gitlab_server.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "path", [HERE, *sys.path]): + spec.loader.exec_module(module) + return module + + +class Lab(BaseHTTPRequestHandler): + """A fake GitLab: an ``empty`` sign-in page, the token's ``admin``, its online ``runners``.""" + empty, admin, runners = False, True, [{"id": 1}] + + def log_message(self, *_): + pass + + def do_GET(self): + url = urlsplit(self.path) + if url.path == "/users/sign_in": + self.answer(200, b"" if self.empty else b"GitLab") + elif self.headers.get("PRIVATE-TOKEN") != TOKEN: + self.answer(401, b"{}") + elif url.path == "/api/v4/user": + self.answer(200, json.dumps({"is_admin": self.admin}).encode()) + elif url.path == "/api/v4/runners/all": + self.answer(200, json.dumps(self.runners).encode()) + else: + self.answer(404, b"{}") + + def answer(self, status, body): + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class Case(unittest.TestCase): + def setUp(self): + self.g = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + Lab.empty, Lab.admin, Lab.runners = False, True, [{"id": 1}] + server = ThreadingHTTPServer(("127.0.0.1", 0), Lab) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + for patch in (mock.patch.object(self.g, "PROXIES", {"http": f"http://127.0.0.1:{server.server_port}"}), + mock.patch.object(self.g.websites, "_resolves", lambda host: {"10.213.0.1"}), + mock.patch.object(self.g, "LOCK", os.path.join(self.tmp, "gitlab.lck")), + mock.patch.object(self.g.websites, "SITES_LOCK", os.path.join(self.tmp, "sites.lck"))): + patch.start() + self.addCleanup(patch.stop) + + +class Option(Case): + def test_no_sites_give_no_url_and_a_task_that_needs_gitlab_is_refused(self): + self.assertIsNone(self.g.url({})) + self.assertIsNone(self.g.needed("001", {})) + self.assertEqual(self.g.env("001", {}), {}) + for eid in sorted(self.g.TASKS): + with self.subTest(eid), self.assertRaises(self.g.GitLabError) as raised: + self.g.env(eid, {SITES: "none"}) + self.assertIn(f"task {eid} needs GitLab", str(raised.exception)) + self.assertIn("Website host suffix", str(raised.exception)) + self.assertEqual(os.listdir(self.tmp), []) + + def test_the_url_and_this_machine_s_token_are_gitlab_s_tasks_alone(self): + given = {SITES: SUFFIX, "NULLIUS_UPSTREAM": os.path.join(self.tmp, "upstreams", "osworld2", "c")} + self.assertEqual(self.g.needed("041", given), URL) + self.assertIsNone(self.g.needed("005", given)) + made = self.g.env("026", given) + self.assertEqual(made, {"GITLAB_URL": URL, self.g.TOKEN: self.g.token(self.g.home(given))}) + self.assertEqual(self.g.env("041", given), made) + self.assertEqual(self.g.env("005", given), {}) + + def test_no_option_and_no_licence_ask_for_gitlab(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + self.assertFalse({"gitlab", "gitlab_write"} & set(bench["options"])) + self.assertFalse(bench.get("environments")) + self.assertFalse(os.path.exists(os.path.join(HERE, "gitlab.Dockerfile"))) + + +class Token(unittest.TestCase): + def setUp(self): + self.g = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.stack = os.path.join(self.tmp, "upstreams", "osworld2", f"gitlab-{self.g.COMMIT}") + + def test_the_stack_is_beside_the_upstream_s_clone(self): + self.assertEqual(self.g.home({"NULLIUS_UPSTREAM": os.path.dirname(self.stack) + "/c/"}), self.stack) + + def test_made_once_beside_the_stack_for_this_user_alone(self): + found, failed = [], [] + + def make(): + try: + found.append(self.g.token(self.stack)) + except Exception as e: + failed.append(e) + threads = [threading.Thread(target=make) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(10) + self.assertEqual((len(found), failed), (8, [])) + self.assertEqual(len(set(found)), 1) + self.assertEqual(self.g.token(self.stack), found[0]) + self.assertGreaterEqual(len(found[0]), self.g.MIN_TOKEN) + self.assertTrue(found[0].isalnum()) + path = self.stack + self.g.TOKEN_FILE + self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600) + self.assertEqual(os.listdir(os.path.dirname(path)), [os.path.basename(path)]) + self.assertNotEqual(self.g.token(self.stack + "-other"), found[0]) + + def test_no_file_left_when_the_token_cannot_be_kept(self): + with mock.patch.object(self.g.os, "link", side_effect=PermissionError), self.assertRaises(PermissionError): + self.g.token(self.stack) + self.assertEqual(os.listdir(os.path.dirname(self.stack)), []) + + def test_a_file_that_holds_no_token_is_refused_without_its_words(self): + os.makedirs(os.path.dirname(self.stack)) + for said in ("secret-but-short", "x" * 30 + "\n"): + with open(self.stack + self.g.TOKEN_FILE, "w") as f: + f.write(said) + with self.subTest(said), self.assertRaises(self.g.GitLabError) as raised: + self.g.token(self.stack) + self.assertNotIn(said.strip(), str(raised.exception)) + + +class Clone(unittest.TestCase): + def test_cloned_at_the_commit_once_and_again_when_changed(self): + g = load() + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + source, stack = os.path.join(tmp, "source"), os.path.join(tmp, "gitlab") + git = lambda *a: subprocess.run(["git", "-C", source, *a], check=True, capture_output=True, text=True).stdout + os.makedirs(source) + git("init", "--quiet") + with open(os.path.join(source, "docker-compose.yml"), "w") as f: + f.write("services: {}\n") + git("add", ".") + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "--quiet", "-m", "c") + with mock.patch.object(g, "REPOSITORY", "file://" + source), mock.patch.object(g, "COMMIT", git("rev-parse", "HEAD").strip()): + self.assertEqual(g._clone(stack), stack) + os.utime(stack, (0, 0)) + g._clone(stack) + self.assertEqual(os.stat(stack).st_mtime, 0) + with open(os.path.join(stack, "docker-compose.yml"), "w") as f: + f.write("changed\n") + g._clone(stack) + with open(os.path.join(stack, "docker-compose.yml")) as f: + self.assertEqual(f.read(), "services: {}\n") + self.assertEqual(sorted(os.listdir(tmp)), ["gitlab", "source"]) + with mock.patch.object(g, "REPOSITORY", "file://" + os.path.join(tmp, "none")), self.assertRaises(g.GitLabError): + g._clone(os.path.join(tmp, "other")) + self.assertEqual(sorted(os.listdir(tmp)), ["gitlab", "source"]) + with mock.patch.object(g, "prepare", lambda *a, **kw: self.fail("prepared")), \ + mock.patch.object(g, "LOCK", os.path.join(tmp, "gitlab.lck")), \ + mock.patch.object(g.websites, "SITES_LOCK", os.path.join(tmp, "sites.lck")), \ + mock.patch("sys.stderr", new=io.StringIO()) as err: + for given in (source, tmp): + self.assertEqual(g.main(["prepare", given, SUFFIX]), 1) + self.assertIn("give the stack Nullius cloned", err.getvalue()) + self.assertTrue(os.path.isfile(os.path.join(source, "docker-compose.yml"))) + + +class Ready(Case): + def test_sign_in_token_and_runner(self): + self.assertEqual(self.g.ready(URL, TOKEN, runner=True), URL) + + def test_an_empty_sign_in_page_is_not_gitlab(self): + Lab.empty = True + with self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN) + self.assertIn("answered no page", str(raised.exception)) + + def test_another_token_or_no_administrator_is_refused(self): + for token, admin in (("x" * 24, True), (TOKEN, False)): + Lab.admin = admin + with self.subTest(admin=admin), self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, token) + self.assertIn("GITLAB_PRIVATE_TOKEN", str(raised.exception)) + self.assertNotIn(token, str(raised.exception)) + + def test_no_runner_online_fails_only_where_one_is_needed(self): + Lab.runners = [] + self.assertEqual(self.g.ready(URL, TOKEN), URL) + with self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN, runner=True) + self.assertIn("no runner online", str(raised.exception)) + + def test_no_answer_says_what_to_do(self): + with mock.patch.object(self.g, "PROXIES", {"http": "http://127.0.0.1:9"}), \ + self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN) + self.assertIn("did not answer", str(raised.exception)) + + +class Held(Case): + def test_gitlab_s_tasks_run_one_at_a_time_and_the_lock_goes_to_the_reaper(self): + inside, released, order = threading.Event(), threading.Event(), [] + + def first(): + with self.g.held(URL, threading.Event()): + order.append("first") + inside.set() + released.wait(10) + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + closed = threading.Event() + closed.set() + with self.assertRaises(self.g.websites.WebsiteError) as raised: + with self.g.held(URL, closed): + pass + self.assertIn("another unit's GitLab task ran", str(raised.exception)) + with self.g.held(None, closed): + order.append("other") + released.set() + t.join(10) + with self.g.held(URL, threading.Event()): + self.assertEqual(len(self.g.websites.held_fds()), 2) + order.append("second") + self.assertEqual(order, ["first", "other", "second"]) + self.assertEqual(self.g.websites.held_fds(), ()) + + +class Shadow(unittest.TestCase): + def test_no_module_of_the_folder_hides_one_the_upstream_imports(self): + """A module here named as one the upstream imports would be loaded in its place by nop.py.""" + here = {n[:-3] for n in os.listdir(HERE) if n.endswith(".py")} + self.assertFalse(here & {"gitlab", "requests", "dotenv", "docker", "desktop_env", "mm_agents", "lib_run_single", + "task_loader"}) + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + if not up: + return + files = glob.glob(os.path.join(up, "*.py")) + _task_classes() + for d in ("desktop_env", "mm_agents"): + files += glob.glob(os.path.join(up, d, "**", "*.py"), recursive=True) + imported = set() + for p in files: + with open(p, encoding="utf-8", errors="replace") as f: + try: + tree = ast.parse(f.read()) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported |= {a.name.split(".")[0] for a in node.names} + elif isinstance(node, ast.ImportFrom) and not node.level and node.module: + imported.add(node.module.split(".")[0]) + self.assertGreater(len(files), 100) + self.assertFalse(here & imported) + + +class Share(unittest.TestCase): + def setUp(self): + self.g = load() + + def test_the_vms_own_limits_and_the_default_where_one_is_none(self): + self.assertIsNone(self.g.share({})) + self.assertEqual(self.g.share({"NULLIUS_OPTION_WRITE_MBPS": "8", "NULLIUS_OPTION_WRITE_IOPS": "80"}), (8, 80)) + self.assertEqual(self.g.share({"NULLIUS_OPTION_WRITE_MBPS": "8"}), (8, 160)) + self.assertEqual(self.g.share({"NULLIUS_OPTION_WRITE_IOPS": "80"}), (16, 80)) + + def test_a_limit_that_leaves_a_service_none_is_refused(self): + with self.assertRaises(self.g.GitLabError): + self.g.share({"NULLIUS_OPTION_WRITE_IOPS": "4"}) + + +class Allocation(unittest.TestCase): + def test_every_service_s_share_within_gitlab_s(self): + g = load() + out = g.allocation(8, 80) + self.assertEqual(set(out), {s for s, _ in g.SHARES}) + self.assertLessEqual(sum(v[0] for v in out.values()), 8 * g.websites.MB) + self.assertLessEqual(sum(v[1] for v in out.values()), 80) + self.assertEqual(out["GITLAB"], (6 * g.websites.MB, 60)) + for bad in ((0, 80), (8, 4)): + with self.subTest(bad), self.assertRaises(g.GitLabError): + g.allocation(*bad) + + +class Started(Exception): + pass + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_gitlab", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class Driver(unittest.TestCase): + """The driver's use of gitlab_server.py.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.g = self.driver.gitlab_server + self.envs, self.held = [], [] + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + @contextlib.contextmanager + def held(value, closed): + self.held.append((value, closed)) + yield + + self.ups, self.order = [], [] + + @contextlib.contextmanager + def warden(dir): + self.order.append(("warden", dir)) + try: + yield + finally: + self.order.append(("let go", dir)) + + for target, name, fake in ((self.driver, "_vm", lambda dir, domain, eid: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_task_file", lambda *a: None), + (self.g, "held", held), + (self.g, "prepare", lambda where, **kw: self.order.append(("prepare", where)) + or self.ups.append(kw["closed"]) or self.order.append(("limits", kw["limits"]))), + (self.g, "unit", lambda where, dir: os.path.join(dir, "gitlab.limits")), + (self.g, "up", lambda *a: self.ups.append(a) or self.order.append(("up",))), + (self.driver, "_warden", warden)): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + self.home = os.path.join(self.tmp, "upstreams", "osworld2", f"gitlab-{self.g.COMMIT}") + p = mock.patch.dict(os.environ, {SITES: SUFFIX, "NULLIUS_UPSTREAM": os.path.join(os.path.dirname(self.home), "c")}) + p.start() + self.addCleanup(p.stop) + p = mock.patch.object(self.driver.websites, "held", lambda *a: contextlib.nullcontext()) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid, **kw): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k", **kw) + + def test_a_task_that_needs_gitlab_is_refused_without_the_sites_before_anything_runs(self): + with mock.patch.dict(os.environ, {SITES: "none"}): + setup = self.driver.EnvironmentSetupError + for start, kind in ((lambda: self.driver.setup("tasks__041", self.tmp), self.g.GitLabError), + (lambda: self.answer("041", judge=JUDGE), setup), + (lambda: self.driver.score("tasks__041", self.tmp, judge=JUDGE), setup)): + with self.assertRaises(kind) as raised: + start() + self.assertEqual(kind is setup, str(raised.exception).startswith("GitLabError: ")) + self.assertEqual(self.envs, []) + self.assertFalse(os.path.exists(self.home + self.g.TOKEN_FILE)) + + def test_the_url_and_token_reach_gitlab_s_tasks_alone_and_gitlab_is_held(self): + for start in (lambda: self.answer("026"), lambda: self.driver.score("tasks__041", self.tmp, judge=JUDGE)): + with self.assertRaises(Started): + start() + self.assertEqual((self.envs[-1]["GITLAB_URL"], self.envs[-1][self.g.TOKEN]), (URL, self.g.token(self.home))) + with self.assertRaises(Started): + self.answer("001") + self.assertNotIn("GITLAB_URL", self.envs[-1]) + self.assertNotIn(self.g.TOKEN, self.envs[-1]) + self.assertEqual(self.held, [(URL, mock.ANY)] * 2 + [(None, mock.ANY)]) + limits = os.path.join(self.tmp, "gitlab.limits") + self.assertEqual(self.ups[1::2], [(self.home, limits, mock.ANY)] * 2) + self.assertIsNotNone(self.held[0][-1].until) + self.assertIsNotNone(self.ups[1][-1].until) + self.assertIs(self.ups[0], self.ups[1][-1]) + self.assertEqual(self.order, [("warden", self.tmp), ("prepare", self.home), ("limits", limits), ("up",), + ("let go", self.tmp)] * 2) + + def test_the_sites_that_serve_gitlab_are_checked_among_what_a_vm_start_needs(self): + for name in ("vm_image", "_provider_image", "_write_limit"): + p = mock.patch.object(self.driver, name, lambda: "image") + p.start() + self.addCleanup(p.stop) + with mock.patch.object(self.driver, "_nat", lambda: True), \ + mock.patch.object(self.driver.os, "listdir", lambda p: ["x"]), \ + mock.patch.object(self.driver.os.path, "isdir", lambda p: True), \ + mock.patch.object(self.driver.websites, "ready", lambda *a, **kw: self.held.append((a, kw))), \ + mock.patch.object(self.driver._startup(), "network", lambda docker: "nullius-osworld2"): + self.driver._ready("tasks", "001") + self.driver._ready("tasks", "026") + self.driver._ready("tasks", "041") + self.assertEqual(self.held, [((SUFFIX,), {"state": False})] * 2) + self.assertEqual(self.order, []) + + def test_a_hosts_snapshot_carries_the_module(self): + from pathlib import Path + + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + for name in ("gitlab_server.py", "gitlab.compose.yml"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + self.assertEqual(json.load(f)["driver"]["command"][-1], "{here}/gitlab.compose.yml") + + +def _stack(): + stack = os.environ.get("NULLIUS_OSWORLD2_GITLAB", "") + return stack if stack and shutil.which("docker") and os.path.isfile(os.path.join(stack, "docker-compose.yml")) else "" + + +@unittest.skipUnless(_stack(), "NULLIUS_OSWORLD2_GITLAB names no Task-Web/gitlab, or no docker compose") +class Override(unittest.TestCase): + def config(self, *args): + g = load() + shares = g.allocation(8, 80) + settings = {"GITLAB_URL": URL, "GITLAB_PRIVATE_TOKEN": g.UNUSED, "OSWORLD2_GITLAB_HOST": URL[7:], + "OSWORLD2_GITLAB_INGRESS": "osworld2-web_web", "OSWORLD2_GITLAB_DISK": "/dev/sda", + "OSWORLD2_GITLAB_RB": "/x/gitlab.rb", + **{f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) for s, v in shares.items() for i, k in enumerate(("BPS", "IOPS"))}} + said = subprocess.run(["docker", "compose", "-p", g.PROJECT, "-f", os.path.join(_stack(), "docker-compose.yml"), + "-f", g.OVERRIDE, *args, "config", "--format", "json"], capture_output=True, text=True, + env=dict(os.environ, **settings), timeout=120) + self.assertEqual(said.returncode, 0, said.stderr) + return g, shares, json.loads(said.stdout)["services"] + + def test_the_project_as_compose_reads_it(self): + g, shares, services = self.config() + self.assertEqual(sorted(services), ["dind", "gitlab", "gitlab-runner"]) + for name, service in services.items(): + share = shares[{"gitlab": "GITLAB", "gitlab-runner": "RUNNER", "dind": "DIND"}[name]] + self.assertEqual(service.get("blkio_config"), { + "device_write_bps": [{"Path": "/dev/sda", "Rate": str(share[0])}], + "device_write_iops": [{"Path": "/dev/sda", "Rate": str(share[1])}]}, name) + self.assertFalse(service.get("ports"), name) + self.assertNotIn("container_name", service, name) + self.assertEqual(service.get("restart"), "no", name) + self.assertRegex(service["image"], r"@sha256:[0-9a-f]{64}$", name) + self.assertFalse([v for v in service.get("volumes") or [] if "docker.sock" in str(v.get("source"))], name) + self.assertEqual(services["gitlab"]["labels"]["caddy"], f"{URL}, http://*.{URL[7:]}") + self.assertEqual(list(services["gitlab"]["networks"]), ["web"]) + self.assertEqual(list(services["gitlab-runner"]["networks"]), ["ci"]) + self.assertTrue(services["dind"]["privileged"]) + + def test_the_upstream_s_token_service_is_disarmed_and_has_no_socket(self): + g, _, services = self.config("--profile", "upstream-init") + self.assertEqual(sorted(services), ["dind", "gitlab", "gitlab-init-token", "gitlab-runner"]) + for name, service in services.items(): + self.assertFalse([v for v in service.get("volumes") or [] if "docker.sock" in str(v.get("source"))], name) + self.assertEqual(services["gitlab-init-token"]["entrypoint"], ["/bin/false"]) + self.assertFalse(services["gitlab-init-token"].get("command")) + + def test_the_upstream_s_token_step_is_the_one_gitlab_server_py_runs(self): + g = load() + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + stack = os.path.join(tmp, "gitlab") + os.symlink(_stack(), stack) + with open(stack + g.ENV_FILE, "w") as f: + f.write(f"GITLAB_URL={URL}\nGITLAB_PRIVATE_TOKEN={g.UNUSED}\n") + script = g._upstream_script(stack) + self.assertEqual(script.count(f'-e {g.TOKEN}="$${g.TOKEN}" gitlab '), 1) + self.assertEqual(script.count("docker exec gitlab "), 1) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + def test_the_tasks_that_need_gitlab(self): + g = load() + found = set() + for p in _task_classes(): + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + if any(isinstance(n, ast.ImportFrom) and (n.module or "").endswith("controllers.gitlab") + for n in ast.walk(tree)): + found.add(os.path.basename(p)[5:-3]) + self.assertEqual(sorted(found), sorted(g.TASKS)) + self.assertLess(g.RUNNER_TASKS, g.TASKS) + + +class Fake: + """A finished process as subprocess.run returns it.""" + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + +class Lifecycle(unittest.TestCase): + """configure, the token step, the runner, the volumes' copy, mark and reap, and the lock, on fakes.""" + + def setUp(self): + self.g = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.stack = os.path.join(self.tmp, "gitlab") + self.calls = [] + p = mock.patch.object(self.g.websites, "SITES_LOCK", os.path.join(self.tmp, "sites.lck")) + p.start() + self.addCleanup(p.stop) + for name, fake in (("LOCK", os.path.join(self.tmp, "gitlab.lck")), + ("_docker", lambda *a, **kw: self.calls.append((a, kw)) or Fake()), + ("_pinned", lambda stack: os.path.abspath(stack)), + ("_container", lambda stack, service: f"id-{service}")): + p = mock.patch.object(self.g, name, fake) + p.start() + self.addCleanup(p.stop) + p = mock.patch.object(self.g, "token", lambda stack: TOKEN) + p.start() + self.addCleanup(p.stop) + + def configure(self): + with mock.patch.object(self.g.websites, "_network", lambda: "nullius-osworld2"), \ + mock.patch.object(self.g.websites, "_root_disk", lambda: ("/var/lib/docker", "/dev/sda")): + return self.g.configure(self.stack, "10.213.0.1.nip.io", 8, 80) + + def test_configure_writes_no_token_and_its_placeholder_is_no_token(self): + path, settings = self.configure() + with open(path, encoding="utf-8") as f: + text = f.read() + self.assertNotIn(TOKEN, text) + self.assertIn(f"GITLAB_PRIVATE_TOKEN={self.g.UNUSED}\n", text) + self.assertEqual(settings["GITLAB_URL"], URL) + self.assertEqual(settings["OSWORLD2_GITLAB_SHARE"], "stack") + + def test_the_token_step_passes_the_token_by_name(self): + script = ('until docker exec gitlab gitlab-rails runner "x"; do sleep 1; done\n' + f'docker exec -e {self.g.TOKEN}="$${self.g.TOKEN}" gitlab gitlab-rails runner "y"\n') + ran = [] + with mock.patch.object(self.g, "_upstream_script", lambda stack: script), \ + mock.patch.object(self.g.subprocess, "run", lambda argv, **kw: ran.append((argv, kw)) or Fake()): + self.g._make_token(self.stack) + (argv, kw), = ran + self.assertNotIn(TOKEN, " ".join(argv)) + self.assertEqual(kw["env"][self.g.TOKEN], TOKEN) + self.assertIn(f"docker exec -e {self.g.TOKEN} id-gitlab gitlab-rails", argv[-1]) + self.assertIn("until docker exec id-gitlab gitlab-rails", argv[-1]) + self.assertNotIn("$$", argv[-1]) + self.assertEqual(argv[:3], ["timeout", "--foreground", str(self.g.START_SECONDS)]) + self.assertGreater(kw["timeout"], self.g.START_SECONDS) + with mock.patch.object(self.g, "_upstream_script", lambda stack: "echo other"), \ + self.assertRaises(self.g.GitLabError): + self.g._make_token(self.stack) + with mock.patch.object(self.g, "_upstream_script", lambda stack: script), \ + mock.patch.object(self.g.subprocess, "run", lambda argv, **kw: Fake(1, stderr=f"x {TOKEN} y\n")), \ + self.assertRaises(self.g.GitLabError) as raised: + self.g._make_token(self.stack) + self.assertNotIn(TOKEN, str(raised.exception)) + + def test_the_runner_runs_jobs_unprivileged_in_the_project_s_docker(self): + asked = [] + + def request(path, value, token=None, data=None, method=None): + asked.append((path, method)) + return 201, json.dumps({"id": 7, "token": "glrt-secret"}).encode() + + with mock.patch.object(self.g, "_request", request): + self.assertEqual(self.g._register(self.stack, URL), 7) + (argv, kw), = self.calls + self.assertNotIn("--docker-privileged", argv) + self.assertNotIn("glrt-secret", " ".join(argv)) + self.assertEqual(kw["env"]["CI_SERVER_TOKEN"], "glrt-secret") + for flag in ("--docker-tlsverify", self.g.DOCKER_HOST, "/certs/client"): + self.assertIn(flag, argv) + with mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(1, stderr="FATAL: no")), \ + self.assertRaises(self.g.GitLabError): + self.g._register(self.stack, URL) + self.assertEqual(asked[-1], ("/api/v4/runners/7", "DELETE")) + + def test_the_copy_runs_under_gitlab_s_limit_with_its_labels(self): + settings = {"OSWORLD2_GITLAB_DISK": "/dev/sda", "OSWORLD2_GITLAB_GITLAB_BPS": "6291456", + "OSWORLD2_GITLAB_GITLAB_IOPS": "60", "OSWORLD2_GITLAB_SHARE": "unit"} + with mock.patch.object(self.g, "compose", lambda *a, **kw: Fake(stdout="gitlab/gitlab-ce@sha256:x\n")): + self.g._copy(self.stack, settings, "tar -x", keep=False) + self.g._copy(self.stack, settings, "tar -c", keep=True) + restore, keep = [a for a, _ in self.calls] + for flag in ("--network none", "--label com.docker.compose.project=osworld2-gitlab", + "--label nullius.osworld2.share=unit", "--device-write-bps /dev/sda:6291456", + "--device-write-iops /dev/sda:60"): + self.assertIn(flag, " ".join(restore)) + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot:ro", restore) + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot", keep) + + def test_a_unit_s_mark_and_limits_and_its_reap(self): + self.configure() + environ = {"NULLIUS_OPTION_WRITE_MBPS": "8", "NULLIUS_OPTION_WRITE_IOPS": "80"} + limits = self.g.unit(self.stack, self.tmp, environ) + settings = self.g._settings(self.stack, limits) + self.assertEqual(settings["OSWORLD2_GITLAB_SHARE"], "unit") + self.assertEqual(settings["OSWORLD2_GITLAB_GITLAB_IOPS"], "60") + gone = [] + with mock.patch.object(self.g, "down", lambda stack, limits=None: gone.append((stack, limits))): + self.assertTrue(self.g.reap(self.tmp)) + self.assertFalse(self.g.reap(self.tmp)) + self.assertEqual(gone, [(self.stack, limits)]) + self.assertIsNone(self.g.unit(self.stack, self.tmp, {})) + + def test_up_down_and_prepare_by_hand_refuse_while_a_unit_holds_gitlab(self): + ran = [] + with mock.patch.object(self.g, "up", lambda *a, **kw: ran.append("up")), \ + mock.patch.object(self.g, "down", lambda *a, **kw: ran.append("down")), \ + mock.patch.object(self.g, "prepare", lambda *a, **kw: ran.append("prepare")): + with self.g.held(URL, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()) as err: + for args in (["up", self.stack], ["down", self.stack], ["prepare", self.stack, SUFFIX]): + self.assertEqual(self.g.main(args), 1) + self.assertIn("a unit's GitLab task runs", err.getvalue()) + self.assertEqual(ran, []) + with mock.patch("sys.stdout", new=io.StringIO()): + self.assertEqual(self.g.main(["down", self.stack]), 0) + self.assertEqual(ran, ["down"]) + + def test_compose_by_hand_changes_nothing_while_a_unit_holds_gitlab(self): + ran = [] + with mock.patch.object(self.g, "compose", lambda stack, args, **kw: ran.append(args) or 0): + with self.g.held(URL, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()): + for args in (["down", "--volumes"], ["--profile", "upstream-init", "up", "-d"], ["-p", "ps", "rm"], []): + self.assertEqual(self.g.main(["compose", self.stack, *args]), 1, args) + for args in (["ps"], ["--profile", "upstream-init", "config"], ["--", "logs", "gitlab"]): + self.assertEqual(self.g.main(["compose", self.stack, *args]), 0, args) + self.assertEqual(self.g.main(["compose", self.stack, "down"]), 0) + self.assertEqual(ran, [["ps"], ["--profile", "upstream-init", "config"], ["logs", "gitlab"], ["down"]]) + + def test_one_gitlab_task_at_a_time_whoever_runs_it(self): + locks = [] + for uid in (1000, 1001): + with mock.patch.object(os, "getuid", lambda: uid): + locks.append(load().LOCK) + self.assertEqual(locks[0], locks[1]) + given = [] + with mock.patch.object(self.g, "_pinned", lambda stack: stack), mock.patch.object(self.g, "_settings", lambda *a: {}), \ + mock.patch.object(self.g.websites, "_compose_version", lambda: None), \ + mock.patch.object(self.g.subprocess, "run", lambda *a, **kw: given.append(kw["pass_fds"]) or mock.Mock()): + with self.g._alone(): + fds = self.g.websites.held_fds() + self.g.compose(self.stack, ["down"], capture=False) + self.assertEqual(len(fds), 2) + self.assertEqual(given, [fds]) + + def test_the_stamp_is_read_from_the_snapshot_read_only(self): + with mock.patch.object(self.g, "_snapshot", lambda: True): + with mock.patch.object(self.g, "_docker", + lambda *a, **kw: self.calls.append(a) or Fake(stdout=f"abc {int(time.time())}")): + self.assertEqual(self.g._made(), "abc") + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot:ro", self.calls[-1]) + self.assertIn("none", self.calls[-1]) + old = int(time.time()) - self.g.TOKEN_DAYS * 86400 - 1 + for said in ("abc", f"abc {old}", "abc x"): + with self.subTest(said), mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(stdout=said)): + self.assertIsNone(self.g._made()) + run = lambda said: lambda *a, **kw: Fake() if a[0] == "image" else said + with mock.patch.object(self.g, "_docker", run(Fake(1, stderr="cat: /snapshot/made: No such file or directory\n"))): + self.assertIsNone(self.g._made()) + with mock.patch.object(self.g, "_docker", run(Fake(1, stderr="cat: /snapshot/made: I/O error\n"))), \ + self.assertRaises(self.g.GitLabError): + self.g._made() + with mock.patch.object(self.g, "_docker", run(Fake(125, stderr="daemon\n"))), \ + self.assertRaises(self.g.GitLabError): + self.g._made() + gone = Fake(1, stderr="Error: No such image: gitlab/gitlab-ce\n") + with mock.patch.object(self.g, "_docker", lambda *a, **kw: self.calls.append(a) or (gone if a[0] == "image" else Fake())): + self.assertIsNone(self.g._made()) + self.assertEqual(self.calls[-1][:2], ("image", "inspect")) + with mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(1, stderr="Cannot connect\n")), \ + self.assertRaises(self.g.GitLabError): + self.g._made() + with mock.patch.object(self.g, "_snapshot", lambda: False): + self.assertIsNone(self.g._made()) + for said in ("Error response from daemon: get x: no such volume\n", "Error: No such volume: x\n"): + with mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(1, stderr=said)): + self.assertFalse(self.g._snapshot()) + with mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(1, stderr="Cannot connect to the Docker daemon\n")), \ + self.assertRaises(self.g.GitLabError): + self.g._snapshot() + + def test_init_keeps_the_stamp_prepare_reads_and_not_the_token(self): + self.configure() + copied = [] + for name, fake in (("compose", lambda *a, **kw: Fake()), ("_until", lambda *a, **kw: 1), + ("_make_token", lambda stack: None), ("ready", lambda *a, **kw: URL), + ("_register", lambda stack, value: 7), ("_volumes", lambda: []), ("_snapshot", lambda: False), + ("_copy", lambda stack, settings, tar, keep: copied.append(tar))): + p = mock.patch.object(self.g, name, fake) + p.start() + self.addCleanup(p.stop) + self.g.init(self.stack) + self.assertIn(f"printf '%s %s' {self.g._stamp(os.path.abspath(self.stack), URL)} ", copied[-1]) + self.assertIn(f" > /snapshot/{self.g.STAMP}", copied[-1]) + self.assertNotIn(TOKEN, copied[-1]) + + def test_init_runs_under_the_unit_s_limits(self): + self.configure() + limits = self.g.unit(self.stack, self.tmp, {"NULLIUS_OPTION_WRITE_MBPS": "8", "NULLIUS_OPTION_WRITE_IOPS": "80"}) + given, copied = [], [] + for name, fake in (("compose", lambda stack, args, **kw: given.append(kw.get("limits")) or Fake()), + ("_until", lambda *a, **kw: 1), ("_make_token", lambda stack: None), + ("ready", lambda *a, **kw: URL), ("_register", lambda stack, value: 7), + ("_volumes", lambda: []), ("_snapshot", lambda: False), + ("_copy", lambda stack, settings, tar, keep: copied.append(settings))): + p = mock.patch.object(self.g, name, fake) + p.start() + self.addCleanup(p.stop) + self.g.init(self.stack, limits=limits) + self.assertEqual(set(given), {limits}) + self.assertEqual((copied[0]["OSWORLD2_GITLAB_GITLAB_IOPS"], copied[0]["OSWORLD2_GITLAB_SHARE"]), ("60", "unit")) + + def test_a_snapshot_made_for_this_stack_host_and_token_is_kept(self): + self.configure() + done = [] + for name in ("_clone", "configure", "compose", "down", "init"): + p = mock.patch.object(self.g, name, lambda *a, n=name, **kw: done.append(n) or Fake()) + p.start() + self.addCleanup(p.stop) + stamp = self.g._stamp(self.stack, URL) + with mock.patch.object(self.g, "_made", lambda: stamp): + self.assertEqual(self.g.prepare(self.stack, {SITES: SUFFIX}), self.stack) + self.assertEqual(done, []) + for made, environ in ((None, {SITES: SUFFIX}), ("other", {SITES: SUFFIX}), + (stamp, {SITES: "10.0.0.1.nip.io"})): + done.clear() + with self.subTest(made=made, environ=environ), mock.patch.object(self.g, "_made", lambda: made), \ + mock.patch.object(self.g, "_snapshot", lambda: True): + self.g.prepare(self.stack, environ) + self.assertEqual(done, ["_clone", "configure", "compose", "down", "init"]) + + def test_made_anew_in_order_under_the_run_s_share(self): + order = [] + for name, fake in (("_made", lambda: None), ("_clone", lambda stack: order.append("clone")), + ("configure", lambda *a: order.append(("configure",) + a[1:])), + ("compose", lambda stack, args, **kw: order.append(args[0]) or Fake()), + ("down", lambda stack: order.append("down")), ("_snapshot", lambda: True), + ("init", lambda stack, say, closed, limits: order.append(("init", limits)))): + p = mock.patch.object(self.g, name, fake) + p.start() + self.addCleanup(p.stop) + self.g.prepare(self.stack, {SITES: SUFFIX}) + self.g.prepare(self.stack, {SITES: SUFFIX}, limits="/unit/gitlab.limits") + self.assertEqual(order, ["clone", ("configure", SUFFIX, 16, 160), "pull", "down", ("init", None), + "clone", ("configure", SUFFIX, 16, 160), "pull", "down", ("init", "/unit/gitlab.limits")]) + self.assertEqual([a for a, _ in self.calls], [("volume", "rm", self.g.SNAPSHOT)] * 2) + with self.assertRaises(self.g.GitLabError): + self.g.prepare(self.stack, {}) + + def test_a_snapshot_that_cannot_be_read_is_kept(self): + def fail(): + raise self.g.GitLabError("docker volume got no answer in 300 s") + done = [] + for name in ("_clone", "configure", "compose", "down", "init"): + p = mock.patch.object(self.g, name, lambda *a, n=name, **kw: done.append(n)) + p.start() + self.addCleanup(p.stop) + with mock.patch.object(self.g, "_made", fail), self.assertRaises(self.g.GitLabError): + self.g.prepare(self.stack, {SITES: SUFFIX}) + self.assertEqual((done, self.calls), ([], [])) + + def test_no_override_is_one_line_saying_what_to_do(self): + with mock.patch.object(self.g, "OVERRIDE", os.path.join(self.tmp, "none.yml")), \ + self.assertRaises(self.g.GitLabError) as raised: + self.g._argv(self.stack, ["ps"]) + self.assertIn("install Nullius again", str(raised.exception)) + + +GIT = """#!/bin/sh +case "$*" in *rev-parse*) echo 8655d651722f4254e59e813de9f68a6732ea525c ;; esac +exit 0 +""" +FAKE_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$*" in + *"version --short"*) echo 2.40.3 ;; + *" down "*) sleep 2; echo "$*" >> "$FAKE_DOCKER_LOG.done" ;; +esac +exit 0 +""" +KILLED = """ +import importlib.util, os, sys, time +here, lock, sites = sys.argv[1:4] +spec = importlib.util.spec_from_file_location("osworld2_driver_killed", os.path.join(here, "driver.py")) +driver = importlib.util.module_from_spec(spec) +sys.argv = ["driver.py"] +sys.path.insert(0, here) +spec.loader.exec_module(driver) +driver.gitlab_server.LOCK, driver.websites.SITES_LOCK = lock, sites +driver.gitlab_server.home = lambda: os.environ["STACK"] +driver.gitlab_server.prepare = lambda *a, **kw: None +driver.gitlab_server.up = lambda *a: (print("up", flush=True), time.sleep(600)) +with driver._gitlab("041", os.environ["UNIT"]): + pass +""" + + +class Warden(unittest.TestCase): + """The driver killed while GitLab starts: its warden removes GitLab, then lets the locks go.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_ = os.path.join(self.tmp, "bin") + os.makedirs(bin_) + for name, text in (("git", GIT), ("docker", FAKE_DOCKER)): + with open(os.path.join(bin_, name), "w") as f: + f.write(text) + os.chmod(os.path.join(bin_, name), 0o755) + self.stack, self.unit = os.path.join(self.tmp, "gitlab"), os.path.join(self.tmp, "unit") + os.makedirs(self.stack) + os.makedirs(self.unit) + with open(self.stack + ".osworld2.env", "w") as f: + f.write(f"GITLAB_URL={URL}\n") + self.log = os.path.join(self.tmp, "docker.log") + self.lock, self.sites = os.path.join(self.tmp, "gitlab.lck"), os.path.join(self.tmp, "sites.lck") + self.env = dict(os.environ, PATH=bin_ + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.log, + NULLIUS_UPSTREAM=self.tmp, STACK=self.stack, UNIT=self.unit, + NULLIUS_OPTION_WEBSITES=SUFFIX) + + def free(self, path, mode): + import fcntl + fd = os.open(path, os.O_RDWR | os.O_CREAT) + try: + fcntl.flock(fd, mode | fcntl.LOCK_NB) + return True + except BlockingIOError: + return False + finally: + os.close(fd) + + def test_the_driver_killed_during_up_leaves_gitlab_to_its_warden(self): + import fcntl + import signal + p = subprocess.Popen([sys.executable, "-c", KILLED, HERE, self.lock, self.sites], env=self.env, + stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: p.poll() is None and p.kill()) + self.assertEqual(p.stdout.readline().strip(), "up") + self.assertFalse(self.free(self.lock, fcntl.LOCK_EX)) + self.assertFalse(self.free(self.sites, fcntl.LOCK_EX)) + p.send_signal(signal.SIGKILL) + p.wait(10) + seen_held = False + t0 = time.time() + while not os.path.exists(self.log + ".done") and time.time() - t0 < 60: + seen_held = seen_held or not self.free(self.lock, fcntl.LOCK_EX) + time.sleep(0.2) + self.assertTrue(os.path.exists(self.log + ".done"), "the warden removed no GitLab") + self.assertTrue(seen_held, "the lock was let go before GitLab was removed") + while not self.free(self.lock, fcntl.LOCK_EX) and time.time() - t0 < 90: + time.sleep(0.5) + self.assertTrue(self.free(self.lock, fcntl.LOCK_EX)) + self.assertTrue(self.free(self.sites, fcntl.LOCK_EX)) + with open(self.log) as f: + said = f.read() + self.assertIn("compose -p osworld2-gitlab", said) + self.assertIn("down --volumes --remove-orphans", said) + self.assertFalse(os.path.exists(os.path.join(self.unit, "gitlab.stack"))) + + +class Sites(Case): + """GitLab's way in is the mock websites' Caddy: a GitLab unit and a hand operation keep a reset away.""" + + def test_a_reset_or_smoke_is_refused_while_a_gitlab_unit_runs(self): + with self.g.held(URL, threading.Event()): + with self.assertRaises(self.g.websites.WebsiteError): + with self.g.websites._alone(): + pass + self.assertEqual(len(self.g.websites.held_fds()), 2) + with self.g.websites._alone(): + pass + + def test_gitlab_by_hand_and_a_unit_wait_while_the_sites_are_reset(self): + closed = threading.Event() + closed.set() + with self.g.websites._alone(): + with self.assertRaises(self.g.websites.WebsiteError) as raised: + with self.g._alone(): + pass + self.assertIn("mock websites are being reset", str(raised.exception)) + with self.assertRaises(self.g.websites.WebsiteError): + with self.g.held(URL, closed): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_guest_disk.py b/app/tests/test_osworld2_guest_disk.py new file mode 100644 index 00000000..ef82dbd1 --- /dev/null +++ b/app/tests/test_osworld2_guest_disk.py @@ -0,0 +1,249 @@ +"""OSWorld2's guest disk on the docker provider: at least GUEST_GB, in nop.py and the Original's runner alike. +A fake desktop_env stands in.""" +from __future__ import annotations + +import importlib.machinery +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +DESKTOP_ENV = """import json, os +MAX_RETRIES = 5 +class EnvironmentSetupError(RuntimeError): + pass +class DesktopEnv: + def __init__(self, provider_name="vmware", region=None, path_to_vm=None, snapshot_name="init_state", + action_space="pyautogui", cache_dir="cache", screen_size=(1920, 1080), headless=False, + require_a11y_tree=True, require_terminal=False, os_type="Ubuntu", enable_proxy=False, + client_password="", instance_type=None, volume_size=None, use_public_ip=False, + force_disable_vnc=False, force_disable_recording=False): + with open(os.environ["FAKE_SIZES"], "a") as f: + f.write(json.dumps([provider_name, volume_size]) + "\\n") + def close(self): + pass +""" +TASK_LOADER = """import json, os +def find_task_class_path(*a): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, **kw): + return dict(json.loads(os.environ["FAKE_TASK"]), instruction="", proxy=False) +""" +LIB_RUN_SINGLE = """def run_single_example(*a, **kw): + pass +""" +RUNNER = """import json, multiprocessing, os, sys +from desktop_env.desktop_env import DesktopEnv +def worker(size): + DesktopEnv(path_to_vm="vm", action_space="pyautogui", provider_name=sys.argv[1], region=None, + snapshot_name="init_state", screen_size=(1920, 1080), headless=True, os_type="Ubuntu", + require_a11y_tree=False, enable_proxy=True, client_password="pw", instance_type=None, + volume_size=size, use_public_ip=False, force_disable_vnc=True, force_disable_recording=True) +if __name__ == "__main__": + size = json.loads(os.environ["FAKE_TASK"]).get("volume_size") + for how in ("fork", "spawn"): + p = multiprocessing.get_context(how).Process(target=worker, args=(size,)) + p.start() + p.join() + assert p.exitcode == 0, how +""" + + +DOCKER_PY = """class ContainerCollection: + def create(self, image, **kwargs): + pass +""" +AGAIN = """import importlib, json, os, runpy, sys +import desktop_env.desktop_env as module +runpy.run_path(os.environ["SITE"], run_name="sitecustomize") +runpy.run_path(os.environ["SITE"], run_name="sitecustomize") +module.DesktopEnv(provider_name="docker") +module = importlib.reload(module) +module.DesktopEnv(provider_name="docker") +from docker.models.containers import ContainerCollection +layers, create = 0, ContainerCollection.create +while hasattr(create, "__wrapped__"): + layers, create = layers + 1, create.__wrapped__ +print(json.dumps({"finders": sum(bool(getattr(f, "_nullius_osworld2", False)) for f in sys.meta_path), + "layers": layers})) +""" + + +def fake_class(): + class DesktopEnv: + def __init__(self, provider_name="docker", volume_size=None): + self.volume_size = volume_size + + return DesktopEnv + + +def load_site(): + spec = importlib.util.spec_from_file_location("osworld2_startup_disk", os.path.join(STARTUP, "sitecustomize.py")) + site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(site) + return site + + +class Size(unittest.TestCase): + def setUp(self): + self.site = load_site() + + def test_the_floor_is_the_aws_providers_root(self): + self.assertEqual(self.site.GUEST_GB, 40) + + def test_docker_gets_the_floor_or_the_tasks_larger_size(self): + for given, wanted in ((None, 40), (0, 40), (30, 40), (40, 40), (60, 60), ("60", 60)): + with self.subTest(given=given): + self.assertEqual(self.site.guest_gb("docker", given), wanted) + self.assertEqual(self.site.guest_gb(" Docker ", None), 40) + + def test_another_provider_is_left_alone(self): + for provider in ("aws", "vmware", "virtualbox"): + for given in (None, 30, 60): + with self.subTest(provider=provider, given=given): + self.assertEqual(self.site.guest_gb(provider, given), given) + + def test_a_module_already_loaded_is_wrapped_and_one_finder_added(self): + module = types.SimpleNamespace(DesktopEnv=fake_class()) + with mock.patch.dict(sys.modules, {self.site.DESKTOP_ENV: module}), mock.patch.object(sys, "meta_path", []): + self.site.wrap_desktop_env() + load_site().wrap_desktop_env() + self.assertEqual(len(sys.meta_path), 1) + self.assertEqual(module.DesktopEnv().volume_size, 40) + + def test_the_finders_loader_is_wrapped_not_changed(self): + """A loader whose methods cannot be set, or that loads other modules too, is left as it was.""" + class Slotted: + __slots__ = () + + def create_module(self, spec): + return None + + def exec_module(self, module): + module.DesktopEnv = fake_class() + + loader = Slotted() + finder = types.SimpleNamespace(find_spec=lambda name, path, target=None: importlib.machinery.ModuleSpec( + name, loader)) + with mock.patch.object(sys, "meta_path", [self.site._Floor(), finder]): + spec = sys.meta_path[0].find_spec(self.site.DESKTOP_ENV, None) + self.assertIsNot(spec.loader, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertEqual(module.DesktopEnv().volume_size, 40) + other = types.ModuleType("other") + loader.exec_module(other) + self.assertIsNone(other.DesktopEnv().volume_size) + + def test_a_class_without_those_arguments_is_left_alone(self): + class DesktopEnv: + def __init__(self, provider="docker", size=None): + pass + + init = DesktopEnv.__init__ + self.site.floor_desktop_env(types.SimpleNamespace(DesktopEnv=DesktopEnv)) + self.assertIs(DesktopEnv.__init__, init) + + def test_the_class_is_wrapped_once_however_it_is_called(self): + made = [] + + class DesktopEnv: + def __init__(self, provider_name="vmware", region=None, path_to_vm=None, volume_size=None): + made.append((provider_name, volume_size, path_to_vm)) + + module = types.SimpleNamespace(DesktopEnv=DesktopEnv) + self.site.floor_desktop_env(module) + wrapped = DesktopEnv.__init__ + self.site.floor_desktop_env(module) + self.assertIs(DesktopEnv.__init__, wrapped) + DesktopEnv("docker", None, "vm") + DesktopEnv("docker", None, "vm", 60) + DesktopEnv(provider_name="docker", volume_size=30) + DesktopEnv() + DesktopEnv("aws", volume_size=None) + self.assertEqual(made, [("docker", 40, "vm"), ("docker", 60, "vm"), ("docker", 40, None), + ("vmware", None, None), ("aws", None, None)]) + + +class BothPaths(unittest.TestCase): + """nop.py and a runner built as run_multienv_m3.py, each with the startup folder first on PYTHONPATH.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.up = os.path.join(self.tmp, "up") + os.makedirs(os.path.join(self.up, "desktop_env")) + for name, text in (("task_loader.py", TASK_LOADER), ("lib_run_single.py", LIB_RUN_SINGLE), + ("runner.py", RUNNER), (os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV)): + with open(os.path.join(self.up, name), "w") as f: + f.write(text) + self.out = os.path.join(self.tmp, "out") + os.makedirs(self.out) + + def sizes(self, argv, task, **env): + log = os.path.join(self.tmp, "sizes") + if os.path.exists(log): + os.remove(log) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + environ.update(PYTHONPATH=os.pathsep.join([STARTUP, self.up]), FAKE_SIZES=log, FAKE_TASK=json.dumps(task), + **env) + done = subprocess.run([sys.executable, *argv], cwd=self.up, env=environ, capture_output=True, text=True, + timeout=60) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertNotIn("Error in sitecustomize", done.stderr) + with open(log) as f: + return [json.loads(ln) for ln in f.read().splitlines()] + + def nop(self, task, **env): + return self.sizes([os.path.join(HERE, "nop.py"), "vm", "pw", "examples", "tasks", "001", self.out], task, + **env) + + def test_the_null_control_and_the_original_setup_get_the_same_disk(self): + for task, wanted in (({}, 40), ({"volume_size": 30}, 40), ({"volume_size": 60}, 60)): + with self.subTest(task=task): + self.assertEqual(self.nop(task), [["docker", wanted]]) + self.assertEqual(self.sizes(["runner.py", "docker"], task), [["docker", wanted]] * 2) + + def test_another_provider_is_left_alone_on_the_runners_path(self): + self.assertEqual(self.sizes(["runner.py", "aws"], {"volume_size": 30}), [["aws", 30]] * 2) + + def test_another_vm_provider_option_wraps_nothing(self): + self.assertEqual(self.nop({}, NULLIUS_OPTION_VM_PROVIDER="aws"), [["docker", None]]) + + +class Again(unittest.TestCase): + """desktop_env imported before the startup module runs, the startup module run twice, then a reload.""" + def test_a_reload_keeps_the_floor_and_a_second_run_wraps_nothing_twice(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + for name, text in ((os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV), + (os.path.join("docker", "__init__.py"), ""), (os.path.join("docker", "models", "__init__.py"), ""), + (os.path.join("docker", "models", "containers.py"), DOCKER_PY), ("again.py", AGAIN)): + os.makedirs(os.path.dirname(os.path.join(tmp, name)), exist_ok=True) + with open(os.path.join(tmp, name), "w") as f: + f.write(text) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + environ.update(PYTHONPATH=tmp, FAKE_SIZES=os.path.join(tmp, "sizes"), + SITE=os.path.join(STARTUP, "sitecustomize.py")) + done = subprocess.run([sys.executable, "again.py"], cwd=tmp, env=environ, capture_output=True, text=True, + timeout=60) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertEqual(json.loads(done.stdout), {"finders": 1, "layers": 1}) + with open(os.path.join(tmp, "sizes")) as f: + self.assertEqual([json.loads(ln) for ln in f.read().splitlines()], [["docker", 40]] * 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_guest_ports.py b/app/tests/test_osworld2_guest_ports.py new file mode 100644 index 00000000..672c20c6 --- /dev/null +++ b/app/tests/test_osworld2_guest_ports.py @@ -0,0 +1,154 @@ +"""OSWorld2's guest ports on 127.0.0.1, one VM at a time per machine, locked until the VM is gone.""" +from __future__ import annotations + +import fcntl +import importlib.util +import os +import shutil +import socket +import sys +import tempfile +import threading +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") + + +def load(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_ports", os.path.join(HERE, "driver.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(module) + return module + + +def unused(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class GuestPorts(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load(self.tmp) + self.port = unused() + self.driver.PORT_LOCK = os.path.join(self.tmp, "port-{}.lck") + self.driver.GUEST_PORTS = {"082": (self.port,)} + self.driver.CLOSED = threading.Event() + self.lock = os.path.join(self.tmp, f"port-{self.port}.lck") + + def held(self): + fd = os.open(self.lock, os.O_RDONLY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return False + except BlockingIOError: + return True + finally: + os.close(fd) + + def test_another_task_takes_no_lock_and_names_no_port(self): + with self.driver._published("001") as env: + self.assertEqual(env, {}) + self.assertEqual(self.driver._PORTS, []) + self.assertFalse(os.path.exists(self.lock)) + + def test_the_tasks_ports_are_named_and_locked_until_the_unit_ends(self): + with self.driver._published("082") as env: + self.assertEqual(env, {"OSWORLD2_GUEST_PORTS": str(self.port)}) + self.assertTrue(self.held()) + self.assertEqual(len(self.driver._PORTS), 1) + self.assertFalse(self.held()) + self.assertEqual(self.driver._PORTS, []) + + def test_the_reaper_keeps_the_lock(self): + with mock.patch.object(self.driver.subprocess, "Popen") as popen, self.driver._published("082"): + held = tuple(self.driver._PORTS) + self.driver._reaper(1, self.tmp, True) + self.assertEqual(len(held), 1) + self.assertEqual(popen.call_args.kwargs["pass_fds"], held) + + def test_a_listener_on_loopback_is_refused_and_the_lock_let_go(self): + with socket.socket() as s: + s.bind(("127.0.0.1", self.port)) + s.listen() + with self.assertRaisesRegex(RuntimeError, rf"task 082 .* port {self.port} .*127\.0\.0\.1"): + with self.driver._published("082"): + self.fail("published while another process listens") + self.assertFalse(self.held()) + self.assertEqual(self.driver._PORTS, []) + + def test_a_listener_on_every_interface_is_refused(self): + with socket.socket() as s: + s.bind(("0.0.0.0", self.port)) + s.listen() + with self.assertRaisesRegex(RuntimeError, rf"task 082 .* port {self.port} "): + with self.driver._published("082"): + self.fail("published while another process listens") + self.assertFalse(self.held()) + + def test_a_listener_on_ipv6_loopback_is_refused(self): + try: + s = socket.socket(socket.AF_INET6) + s.bind(("::1", self.port)) + except OSError: + self.skipTest("no IPv6 loopback here") + with s: + s.listen() + with self.assertRaisesRegex(RuntimeError, r"listens on ::1"): + with self.driver._published("082"): + self.fail("published while another process listens") + + def test_a_connection_closing_on_the_port_is_no_listener(self): + with socket.socket() as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", self.port)) + server.listen() + client = socket.create_connection(("127.0.0.1", self.port)) + accepted, _ = server.accept() + accepted.close() + client.close() + with self.driver._published("082") as env: + self.assertEqual(env, {"OSWORLD2_GUEST_PORTS": str(self.port)}) + + def test_another_unit_waits_until_its_time_runs_out(self): + fd = os.open(self.lock, os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + threading.Timer(1.5, self.driver.CLOSED.set).start() + with self.assertRaisesRegex(RuntimeError, rf"time ran out while another unit's VM held port {self.port}"): + with self.driver._published("082"): + self.fail("published while another unit holds the port") + self.assertEqual(self.driver._PORTS, []) + + def test_a_fifo_in_the_locks_place_is_refused_at_once(self): + os.mkfifo(self.lock) + with self.assertRaisesRegex(RuntimeError, "cannot be opened"): + with self.driver._published("082"): + self.fail("published through a FIFO") + self.assertEqual(self.driver._PORTS, []) + + def test_the_lock_is_readable_by_every_user_whatever_the_umask(self): + mask = os.umask(0o077) + try: + with self.driver._published("082"): + pass + finally: + os.umask(mask) + self.assertEqual(os.stat(self.lock).st_mode & 0o777, 0o644) + + def test_a_link_in_the_locks_place_is_refused(self): + os.symlink(os.path.join(self.tmp, "elsewhere"), self.lock) + with self.assertRaisesRegex(RuntimeError, "cannot be opened"): + with self.driver._published("082"): + self.fail("published through a link") + self.assertFalse(os.path.exists(os.path.join(self.tmp, "elsewhere"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_judge.py b/app/tests/test_osworld2_judge.py new file mode 100644 index 00000000..a0386f18 --- /dev/null +++ b/app/tests/test_osworld2_judge.py @@ -0,0 +1,657 @@ +"""OSWorld2's LLM grader, simulated user and judge route, and its pinned dataset lists. +With NULLIUS_OSWORLD2_UPSTREAM set, the lists are also checked against the task classes, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import providers +from nullius.standard import definitions, relay, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +OFFLINE36 = ("001 002 004 010 012 022 029 030 040 042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 " + "076 080 084 085 088 094 103 104 105 106 107 108").split() +MORE27 = "003 006 009 011 014 017 023 024 025 027 032 033 034 071 074 078 083 086 087 089 090 092 093 095 096 097 099".split() +PROXY = "036 037 050 055 056 062 075 098".split() +CATCHING = "003 008 019 025 031 041 077 079 082 090 092".split() +SONNET = "anthropic/claude-sonnet-4.6" + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_judge", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def bench(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f) + + +class Started(Exception): + pass + + +class Driver(unittest.TestCase): + def setUp(self): + self.job = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.job, ignore_errors=True) + trial = os.path.join(self.job, "u1") + self.tmp = os.path.join(trial, "workspace") + self.relayed = os.path.join(trial, ".work", "relay", "u1-judge.jsonl") + os.makedirs(os.path.dirname(self.relayed)) + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.judge = {"model": SONNET, "base_url": "http://127.0.0.1:9/u1-judge/v1", "key": relay.PLACEHOLDER_KEY} + self.vms, self.envs = [], [] + + def vm(dir, domain, eid): + self.vms.append(eid) + return os.path.join(dir, "vm.qcow2") + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + for name, fake in (("_vm", vm), ("_run", run), ("_task", lambda *_: {"instruction": "", "proxy": False}), + ("_assets", lambda *a: "assets")): + p = mock.patch.object(self.driver, name, fake) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid, **kw): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", + relay.PLACEHOLDER_KEY, **kw) + + def nop(self, eid, **kw): + return self.driver.score(f"tasks__{eid}", self.tmp, **kw) + + def test_the_grader_gets_the_judge_route_in_the_original_and_the_null_control(self): + for start in (self.answer, self.nop): + with self.subTest(start.__name__), self.assertRaises(Started): + start("003", judge=self.judge) + env = self.envs[-1] + self.assertEqual({k: v for k, v in env.items() if k.startswith("OSWORLD_EVAL_MODEL_")}, + {"OSWORLD_EVAL_MODEL_PROVIDER": "anthropic", "OSWORLD_EVAL_MODEL_NAME": SONNET, + "OSWORLD_EVAL_MODEL_BASE_URL": "http://127.0.0.1:9/u1-judge", + "OSWORLD_EVAL_MODEL_API_KEY": relay.PLACEHOLDER_KEY}) + + def test_only_a_trailing_v1_leaves_the_route(self): + with self.assertRaises(Started): + self.nop("001", judge=dict(self.judge, base_url="http://h:1/v1-judge/v1")) + self.assertEqual(self.envs[-1]["OSWORLD_EVAL_MODEL_BASE_URL"], "http://h:1/v1-judge") + + def test_the_simulated_user_asks_sonnet_on_the_units_own_route(self): + with self.assertRaises(Started): + self.answer("024", judge=self.judge) + env = self.envs[-1] + self.assertEqual({k: v for k, v in env.items() if k.startswith("OSWORLD_USER_SIM_")}, + {"OSWORLD_USER_SIM_PROVIDER": "anthropic", "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "http://127.0.0.1:9/u1", + "OSWORLD_USER_SIM_API_KEY": relay.PLACEHOLDER_KEY}) + self.assertEqual(env["ANTHROPIC_BASE_URL"], "http://127.0.0.1:9/u1") + + def test_a_graded_task_with_no_judge_route_is_refused_before_its_vm(self): + for start in (self.answer, self.nop): + for eid in ("003", "093"): + with self.subTest(f"{start.__name__} {eid}"), self.assertRaises(RuntimeError) as raised: + start(eid) + self.assertIn(f"task {eid} is scored by the LLM grader", str(raised.exception)) + self.assertEqual(self.vms, []) + + def test_a_bad_judge_route_is_refused_before_the_vm(self): + with self.assertRaises(RuntimeError) as raised: + self.nop("001", judge={"model": SONNET, "base_url": ""}) + self.assertIn("judge route needs", str(raised.exception)) + self.assertEqual(self.vms, []) + + def test_an_ungraded_task_runs_with_no_judge_route_and_no_grader_settings(self): + for start in (self.answer, self.nop): + with self.subTest(start.__name__), self.assertRaises(Started): + start("001") + self.assertFalse([k for k in self.envs[-1] if k.startswith("OSWORLD_EVAL_MODEL_")]) + self.assertEqual(self.vms, ["001", "001"]) + + def test_a_simulated_user_with_no_route_is_refused_before_its_vm(self): + agents = json.loads(json.dumps(self.driver.AGENTS)) + for k in [k for k in agents["agents"]["m3"]["env"] if k.startswith("OSWORLD_USER_SIM_")]: + del agents["agents"]["m3"]["env"][k] + with mock.patch.object(self.driver, "AGENTS", agents): + with self.assertRaises(RuntimeError) as raised: + self.answer("095", judge=self.judge) + self.assertIn("task 095 has a simulated user", str(raised.exception)) + self.assertEqual(self.vms, []) + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + + def result(self, where, value="0.0"): + os.makedirs(where, exist_ok=True) + with open(os.path.join(where, "result.txt"), "w") as f: + f.write(value) + + def log(self, path, *lines): + with open(path, "a") as f: + f.write("".join(ln + "\n" for ln in lines)) + + def test_a_failed_grader_is_the_original_scores_error_not_a_0(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 0.0) + failed = "\x1b[1;33m[2026-11-06 01:02:03 \x1b[31mERROR \x1b[32mllm_metrics/388-Worker\x1b[1;33m] \x1b[0m" \ + "Error in compare_text_with_llm: Error code: 402 - insufficient credits" + for log in (os.path.join(where, "eval.log"), os.path.join(self.tmp, "run_multienv_m3.log")): + with self.subTest(os.path.basename(log)): + self.log(log, "[2026-11-06 01:02:00 INFO desktop_env/1] evaluating", failed) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + said = str(raised.exception) + self.assertIn("the LLM grader failed in compare_text_with_llm (HTTP 402), and the upstream", said) + self.assertIn("see the judge's calls", said) + self.assertNotIn("insufficient credits", said) + self.assertNotIn("\x1b", said) + os.remove(log) + + def test_an_earlier_runs_results_are_moved_aside_before_the_runner(self): + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001"), "1.0") + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + self.assertFalse(os.path.lexists(os.path.join(self.tmp, "results"))) + (earlier,) = glob.glob(os.path.join(self.tmp, "results.*")) + self.assertTrue(glob.glob(os.path.join(earlier, "**", "result.txt"), recursive=True)) + + def test_an_earlier_runs_logs_are_moved_aside_and_its_grader_failure_is_not_this_runs(self): + failed = "[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: Error code: 402 - x" + for name in ("run_multienv_m3.log", "nop.log", "setup.log"): + self.log(os.path.join(self.tmp, name), failed) + with self.assertRaises(Started): + self.answer("003", judge=self.judge) + self.assertEqual(sorted(os.path.basename(p).rsplit(".", 1)[0] for p in glob.glob(os.path.join(self.tmp, "*.log.*"))), + ["nop.log", "run_multienv_m3.log", "setup.log"]) + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003"), "1.0") + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 1.0) + self.log(os.path.join(self.tmp, "nop.log"), failed) + with contextlib.suppress(RuntimeError): + self.driver.setup("tasks__003", self.tmp) + nop = os.path.join(self.tmp, "results", "nop", "tasks", "003") + with mock.patch.object(self.driver, "_run", lambda *a, **kw: self.result(nop, "1.0")): + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 1.0) + + def test_a_unit_an_earlier_run_or_its_reaper_still_holds_is_not_begun(self): + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, os, sys, time\n" + "fd = os.open(sys.argv[1], os.O_RDONLY | os.O_CREAT)\nfcntl.flock(fd, fcntl.LOCK_EX)\n" + "print(flush=True)\ntime.sleep(30)", os.path.join(self.tmp, "unit.lck")], + stdout=subprocess.PIPE) + self.addCleanup(holder.kill) + holder.stdout.readline() + with mock.patch.object(self.driver, "HELD_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.answer("001", judge=self.judge) + self.assertIn("an earlier run of this unit, or its reaper, still holds its folder", str(raised.exception)) + holder.kill() + holder.wait() + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + self.assertEqual(self.driver._OWN, []) + self.driver._fresh(self.tmp) + with mock.patch.object(self.driver.subprocess, "Popen") as started: + self.driver._arm(os.getpid(), self.tmp, True) + os.close(self.driver._ARMED.pop(os.getpid())) + self.assertIn(self.driver._OWN[0], started.call_args.kwargs["pass_fds"]) + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001"), "1.0") + self.driver.score("tasks__001", self.tmp, setup="original") + self.assertEqual(self.driver._OWN, []) + + def test_a_result_outside_0_to_1_is_no_score(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + for value in ("nan", "inf", "-inf", "-0.5", "1.5"): + with self.subTest(value): + self.result(where, value) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__001", self.tmp, setup="original") + self.assertIn("not a score from 0 to 1", str(raised.exception)) + for value in ("0", "0.25", "1.0", "1.0000000000000002"): + self.result(where, value) + self.assertEqual(self.driver.score("tasks__001", self.tmp, setup="original")["reward"], float(value)) + + def test_a_failed_graders_message_says_its_status_and_class_never_its_words(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + words = "the task's own words" + records = {"Error code: 429 - " + words: "HTTP 429", + "RateLimitError: " + words: "RateLimitError", + "anthropic.APIStatusError: status code 529 " + words: "anthropic.APIStatusError, HTTP 529", + "openai.BadRequestError: " + words: "openai.BadRequestError", + "anthropic._exceptions.OverloadedError: " + words: "anthropic.OverloadedError", + "httpx.ReadTimeout " + words: "httpx.ReadTimeout", + "TimeoutError: " + words: "TimeoutError", + words + " 500 times": "no HTTP status or known error class in its record"} + log = os.path.join(where, "eval.log") + for record, said in records.items(): + with self.subTest(said): + self.log(log, f"[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: {record}") + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + self.assertIn(f"compare_text_with_llm ({said})", str(raised.exception)) + self.assertNotIn(words, str(raised.exception)) + os.remove(log) + + def test_a_word_shaped_like_a_class_is_never_said(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + log = os.path.join(where, "eval.log") + for record in ("SyntheticPrivateDocumentError: the rest", "PrivateLedgerException the rest", + "anthropic.SyntheticPrivateError: x", "SyntheticPrivate.RateLimitError: x", + "secret.module.ValueError: x", "RateLimitErrorOfTheTask: x", "rate_limit_error: x", + "anthropic.RateLimitErrorXYZ: x"): + with self.subTest(record): + self.log(log, f"[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: {record}") + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + said = str(raised.exception) + self.assertIn("compare_text_with_llm (no HTTP status or known error class in its record)", said) + for word in ("Synthetic", "Private", "secret", "XYZ", "OfTheTask", "rate_limit"): + self.assertNotIn(word, said) + os.remove(log) + + def test_a_refusing_endpoints_error_text_is_not_said(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + self.calls({"status": 400, "request": {"n": 1}, "error": "status 400: the task's own words"}) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + self.assertIn("answered 400", str(raised.exception)) + self.assertNotIn("own words", str(raised.exception)) + os.remove(self.relayed) + + def test_the_marks_words_outside_the_graders_error_record_change_nothing(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where, "1.0") + words = "Error in compare_text_with_llm: Error code: 402" + self.log(os.path.join(self.tmp, "run_multienv_m3.log"), + f"\x1b[1;33m[2026-11-06 01:02:03,004 \x1b[31mINFO \x1b[32mrun_multienv_m3/512-Worker\x1b[1;33m] \x1b[0m" + f"Step 3: pyautogui.typewrite('{words}')", + f"[2026-11-06 01:02:03,004 ERROR lib_run_single/88-Worker] step failed: {words}", + f"[2026-11-06 01:02:03,004 ERROR run_multienv_m3/600-Worker] {words}", + words) + self.log(os.path.join(where, "eval.log"), f"[2026-11-06 01:02:03,004 INFO llm_metrics/40-Worker] {words}") + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 1.0) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + self.log(log, f"[2026-11-06 01:02:04,005 INFO desktopenv.env] user said: {words}", + f"[2026-11-06 01:02:04,005 ERROR desktopenv.experiment] {words}") + + with mock.patch.object(self.driver, "_run", run): + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + + def calls(self, *calls): + with open(self.relayed, "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls) + "not json\n") + + def test_a_grader_endpoints_failure_no_retry_recovered_refuses_the_score(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + + asked, other = {"n_messages": 1, "last": ["grade this"], "sha256": "a"}, {"n_messages": 1, "last": ["grade that"], "sha256": "b"} + undigested = {"n_messages": 1, "last": ["grade this"]} + ok = lambda req: {"status": 200, "request": req, "reply": {"text": "YES"}} + bad = lambda status, req: {"status": status, "request": req, "error": f"status {status}: credits"} + refused = {"402 then another call's 200": [bad(402, asked), ok(other)], + "429 three times": [bad(429, asked), bad(429, asked), bad(429, asked)], + "500 last": [ok(other), bad(500, asked)], + "400 then 200 on another request": [bad(400, other), ok(asked)], + "402, another call, then 200 to its words": [bad(402, asked), ok(other), ok(asked)], + "no request recorded": [{"status": 500, "error": "x"}, {"status": 200}], + "a late 200 recorded after the last failure": [dict(bad(500, asked), t0=2), dict(ok(asked), t0=1)], + "no digest of the request": [bad(429, undigested), ok(undigested)], + "a failure with no start time": [dict(ok(asked), t0=1), bad(429, asked)]} + recovered = {"none": [], "429 then 200": [bad(429, asked), ok(asked)], + "503, 503, 200": [bad(503, asked), bad(503, asked), ok(asked), ok(other)], + "an abandoned 502 recorded after its retry's 200": [dict(ok(asked), t0=2), dict(bad(502, asked), t0=1)]} + with mock.patch.object(self.driver, "_run", run): + for name, calls in refused.items(): + self.calls(*calls) + for setup, start in (("original", lambda: self.driver.score("tasks__003", self.tmp, setup="original", + judge=self.judge)), + ("nop", lambda: self.nop("003", judge=self.judge))): + with self.subTest(f"{name} {setup}"), self.assertRaises(RuntimeError) as raised: + start() + self.assertIn("the LLM grader's endpoint answered", str(raised.exception)) + self.assertIsInstance(raised.exception, self.driver.VerifierError) + self.assertIn("see the judge's calls", str(raised.exception)) + for name, calls in recovered.items(): + self.calls(*calls) + with self.subTest(name): + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 0.0) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + os.remove(self.relayed) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + self.calls(bad(402, asked)) + self.assertEqual(self.nop("001")["reward"], 0.0) + self.assertEqual(self.nop("001", judge=dict(self.judge, base_url="http://h:1/u1/v1"))["reward"], 0.0) + + def test_the_agents_file_refused_keeps_the_upstreams_score_by_the_images_sent_never_the_words(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + asked = {"n_messages": 1, "sha256": "a"} + whole, broken = {"type": "image/png", "is": "image/png", "bytes": 4000}, {"type": "image/png", "is": None, "bytes": 4000} + big = dict(whole, bytes=self.driver.IMAGE_BYTES + 1) + said = "messages.0.content.1.image.source.base64.data: could not process image, too large" + bad = lambda status, *images: dict({"status": status, "request": asked, "error": f"HTTP {status}: {said}"}, + **({"images": list(images)} if images else {})) + file, size = bad(400, whole, broken), bad(413, whole, big) + record = "[2026-11-06 01:02:04 ERROR desktopenv.env] Error in compare_images_with_llm: Error code: {} - x" + score = lambda: self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + + def nop(): + try: + return self.nop("003", judge=self.judge) + finally: + shutil.rmtree(os.path.join(self.tmp, "results", "nop"), ignore_errors=True) + + with mock.patch.object(self.driver, "_run", run): + for calls, refused in (([file], ["agent file"]), ([size, file], ["agent file", "agent file size"]), + ([bad(422, {"type": "image/bmp", "is": "image/bmp", "bytes": 10})], ["agent file"])): + self.calls(*calls) + with open(os.path.join(where, "eval.log"), "w") as f: + f.write("".join(record.format(c["status"]) + "\n" for c in calls)) + with self.subTest(refused=refused): + self.assertEqual((score()["reward"], score()["grader_refused"]), (0.0, refused)) + self.assertEqual(dict(nop(), detail=None), {"reward": 0.0, "detail": None, "grader_refused": refused}) + os.remove(os.path.join(where, "eval.log")) + for name, calls in (("the provider's words alone", [bad(400)]), ("whole images", [bad(400, whole, whole)]), + ("a whole image said to be another type", [bad(400, dict(whole, type="image/jpeg"))]), + ("a 413 of whole images", [bad(413, whole)]), ("a 413 of a broken one", [bad(413, broken)]), + ("a 401", [bad(401, broken)]), ("with a 429", [file, bad(429)])): + self.calls(*calls) + for setup, start in (("original", score), ("nop", nop)): + with self.subTest(f"{name} {setup}"), self.assertRaises(self.driver.VerifierError) as raised: + start() + self.assertIn("the LLM grader's endpoint answered", str(raised.exception)) + self.calls(file) + for status in (429, 400): + with open(os.path.join(where, "eval.log"), "w") as f: + f.write(record.format(400) + "\n" + record.format(status) + "\n") + with self.subTest(status=status), self.assertRaises(self.driver.VerifierError) as raised: + score() + self.assertIn(f"compare_images_with_llm (HTTP {status})", str(raised.exception)) + os.remove(os.path.join(where, "eval.log")) + other = os.path.join(self.job, "u2", "workspace") + self.result(os.path.join(other, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "005")) + shutil.copytree(os.path.dirname(os.path.dirname(self.relayed)), os.path.join(self.job, "u2", ".work")) + for eid, dir in (("005", other), ("003", self.tmp), ("005", other)): + with self.subTest(eid=eid): + self.assertEqual(self.driver.score(f"tasks__{eid}", dir, setup="original", judge=self.judge)["grader_refused"], + ["agent file"]) + + def test_a_failed_grader_is_the_null_controls_error_not_a_0(self): + marks = list(self.driver.GRADER_FAILED) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + self.log(log, "[2026-11-06 01:02:03 INFO desktopenv.env] evaluating", + *([f"[2026-11-06 01:02:04 ERROR desktopenv.env] {marks.pop(0)}Error code: 429"] if marks else [])) + + with mock.patch.object(self.driver, "_run", run): + for mark in list(marks): + with self.subTest(mark), self.assertRaises(RuntimeError) as raised: + self.nop("003", judge=self.judge) + self.assertIn(f"{mark[len('Error in '):-2]} (HTTP 429)", str(raised.exception)) + os.remove(os.path.join(self.tmp, "nop.log")) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + + def test_the_lists(self): + lists = self.driver.LISTS + self.assertEqual(lists["offline36"], OFFLINE36) + self.assertEqual(lists["no_services63"], sorted(OFFLINE36 + MORE27)) + self.assertEqual(len(set(lists["no_services63"])), 63) + self.assertFalse(set(lists["no_services63"]) & set(PROXY)) + self.assertEqual(len(self.driver.JUDGED), 21) + self.assertEqual(len(self.driver.ASKS_USER), 6) + self.assertFalse((self.driver.JUDGED | self.driver.ASKS_USER) & set(OFFLINE36)) + with mock.patch.dict(os.environ, {"NULLIUS_DATASET": "", "NULLIUS_DATASET_ID": "no_services63"}): + self.assertEqual(self.driver.units()["units"], [f"tasks__{e}" for e in sorted(OFFLINE36 + MORE27)]) + counted = {d["id"]: d["units"] for d in bench()["datasets"]} + self.assertEqual((counted["offline36"], counted["no_services63"]), (36, 63)) + + +class Definition(unittest.TestCase): + def test_the_judge_and_the_simulated_user_are_sonnet_as_openrouter_names_it(self): + b = bench() + self.assertEqual(definitions.problems(b, "benchmark"), []) + self.assertEqual(providers.served_as(b["score"]["judge"]["model"], "openrouter"), SONNET) + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + asked = json.load(f)["agents"]["m3"]["env"]["OSWORLD_USER_SIM_MODEL"] + other, why = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.assertEqual(why, "") + self.assertEqual(other["sent"], "minimax/minimax-m3") + self.assertEqual(other["model"]["side_models"], [f"{asked}={SONNET}"]) + + +class Provider(BaseHTTPRequestHandler): + seen = [] + status = {} + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + Provider.seen.append((self.path, body["model"])) + code = int(self.headers.get("X-Status") or 200) + out = (json.dumps({"type": "message", "role": "assistant", "content": [{"type": "text", "text": "ok"}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) if code == 200 + else json.dumps({"error": {"message": f"status {code}"}})).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + +class Routes(unittest.TestCase): + """The unit's relay as an Original run of OSWorld2 on OpenRouter starts it, before a fake provider.""" + + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + b = bench() + other, _ = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.judge_model = providers.served_as(b["score"]["judge"]["model"], "openrouter") + self.trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.trial, ignore_errors=True) + self.relay = relay.Relay(f"http://127.0.0.1:{self.up.server_address[1]}/api/v1", "sk-real", + os.path.join(self.trial, ".work", "relay"), + model=other["sent"], side_models=other["model"]["side_models"], + key_header=run.key_header("openrouter", other, {})).start() + self.addCleanup(self.relay.stop) + self.judge_url = self.relay.url("u1-judge", model=self.judge_model) + + def post(self, url, model, status=200, words=""): + messages = [{"role": "user", "content": words}] if words else [] + req = urllib.request.Request(url + "/messages", data=json.dumps({"model": model, "max_tokens": 8, + "messages": messages}).encode(), + headers={"Content-Type": "application/json", "x-api-key": relay.PLACEHOLDER_KEY, + "X-Status": str(status), + "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status + except urllib.error.HTTPError as e: + e.close() + return e.code + + def test_sonnet_is_never_sent_as_m3_and_the_graders_calls_are_not_the_agents(self): + unit = self.relay.url("u1") + self.post(unit, "MiniMax-M3") + self.post(unit, "claude-sonnet-4-6") + self.post(self.judge_url, "claude-sonnet-4-6") + self.assertEqual([m for _, m in Provider.seen[-3:]], ["minimax/minimax-m3", SONNET, SONNET]) + self.assertTrue(all(p == "/api/v1/messages" for p, _ in Provider.seen[-3:])) + agent, user = self.relay.calls("u1") + self.assertEqual((agent["model"], agent.get("side")), ("minimax/minimax-m3", None)) + self.assertEqual((user["model"], user["model_asked"], user.get("side")), (SONNET, "claude-sonnet-4-6", True)) + self.assertEqual([c["model"] for c in self.relay.calls("u1-judge")], [SONNET]) + + def test_the_providers_refusals_are_the_judge_routes_failure(self): + for status, kind in ((402, "ApiUsageLimitError"), (429, "RateLimitError"), (500, "ApiError"), (503, "ApiError")): + with self.subTest(status): + self.assertEqual(self.post(self.judge_url, SONNET, status), status) + self.assertEqual(relay.failure(self.relay.calls("u1-judge"))[0], kind) + self.assertEqual(self.post(self.judge_url, SONNET), 200) + self.assertIsNone(relay.failure(self.relay.calls("u1-judge"))) + self.assertEqual(self.relay.calls("u1"), []) + + def test_the_driver_reads_the_relays_own_record_of_the_judge_route(self): + """The driver reads the relay's own record of the judge route, as run.Run writes it.""" + other, _ = definitions.setup(bench(), "original", "MiniMax-M3", "openrouter") + unit = types.SimpleNamespace(base=f"http://127.0.0.1:{self.up.server_address[1]}/api/v1", key="sk-real", call_seconds=relay.TIMEOUT, max_calls=None, + other=other, provider="openrouter", backend="local", model=other["sent"], + judge_model=self.judge_model, key_header=run.key_header("openrouter", other, {}), + serving={self.judge_model: ["anthropic"]}, ledger_file=lambda: None) + trial = os.path.join(self.trial, "unit") + rl = run.Run.unit_relay(unit, "u1", trial, {}) + self.addCleanup(rl.stop) + judge = run.Run.judge(unit, rl, "u1")["judge"] + driver = load_driver(self.trial) + ws = os.path.join(trial, "workspace") + driver._grader_refused(ws, judge) + self.post(judge["base_url"], SONNET, 429, "first") + self.post(judge["base_url"], SONNET, 200, "first") + self.post(judge["base_url"], SONNET, 200, "second") + driver._grader_refused(ws, judge) + self.assertEqual([c["status"] for c in rl.calls("u1-judge")], [429, 200, 200]) + self.assertEqual({c["request"]["routing"]["order"][0] for c in rl.calls("u1-judge")}, {"anthropic"}) + self.post(judge["base_url"], SONNET, 402, "third") + self.post(judge["base_url"], SONNET, 200, "fourth") + with self.assertRaises(RuntimeError) as raised: + driver._grader_refused(ws, judge) + self.assertIn("answered 402, and the task was scored without it", str(raised.exception)) + self.assertNotIn("status 402", str(raised.exception)) + + def test_a_retry_is_the_whole_request_again_not_only_its_words(self): + shot = lambda data: [{"type": "text", "text": "the same words"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": data}}] + self.post(self.judge_url, SONNET, 429, shot("QQ==")) + self.post(self.judge_url, SONNET, 200, shot("Qg==")) + first, second = self.relay.calls("u1-judge") + self.assertNotEqual(first["request"], second["request"]) + with self.assertRaises(RuntimeError): + load_driver(self.trial)._grader_refused(os.path.join(self.trial, "workspace"), {"base_url": self.judge_url}) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +def _metrics(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + path = os.path.join(up, "desktop_env", "evaluators", "metrics", "llm_metrics.py") + return path if up and os.path.isfile(path) else "" + + +@unittest.skipUnless(_metrics(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream") +class Metrics(unittest.TestCase): + def test_each_mark_is_what_the_upstreams_llm_metrics_log(self): + with open(_metrics(), encoding="utf-8") as f: + source = f.read() + driver = load_driver(os.environ["NULLIUS_OSWORLD2_UPSTREAM"]) + for mark in driver.GRADER_FAILED: + self.assertEqual(source.count(f'logger.error("{mark}%s", exc)'), 1, mark) + self.assertEqual(source.count('logger.error("Error in '), len(driver.GRADER_FAILED)) + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + """What each task class asks for, read from its syntax tree.""" + + def test_the_drivers_lists_are_the_task_classes(self): + graded, users, hosted, proxied = set(), set(), set(), set() + for p in _task_classes(): + n = os.path.basename(p)[5:-3] + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + if isinstance(node, ast.Call): + f = node.func + name = f.id if isinstance(f, ast.Name) else f.attr if isinstance(f, ast.Attribute) else "" + if name in ("generate_text", "generate_chat") or name.endswith("_with_llm"): + graded.add(n) + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [a.name for a in node.names] if isinstance(node, ast.Import) else []) + if any("llm_metrics" in m or "model_client" in m for m in modules): + graded.add(n) + if any(m.endswith(("controllers.website", "controllers.gitlab")) for m in modules) or \ + (isinstance(node, ast.ImportFrom) and (node.module or "").endswith("controllers") + and {a.name for a in node.names} & {"website", "gitlab"}): + hosted.add(n) + if isinstance(node, ast.ClassDef): + for st in node.body: + targets = st.targets if isinstance(st, ast.Assign) else [st.target] if isinstance(st, ast.AnnAssign) else [] + names = {t.id for t in targets if isinstance(t, ast.Name)} + value = getattr(st, "value", None) + if "proxy" in names and isinstance(value, ast.Constant) and value.value is True: + proxied.add(n) + if "user_simulator" in names and isinstance(value, ast.Dict): + if any(isinstance(k, ast.Constant) and k.value == "type" and isinstance(v, ast.Constant) + and v.value == "llm" for k, v in zip(value.keys, value.values)): + users.add(n) + driver = load_driver(os.environ["NULLIUS_OSWORLD2_UPSTREAM"]) + self.assertEqual(sorted(graded), sorted(driver.JUDGED)) + self.assertEqual(sorted(users), sorted(driver.ASKS_USER)) + self.assertEqual(sorted(proxied), PROXY) + self.assertFalse(set(driver.LISTS["no_services63"]) & (hosted | proxied)) + self.assertEqual(len(_task_classes()) - len(hosted | proxied), 63) + + def test_the_graded_tasks_that_catch_a_grader_error_themselves(self): + """The graded tasks that catch a grader error themselves; pinned, so a change shows.""" + catching = set() + for p in _task_classes(): + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + if not isinstance(node, ast.Try) or all(any(isinstance(x, ast.Raise) for x in ast.walk(h)) + for h in node.handlers): + continue + if any(isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute)) + and (c.func.id if isinstance(c.func, ast.Name) else c.func.attr) in ("generate_text", "generate_chat") + for part in node.body for c in ast.walk(part)): + catching.add(os.path.basename(p)[5:-3]) + self.assertEqual(sorted(catching), CATCHING) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_mail_profiles.py b/app/tests/test_osworld2_mail_profiles.py new file mode 100644 index 00000000..f4d5ebe1 --- /dev/null +++ b/app/tests/test_osworld2_mail_profiles.py @@ -0,0 +1,136 @@ +"""OSWorld2's mail tasks get their profiles signed out of the live mailboxes their assets ship, and started offline.""" +from __future__ import annotations + +import contextlib +import io +import os +import tarfile +import tempfile +import unittest +import zlib +from unittest import mock + +from tests.test_osworld2_driver import load + +PROFILE = ".thunderbird/abc.default-release/" +PREFS = ('user_pref("mail.server.server1.hostname", "mail.example.com");\n' + 'user_pref("mail.server.server1.login_at_startup", true);\n' + 'user_pref("mail.server.server1.check_new_mail", true);\n' + 'user_pref("mail.server.server1.download_on_biff", true);\n' + ' user_pref("mail.server.server2.check_new_mail",true);\r\n' + 'user_pref("offline.startup_state", 2);\n') +SIGNED_IN = ("logins.json", "logins-backup.json", "signons.sqlite", "cookies.sqlite", "cookies.sqlite-journal", "user.js", + "storage/default/https+++login.example.com/ls/data.sqlite") + + +def archive(path, files): + with tarfile.open(path, "w:gz") as t: + for name, data in files.items(): + info = tarfile.TarInfo(name) + info.size, info.mtime = len(data), 1700000000 + t.addfile(info, io.BytesIO(data)) + + +def members(path): + with tarfile.open(path) as t: + return {m.name: t.extractfile(m).read() for m in t if m.isfile()} + + +class SignedOut(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + up = os.path.join(self.tmp.name, "up") + assets = os.path.join(up, "cache", "osworld_v2_assets") + for d in ("task_006", "task_007", "shared"): + os.makedirs(os.path.join(assets, d)) + self.files = {PROFILE + "prefs.js": PREFS.encode(), PROFILE + "key4.db": b"k", + PROFILE + "Mail/host/Inbox": b"From a\n\nbody\n", PROFILE + "storage/permanent/chrome/idb/x": b"i", + **{PROFILE + n: b"fake" for n in SIGNED_IN}} + archive(os.path.join(assets, "task_006", "thunderbird-profile.tar.gz"), self.files) + with open(os.path.join(assets, "task_006", "sheet.xlsx"), "wb") as f: + f.write(b"sheet") + self.driver = load(up) + self.unit = os.path.join(self.tmp.name, "unit") + os.makedirs(self.unit) + + def tearDown(self): + self.tmp.cleanup() + + def test_other_tasks_keep_the_assets_folder(self): + self.assertEqual(self.driver._assets(self.unit, "007"), self.driver.ASSETS) + self.assertFalse(os.path.exists(os.path.join(self.unit, "assets"))) + + def test_the_profile_loses_its_sign_ins_and_keeps_its_mail(self): + root = self.driver._assets(self.unit, "006") + got = members(os.path.join(root, "task_006", "thunderbird-profile.tar.gz")) + self.assertEqual(sorted(got), sorted(n for n in self.files if n[len(PROFILE):] not in SIGNED_IN)) + self.assertEqual(got[PROFILE + "Mail/host/Inbox"], self.files[PROFILE + "Mail/host/Inbox"]) + prefs = got[PROFILE + "prefs.js"].decode() + self.assertNotIn("true", prefs) + self.assertIn('"mail.server.server1.hostname", "mail.example.com"', prefs) + self.assertTrue(prefs.endswith('user_pref("offline.autoDetect", false);\n' + 'user_pref("offline.startup_state", 3);\n')) + + def test_the_rest_of_the_assets_are_the_originals(self): + root = self.driver._assets(self.unit, "006") + for rel in ("shared", "task_007", os.path.join("task_006", "sheet.xlsx")): + self.assertEqual(os.path.realpath(os.path.join(root, rel)), + os.path.realpath(os.path.join(self.driver.ASSETS, rel))) + self.assertNotIn("thunderbird-profile.tar.gz.part", os.listdir(os.path.join(root, "task_006"))) + + def test_the_copy_is_the_same_each_time(self): + root = self.driver._assets(self.unit, "006") + with open(os.path.join(root, "task_006", "thunderbird-profile.tar.gz"), "rb") as f: + first = f.read() + self.driver._assets(self.unit, "006") + with open(os.path.join(root, "task_006", "thunderbird-profile.tar.gz"), "rb") as f: + self.assertEqual(f.read(), first) + + def test_a_missing_profile_is_refused(self): + os.remove(os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz")) + with self.assertRaisesRegex(RuntimeError, "no thunderbird-profile.tar.gz"): + self.driver._assets(self.unit, "006") + + def test_a_hard_link_is_refused(self): + path = os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz") + with tarfile.open(path, "w:gz") as t: + link = tarfile.TarInfo(PROFILE + "prefs.js") + link.type, link.linkname = tarfile.LNKTYPE, PROFILE + "key4.db" + t.addfile(link) + with self.assertRaisesRegex(RuntimeError, "does not rewrite"): + self.driver._assets(self.unit, "006") + + def test_a_broken_profile_archive_is_a_setup_error(self): + path = os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz") + with open(path, "rb") as f: + whole = f.read() + patches = {"_vm": lambda *a: "vm", "_task": lambda *_: {"instruction": "", "proxy": False}} + for name, data, fault in (("not an archive", b"x" * 600, None), ("cut short", whole[:len(whole) // 2], None), + ("bad deflate data", whole, zlib.error("invalid stored block lengths"))): + with open(path, "wb") as f: + f.write(data) + with self.subTest(name), mock.patch.multiple(self.driver, **patches), \ + mock.patch.object(self.driver, "_signed_out", side_effect=fault) if fault else contextlib.nullcontext(), \ + self.assertRaises(self.driver.EnvironmentSetupError): + self.driver.score("tasks__006", self.unit) + + def test_the_null_control_and_the_original_get_the_copy(self): + envs = [] + + def run(argv, cwd, env, *a, **kw): + envs.append(env) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", run), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}): + for start in (lambda: self.driver.score("tasks__006", self.unit), + lambda: self.driver.answer("tasks__006", self.unit, "original", "MiniMax-M3", + "http://127.0.0.1:9/v1", "k")): + with self.assertRaises(KeyboardInterrupt): + start() + self.assertEqual([e["OSWORLD_FILE_BASE_URL"] for e in envs], [os.path.join(self.unit, "assets")] * 2) + self.assertEqual(self.driver._env("007")["OSWORLD_FILE_BASE_URL"], self.driver.ASSETS) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_muse.py b/app/tests/test_osworld2_muse.py new file mode 100644 index 00000000..4373c4e3 --- /dev/null +++ b/app/tests/test_osworld2_muse.py @@ -0,0 +1,402 @@ +"""OSWorld2's Muse Spark agent beside M3: its arguments, its relay route, its tool calls, its call timeout. +With NULLIUS_OSWORLD2_UPSTREAM set, every flag is checked against the runner's parser, read as a syntax tree.""" +from __future__ import annotations + +import ast +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius.standard import container, definitions, job, jsonl, recipe, relay, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +MUSE, SENT, SONNET = "muse-spark-1.3", "meta/muse-spark-1.3", "anthropic/claude-sonnet-4.6" +UNIT = "http://127.0.0.1:9/u1/v1" + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_muse", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def bench(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f) + + +class Started(Exception): + pass + + +class Driver(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.vms, self.runs = [], [] + + def vm(dir, domain, eid): + self.vms.append(eid) + return os.path.join(dir, "vm.qcow2") + + def started(argv, cwd, env, *a, **kw): + self.runs.append((argv, env)) + raise Started() + + for name, fake in (("_vm", vm), ("_run", started), ("_task", lambda *_: {"instruction": "", "proxy": False})): + p = mock.patch.object(self.driver, name, fake) + p.start() + self.addCleanup(p.stop) + + def answer(self, agent, model, eid="045", **options): + env = {"NULLIUS_OPTION_AGENT": agent, **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()}} + with mock.patch.dict(os.environ, env): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", model, UNIT, relay.PLACEHOLDER_KEY, + judge={"model": SONNET, "base_url": "http://127.0.0.1:9/u1-judge/v1", + "key": relay.PLACEHOLDER_KEY}) + + def test_muse_spark_runs_its_script_with_its_published_runs_settings_on_the_relay(self): + with self.assertRaises(Started): + self.answer("muse_spark", MUSE, max_steps="500", thinking="adaptive") + argv, env = self.runs[-1] + self.assertEqual(argv[1], os.path.join(self.tmp, "scripts", "python", "run_multienv_muse_spark.py")) + flags = {w: v for w, v in zip(argv[2:], argv[3:]) if w.startswith("--") and not v.startswith("--")} + self.assertEqual({k: flags[k] for k in ("--model", "--base_url", "--reasoning_effort", "--max_tokens", "--max_steps", + "--provider_name", "--num_envs")}, + {"--model": MUSE, "--base_url": UNIT, "--reasoning_effort": "max", "--max_tokens": "262144", + "--max_steps": "500", "--provider_name": "docker", "--num_envs": "1"}) + self.assertEqual(argv[-1], "--save_model_eval_raw_info") + self.assertNotIn("--api_key", argv) + self.assertNotIn("--truncation", argv) + self.assertEqual({k: env[k] for k in ("MUSE_SPARK_API_KEY", "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT")}, + {"MUSE_SPARK_API_KEY": relay.PLACEHOLDER_KEY, "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1"}) + self.assertFalse([k for k in env if k.startswith("M3_")]) + + def test_the_simulated_user_and_the_grader_are_m3s(self): + envs = [] + for agent, model in (("m3", "MiniMax-M3"), ("muse_spark", MUSE)): + with self.subTest(agent), self.assertRaises(Started): + self.answer(agent, model, eid="024") + envs.append({k: v for k, v in self.runs[-1][1].items() if k.startswith(("OSWORLD_USER_SIM_", "OSWORLD_EVAL_MODEL_"))}) + self.assertEqual(envs[0], envs[1]) + self.assertEqual(envs[1]["OSWORLD_USER_SIM_BASE_URL"], "http://127.0.0.1:9/u1") + self.assertEqual(envs[1]["OSWORLD_EVAL_MODEL_NAME"], SONNET) + + def test_m3_keeps_its_thinking_option(self): + with self.assertRaises(Started): + self.answer("m3", "MiniMax-M3", thinking="adaptive") + self.assertEqual(self.runs[-1][1]["M3_THINKING_MODE"], "adaptive") + + def test_every_agent_takes_the_steps_option_and_its_own_by_default(self): + own = bench()["options"]["max_steps"]["by"]["agent"] + self.assertEqual(own, {"m3": "100", "muse_spark": "500"}) + for agent, model in (("m3", "MiniMax-M3"), ("muse_spark", MUSE)): + for steps, sent in (("default", own[agent]), ("50", "50")): + with self.subTest(agent, steps=steps), self.assertRaises(Started): + self.answer(agent, model, max_steps=steps) + argv = self.runs[-1][0] + self.assertEqual(argv[argv.index("--max_steps") + 1], sent) + spec = self.driver.AGENTS["agents"]["m3"] + with mock.patch.dict(spec, argv=[a for a in spec["argv"] if a not in ("--max_steps", "100")]), self.assertRaises(Started): + self.answer("m3", "MiniMax-M3", max_steps="50") + self.assertEqual(self.runs[-1][0][-2:], ["--max_steps", "50"]) + + def test_a_model_of_the_other_agent_is_refused_before_the_vm_naming_its_agent(self): + for agent, model, other in (("m3", MUSE, "muse_spark"), ("muse_spark", "MiniMax-M3", "m3")): + with self.subTest(agent), self.assertRaises(RuntimeError) as raised: + self.answer(agent, model) + self.assertIn(f"{model} is not one of the {agent} agent's models", str(raised.exception)) + self.assertIn(f"choose the agent {other} for it", str(raised.exception)) + with self.assertRaises(RuntimeError) as raised: + self.answer("muse_spark", "gpt-5") + self.assertTrue(str(raised.exception).endswith("choose one of them")) + self.assertEqual((self.vms, self.runs), ([], [])) + + def test_the_option_offers_every_agent_and_thinking_only_with_m3(self): + opts = bench()["options"] + self.assertEqual(opts["agent"]["values"], list(self.driver.AGENTS["agents"])) + self.assertEqual(opts["thinking"]["when"], {"agent": ["m3"]}) + models = [m for spec in self.driver.AGENTS["agents"].values() for m in spec["models"]] + self.assertEqual(models, bench()["setups"]["original"]["models"]) + + +class Definition(unittest.TestCase): + def test_muse_spark_is_sent_as_meta_and_not_compared_with_the_paper(self): + b = bench() + self.assertEqual(definitions.problems(b, "benchmark"), []) + other, why = definitions.setup(b, "original", MUSE, "openrouter") + self.assertEqual(why, "") + self.assertEqual(other["sent"], SENT) + self.assertNotIn("pins", other) + self.assertEqual(other["model"]["side_models"], [f"claude-sonnet-4-6={SONNET}"]) + self.assertFalse(other["comparable"]) + self.assertTrue(definitions.setup(b, "original", "MiniMax-M3", "openrouter")[0]["comparable"]) + + def test_comparable_names_some_of_the_setups_models(self): + for value, ok in ((["MiniMax-M3"], True), (True, True), (False, True), ([MUSE], False), (["gpt-5"], False), ([], False), + (["MiniMax-M3", "MiniMax-M3"], False)): + b = bench() + b["setups"]["original"]["comparable"] = value + with self.subTest(value): + self.assertEqual(not [p for p in definitions.problems(b, "benchmark") if "comparable" in p], ok) + + def test_the_builder_offers_like_the_paper_with_m3_only_and_runs_muse_spark_as_any_model(self): + page, = recipe.page_of("osworld2")["setups"] + self.assertEqual((page["models"], page["sent"]["openrouter"]), (["MiniMax-M3"], {"MiniMax-M3": "minimax/minimax-m3"})) + r = {"version": 1, "harness": recipe.HARNESS, "name": "x", "title": "x", "benchmark": "osworld2", "datasets": ["test_v2"], + "others": ["original"], "provider": "openrouter", "limit": 1, "concurrency": 1, "unsandboxed": False, + "where": "local", "options": {}, "bench_options": {"agent": "muse_spark"}, "model": MUSE} + self.assertIn("Any of your choice", recipe.compile(dict(r, model_choice="paper"))["errors"][0]["message"]) + out = recipe.compile(dict(r, model_choice="any")) + self.assertEqual(out["errors"], []) + self.assertIn(f"run osworld2 original --model {MUSE} --provider openrouter", out["script"]) + self.assertIn("--bench-option agent=muse_spark", out["script"]) + picked = recipe.compile(dict(r, model_choice="any", model=SENT)) + self.assertEqual(picked["errors"], []) + self.assertIn(f"run osworld2 original --model {MUSE} --provider openrouter", picked["script"]) + + def test_the_relay_waits_longer_than_muse_spark(self): + b = bench() + self.assertEqual(b["limits"]["call_seconds"], 3300) + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + self.assertNotIn("--request_timeout", json.load(f)["agents"]["muse_spark"]["argv"]) + for bad in (0, -1, 1.5, True, "3300"): + b["limits"]["call_seconds"] = bad + with self.subTest(bad): + self.assertIn("limits.call_seconds: a whole number from 1", definitions.problems(b, "benchmark")) + + +class OpenRouter(BaseHTTPRequestHandler): + """A fake OpenRouter /responses; ``slow`` holds the reply.""" + seen = [] + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + OpenRouter.seen.append({"path": self.path, "body": body}) + time.sleep(float(body.get("slow") or 0)) + out = {"id": "gen-42", "object": "response", "status": "completed", "model": body["model"], + "output": [{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "e"}, + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "computer.stop", + "arguments": json.dumps({"answer": "done"})}], + "usage": {"input_tokens": 100, "input_tokens_details": {"cached_tokens": 90}, "output_tokens": 7, + "output_tokens_details": {"reasoning_tokens": 5}, "cost": 0.0002}} + if body.get("no_cost"): + del out["usage"]["cost"] + if self.headers.get("X-OpenRouter-Metadata") == "enabled": + out["openrouter_metadata"] = {"requested": body["model"], "endpoints": {"available": [ + {"provider": "Meta", "model": body["model"], "selected": True}]}} + data = json.dumps(out).encode() + try: + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + except OSError: + pass + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, json.loads(r.read()) + except urllib.error.HTTPError as e: + try: + return e.code, json.loads(e.read()) + finally: + e.close() + + +class Responses(unittest.TestCase): + """The unit's relay as an Original run of Muse Spark on OpenRouter with --served-by meta starts it, on the local backend.""" + + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def unit(self, seconds, prices=None): + b = bench() + other, _ = definitions.setup(b, "original", MUSE, "openrouter") + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + ns = types.SimpleNamespace(base=self.base, key="sk-real", other=other, provider="openrouter", backend="local", max_calls=None, + key_header=run.key_header("openrouter", other, {}), + model=other["sent"], judge_model=SONNET, call_seconds=seconds, + serving={SENT: ["meta"]}, prices=prices, ledger_file=lambda: None) + rl = run.Run.unit_relay(ns, "u1", trial, {}) + self.addCleanup(rl.stop) + return rl + + def test_a_call_goes_to_responses_served_by_meta_and_its_cache_is_recorded(self): + OpenRouter.seen.clear() + rl = self.unit(3300) + self.assertEqual(rl.pool.timeout, 3300) + status, reply = post(rl.url("u1") + "/responses", {"model": MUSE, "input": [], "store": False, + "include": ["reasoning.encrypted_content"], + "reasoning": {"effort": "max", "summary": "auto"}}) + self.assertEqual((status, [i["type"] for i in reply["output"]]), (200, ["reasoning", "function_call"])) + sent = OpenRouter.seen[-1] + self.assertEqual(sent["path"], "/api/v1/responses") + self.assertEqual((sent["body"]["model"], sent["body"]["provider"]), (SENT, {"order": ["meta"], "allow_fallbacks": False})) + self.assertEqual(sent["body"]["reasoning"], {"effort": "max", "summary": "auto"}) + call, = rl.calls("u1") + self.assertEqual(call["request"]["routing"], {"order": ["meta"], "allow_fallbacks": False}) + self.assertEqual({k: call["reply"]["usage"][k] for k in ("input", "cached", "output", "reasoning")}, + {"input": 100, "cached": 90, "output": 7, "reasoning": 5}) + self.assertEqual(call["reply"]["tool_calls"][0]["name"], "computer.stop") + + def test_a_reply_with_no_cost_is_priced_from_the_price_list_the_run_has(self): + rl = self.unit(30, {SENT: {"input": 1.25, "cached": 0.15, "output": 4.25}}) + post(rl.url("u1") + "/responses", {"model": MUSE, "input": [], "no_cost": True}) + call = rl.calls("u1")[-1] + self.assertEqual(call["cost_from"], "estimate") + self.assertAlmostEqual(call["cost_usd"], (10 * 1.25 + 90 * 0.15 + 7 * 4.25) / 1e6) + + def test_a_reply_slower_than_the_relays_wait_is_a_502_and_one_within_it_goes_through(self): + status, reply = post(self.unit(0.3).url("u1") + "/responses", {"model": MUSE, "input": [], "slow": 1}) + self.assertEqual(status, 502) + self.assertIn("could not reach the endpoint", reply["error"]["message"]) + self.assertEqual(post(self.unit(5).url("u1") + "/responses", {"model": MUSE, "input": [], "slow": 1})[0], 200) + + def test_a_relay_container_is_given_the_benchmarks_wait(self): + argv = container.relay_argv("img", name="n", job="j", app="/app", record="/r", key_file="/k", base="b", model="m", + routes={}, timeout=3300) + self.assertEqual(argv[argv.index("--timeout") + 1], "3300") + self.assertNotIn("--timeout", container.relay_argv("img", name="n", job="j", app="/app", record="/r", key_file="/k", base="b", + model="m", routes={})) + + def test_a_run_reads_the_wait_from_the_benchmarks_limits(self): + b = definitions.get("benchmark", "osworld2") + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.call_seconds, 3300) + del b["limits"]["call_seconds"] + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.call_seconds, relay.TIMEOUT) + + +class Upstream(unittest.TestCase): + """What a unit of the Original keeps of its agent's calls: Muse Spark answers with tool calls and no text.""" + + ACTION = {"t0": 1, "t1": 2, "status": 200, "model": SENT, "request": {}, + "reply": {"text": "", "tool_calls": [{"name": "computer.computer", "arguments": '{"actions": []}'}], + "usage": {"input": 9, "output": 3, "cached": 0, "reasoning": 2}}} + + def drive(self, calls, stop=None, said="answer: RuntimeError: wrote no result.txt"): + record = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, record, ignore_errors=True) + if stop is not None: + with open(os.path.join(record, relay.STOP), "w") as f: + f.write(stop) + failing = types.SimpleNamespace(request=mock.Mock(side_effect=jsonl.Failed(said))) + ns = types.SimpleNamespace(other={"key": "original", "label": "OSWorld2 original"}, asked=MUSE, model=SENT, + name="j", bench={"label": "OSWorld2"}, agent_limit=lambda setup: 10, driver=lambda: failing, + attempt_kw=lambda: {}, simulator=lambda rl, tag: {}, judge=lambda rl, tag: {}) + rl = types.SimpleNamespace(calls=lambda tag: calls, url=lambda tag: "http://127.0.0.1:9/u1/v1", dir=record) + with self.assertRaises(run._Ended) as ended: + run.Run.drive_upstream(ns, "tasks__045", "/ws", "u1", [], rl) + return ended.exception + + def test_its_actions_are_the_trajectorys_tool_calls(self): + steps = job.trajectory_from_exchanges([], [self.ACTION], {"key": "original"}, SENT, "s")["steps"] + self.assertEqual(steps[-1]["tool_calls"], [{"tool_call_id": "call-2-1", "function_name": "computer.computer", + "arguments": {"actions": []}}]) + cut = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[{"name": "computer.computer", "arguments": '{"act'}])) + self.assertEqual(job.trajectory_from_exchanges([], [cut], {"key": "original"}, SENT, "s")["steps"][-1]["tool_calls"][0]["arguments"], + {"raw": '{"act'}) + bare = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[{"name": "computer.stop", "arguments": None}])) + self.assertEqual(job.trajectory_from_exchanges([], [bare], {"key": "original"}, SENT, "s")["steps"][-1]["tool_calls"][0]["arguments"], {}) + + def test_a_runner_that_fails_after_an_action_is_not_an_answer_the_model_ran_out_of(self): + ended = self.drive([self.ACTION]) + self.assertEqual(ended.kind, "NonZeroAgentExitCodeError") + self.assertIn("wrote no result.txt", ended.said) + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + self.assertEqual(self.drive([self.ACTION, silent]).kind, "OutputLengthExceededError") + self.assertEqual(self.drive([silent, self.ACTION]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(self.drive([self.ACTION, silent, dict(self.ACTION, side=True)]).kind, "OutputLengthExceededError") + self.assertEqual(self.drive([self.ACTION, dict(silent, side=True)]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(self.drive([self.ACTION, silent], stop=relay.OVERFLOW + "\n").kind, "OutputLengthExceededError") + + def test_the_drivers_setup_or_verifier_error_and_a_failed_last_reply_keep_their_kinds(self): + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + for kind in ("EnvironmentSetupError", "VerifierError"): + ended = self.drive([self.ACTION, silent], said=f"answer: {kind}: x") + self.assertEqual((ended.kind, ended.said), (kind, f"OSWorld2 original: answer: {kind}: x")) + failed = dict(silent, reply=dict(silent["reply"], end="failed", error={"code": "server_error"})) + ended = self.drive([self.ACTION, failed, dict(self.ACTION, side=True)]) + self.assertEqual(ended.kind, "ApiError") + self.assertIn("meta/muse-spark-1.3's last reply failed (server_error)", ended.said) + self.assertEqual(self.drive([failed, self.ACTION]).kind, "NonZeroAgentExitCodeError") + for late in (failed, silent): + self.assertEqual(self.drive([dict(self.ACTION, t0=3), dict(late, t0=2)]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(run._kind(jsonl.Failed("score: EnvironmentSetupError: x"), "VerifierError"), "EnvironmentSetupError") + self.assertEqual(run._kind(jsonl.Failed("score: RuntimeError: EnvironmentSetupError: x"), "VerifierError"), + "VerifierError") + self.assertEqual(run._kind(jsonl.Failed("score: CancelledError: x"), "VerifierError"), "CancelledError") + self.assertEqual(run._kind(jsonl.Failed("answer: CauseUnknownError: x"), "NonZeroAgentExitCodeError"), "CauseUnknownError") + for kind in ("EnvironmentSetupError", "VerifierError"): + ended = self.drive([self.ACTION], stop="turn limit\n", said=f"answer: {kind}: x") + self.assertEqual((ended.kind, ended.said), (kind, f"OSWorld2 original: answer: {kind}: x")) + + def test_a_runner_that_fails_after_a_guards_stop_is_cancelled_not_out_of_output(self): + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + for stop, said in (("spent $2.39 >= $2.40\n", "spent $2.39 >= $2.40"), ("", relay.STOP)): + with self.subTest(said): + ended = self.drive([silent, self.ACTION, silent], stop=stop) + self.assertEqual(ended.kind, run.CANCELLED) + self.assertIn(f"a guard asked the unit to stop ({said})", ended.said) + self.assertIn("wrote no result.txt", ended.said) + self.assertTrue(ended.trajectory["steps"]) + + +def _runner(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + path = os.path.join(up, "scripts", "python", "run_multienv_muse_spark.py") + return path if up and os.path.isfile(path) else None + + +@unittest.skipUnless(_runner(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with run_multienv_muse_spark.py") +class Runner(unittest.TestCase): + def test_every_flag_passed_is_one_the_runners_parser_has(self): + with open(_runner(), encoding="utf-8") as f: + tree = ast.parse(f.read()) + known = {a.value for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "attr", "") == "add_argument" + for a in n.args if isinstance(a, ast.Constant) and str(a.value).startswith("--")} + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + agents = json.load(f) + passed = {w for w in agents["common"] + agents["agents"]["muse_spark"]["argv"] if w.startswith("--")} + self.assertEqual(passed - known, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_nop.py b/app/tests/test_osworld2_nop.py new file mode 100644 index 00000000..734e0e0d --- /dev/null +++ b/app/tests/test_osworld2_nop.py @@ -0,0 +1,251 @@ +"""OSWorld2's null control: two setup attempts in nop.py, and a failed setup the score's error, never a 0. +A fake upstream and fake nop.py scripts stand in.""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +from nullius.standard import definitions, run + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +WORDS = "the task's own words" +FAILED = {"error": "EnvironmentSetupError", "cause": "TimeoutError", "attempts": 2} +TASK_LOADER = """def find_task_class_path(*a): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, **kw): + return {"instruction": "", "proxy": False} +""" +DESKTOP_ENV = """import json, os +MAX_RETRIES = 5 +class EnvironmentSetupError(RuntimeError): + pass +class DesktopEnv: + def __init__(self, **kw): + pass + def close(self): + out = os.environ["FAKE_OUT"] + with open(os.path.join(out, "closed"), "w") as f: + json.dump(os.path.exists(os.path.join(out, "setup_failed.json")), f) +""" +LIB_RUN_SINGLE = """import os +from desktop_env import desktop_env +def run_single_example(agent, env, example, max_steps, instruction, args, out, scores): + with open(os.path.join(out, "attempts"), "w") as f: + f.write(str(desktop_env.MAX_RETRIES)) + how = os.environ["FAKE_RUN"] + if how == "setup": + try: + raise TimeoutError(%(words)r) + except TimeoutError as e: + raise desktop_env.EnvironmentSetupError(%(words)r) from e + if how == "other": + raise ValueError(%(words)r) + with open(os.path.join(out, "result.txt"), "w") as f: + f.write("0.0") +""" % {"words": WORDS} +FAKE_NOP = """import json, os, sys, time +print("Started container with ports", flush=True) +print("[2026-11-07 01:02:03,456 ERROR desktopenv.setup] guest said: %(words)s", flush=True) +print("[2026-11-07 01:02:04,567 ERROR desktopenv.env] Environment setup failed with exception, retrying (1/2): " + "%(words)s", flush=True) +out = sys.argv[-1] +if os.environ.get("FAKE_FAILED"): + with open(os.path.join(out, "setup_failed.json"), "w") as f: + json.dump(%(failed)r, f) +with open(os.path.join(out, "result.txt"), "w") as f: + f.write("0.0") +time.sleep(float(os.environ.get("FAKE_HANG", "0"))) +sys.exit(int(os.environ.get("FAKE_EXIT", "0"))) +""" % {"failed": FAILED, "words": WORDS} + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_nop", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class ScoreLimit(unittest.TestCase): + def test_the_score_step_waits_an_hour(self): + b = definitions.get("benchmark", "osworld2") + self.assertEqual(b["limits"]["score_seconds"], 3600) + self.assertFalse([p for p in definitions.problems(b, "benchmark") if "limits" in p]) + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.score_seconds, 3600) + + +class ScoringPrograms(unittest.TestCase): + def test_the_control_and_the_original_refuse_without_them(self): + b = definitions.get("benchmark", "osworld2") + for setup in ("yours", "original"): + with self.assertRaisesRegex(run.Refused, r"scoring_programs \(tesseract not on PATH\)"): + run.resolve_bench_options(b, "test_v2", {}, setup, lambda p: None if p == "tesseract" else "/bin/" + p) + self.assertIn("scoring_programs", run.resolve_bench_options(b, "test_v2", {}, setup, lambda p: "/bin/" + p)[1]["host"]) + self.assertEqual(b["host"]["scoring_programs"]["check"], ["identify", "convert", "ffmpeg", "ffprobe", "tesseract"]) + + +class TwoAttempts(unittest.TestCase): + """nop.py itself, on a fake upstream.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + up = os.path.join(self.tmp, "up") + os.makedirs(os.path.join(up, "desktop_env")) + for name, text in (("task_loader.py", TASK_LOADER), ("lib_run_single.py", LIB_RUN_SINGLE), + (os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV)): + with open(os.path.join(up, name), "w") as f: + f.write(text) + self.up = up + self.out = os.path.join(self.tmp, "out") + os.makedirs(self.out) + + def nop(self, how): + env = dict(os.environ, PYTHONPATH=self.up, FAKE_OUT=self.out, FAKE_RUN=how) + return subprocess.run([sys.executable, os.path.join(HERE, "nop.py"), "vm", "pw", "examples", "tasks", "001", + self.out], env=env, capture_output=True, text=True, timeout=60) + + def read(self, name): + with open(os.path.join(self.out, name)) as f: + return f.read() + + def test_a_failed_setup_is_said_before_the_close_with_no_words_of_the_task(self): + done = self.nop("setup") + self.assertEqual(done.returncode, 1) + self.assertEqual(json.loads(self.read("setup_failed.json")), FAILED) + self.assertNotIn(WORDS, self.read("setup_failed.json")) + self.assertEqual(self.read("attempts"), "2") + self.assertEqual(json.loads(self.read("closed")), True) + self.assertFalse(os.path.exists(os.path.join(self.out, "result.txt"))) + self.assertIn("the null control's setup of tasks__001 failed: EnvironmentSetupError", done.stderr) + + def test_another_failure_is_not_the_setups(self): + self.assertEqual(self.nop("other").returncode, 1) + self.assertFalse(os.path.exists(os.path.join(self.out, "setup_failed.json"))) + self.assertEqual(json.loads(self.read("closed")), False) + + def test_a_run_that_sets_up_scores(self): + self.assertEqual(self.nop("ok").returncode, 0) + self.assertEqual(self.read("attempts"), "2") + self.assertEqual(self.read("result.txt"), "0.0") + self.assertFalse(os.path.exists(os.path.join(self.out, "setup_failed.json"))) + + def test_the_driver_reads_the_file_nop_writes(self): + driver = load_driver(self.tmp) + spec = importlib.util.spec_from_file_location("osworld2_nop_constants", os.path.join(HERE, "nop.py")) + with open(spec.origin) as f: + self.assertIn(f'SETUP_FAILED = "{driver.SETUP_FAILED}"', f.read()) + + +class EndsEarly(unittest.TestCase): + """The driver's score of the null control, with a fake nop.py whose log quotes the task.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + self.driver = load_driver(self.tmp) + self.driver.START_LOCK = os.path.join(self.tmp, "start.lck") + self.driver.CLOSE_SECONDS = 1 + fake = os.path.join(self.tmp, "nop.py") + with open(fake, "w") as f: + f.write(FAKE_NOP) + self.reaped, started = [], [] + + def popen(*a, **kw): + started.append(subprocess.Popen(*a, **kw)) + return started[-1] + + def reaper(pgid, *_): + self.reaped.append(time.time()) + try: + os.killpg(pgid, signal.SIGKILL) + except OSError: + pass + + def end(): + for p in started: + if p.poll() is None: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + + self.addCleanup(end) + self.driver.subprocess = types.SimpleNamespace(**{k: getattr(subprocess, k) for k in dir(subprocess) + if not k.startswith("__") and k != "Popen"}, Popen=popen) + + for name, value in (("NOP", fake), ("_reaper", reaper), ("_remove_vms", lambda *a: None), + ("_noting", lambda *a: None), ("_vm", lambda *a: os.path.join(self.unit, "vm.qcow2")), + ("_task", lambda *_: {"instruction": "", "proxy": False}), ("_assets", lambda *a: "assets")): + p = mock.patch.object(self.driver, name, value) + p.start() + self.addCleanup(p.stop) + + def nop(self, **env): + with mock.patch.dict(os.environ, **env): + return self.driver.score("tasks__001", self.unit) + + def test_a_failed_setup_is_the_scores_error_not_a_0(self): + with self.assertRaises(RuntimeError) as raised: + self.nop(FAKE_FAILED="1") + said = str(raised.exception) + self.assertIn("the environment's setup failed (EnvironmentSetupError from TimeoutError, after 2 attempts), " + "and the task was not scored", said) + self.assertIsInstance(raised.exception, self.driver.EnvironmentSetupError) + self.assertNotIn(WORDS, said) + self.assertEqual(len(self.reaped), 1) + + def test_a_process_that_does_not_end_after_its_failure_is_handed_to_the_reaper(self): + t0 = time.time() + with self.assertRaises(RuntimeError) as raised: + self.nop(FAKE_FAILED="1", FAKE_HANG="30") + self.assertLess(time.time() - t0, 10) + self.assertIn("did not end 1 s after its failure, and was stopped", str(raised.exception)) + self.assertIn("EnvironmentSetupError", str(raised.exception)) + self.assertNotIn(WORDS, str(raised.exception)) + self.assertTrue(self.reaped) + + def test_a_failure_said_by_an_earlier_run_is_not_this_ones(self): + where = os.path.join(self.unit, "results", "nop", "tasks", "001") + os.makedirs(where) + with open(os.path.join(where, "setup_failed.json"), "w") as f: + json.dump(FAILED, f) + self.assertEqual(self.nop()["reward"], 0.0) + + def test_a_setup_that_succeeds_scores(self): + self.assertEqual(self.nop()["reward"], 0.0) + self.assertEqual(len(self.reaped), 1) + + def test_the_original_setup_is_never_ended_by_the_null_controls_file(self): + given = [] + + def run(argv, cwd, env, log, what, **kw): + given.append(kw) + raise RuntimeError("stop here") + + with mock.patch.object(self.driver, "_run", run), \ + mock.patch.object(self.driver, "_command", lambda *a: ({"script": "run.py", "env": {}}, os.path.join( + self.unit, "meta.json"), ["run.py"], {})), \ + self.assertRaises(RuntimeError): + self.driver.answer("tasks__001", self.unit, "original", "MiniMax-M3", "http://127.0.0.1:9/v1", "k") + self.assertEqual([{k: v for k, v in kw.items() if k != "until"} for kw in given], [{"stagger": True}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_proxy.py b/app/tests/test_osworld2_proxy.py new file mode 100644 index 00000000..9ab38aeb --- /dev/null +++ b/app/tests/test_osworld2_proxy.py @@ -0,0 +1,725 @@ +"""OSWorld2's proxy for the tasks that ask for one: tinyproxy in a container of the unit's own, removed however +the driver ends. A fake docker on PATH and a fake task_loader stand in.""" +from __future__ import annotations + +import contextlib +import importlib.util +import json +import logging +import os +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +DOCKER = """#!/bin/sh +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + image) [ -z "$FAKE_NO_IMAGE" ] || exit 1; printf '%s\n' "$FAKE_IMAGE" ;; + run) case "$*" in *"--entrypoint apk"*) echo "tinyproxy-1.11.3-r0 x86_64 {tinyproxy} (GPL-2.0-or-later) [installed]"; exit 0 ;; esac + [ -z "$FAKE_RUN_SLEEP" ] || exec sleep "$FAKE_RUN_SLEEP" + [ -z "$FAKE_RUN_FAIL" ] || { printf 'docker: Error: no such network\n\nRun %sdocker run --help%s for more information\n' "'" "'" >&2; exit 1; } + printf '%s' "$TINYPROXY_CONF" > "$FAKE_CONF"; echo cid ;; + inspect) [ -z "$FAKE_INSPECT_SLEEP" ] || sleep "$FAKE_INSPECT_SLEEP" + if [ -s "$FAKE_INSPECT_SEQ" ]; then head -n 1 "$FAKE_INSPECT_SEQ"; sed -i 1d "$FAKE_INSPECT_SEQ" + else printf '%s\\n' "$FAKE_INSPECT"; fi ;; + ps) case "$*" in *--format*) [ -z "$FAKE_NAMES" ] || printf '%s\\n' "$FAKE_NAMES" ;; + *) case "$FAKE_PS" in hang) exec sleep 5 ;; fail) echo "Error: daemon busy" >&2; exit 1 ;; esac + printf '%s\\n' $FAKE_IDS ;; esac ;; + rm) [ -z "$FAKE_RM_FAIL" ] || { echo "Error: busy" >&2; exit 1; } ;; + build) cat > "$FAKE_BUILD" ;; +esac +""" +LOADER = """import json, os +def find_task_class_path(eid, base, domain, version): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, task_id, **kw): + return json.loads(os.environ["FAKE_TASK"]) +""" + + +def load(name, path, up=None): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up or HERE): + spec.loader.exec_module(module) + return module + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class Proxy(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + self.calls = os.path.join(self.tmp, "calls") + self.conf = os.path.join(self.tmp, "conf") + self.listener = socket.socket() + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen() + self.addCleanup(self.listener.close) + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_CONF=self.conf, FAKE_INSPECT="running\t0\t127.0.0.1", FAKE_NAMES="", FAKE_IDS="", + FAKE_BUILD=os.path.join(self.tmp, "built"), + FAKE_INSPECT_SEQ=os.path.join(self.tmp, "inspect"), + FAKE_IMAGE='sha256:fake\t{"Labels":{"nullius.osworld2.proxy.package":"tinyproxy-1.11.3-r0"}}') + env.start() + self.addCleanup(env.stop) + for name in ("FAKE_NO_IMAGE", "FAKE_RUN_FAIL", "FAKE_RUN_SLEEP", "FAKE_RM_FAIL", "FAKE_PS", "FAKE_INSPECT_SLEEP", + "TINYPROXY_CONF"): + os.environ.pop(name, None) + self.proxy = load("osworld2_proxy", os.path.join(HERE, "proxy.py")) + self.proxy.PORT = self.listener.getsockname()[1] + self.proxy.START_SECONDS = 3 + self.later, self.watchers = [], [] + self.proxy._later = lambda dir, name=None: self.later.append(dir) + self.proxy._watcher = lambda dir, fd, name: self.watchers.append( + (dir, os.path.exists(self.pool()), [ln for ln in self.lines() if ln.startswith("run ")])) + + def lines(self): + try: + with open(self.calls) as f: + return f.read().splitlines() + except OSError: + return [] + + def ran(self): + return [ln for ln in self.lines() if ln.startswith("run ")][-1] + + def pool(self): + return os.path.join(self.unit, self.proxy.FILE) + + def held_open(self): + fd = os.open(self.pool(), os.O_RDONLY) + self.addCleanup(os.close, fd) + return fd + + def log(self): + try: + with open(os.path.join(self.unit, self.proxy.LOG)) as f: + return f.read().splitlines() + except OSError: + return [] + + def test_a_proxy_is_on_the_vms_network_unpublished_read_only_and_marked_with_its_unit(self): + path = self.proxy.start(self.unit) + argv = self.ran().split() + for flag in (["--network", "nullius-osworld2"], ["--read-only"], ["--cap-drop", "ALL"], ["--security-opt", "no-new-privileges"], + ["--log-driver", "local", "--log-opt", "max-size=1m", "--log-opt", "max-file=1", + "--log-opt", "compress=false"], + ["--label", f"{self.proxy.UNIT}={self.proxy._mark(self.unit)}"], + ["-e", self.proxy.CONF, self.proxy.IMAGE]): + self.assertIn(" ".join(flag), " ".join(argv)) + self.assertTrue(any(a.startswith("/tmp:") for a in argv)) + self.assertFalse({"-p", "--publish", "-P", "--publish-all", "--privileged", "-v", "--volume"} & set(argv)) + self.assertNotIn("--device-write-bps", argv) + self.assertTrue(argv[argv.index("--name") + 1].startswith(f"{self.proxy.NAME}{os.getuid()}-{os.getpid()}-")) + self.assertEqual(path, self.pool()) + + def test_the_network_is_the_vms_own_and_the_address_is_read_on_it(self): + site = load("osworld2_startup_proxy", os.path.join(HERE, "startup", "sitecustomize.py")) + self.assertEqual(self.proxy.NETWORK, site.NETWORK) + self.proxy.start(self.unit) + inspect = [ln for ln in self.lines() if ln.startswith("inspect ")][-1] + self.assertIn('{{with index .NetworkSettings.Networks "nullius-osworld2"}}{{.IPAddress}}{{end}}', inspect) + self.assertNotIn("bridge", inspect) + + def test_the_pool_file_names_the_container_and_the_units_password_and_only_its_owner_reads_it(self): + self.proxy.start(self.unit) + self.assertEqual(stat.S_IMODE(os.stat(self.pool()).st_mode), 0o600) + with open(self.pool()) as f: + entry, = json.load(f) + self.assertEqual({k: entry[k] for k in ("host", "port", "username", "protocol")}, + {"host": "127.0.0.1", "port": self.proxy.PORT, "username": self.proxy.USER, "protocol": "http"}) + self.assertRegex(entry["password"], r"^[0-9a-f]{32}$") + with open(self.conf) as f: + conf = f.read() + self.assertIn(f"BasicAuth {self.proxy.USER} {entry['password']}\n", conf) + self.assertNotIn(entry["password"], "\n".join(self.lines())) + self.proxy.remove(self.unit) + self.proxy.start(self.unit) + with open(self.pool()) as f: + self.assertNotEqual(json.load(f)[0]["password"], entry["password"]) + + def test_no_log_record_of_a_process_the_driver_starts_shows_the_units_password(self): + self.proxy.start(self.unit) + with open(self.pool()) as f: + word = json.load(f)[0]["password"] + out = os.path.join(self.unit, "file.log") + code = ("import logging, sys\n" + "a = logging.getLogger('desktopenv'); a.setLevel(logging.INFO)\n" + "a.addHandler(logging.StreamHandler(sys.stdout)); a.addHandler(logging.FileHandler(sys.argv[2]))\n" + "a.handlers[0].setFormatter(logging.Formatter('[%(levelname)s] %(message)s'))\n" + "a.info('Command executed successfully: %s', 'Upstream http nullius:' + sys.argv[1] + '@h:1')\n" + "a.info(f'export http_proxy=http://nullius:{sys.argv[1]}@h:1')\n" + "try:\n raise RuntimeError(sys.argv[1])\nexcept RuntimeError:\n a.exception('failed')\n") + env = dict(os.environ, PYTHONPATH=os.path.join(HERE, "startup"), PROXY_CONFIG_FILE=self.pool()) + said = subprocess.run([sys.executable, "-c", code, word, out], env=env, capture_output=True, text=True) + with open(out) as f: + written = said.stdout + said.stderr + f.read() + self.assertEqual(said.returncode, 0, said.stderr) + self.assertNotIn(word, written) + self.assertEqual(written.count(""), 6) + env.pop("PROXY_CONFIG_FILE") + said = subprocess.run([sys.executable, "-c", code, word, out], env=env, capture_output=True, text=True) + self.assertEqual(said.stdout.count(word), 3) + + def test_a_pool_file_without_a_password_hides_nothing_and_a_second_look_wraps_once(self): + site = load("osworld2_startup_hide", os.path.join(HERE, "startup", "sitecustomize.py")) + self.addCleanup(setattr, logging.Formatter, "format", logging.Formatter.format) + self.addCleanup(setattr, logging, "raiseExceptions", logging.raiseExceptions) + plain, path = logging.Formatter.format, os.path.join(self.unit, "pool.json") + for text in ("{", "{}", '[{"password": 1}]', '[{"password": ""}]', "[1]"): + with open(path, "w") as f: + f.write(text) + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + self.assertIs(logging.Formatter.format, plain) + site.hide_proxy_password({"PROXY_CONFIG_FILE": os.path.join(self.unit, "none.json")}) + site.hide_proxy_password({}) + self.assertIs(logging.Formatter.format, plain) + with open(path, "w") as f: + json.dump([{"password": "p" * 32}], f) + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + hidden = logging.Formatter.format + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + self.assertIs(logging.Formatter.format, hidden) + self.assertFalse(logging.raiseExceptions) + record = logging.LogRecord("a", logging.INFO, "", 0, "x %s", ("p" * 32,), None) + self.assertEqual(logging.Formatter().format(record), "x ") + + def test_tinyproxy_logs_warnings_only_to_its_own_output_and_goes_out_directly(self): + lines = self.proxy.config("t" * 32).splitlines() + self.assertIn("LogLevel Warning", lines) + self.assertIn(f"Port {self.proxy.PORT}", lines) + self.assertFalse([ln for ln in lines if ln.split()[0] in ("Upstream", "LogFile", "Syslog", "Listen")]) + + def test_connect_goes_to_the_webs_ports_only_and_clients_with_the_main_thread_stay_below_its_processes(self): + lines = self.proxy.config("t" * 32).splitlines() + self.assertEqual(sorted(ln for ln in lines if ln.startswith("ConnectPort ")), ["ConnectPort 443", "ConnectPort 80"]) + self.assertIn(f"MaxClients {self.proxy.CLIENTS}", lines) + self.proxy.start(self.unit) + argv = self.ran().split() + self.assertEqual(int(argv[argv.index("--pids-limit") + 1]), self.proxy.PIDS) + self.assertLess(self.proxy.CLIENTS + 1, self.proxy.PIDS, + "a thread per client and tinyproxy's main thread all count against the container's pids limit") + + def test_the_image_is_alpine_pinned_by_digest_named_by_its_text_and_built_only_by_hand(self): + self.assertRegex(self.proxy.DOCKERFILE.splitlines()[0], r"^FROM alpine:[\d.]+@sha256:[0-9a-f]{64}$") + self.assertIn("tinyproxy -d -c /tmp/tinyproxy.conf", self.proxy.DOCKERFILE) + self.assertIn("USER 65534", self.proxy.DOCKERFILE) + os.environ["FAKE_NO_IMAGE"] = "1" + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("python3 proxy.py --build", str(raised.exception)) + self.assertFalse([ln for ln in self.lines() if ln.startswith(("run ", "build "))]) + self.assertFalse(os.path.exists(self.pool())) + built = subprocess.run([sys.executable, os.path.join(HERE, "proxy.py"), "--build"], capture_output=True, text=True) + self.assertEqual(built.returncode, 0, built.stderr) + builds = [ln for ln in self.lines() if ln.startswith("build ")] + self.assertEqual(builds, [f"build -t {self.proxy.IMAGE} -", + f"build --label {self.proxy.PACKAGE}=tinyproxy-1.11.3-r0 -t {self.proxy.IMAGE} -"]) + self.assertIn(f"run --rm --network none --entrypoint apk {self.proxy.IMAGE} list --installed tinyproxy", self.lines()) + self.assertEqual(built.stdout.split(), [self.proxy.IMAGE, "tinyproxy-1.11.3-r0"]) + with open(os.environ["FAKE_BUILD"]) as f: + self.assertEqual(f.read(), self.proxy.DOCKERFILE) + + def test_each_start_writes_the_images_id_and_tinyproxy_package_into_the_units_log(self): + self.proxy.start(self.unit) + self.assertRegex(self.log()[-1], r"\] proxy image sha256:fake, tinyproxy-1\.11\.3-r0$") + proxy = load("osworld2_proxy_unlabelled", os.path.join(HERE, "proxy.py")) + proxy.PORT, proxy._watcher = self.proxy.PORT, self.proxy._watcher + os.environ["FAKE_IMAGE"] = 'sha256:old\t{"User":"65534:65534"}' + proxy.start(self.unit) + self.assertIn("proxy image sha256:old, its tinyproxy package not labelled", self.log()[-1]) + + def test_a_process_looks_at_the_image_and_its_users_orphans_at_its_first_start_only(self): + for _ in range(2): + self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("image inspect")]), 1) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("ps ") and "--format" in ln]), 1) + self.assertEqual(len([ln for ln in self.log() if "proxy image" in ln]), 2) + + def test_the_proxy_and_its_units_vm_write_no_more_together_than_the_options_give_the_vm(self): + site = load("osworld2_startup_proxy", os.path.join(HERE, "startup", "sitecustomize.py")) + options = {"NULLIUS_OPTION_WRITE_MBPS": "8", "NULLIUS_OPTION_WRITE_IOPS": "80"} + with self.proxy.held(self.unit, True, lambda: "/dev/sda") as routed: + argv = self.ran().split() + vm = {key: rate for key, rate, _, _ in site.wanted(dict(options, **routed))} + own = {key: int(argv[argv.index(flag) + 1].rpartition(":")[2]) + for key, flag in (("device_write_bps", "--device-write-bps"), ("device_write_iops", "--device-write-iops"))} + self.assertEqual(own, {"device_write_bps": 52428, "device_write_iops": 1}) + self.assertEqual({k: vm[k] + own[k] for k in vm}, {"device_write_bps": 8 * 1024 * 1024, "device_write_iops": 80}) + self.assertEqual({key: rate for key, rate, _, _ in site.wanted(options)}, + {"device_write_bps": 8 * 1024 * 1024, "device_write_iops": 80}) + with self.proxy.held(self.unit, True, lambda: None) as routed: + self.assertEqual(routed, {"PROXY_CONFIG_FILE": self.pool()}) + self.assertNotIn("--device-write-bps", self.ran()) + for iops, taken in (("1", "1"), ("80", "0x")): + with self.subTest(iops=iops, taken=taken), self.assertRaises(site.WriteLimitError): + site.wanted(dict(options, NULLIUS_OPTION_WRITE_IOPS=iops, OSWORLD2_PROXY_WRITE_IOPS=taken)) + + def test_the_machines_limits_together_stay_within_its_budget_with_the_sites_and_a_proxy_per_unit(self): + site = load("osworld2_startup_budget", os.path.join(HERE, "startup", "sitecustomize.py")) + web = load("osworld2_websites_budget", os.path.join(HERE, "websites.py")) + mb = 1024 * 1024 + for (budget, units, vm, stack) in (((120, 1200), 7, (16, 160), (8, 80)), ((30, 300), 1, (22, 220), (8, 80)), + ((30, 300), 2, (11, 110), (8, 80))): + options = {"NULLIUS_OPTION_WRITE_MBPS": str(vm[0]), "NULLIUS_OPTION_WRITE_IOPS": str(vm[1])} + tiers = web.allocation(*stack) + sites = web.limits({f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() + for i, k in enumerate(("BPS", "IOPS"))})[1] + for routed in ({}, self.proxy.TAKEN): + with self.subTest(budget=budget, units=units, proxy=bool(routed)): + own = {key: rate for key, rate, _, _ in site.wanted(dict(options, **routed))} + each = (own["device_write_bps"] + (self.proxy.WRITE_BPS if routed else 0), + own["device_write_iops"] + (self.proxy.WRITE_IOPS if routed else 0)) + total = (units * each[0] + sites[0], units * each[1] + sites[1]) + self.assertLessEqual(total[0], budget[0] * mb) + self.assertLessEqual(total[1], budget[1]) + + def test_a_removal_that_fails_on_the_way_out_is_a_line_in_the_units_log(self): + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + with self.proxy.held(self.unit, True): + pass + lines = [ln for ln in self.log() if "proxy image" not in ln] + self.assertEqual(len(lines), 1) + self.assertRegex(lines[0], r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\] the unit's proxy was not removed: .*busy") + self.assertTrue(os.path.exists(self.pool())) + + def test_a_proxy_that_stops_or_never_answers_is_an_error_and_is_removed(self): + os.environ["FAKE_INSPECT"] = "exited\t1\t" + os.environ["FAKE_IDS"] = "cid" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertIn("exit code 1", str(raised.exception)) + self.assertFalse([ln for ln in self.lines() if ln.startswith("logs ")]) + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + os.environ["FAKE_INSPECT"] = "running\t0\t127.0.0.1" + self.proxy.PORT = free_port() + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("did not answer", str(raised.exception)) + + def test_a_refused_container_is_an_error_with_dockers_words(self): + os.environ["FAKE_RUN_FAIL"] = "1" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertTrue(str(raised.exception).endswith("docker: Error: no such network"), str(raised.exception)) + self.assertFalse(os.path.exists(self.pool())) + + def test_held_gives_the_pool_while_the_body_runs_and_removes_the_proxy_however_it_ends(self): + os.environ["FAKE_IDS"] = "cid" + with self.proxy.held(self.unit, True, lambda: None) as routed: + self.assertEqual(routed, {"PROXY_CONFIG_FILE": self.pool()}) + self.assertTrue(os.path.exists(self.pool())) + self.assertFalse(os.path.exists(self.pool())) + with self.assertRaises(KeyError), self.proxy.held(self.unit, True): + raise KeyError("the run") + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual([ln for ln in self.lines() if ln.startswith("rm ")], ["rm -f cid", "rm -f cid"]) + + def test_a_task_without_a_proxy_costs_no_docker_call_nor_the_disk(self): + def disk(): + raise AssertionError("asked") + + with self.proxy.held(self.unit, False, disk) as routed: + self.assertEqual(routed, {}) + self.proxy.remove(self.unit) + self.assertEqual(self.lines(), []) + + def test_a_proxy_docker_did_not_remove_keeps_the_pool_file_for_the_next_sweep(self): + self.proxy.start(self.unit) + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + with self.assertRaises(RuntimeError): + self.proxy.remove(self.unit) + self.assertTrue(os.path.exists(self.pool())) + del os.environ["FAKE_RM_FAIL"] + self.proxy.remove(self.unit) + self.assertFalse(os.path.exists(self.pool())) + search = [ln for ln in self.lines() if ln.startswith("ps -a -q")][-1] + self.assertIn(f"label={self.proxy.UNIT}={self.proxy._mark(self.unit)}", search) + + def test_a_start_removes_this_users_proxies_whose_driver_is_gone_and_only_those(self): + ended = subprocess.Popen([sys.executable, "-c", ""]) + ended.wait() + prefix = f"{self.proxy.NAME}{os.getuid()}-" + dead, alive = f"{prefix}{ended.pid}-1", f"{prefix}{os.getpid()}-2" + other = f"{self.proxy.NAME}{os.getuid() + 1}-{ended.pid}-3" + os.environ["FAKE_NAMES"] = "\n".join([dead, alive, other, f"{prefix}junk"]) + self.proxy.start(self.unit) + self.assertIn(f"rm -f {dead}", self.lines()) + self.assertFalse([ln for ln in self.lines() if ln.startswith("rm ") and (alive in ln or other in ln)]) + + def test_a_container_the_daemon_creates_after_a_start_it_did_not_answer_is_still_removed(self): + self.proxy.DOCKER_SECONDS = 0.5 + os.environ["FAKE_RUN_SLEEP"] = "5" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertIn("docker run got no answer in 0.5 s", str(raised.exception)) + self.assertIn("has made no container for the proxy yet; the unit's sweeper removes one it makes later", + str(raised.exception)) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("run ")]), 1) + self.assertEqual(os.path.getsize(self.pool()), 0) + self.assertEqual(self.later, [self.unit]) + del os.environ["FAKE_RUN_SLEEP"] + self.proxy.remove(self.unit) + self.assertTrue(os.path.exists(self.pool())) + os.environ["FAKE_IDS"] = "late" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, self.held_open()) + self.assertIn("rm -f late", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def sweep_with_a_clock(self, docker_answers_at=None): + clock = [0.0] + + def sleep(seconds): + clock[0] += seconds + if docker_answers_at is not None and clock[0] >= docker_answers_at: + os.environ.pop("FAKE_RM_FAIL", None) + + self.proxy.time = types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0], strftime=time.strftime) + with open(self.pool(), "w") as f: + f.write("[]") + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + self.proxy.sweep(self.unit, self.held_open(), "name") + return clock[0], [ln for ln in self.lines() if ln.startswith("rm ")] + + def test_a_sweeper_keeps_removing_while_docker_refuses_and_stops_once_it_succeeds(self): + ended, removals = self.sweep_with_a_clock(docker_answers_at=sum(self.proxy.SWEEPS) + 2 * self.proxy.RETRY_SECONDS) + self.assertFalse(os.path.exists(self.pool())) + self.assertFalse(os.path.exists(os.path.join(self.unit, self.proxy.LOG))) + self.assertEqual(ended, sum(self.proxy.SWEEPS) + 2 * self.proxy.RETRY_SECONDS) + self.assertEqual(len(removals), len(self.proxy.SWEEPS) + 2) + + def test_a_sweeper_whose_units_record_is_gone_waits_for_no_further_pass(self): + ended, removals = self.sweep_with_a_clock(docker_answers_at=0) + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual(ended, self.proxy.SWEEPS[0]) + self.assertEqual(len(removals), 1) + + def test_a_sweeper_docker_refuses_throughout_gives_up_in_time_with_a_line_in_the_units_log(self): + ended, removals = self.sweep_with_a_clock() + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS) + self.assertGreater(ended, self.proxy.SWEEP_SECONDS - self.proxy.RETRY_SECONDS) + self.assertGreater(len(removals), len(self.proxy.SWEEPS)) + self.assertTrue(os.path.exists(self.pool())) + lines = self.log() + self.assertEqual(len(lines), 1) + self.assertRegex(lines[0], rf"the unit's proxy was not removed in {self.proxy.SWEEP_SECONDS // 60} min: .*busy$") + + def test_a_start_docker_answers_late_goes_on_with_its_container_and_never_runs_a_second(self): + self.proxy.DOCKER_SECONDS = 0.5 + os.environ.update(FAKE_RUN_SLEEP="5", FAKE_IDS="cid") + path = self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("run ")]), 1) + name = self.ran().split()[self.ran().split().index("--name") + 1] + self.assertIn(f"ps -a -q --filter label={self.proxy.UNIT}={self.proxy._mark(self.unit)} --filter name={name}", + self.lines()) + with open(path) as f: + self.assertEqual(json.load(f)[0]["host"], "127.0.0.1") + + def sweep_with_slow_docker(self, found): + clock, calls = [0.0], [] + + def sleep(seconds): + clock[0] += seconds + + def run(argv, timeout=None, **kw): + calls.append((clock[0], timeout)) + if argv[1] == "ps" and timeout >= 119: + clock[0] += 119 + return subprocess.CompletedProcess(argv, 0, found, "") + clock[0] += timeout + raise subprocess.TimeoutExpired(argv, timeout) + + self.proxy.time = types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0], strftime=time.strftime) + self.proxy.subprocess = types.SimpleNamespace(run=run, TimeoutExpired=subprocess.TimeoutExpired, + CompletedProcess=subprocess.CompletedProcess) + open(self.pool(), "w").close() + self.proxy.sweep(self.unit, self.held_open(), "first") + for began, timeout in calls: + self.assertLessEqual(timeout, self.proxy.DOCKER_SECONDS) + self.assertLessEqual(began + timeout, self.proxy.SWEEP_SECONDS + 0.1) + return clock[0], calls + + def test_a_sweep_whose_searches_take_minutes_and_removals_never_answer_ends_in_its_time(self): + ended, calls = self.sweep_with_slow_docker("cid\n") + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS + 0.1) + self.assertGreater(len(calls), 2 * len(self.proxy.SWEEPS) - 2) + self.assertTrue(os.path.exists(self.pool())) + self.assertRegex(self.log()[-1], rf"was not removed in {self.proxy.SWEEP_SECONDS // 60} min: docker ") + + def test_a_sweep_whose_searches_take_minutes_still_removes_its_record_in_its_time(self): + ended, calls = self.sweep_with_slow_docker("") + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS) + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual(self.log(), []) + + def test_a_sweeper_leaves_a_later_start_in_the_same_folder_its_proxy_and_its_record(self): + with open(self.pool(), "w") as f: + f.write("[]") + own = self.held_open() + os.remove(self.pool()) + with open(self.pool(), "w") as f: + f.write('[{"host": "later"}]') + os.environ["FAKE_IDS"] = "later" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, own, "first") + self.assertFalse([ln for ln in self.lines() if ln.startswith(("rm ", "ps -a -q"))]) + self.assertTrue(os.path.exists(self.pool())) + with self.proxy._locked(self.unit): + self.assertFalse(self.proxy._ours(self.unit, own)) + + def test_a_sweeper_removes_its_own_starts_container_by_its_name(self): + with open(self.pool(), "w") as f: + f.write("[]") + os.environ["FAKE_IDS"] = "cid" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, self.held_open(), "first") + self.assertIn(f"ps -a -q --filter label={self.proxy.UNIT}={self.proxy._mark(self.unit)} --filter name=first", + self.lines()) + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def test_a_start_whose_run_and_search_docker_both_leave_unanswered_is_an_error_for_its_sweeper(self): + self.proxy.DOCKER_SECONDS = 0.5 + for said, words in (("hang", "docker ps got no answer in 0.5 s: is its daemon running?: whether it made the " + "proxy's container is unknown"), + ("fail", "could not say whether it made the proxy's container: Error: daemon busy")): + with self.subTest(said=said): + os.environ.update(FAKE_RUN_SLEEP="5", FAKE_PS=said) + with contextlib.suppress(FileNotFoundError): + os.remove(self.pool()) + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("docker run got no answer in 0.5 s", str(raised.exception)) + self.assertIn(words, str(raised.exception)) + self.assertTrue(str(raised.exception).endswith("the unit's sweeper removes one it makes later")) + self.assertEqual(os.path.getsize(self.pool()), 0) + + def test_a_start_with_slow_answers_ends_within_its_time_however_docker_answers(self): + self.proxy.START_SECONDS = 1.5 + os.environ["FAKE_INSPECT_SLEEP"] = "0.4" + os.environ["FAKE_INSPECT"] = "created\t0\t" + t0 = time.monotonic() + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertLess(time.monotonic() - t0, self.proxy.START_SECONDS + 0.5) + self.assertRegex(str(raised.exception), "did not answer in its start's 1.5 s|the time given for it ran out|" + "got no answer in") + + def test_a_container_created_and_not_yet_started_is_waited_for(self): + with open(os.environ["FAKE_INSPECT_SEQ"], "w") as f: + f.write("created\t0\t\ncreated\t0\t\n") + self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("inspect ")]), 3) + + def test_the_waits_are_sized_for_a_shared_daemon_that_takes_minutes_to_answer(self): + proxy = load("osworld2_proxy_waits", os.path.join(HERE, "proxy.py")) + self.assertGreaterEqual(proxy.DOCKER_SECONDS, 120) + self.assertGreater(proxy.START_SECONDS, 2 * proxy.DOCKER_SECONDS) + self.assertGreaterEqual(proxy.RETRY_SECONDS, proxy.DOCKER_SECONDS) + self.assertGreaterEqual(sum(proxy.SWEEPS), 900) + self.assertGreater(proxy.SWEEP_SECONDS, sum(proxy.SWEEPS) + proxy.RETRY_SECONDS) + + def test_a_late_container_whose_unit_holds_no_record_is_an_orphan_while_its_driver_lives(self): + prefix = f"{self.proxy.NAME}{os.getuid()}-{os.getpid()}" + kept, late = f"{prefix}-1", f"{prefix}-2" + os.makedirs(os.path.join(self.tmp, "other")) + open(os.path.join(self.tmp, "other", self.proxy.FILE), "w").close() + os.environ["FAKE_NAMES"] = f"{kept}\t{os.path.join(self.tmp, 'other')}\n{late}\t{self.unit}" + self.proxy._orphans() + self.assertIn(f"rm -f {late}", self.lines()) + self.assertFalse([ln for ln in self.lines() if ln.startswith("rm ") and kept in ln]) + + def test_a_proxy_that_started_and_was_removed_needs_no_sweeper(self): + with self.proxy.held(self.unit, True): + pass + self.assertEqual(self.later, []) + self.assertIn(f"--label {self.proxy.FOLDER}={self.unit}", self.ran()) + + def test_the_sweeper_is_a_session_of_its_own_holding_its_starts_record_and_naming_its_container(self): + proxy = load("osworld2_proxy_later", os.path.join(HERE, "proxy.py")) + open(self.pool(), "w").close() + with mock.patch.object(proxy.subprocess, "Popen") as popen: + proxy._later(self.unit, "first") + argv, kw = popen.call_args + fd, = kw["pass_fds"] + self.assertEqual(argv[0][1:], [os.path.join(HERE, "proxy.py"), "--sweep", self.unit, str(fd), "first"]) + self.assertTrue(kw["start_new_session"]) + + def test_a_start_arms_its_watcher_once_its_record_exists_and_before_docker_is_asked(self): + self.proxy.start(self.unit) + self.assertEqual(self.watchers, [(self.unit, True, [])]) + + def test_the_watcher_sweeps_the_units_proxy_once_its_driver_is_gone_and_only_then(self): + ended = subprocess.Popen([sys.executable, "-c", ""]) + ended.wait() + self.proxy.SWEEPS = (0, 0) + open(self.pool(), "w").close() + gone = self.held_open() + os.remove(self.pool()) + self.proxy.watch(self.unit, ended.pid, gone, "first") + self.assertEqual(self.lines(), []) + open(self.pool(), "w").close() + os.environ["FAKE_IDS"] = "cid" + self.proxy.watch(self.unit, ended.pid, self.held_open(), "first") + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def test_a_driver_killed_while_its_proxy_starts_leaves_the_proxy_to_its_watcher(self): + os.environ.update(FAKE_RUN_SLEEP="60", FAKE_IDS="cid") + driver = subprocess.Popen([sys.executable, "-c", "import sys; sys.path.insert(0, sys.argv[1]); import proxy; " + "proxy.start(sys.argv[2])", HERE, self.unit], start_new_session=True) + self.addCleanup(self.end_watchers) + t0 = time.time() + while not any(ln.startswith("run ") for ln in self.lines()) and time.time() - t0 < 20: + time.sleep(0.1) + self.assertTrue(os.path.exists(self.pool())) + os.killpg(driver.pid, signal.SIGKILL) + while "rm -f cid" not in self.lines() and time.time() - t0 < 40: + time.sleep(0.2) + driver.wait() + self.assertIn("rm -f cid", self.lines()) + + def end_watchers(self): + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + argv = f.read().split(b"\0") + except OSError: + continue + if b"--watch" in argv and self.unit.encode() in argv: + with contextlib.suppress(OSError): + os.kill(int(pid), signal.SIGKILL) + + def test_an_attempts_leftover_proxy_is_removed_before_a_new_one(self): + open(self.pool(), "w").close() + os.environ["FAKE_IDS"] = "old" + self.proxy.start(self.unit) + lines = self.lines() + self.assertLess(lines.index("rm -f old"), lines.index(self.ran())) + + +class Driver(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load("osworld2_driver_proxy", os.path.join(HERE, "driver.py"), self.tmp) + self.envs, self.held, self.asked = [], [], [] + + def start(dir, disk=None): + self.held.append(("start", disk)) + return os.path.join(dir, "proxy.json") + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + + for target, name, fake in ((self.driver, "_vm", lambda dir, d, e: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_disk", lambda: self.asked.append(1) or "/dev/sda"), + (self.driver.proxy, "start", start), + (self.driver.proxy, "remove", lambda dir: self.held.append(("remove", dir)))): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + + def unit(self, asks): + p = mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": asks}) + p.start() + self.addCleanup(p.stop) + + def both(self): + with self.assertRaises(RuntimeError): + self.driver.answer("tasks__037", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + with self.assertRaises(RuntimeError): + self.driver.score("tasks__037", self.tmp) + + def test_a_task_that_asks_for_a_proxy_runs_with_its_own_in_the_original_and_the_null_control(self): + self.unit(True) + self.both() + self.assertEqual([e.get("PROXY_CONFIG_FILE") for e in self.envs], [os.path.join(self.tmp, "proxy.json")] * 2) + self.assertEqual(self.held, [("start", "/dev/sda"), ("remove", self.tmp)] * 2) + + def test_a_task_that_asks_for_none_gets_no_proxy_nor_the_variable(self): + self.unit(False) + with mock.patch.dict(os.environ): + os.environ.pop("PROXY_CONFIG_FILE", None) + self.both() + self.assertEqual([e.get("PROXY_CONFIG_FILE") for e in self.envs], [None, None]) + self.assertEqual((self.held, self.asked), ([], [])) + + def test_the_vms_removal_removes_the_units_proxy(self): + with mock.patch.object(self.driver, "_mine", lambda dir: ([], [])): + self.driver._remove_vms(self.tmp) + self.assertEqual(self.held, [("remove", self.tmp)]) + + def test_a_write_limit_that_leaves_the_vm_nothing_beside_the_proxy_is_refused_before_it_starts(self): + driver = load("osworld2_driver_share", os.path.join(HERE, "driver.py"), self.tmp) + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="8", NULLIUS_OPTION_WRITE_IOPS="1"), \ + mock.patch.object(driver, "_asked", mock.Mock(side_effect=AssertionError("asked"))): + with self.assertRaises(driver._startup().WriteLimitError) as raised: + driver._disk() + self.assertIn("Disk write IOPS is 1, no more than the unit's proxy takes", str(raised.exception)) + + def test_the_proxys_disk_is_the_vms_one_only_with_a_write_limit_set(self): + driver = load("osworld2_driver_disk", os.path.join(HERE, "driver.py"), self.tmp) + site = mock.Mock() + site.root_disk.return_value = "/dev/sdb" + asked = mock.Mock(return_value=subprocess.CompletedProcess([], 0, "/var/lib/docker\n", "")) + with mock.patch.object(driver, "_startup", lambda: site), mock.patch.object(driver, "_asked", asked): + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="none", NULLIUS_OPTION_WRITE_IOPS="none"): + self.assertIsNone(driver._disk()) + asked.assert_not_called() + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="30", NULLIUS_OPTION_WRITE_IOPS="none"): + self.assertEqual(driver._disk(), "/dev/sdb") + site.root_disk.assert_called_once_with("/var/lib/docker") + + +class Flag(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + with open(os.path.join(self.tmp, "task_loader.py"), "w") as f: + f.write(LOADER) + + def loaded(self, task): + out = os.path.join(self.tmp, "out.json") + done = subprocess.run([sys.executable, os.path.join(HERE, "nop.py"), "--instruction", self.tmp, "tasks", "037", out], + env=dict(os.environ, PYTHONPATH=self.tmp, FAKE_TASK=json.dumps(task)), + capture_output=True, text=True) + self.assertEqual(done.returncode, 0, done.stderr) + with open(out) as f: + return json.load(f) + + def test_nop_says_whether_the_task_asks_for_a_proxy_as_the_upstream_reads_it(self): + self.assertEqual(self.loaded({"instruction": "x", "proxy": True}), {"instruction": "x", "proxy": True}) + self.assertEqual(self.loaded({"instruction": "x"}), {"instruction": "x", "proxy": False}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_stop.py b/app/tests/test_osworld2_stop.py new file mode 100644 index 00000000..9c984674 --- /dev/null +++ b/app/tests/test_osworld2_stop.py @@ -0,0 +1,420 @@ +"""OSWorld2's episode ends with DONE and is scored once the unit's relay holds its stop file: the agent's own call +refused for its context window, or a spend guard that asks. A fake agent and episode stand in for the upstream's, +a fake provider on loopback for the endpoint: nothing starts a VM or calls a model.""" +from __future__ import annotations + +import contextlib +import fcntl +import importlib.util +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius.standard import relay + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +EPISODE = """def run_single_example(agent, env, max_steps): + taken, done, step = [], False, 0 + while not done and step < max_steps: + response, actions = agent.predict("", {}) + if not actions: + raise RuntimeError("the simulated user was asked nothing") + for action in actions: + taken.append(action) + if action in ("DONE", "FAIL"): + done = True + break + step += 1 + env.append(taken) +""" +RUNNER = """import json, os, sys +import lib_run_single +class Agent: + def predict(self, instruction, obs): + open(os.environ["OSWORLD2_STOP"], "a").close() + return "", [] +env = [] +lib_run_single.run_single_example(Agent(), env, 5) +print(json.dumps(env)) +""" + + +def load(name, path, **env): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [os.path.dirname(path), *sys.path]), \ + mock.patch.dict(os.environ, env): + spec.loader.exec_module(module) + return module + + +class Agent: + def __init__(self, *steps): + self.steps, self.calls = list(steps), 0 + + def predict(self, instruction, obs): + self.calls += 1 + step = self.steps.pop(0) + return step() if callable(step) else step + + +class Episode(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.stop = os.path.join(self.tmp, "stop") + self.site = load("osworld2_startup_stop", os.path.join(STARTUP, "sitecustomize.py")) + self.module = types.ModuleType("lib_run_single") + exec(EPISODE, self.module.__dict__) + with mock.patch.dict(os.environ, OSWORLD2_STOP=self.stop): + self.site.stop_when_asked(self.module) + + def asked(self, *_): + open(self.stop, "a").close() + + def ran(self, agent, steps=5): + env = [] + self.module.run_single_example(agent, env, steps) + return env[0] + + def test_with_no_stop_the_agents_steps_run_as_they_are(self): + agent = Agent(*[("", ["click"])] * 3) + self.assertEqual(self.ran(agent, 3), ["click"] * 3) + self.assertEqual(agent.calls, 3) + + def test_an_overflow_during_the_call_ends_the_episode_with_done(self): + agent = Agent(("", ["click"]), lambda: (self.asked(), ("", []))[1], ("", ["click"])) + self.assertEqual(self.ran(agent), ["click", "DONE"]) + self.assertEqual(agent.calls, 2) + self.assertTrue(os.path.exists(self.stop + ".taken")) + + def test_the_action_already_bought_is_taken_before_done(self): + agent = Agent(lambda: (self.asked(), ("", ["type"]))[1]) + self.assertEqual(self.ran(agent), ["type", "DONE"]) + os.remove(self.stop) + agent = Agent(lambda: (self.asked(), ("", ["FAIL"]))[1]) + self.assertEqual(self.ran(agent), ["FAIL"]) + os.remove(self.stop) + agent = Agent(lambda: (self.asked(), ({"text": ""}, [{"action_type": "CLICK"}]))[1]) + self.assertEqual(self.ran(agent), [{"action_type": "CLICK"}, "DONE"]) + + def test_a_call_that_raised_after_the_stop_ends_with_done(self): + def raised(): + self.asked() + raise RuntimeError("the endpoint failed too many times") + self.assertEqual(self.ran(Agent(raised)), ["DONE"]) + self.assertTrue(os.path.exists(self.stop + ".taken")) + + def test_a_call_that_raised_with_no_stop_is_the_upstreams(self): + def raised(): + raise RuntimeError("boom") + with self.assertRaises(RuntimeError): + self.ran(Agent(raised)) + + def test_only_a_stop_that_ended_a_step_is_recorded_with_its_reasons(self): + taken = self.stop + ".taken" + self.assertEqual(self.ran(Agent(("", ["DONE"]))), ["DONE"]) + with open(self.stop, "w") as f: + f.write("batch cap: spent\n") + self.assertFalse(os.path.exists(taken)) + self.assertEqual(self.ran(Agent(("", ["click"]))), ["DONE"]) + with open(self.stop, "a") as f: + f.write("turn limit\n") + self.assertEqual(self.ran(Agent(("", ["click"]))), ["DONE"]) + with open(taken) as f: + self.assertEqual(f.read(), "batch cap: spent\n") + + def test_a_stop_before_the_step_asks_the_model_nothing(self): + self.asked() + agent = Agent(("", ["click"])) + self.assertEqual(self.ran(agent), ["DONE"]) + self.assertEqual(agent.calls, 0) + + def test_wrapped_once_and_not_at_all_without_a_stop_path(self): + wrapped = self.module.run_single_example + with mock.patch.dict(os.environ, OSWORLD2_STOP=self.stop): + self.site.stop_when_asked(self.module) + self.assertIs(self.module.run_single_example, wrapped) + agent = Agent(("", ["click"]), ("", ["click"])) + self.ran(agent, 1) + predict = agent.predict + self.ran(agent, 1) + self.assertIs(agent.predict, predict) + bare = types.ModuleType("lib_run_single") + exec(EPISODE, bare.__dict__) + run = bare.run_single_example + with mock.patch.dict(os.environ, {"OSWORLD2_STOP": ""}): + self.site.stop_when_asked(bare) + self.assertIs(bare.run_single_example, run) + + +class Imported(unittest.TestCase): + def test_the_runners_import_is_wrapped_as_the_driver_starts_it(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + for name, text in (("lib_run_single.py", EPISODE), ("runner.py", RUNNER)): + with open(os.path.join(tmp, name), "w") as f: + f.write(text) + env = dict(os.environ, PYTHONPATH=os.pathsep.join([STARTUP, tmp]), OSWORLD2_STOP=os.path.join(tmp, "stop"), + NULLIUS_OPTION_VM_PROVIDER="docker") + out = subprocess.run([sys.executable, os.path.join(tmp, "runner.py")], env=env, capture_output=True, text=True, + timeout=60) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertEqual(json.loads(out.stdout), [["DONE"]]) + + +class Driver(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.dir = os.path.join(trial, "workspace") + self.stop = os.path.join(trial, ".work", "relay", "stop") + os.makedirs(os.path.dirname(self.stop)) + os.makedirs(self.dir) + self.driver = load("osworld2_driver_stop", os.path.join(HERE, "driver.py"), NULLIUS_UPSTREAM=self.dir) + + def test_the_runner_is_told_the_units_stop_file(self): + envs = [] + + def started(argv, cwd, env, *a, **kw): + envs.append(env) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", started), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}), \ + mock.patch.object(self.driver, "_assets", lambda *a: "assets"), self.assertRaises(KeyboardInterrupt): + self.driver.answer("tasks__001", self.dir, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertEqual(envs[0]["OSWORLD2_STOP"], self.stop) + + def test_a_stopped_episodes_score_says_why_and_keeps_its_result(self): + where = os.path.join(self.dir, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + os.makedirs(where) + with open(os.path.join(where, "result.txt"), "w") as f: + f.write("0.5") + scored = self.driver.score("tasks__001", self.dir, setup="original") + self.assertEqual(scored["reward"], 0.5) + self.assertNotIn("stopped", scored) + with open(self.stop, "w") as f: + f.write("turn limit\n") + self.assertNotIn("stopped", self.driver.score("tasks__001", self.dir, setup="original")) + with open(self.stop + ".taken", "w") as f: + f.write("context window\ncontext window\nbatch cap: spent $1\n") + relayed = os.path.dirname(self.stop) + for name, calls in (("u0001", []), ("u0001", [{"status": 200, "side": True}]), ("u0001-judge", [{"status": 200}])): + with open(os.path.join(relayed, name + ".jsonl"), "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls)) + with self.subTest(name=name, calls=calls), self.assertRaises(self.driver.CancelledError) as raised: + self.driver.score("tasks__001", self.dir, setup="original") + self.assertIn("stopped (context window) before the agent's first step", str(raised.exception)) + with open(os.path.join(relayed, "u0001.jsonl"), "a") as f: + f.write(json.dumps({"status": 200}) + '\n{"stat\n') + scored = self.driver.score("tasks__001", self.dir, setup="original") + self.assertEqual((scored["reward"], scored["stopped"], scored["stopped_by"]), + (0.5, "context window", ["context window", "money"])) + with open(self.stop + ".taken", "w") as f: + f.write("forecast $2 >= $1\nturn limit\nsomething else\n") + self.assertEqual(self.driver.score("tasks__001", self.dir, setup="original")["stopped_by"], + ["money", "time", "other"]) + open(self.stop + ".taken", "w").close() + with open(self.stop, "w") as f: + f.write("turn limit\n") + self.assertEqual(self.driver.score("tasks__001", self.dir, setup="original")["stopped_by"], ["time"]) + with open(os.path.join(relayed, "u0001.jsonl"), "w") as f: + f.write(json.dumps({"status": 200, "side": True, "said": "a\u2028b"}, ensure_ascii=False) + "\n") + with open(self.stop + ".taken", "w") as f: + f.write("turn limit\n") + with self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.dir, setup="original") + self.assertIn("stopped (turn limit) before the agent's first step", str(raised.exception)) + self.assertEqual(len(self.driver._traj(relayed, "u0001.jsonl")), 1) + + def test_the_episode_is_stopped_at_its_own_limit_counted_from_the_runners_start(self): + argv = [sys.executable, "-c", "import os, sys, time\nt = time.time()\n" + "while not os.path.exists(sys.argv[1]) and time.time() - t < 20: time.sleep(0.05)", self.stop] + log = os.path.join(self.dir, "run.log") + with mock.patch.object(self.driver, "_reaper", lambda *a, **kw: None): + for episode, until in ((1, time.time() + 600), (600, time.time() + 1), (1, None)): + if until is None: + shutil.rmtree(os.path.dirname(self.stop)) + until = time.time() + 600 + with self.subTest(episode=episode), mock.patch.object(self.driver, "EPISODE_SECONDS", episode): + self.driver._run(argv, self.dir, dict(os.environ), log, "the runner", until=until) + with open(self.stop) as f: + self.assertEqual(f.read(), "turn limit\n") + os.remove(self.stop) + + def test_the_answers_limit_leaves_the_turn_time_to_evaluate(self): + kws = [] + + def started(*a, **kw): + kws.append(kw) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", started), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}), \ + mock.patch.object(self.driver, "_assets", lambda *a: "assets"), self.assertRaises(KeyboardInterrupt): + self.driver.answer("tasks__001", self.dir, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + left = kws[0]["until"] - time.time() + self.assertAlmostEqual(left, self.driver._turn_seconds() - self.driver.EVALUATE_SECONDS - self.driver.CLOSE_SECONDS, + delta=5) + self.assertGreater(left, self.driver.EPISODE_SECONDS) + + def test_a_runner_alive_an_evaluations_time_after_its_turn_limit_is_reaped_unscored(self): + argv = [sys.executable, "-c", "import time; time.sleep(15)"] + log, reaped = os.path.join(self.dir, "run.log"), [] + + def reaper(pgid, *a, **kw): + reaped.append(pgid) + with contextlib.suppress(OSError): + os.killpg(pgid, signal.SIGKILL) + + with mock.patch.object(self.driver, "_reaper", reaper): + for calls, kind in (([{"status": 200, "side": True}], self.driver.EnvironmentSetupError), + ([{"status": 200}], self.driver.VerifierError)): + with open(os.path.join(os.path.dirname(self.stop), "u0001.jsonl"), "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls)) + t0 = time.time() + with self.subTest(calls=calls), self.assertRaises(kind) as raised: + self.driver._run(argv, self.dir, dict(os.environ), log, "the runner", until=time.time() + 0.5, grace=1) + self.assertLess(time.time() - t0, 10) + self.assertIn("the runner did not end 1 s after the unit's turn limit", str(raised.exception)) + self.assertTrue(reaped and self.driver._gone(reaped[-1], 5)) + with open(os.path.join(os.path.dirname(self.stop), "u0001.jsonl"), "w") as f: + f.write(json.dumps({"status": 200, "side": True}) + "\n") + done = "[2026-11-10 20:00:00,000 INFO desktopenv.env/1-EnvProcess-1] Environment setup complete." + for said, kind in (("", self.driver.EnvironmentSetupError), (done, self.driver.VerifierError)): + with self.subTest(said=said), self.assertRaises(kind): + self.driver._run([sys.executable, "-c", f"import time; print({said!r}, flush=True); time.sleep(15)"], self.dir, + dict(os.environ), log, "the null control's run", failed=os.path.join(self.dir, "failed"), + until=time.time() + 0.5, grace=1) + + def test_a_reaper_that_does_not_start_is_a_setup_error_after_the_start_lock_is_let_go(self): + argv = [sys.executable, "-c", "import time; time.sleep(15)"] + lock, held, groups = os.path.join(self.dir, "start.lck"), [], [] + for failing in (None, RuntimeError("no thread")): + def reap(pgid, *a, **kw): + groups.append(pgid) + with open(lock) as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + held.append(False) + except BlockingIOError: + held.append(True) + if failing: + raise failing + os.killpg(pgid, signal.SIGKILL) + + with self.subTest(failing=failing), mock.patch.object(self.driver, "START_LOCK", lock), \ + mock.patch.object(self.driver, "_remove_vms") as removed, \ + mock.patch.object(self.driver, "_arm", mock.Mock(side_effect=ValueError("no pipe"))), \ + mock.patch.object(self.driver, "reap", reap), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(argv, self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", stagger=True) + self.assertIn("the runner's reaper did not start: ValueError: no pipe", str(raised.exception)) + self.assertTrue(self.driver._gone(groups[-1], 5)) + self.assertEqual(removed.call_count, 1 if failing else 0) + self.assertEqual(held, [False, False]) + + def test_a_start_lock_that_is_no_plain_file_is_a_setup_error(self): + lock = os.path.join(self.dir, "start.lck") + os.mkdir(lock) + with mock.patch.object(self.driver, "START_LOCK", lock), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(["true"], self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", stagger=True) + self.assertIn("OSError: [Errno 22] not a plain file", str(raised.exception)) + + def test_a_start_lock_held_past_the_units_turn_limit_is_a_setup_error(self): + lock = os.path.join(self.dir, "start.lck") + fd = os.open(lock, os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + closing = threading.Timer(10, self.driver.CLOSED.set) + closing.start() + self.addCleanup(closing.cancel) + with mock.patch.object(self.driver, "START_LOCK", lock), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(["true"], self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", + stagger=True, until=time.time() + 1) + self.assertIn("the runner was not started: the unit's time ran out", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + + +class Provider(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + said = {"long": b'{"error":{"message":"This endpoint\'s maximum context length is 10 tokens"}}', + "image": b'{"error":{"message":"invalid image"}}'}.get(body["messages"][0]["content"]) + self.send_response(400 if said else 200) + said = said or json.dumps({"choices": [{"message": {"content": "ok"}}], "usage": {}}).encode() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(said))) + self.end_headers() + self.wfile.write(said) + + +class Relayed(unittest.TestCase): + def setUp(self): + self.up = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + threading.Thread(target=self.up.serve_forever, daemon=True).start() + self.tmp = tempfile.mkdtemp() + self.relay = relay.Relay(f"http://127.0.0.1:{self.up.server_address[1]}/v1", "sk", self.tmp, model="m", + side_models=["sim"]).start() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(self.up.shutdown) + self.addCleanup(self.relay.stop) + self.stop = os.path.join(self.tmp, relay.STOP) + + def post(self, url, model, text): + req = urllib.request.Request(url + "/chat/completions", headers={"Content-Type": "application/json"}, + data=json.dumps({"model": model, "messages": [{"role": "user", "content": text}]}).encode()) + try: + with urllib.request.urlopen(req) as r: + return r.status + except urllib.error.HTTPError as e: + e.close() + return e.code + + def test_only_the_agents_own_overflow_leaves_the_stop_file(self): + self.assertEqual(self.post(self.relay.url("u1"), "m", "image"), 400) + self.assertEqual(self.post(self.relay.url("u1"), "sim", "long"), 400) + self.assertEqual(self.post(self.relay.url("u1-judge", model="j"), "j", "long"), 400) + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 200) + self.assertFalse(os.path.exists(self.stop)) + self.assertEqual(self.post(self.relay.url("u1"), "m", "long"), 400) + with open(self.stop) as f: + self.assertEqual(f.readline(), "context window\n") + self.assertEqual(len(self.relay.calls("u1")), 4) + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 200) + + def test_a_stop_asked_after_an_overflow_is_heeded(self): + with open(self.stop, "w") as f: + f.write("context window\nbatch cap\n") + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 400) + self.assertEqual(self.relay.calls("u1"), []) + + def test_once_asked_from_outside_the_agents_own_calls_are_not_sent(self): + with open(self.stop, "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 400) + self.assertEqual(self.relay.calls("u1"), []) + self.assertEqual(self.post(self.relay.url("u1"), "sim", "fine"), 200) + self.assertEqual(self.post(self.relay.url("u1-judge", model="j"), "j", "fine"), 200) + self.assertEqual(len(self.relay.calls("u1")) + len(self.relay.calls("u1-judge")), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_websites.py b/app/tests/test_osworld2_websites.py new file mode 100644 index 00000000..3b550fd2 --- /dev/null +++ b/app/tests/test_osworld2_websites.py @@ -0,0 +1,821 @@ +"""OSWorld2's mock websites, as websites.py and the driver reach them, before a fake proxy and a fake docker. +With NULLIUS_OSWORLD2_UPSTREAM or NULLIUS_OSWORLD2_WEB set, also checked against them, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock +from urllib.parse import parse_qs, urlsplit + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +GATEWAY = "10.213.0.1" +SUFFIX = f"{GATEWAY}.nip.io" +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + ps) [ -n "$FAKE_ROW" ] && printf '%s\\n' "$FAKE_ROW" ;; + compose) [ -n "$FAKE_UP" ] && touch "$FAKE_UP" ;; +esac +exit 0 +""" + + +def load(): + spec = importlib.util.spec_from_file_location("osworld2_websites", os.path.join(HERE, "websites.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Sites(BaseHTTPRequestHandler): + """Each site by the host asked for: ``down`` 502, ``empty`` an empty page, ``stateless`` no state.""" + down, empty, stateless, overleaf_up = set(), set(), set(), None + + def log_message(self, *_): + pass + + def do_GET(self): + url = urlsplit(self.path) + name = url.hostname[:-len(SUFFIX) - 1] + if name in self.down or (name == "overleaf" and url.path == "/login" and self.overleaf_up + and not os.path.exists(self.overleaf_up)): + self.send_response(502) + self.end_headers() + return + if url.path == "/api/state": + cookie = parse_qs(url.query).get("cookie", [""])[0] + body = b"{}" if name in self.stateless else json.dumps({"user_id": cookie, "state": {"data": {}}}).encode() + else: + body = b"" if name in self.empty else b"site" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class Case(unittest.TestCase): + def setUp(self): + self.w = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + Sites.down, Sites.empty, Sites.stateless, Sites.overleaf_up = set(), set(), set(), None + server = ThreadingHTTPServer(("127.0.0.1", 0), Sites) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + self.unknown, self.elsewhere = set(), set() + + def resolves(host): + name = host[:-len(SUFFIX) - 1] + if not host.endswith(SUFFIX) or name in self.unknown: + return set() + return {"172.17.0.1"} if name in self.elsewhere else {GATEWAY} + + for patch in (mock.patch.object(self.w, "PROXIES", {"http": f"http://127.0.0.1:{server.server_port}"}), + mock.patch.object(self.w, "_resolves", resolves), + mock.patch.object(self.w, "POLL", 0.2), + mock.patch.object(self.w, "LOCK", os.path.join(self.tmp, "overleaf.lck")), + mock.patch.object(self.w, "SITES_LOCK", os.path.join(self.tmp, "websites.lck"))): + patch.start() + self.addCleanup(patch.stop) + + +class Option(Case): + def test_none_gives_no_suffix_and_a_task_that_needs_the_sites_is_refused(self): + self.assertEqual(self.w.env({}), {}) + self.assertIsNone(self.w.needed("001", {})) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.needed("005", {self.w.OPTION: "none"}) + self.assertIn("task 005 needs the mock websites", str(raised.exception)) + + def test_a_suffix_is_every_process_s_website_host_suffix(self): + given = {self.w.OPTION: SUFFIX} + self.assertEqual(self.w.env(given), {"WEBSITE_HOST_SUFFIX": SUFFIX}) + self.assertEqual(self.w.needed("057", given), SUFFIX) + self.assertIsNone(self.w.needed("001", given)) + + def test_a_value_that_is_no_host_suffix_is_refused(self): + for bad in ("http://x.nip.io", "x", "a b.io", "x.nip.io/", "-x.io", "X.IO"): + with self.subTest(bad), self.assertRaises(self.w.WebsiteError): + self.w.env({self.w.OPTION: bad}) + + def test_the_tasks(self): + self.assertEqual(len(self.w.TASKS), 40) + self.assertLess(self.w.OVERLEAF_TASKS, self.w.TASKS) + self.assertEqual(len(self.w.NAMES), 25) + self.assertEqual(len(set(self.w.NAMES)), 25) + + +class Ready(Case): + def test_every_name_and_each_stateful_site_answers(self): + self.assertEqual(self.w.ready(SUFFIX, overleaf=True), 25) + + def test_a_unit_s_check_leaves_overleaf_to_its_own_tasks(self): + Sites.down.add("overleaf") + self.assertEqual(self.w.ready(SUFFIX), 24) + with self.assertRaises(self.w.WebsiteError): + self.w.ready(SUFFIX, overleaf=True) + + def test_a_name_that_does_not_resolve(self): + self.unknown.add("trippza") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"trippza.{SUFFIX} does not resolve", str(raised.exception)) + + def test_a_suffix_whose_names_resolve_past_the_gateway_is_refused(self): + self.elsewhere.add("awsconsole") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"awsconsole.{SUFFIX} resolves to 172.17.0.1, not {GATEWAY}", str(raised.exception)) + self.assertIn(f"such as {GATEWAY}.nip.io", str(raised.exception)) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.suffix({self.w.OPTION: "x"}) + self.assertIn(f"such as {GATEWAY}.nip.io", str(raised.exception)) + + def test_a_site_that_fails(self): + Sites.down.add("mailhub") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"http://mailhub.{SUFFIX}/ answered 502", str(raised.exception)) + self.assertIn("websites.py compose STACK up -d", str(raised.exception)) + + def test_an_empty_page_is_a_name_the_stack_does_not_serve(self): + Sites.empty.add("interfacinglinux") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn("serves another suffix", str(raised.exception)) + + def test_a_site_with_no_state(self): + Sites.stateless.add("wandb") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"wandb.{SUFFIX}/api/state?cookie={self.w.COOKIE} answered no state", str(raised.exception)) + + +class Overleaf(Case): + def setUp(self): + super().setUp() + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.calls = os.path.join(self.tmp, "calls") + self.settings = os.path.join(self.tmp, "OSWorld-web.osworld2.env") + with open(self.settings, "w") as f: + f.write(f"HOST_SUFFIX={SUFFIX}\nCADDY_SCHEME=http://\n") + self.up = os.path.join(self.tmp, "up") + Sites.overleaf_up = self.up + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_UP=self.up, FAKE_ROW=f"/s/docker-compose.yml,/s.osworld2.compose.yml\t/s\t{self.settings}") + env.start() + self.addCleanup(env.stop) + + def docker(self): + with open(self.calls) as f: + return f.read().splitlines() + + def test_any_other_task_holds_nothing(self): + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertFalse(os.path.exists(self.calls)) + + def test_overleaf_starts_anew_as_the_project_runs(self): + with self.w.held("057", SUFFIX, threading.Event()): + calls = self.docker() + compose = f"compose -p osworld2-web --project-directory /s -f /s/docker-compose.yml -f /s.osworld2.compose.yml --env-file {self.settings}" + self.assertEqual(calls[1:], [ + f"{compose} rm --stop --force -v " + " ".join(self.w.OVERLEAF), + "volume rm --force osworld2-web_overleaf_web_data osworld2-web_overleaf_web_mongo_data osworld2-web_overleaf_web_redis_data", + f"{compose} up -d --no-build --pull never overleaf_web_app"]) + self.assertIn("label=com.docker.compose.project=osworld2-web", calls[0]) + self.assertNotIn("service=", calls[0]) + + def test_a_stack_for_another_suffix_is_not_touched(self): + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", "10.0.0.5.nip.io", threading.Event()): + pass + self.assertIn(f"the stack serves {SUFFIX}", str(raised.exception)) + self.assertEqual(len(self.docker()), 1) + + def test_no_overleaf_running(self): + with mock.patch.dict(os.environ, FAKE_ROW=""), self.assertRaises(self.w.WebsiteError) as raised: + self.w.reset_overleaf(SUFFIX) + self.assertIn("no container of the osworld2-web project", str(raised.exception)) + + def test_overleaf_is_waited_on_until_its_login_page_answers(self): + with mock.patch.object(self.w, "OVERLEAF_SECONDS", 1), mock.patch.dict(os.environ, FAKE_UP=""): + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.reset_overleaf(SUFFIX) + self.assertIn("Overleaf did not come up in 1 s", str(raised.exception)) + + def test_the_next_overleaf_task_waits_for_the_lock(self): + order, released = [], threading.Event() + + def first(): + with self.w.held("057", SUFFIX, threading.Event()): + order.append("057 in") + released.wait(10) + order.append("057 out") + + t = threading.Thread(target=first) + t.start() + while not order: + time.sleep(0.05) + threading.Timer(1.0, released.set).start() + with self.w.held("072", SUFFIX, threading.Event()): + order.append("072 in") + t.join(10) + self.assertEqual(order, ["057 in", "057 out", "072 in"]) + + def test_a_process_given_the_locks_keeps_them_after_the_unit_lets_go(self): + with self.w.held("057", SUFFIX, threading.Event()): + fds = self.w.held_fds() + self.assertEqual(len(fds), 2) + child = subprocess.Popen([sys.executable, "-c", "import sys; sys.stdin.read()"], stdin=subprocess.PIPE, + pass_fds=fds) + self.addCleanup(child.wait, 10) + self.assertEqual(self.w.held_fds(), ()) + with self.assertRaises(self.w.WebsiteError): + with self.w._alone(): + pass + closed = threading.Event() + threading.Timer(1.0, closed.set).start() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", SUFFIX, closed): + pass + self.assertIn("another unit's Overleaf task ran", str(raised.exception)) + child.stdin.close() + child.wait(10) + with self.w.held("072", SUFFIX, threading.Event()): + pass + + def test_reset_and_smoke_refuse_while_a_unit_uses_the_sites(self): + stack = os.path.join(self.tmp, "OSWorld-web") + with self.w.held("005", SUFFIX, threading.Event()), mock.patch.object(self.w, "_pinned", os.path.abspath): + for start in (self.w.reset, self.w.smoke): + with self.assertRaises(self.w.WebsiteError) as raised: + start(stack) + self.assertIn("a unit is using the mock websites", str(raised.exception)) + self.assertFalse(os.path.exists(self.calls)) + + def test_compose_by_hand_changes_nothing_while_a_unit_uses_the_sites(self): + ran = [] + with mock.patch.object(self.w, "compose", lambda stack, args, **kw: ran.append(args) or 0): + with self.w.held("005", SUFFIX, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()) as err: + for args in (["down", "--volumes"], ["up", "-d", "--force-recreate"], ["-p", "ps", "rm"], []): + self.assertEqual(self.w.main(["compose", self.tmp, *args]), 1, args) + for args in (["ps"], ["--", "logs", "caddy"], ["stats", "--no-stream"]): + self.assertEqual(self.w.main(["compose", self.tmp, *args]), 0, args) + self.assertIn("a unit is using the mock websites", err.getvalue()) + self.assertEqual(self.w.main(["compose", self.tmp, "down"]), 0) + self.assertEqual(ran, [["ps"], ["logs", "caddy"], ["stats", "--no-stream"], ["down"]]) + + def test_the_locks_are_the_machine_s_whoever_runs(self): + names = [] + for uid in (1000, 1001): + with mock.patch.object(os, "getuid", lambda: uid): + w = load() + names.append((w.LOCK, w.SITES_LOCK)) + self.assertEqual(names[0], names[1]) + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertEqual(os.stat(self.w.SITES_LOCK).st_mode & 0o777, 0o644) + os.chmod(self.w.SITES_LOCK, 0o444) + other = subprocess.Popen([sys.executable, "-c", "import fcntl, os, sys; fd = os.open(sys.argv[1], os.O_RDONLY); " + "fcntl.flock(fd, fcntl.LOCK_SH); print(flush=True); sys.stdin.read()", + self.w.SITES_LOCK], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(other.wait, 10) + self.addCleanup(other.stdout.close) + self.addCleanup(other.kill) + other.stdout.readline() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w._alone(): + pass + self.assertIn("a unit is using the mock websites", str(raised.exception)) + with self.w.held("005", SUFFIX, threading.Event()): + pass + other.stdin.close() + other.wait(10) + with self.w._alone(): + pass + os.remove(self.w.SITES_LOCK) + os.symlink(os.path.join(self.tmp, "elsewhere"), self.w.SITES_LOCK) + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertIn("cannot be opened", str(raised.exception)) + self.assertFalse(os.path.exists(os.path.join(self.tmp, "elsewhere"))) + + def test_one_overleaf_task_at_a_time_and_no_longer_than_its_unit(self): + inside, waited = threading.Event(), threading.Event() + released = threading.Event() + + def first(): + with self.w.held("057", SUFFIX, threading.Event()): + inside.set() + released.wait(10) + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + closed = threading.Event() + threading.Timer(1.5, closed.set).start() + t0 = time.time() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", SUFFIX, closed): + waited.set() + self.assertFalse(waited.is_set()) + self.assertIn("time ran out", str(raised.exception)) + self.assertLess(time.time() - t0, 5) + released.set() + t.join(10) + with self.w.held("072", SUFFIX, threading.Event()): + waited.set() + self.assertTrue(waited.is_set()) + + +class Stack(Case): + def test_a_folder_not_at_the_release_is_refused(self): + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.configure(self.tmp, SUFFIX, 2, 20) + self.assertIn(self.w.COMMIT, str(raised.exception)) + + def test_the_stack_listens_on_the_vms_network_s_gateway_alone(self): + stack, made = os.path.join(self.tmp, "OSWorld-web"), [] + with mock.patch.object(self.w, "_pinned", lambda stack: stack), \ + mock.patch.object(self.w, "_root_disk", lambda: ("/var/lib/docker", "/dev/sda")), \ + mock.patch.object(self.w, "_network", lambda: made.append(1) or "nullius-osworld2"): + with self.assertRaises(self.w.WebsiteError): + self.w.configure(stack, SUFFIX, 0, 80) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.configure(stack, SUFFIX, 8, 40) + self.assertIn("at least 80 IOPS", str(raised.exception)) + self.assertEqual(made, []) + path, settings = self.w.configure(stack, SUFFIX, 8, 80) + self.assertEqual((settings["OSWORLD2_WEB_ADDRESS"], made), (GATEWAY, [1])) + with open(path) as f: + self.assertIn(f"OSWORLD2_WEB_ADDRESS={GATEWAY}\n", f.read()) + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + self.w.main(["configure", stack, SUFFIX, "--address", "172.17.0.1", "--write-mbps", "8", "--write-iops", "80"]) + + def test_the_sum_of_the_services_limits_never_passes_the_stack_s_share(self): + for mbps, iops in ((8, 80), (10, 100), (9, 81), (30, 300), (1, 80)): + tiers = self.w.allocation(mbps, iops) + settings = {f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))} + each, total = self.w.limits(settings) + self.assertEqual(sorted(each), sorted(self.w.SERVICES)) + self.assertLessEqual(total, (mbps * self.w.MB, iops)) + self.assertTrue(all(b > 0 and i > 0 for b, i in each.values())) + writers = [each[s] for s in self.w.WEIGHTS] + self.assertTrue(all(w >= each["awsconsole"] for w in writers)) + if (mbps, iops) == (8, 80): + self.assertEqual(total[1], 80) + self.assertGreater(total[0], 8 * self.w.MB - 1024) + + def test_the_groups_build_every_service_once(self): + self.assertEqual(len(self.w.SERVICES), 40) + self.assertEqual(len(set(self.w.SERVICES)), 40) + + def test_every_service_of_the_override_is_limited_and_caddy_pinned_on_one_address(self): + with open(os.path.join(HERE, "websites.compose.yml"), encoding="utf-8") as f: + text = f.read() + services = re.findall(r"^ ([a-z0-9_]+):$", text.split("\nservices:\n", 1)[1], re.M) + self.assertEqual(sorted(services), sorted(self.w.SERVICES)) + self.assertEqual(text.count(" blkio_config: *blkio\n logging: *logs\n"), 40 - len(self.w.WEIGHTS)) + self.assertEqual(text.count(" logging: *logs\n"), 40) + self.assertEqual(set(re.findall(r"\$\{(OSWORLD2_WEB_\w+|HOST_SUFFIX|CADDY_SCHEME)", text)) - {"HOST_SUFFIX", "CADDY_SCHEME"}, + set(self.w.SETTINGS) - {"HOST_SUFFIX", "CADDY_SCHEME"}) + for service in self.w.WEIGHTS: + block = re.split(r"\n (?=\S)", text.split(f"\n {service}:\n", 1)[1], maxsplit=1)[0] + self.assertIn(f"OSWORLD2_WEB_{service.upper()}_BPS", block) + self.assertRegex(self.w.CADDY.search(text).group(1), r"@sha256:[0-9a-f]{64}$") + self.assertIn(" ports: !override\n - ${OSWORLD2_WEB_ADDRESS:", text) + self.assertNotIn("443", text) + + +BUILD_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1 $2" in + "network inspect") echo "1|10.213.0.0/24 10.213.0.1" ;; + "inspect -f") printf '%s\\n' "$FAKE_HOSTCONFIG" ;; + "buildx inspect") exit "${FAKE_NO_BUILDER:-0}" ;; + "image inspect") echo "sha256:$(echo "$5" | md5sum | cut -c1-12)" ;; + "compose version") echo "${FAKE_COMPOSE:-2.40.3+ds1-0ubuntu1}" ;; +esac +case "$*" in *" config --format json") printf '%s\\n' "$FAKE_CONFIG" ;; esac +exit 0 +""" + + +class Build(Case): + """The stack's own BuildKit with a write limit, and the build group by group, on a fake docker.""" + + def setUp(self): + super().setUp() + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(BUILD_DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.stack = os.path.join(self.tmp, "OSWorld-web") + os.makedirs(self.stack) + with open(self.stack + ".osworld2.env", "w") as f: + f.write(f"HOST_SUFFIX={SUFFIX}\n") + self.calls = os.path.join(self.tmp, "calls") + limits = {"BlkioDeviceWriteBps": [{"Path": "/dev/dm-0", "Rate": 20 * 1024 * 1024}], + "BlkioDeviceWriteIOps": [{"Path": "/dev/dm-0", "Rate": 200}]} + config = {"services": {"caddy": {"image": "lucaslorentz/caddy-docker-proxy@sha256:" + "c" * 64}, + "overleaf_web_mongo": {"image": "mongo:8.0.20"}}} + self.free = [100.0] + for patch in (mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_HOSTCONFIG=json.dumps(limits), FAKE_CONFIG=json.dumps(config)), + mock.patch.object(self.w, "_pinned", os.path.abspath), + mock.patch.object(self.w, "_root_disk", lambda: ("/var/lib/docker", "/dev/dm-0")), + mock.patch.object(self.w, "_free_gb", lambda root: self.free.pop(0) if len(self.free) > 1 else self.free[0])): + patch.start() + self.addCleanup(patch.stop) + + def docker(self): + with open(self.calls) as f: + return f.read().splitlines() + + def test_the_builder_writes_under_its_limit(self): + self.w.builder(self.stack, "up", 20, 200) + run, inspect, create = self.docker() + self.assertIn("--privileged", run) + self.assertIn("--device-write-bps /dev/dm-0:20971520 --device-write-iops /dev/dm-0:200", run) + self.assertTrue(run.endswith(self.w.BUILDKIT)) + self.assertEqual(create, f"buildx create --name {self.w.BUILDER} --driver remote docker-container://{self.w.BUILDKITD}") + with open(self.stack + ".osworld2.buildkitd.toml") as f: + self.assertIn("max-parallelism = 4", f.read()) + + def test_a_builder_docker_kept_no_limit_for_is_removed_and_never_used(self): + with mock.patch.dict(os.environ, FAKE_HOSTCONFIG="{}"), self.assertRaises(self.w.WebsiteError) as raised: + self.w.builder(self.stack, "up", 20, 200) + self.assertIn("no write limit", str(raised.exception)) + self.assertEqual(self.docker()[-1], f"rm -f {self.w.BUILDKITD}") + self.assertFalse([c for c in self.docker() if c.startswith("buildx create")]) + + def test_the_groups_one_after_another_each_cache_removed(self): + said = [] + out = self.w.build(self.stack, say=said.append) + calls = self.docker() + steps = [c for c in calls if " pull " in c or " build " in c or c.startswith("buildx prune")] + self.assertEqual(len(steps), 1 + 2 * (len(self.w.GROUPS) - 1)) + self.assertTrue(steps[0].endswith("pull " + " ".join(self.w.GROUPS[0][1]))) + for (name, services, _), (built, pruned) in zip(self.w.GROUPS[1:], zip(steps[1::2], steps[2::2])): + self.assertTrue(built.endswith("build " + " ".join(services)), name) + self.assertEqual(pruned, f"buildx prune --builder {self.w.BUILDER} --all --force") + self.assertEqual([json.loads(s)["group"] for s in said], [g[0] for g in self.w.GROUPS]) + with open(out["images"]) as f: + images = json.load(f)["images"] + self.assertEqual(len(images), 40) + self.assertEqual(images["overleaf_web_mongo"]["image"], "mongo:8.0.20") + self.assertEqual(images["wandb"]["image"], "osworld2-web-wandb") + + def test_the_build_stops_before_a_group_that_would_eat_the_reserve(self): + self.free = [40.0, 30.0] + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.build(self.stack, say=lambda line: None) + self.assertIn("stopped before overleaf: 30.0 GB free", str(raised.exception)) + self.assertIn("done: pulled", str(raised.exception)) + self.assertFalse([c for c in self.docker() if " build " in c]) + + def test_a_compose_without_override_tags_is_refused(self): + for old in ("2.20.3", "2.24.3", "v2.24.0"): + with self.subTest(old), mock.patch.dict(os.environ, FAKE_COMPOSE=old), \ + self.assertRaises(self.w.WebsiteError) as raised: + self.w.compose(self.stack, ["config"], capture=True) + self.assertIn("Docker Compose 2.24.4 or later", str(raised.exception)) + with mock.patch.dict(os.environ, FAKE_COMPOSE="2.24.4"): + self.assertEqual(self.w.compose(self.stack, ["config"], capture=True).returncode, 0) + + def test_compose_hands_the_locks_it_holds_to_docker(self): + given = [] + with mock.patch.object(self.w, "_compose_version", lambda: None), mock.patch.object(self.w, "_network", lambda: None), \ + mock.patch.object(self.w.subprocess, "run", lambda *a, **kw: given.append(kw["pass_fds"]) or mock.Mock()): + with self.w._alone(): + fds = self.w.held_fds() + self.w.compose(self.stack, ["down"]) + self.w.compose(self.stack, ["up", "-d"], capture=True) + self.assertEqual(len(fds), 1) + self.assertEqual(given, [fds, fds]) + + def test_every_compose_makes_the_vms_network_first(self): + self.w.compose(self.stack, ["up", "-d"], capture=True) + calls = self.docker() + made = next(i for i, c in enumerate(calls) if c.startswith("network inspect ") and c.endswith(" nullius-osworld2")) + self.assertLess(made, next(i for i, c in enumerate(calls) if c.endswith(" up -d"))) + + def test_the_names_are_tried_from_the_vms_network(self): + self.w._from_vms(SUFFIX) + run = [c for c in self.docker() if c.startswith("run ")][-1].split() + self.assertEqual(run[run.index("--network") + 1], "nullius-osworld2") + self.assertEqual(run[-len(self.w.NAMES):], [f"{n}.{SUFFIX}" for n in self.w.NAMES]) + + def test_a_missing_override_says_so(self): + with mock.patch.object(self.w, "OVERRIDE", os.path.join(self.tmp, "none.yml")), \ + self.assertRaises(self.w.WebsiteError) as raised: + self.w.compose(self.stack, ["config"], capture=True) + self.assertIn("this benchmark's folder is incomplete", str(raised.exception)) + + def test_no_builder_or_an_unknown_group_builds_nothing(self): + with self.assertRaises(self.w.WebsiteError): + self.w.build(self.stack, groups=["nope"]) + with mock.patch.dict(os.environ, FAKE_NO_BUILDER="1"), self.assertRaises(self.w.WebsiteError) as raised: + self.w.build(self.stack) + self.assertIn("no builder", str(raised.exception)) + self.assertFalse([c for c in self.docker() if " pull " in c or " build " in c]) + + +class Started(Exception): + pass + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_websites", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class Driver(unittest.TestCase): + """The driver's use of websites.py.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.w = self.driver.websites + self.envs, self.checked, self.held = [], [], [] + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + @contextlib.contextmanager + def held(eid, value, closed): + self.held.append((eid, value, closed)) + yield + + for target, name, fake in ((self.driver, "_vm", lambda dir, domain, eid: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_task_file", lambda *a: None), + (self.w, "held", held), (self.w, "ready", lambda *a, **kw: self.checked.append((a, kw)))): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + + def option(self, value): + p = mock.patch.dict(os.environ, {self.w.OPTION: value}) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + + def test_a_task_that_needs_the_sites_is_refused_without_a_suffix_before_anything_runs(self): + self.option("none") + setup = self.driver.EnvironmentSetupError + for start, kind in ((lambda: self.driver.setup("tasks__005", self.tmp), self.w.WebsiteError), + (lambda: self.answer("005"), setup), (lambda: self.driver.score("tasks__005", self.tmp), setup)): + with self.assertRaises(kind) as raised: + start() + self.assertIn(("WebsiteError: " if kind is setup else "") + "task 005 needs the mock websites", str(raised.exception)) + self.assertEqual(self.envs, []) + + def test_the_suffix_reaches_every_process_and_overleaf_is_held_around_the_run(self): + self.option(SUFFIX) + for start in (lambda: self.answer("057"), lambda: self.driver.score("tasks__057", self.tmp), + lambda: self.driver.setup("tasks__001", self.tmp)): + with self.assertRaises(Started): + start() + self.assertEqual(self.envs[-1]["WEBSITE_HOST_SUFFIX"], SUFFIX) + self.assertEqual(self.held, [("057", SUFFIX, mock.ANY)] * 2) + self.assertIsNotNone(self.held[0][-1].until) + + def test_no_suffix_is_no_variable(self): + self.option("none") + with self.assertRaises(Started): + self.answer("001") + self.assertNotIn("WEBSITE_HOST_SUFFIX", self.envs[-1]) + self.assertEqual(self.held, [("001", None, mock.ANY)]) + + def test_the_sites_are_checked_among_what_a_vm_start_needs(self): + for name in ("vm_image", "_provider_image", "_write_limit"): + p = mock.patch.object(self.driver, name, lambda: "image") + p.start() + self.addCleanup(p.stop) + made = [] + with mock.patch.object(self.driver, "_nat", lambda: True), mock.patch.object(self.driver.os, "listdir", lambda p: ["x"]), \ + mock.patch.object(self.driver.os.path, "isdir", lambda p: True), \ + mock.patch.object(self.driver._startup(), "network", + lambda docker: made.append((docker, len(self.checked))) or "nullius-osworld2"): + self.option(SUFFIX) + self.driver._ready("tasks", "001") + self.driver._ready("tasks", "081") + self.assertEqual(self.checked, [((SUFFIX,), {"state": False})]) + self.assertEqual(made, [(self.driver._asked, 0), (self.driver._asked, 0)]) + self.assertEqual(self.w.SITE, self.driver.SITE) + self.option("none") + with self.assertRaises(self.w.WebsiteError): + self.driver._ready("tasks", "081") + + def test_a_hosts_snapshot_carries_the_module_and_the_override(self): + from pathlib import Path + + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + manifest = json.loads((Path(root) / "manifest.json").read_text()) + for name in ("websites.py", "websites.compose.yml"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + self.assertIn(f"catalogue/benchmarks/osworld2/{name}", manifest["files"]) + + def test_the_override_is_the_one_the_command_names(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + command = json.load(f)["driver"]["command"] + self.assertEqual(command[-2], "{here}/websites.compose.yml") + self.assertEqual(self.w.OVERRIDE, os.path.join(HERE, "websites.compose.yml")) + given = os.path.join(self.tmp, "snapshot", "websites.compose.yml") + spec = importlib.util.spec_from_file_location("osworld2_driver_argv", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + files = [os.path.join(HERE, n) for n in ("agents.json", "nop.py", os.path.join("startup", "sitecustomize.py"))] + with mock.patch.object(sys, "argv", ["driver.py", *files, given]), \ + mock.patch.object(sys, "path", [HERE, *sys.path]), mock.patch.dict(os.environ, NULLIUS_UPSTREAM=self.tmp): + spec.loader.exec_module(driver) + self.addCleanup(setattr, driver.websites, "OVERRIDE", os.path.join(HERE, "websites.compose.yml")) + self.assertEqual(driver.websites.OVERRIDE, given) + + def test_the_lists(self): + lists = self.driver.LISTS + gitlab_or_proxy = set("026 036 050 056 098".split()) + self.assertEqual(lists["websites35"], sorted(self.w.TASKS - gitlab_or_proxy)) + self.assertEqual(lists["websites98"], sorted(lists["no_services63"] + lists["websites35"])) + self.assertEqual(len(set(lists["websites98"])), 98) + self.assertFalse(set(lists["no_services63"]) & self.w.TASKS) + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + counted = {d["id"]: d["units"] for d in bench["datasets"]} + self.assertEqual((counted["websites35"], counted["websites98"]), (35, 98)) + self.assertEqual(bench["options"]["websites"]["pattern"], "none|" + self.w.SUFFIX.pattern) + with mock.patch.dict(os.environ, {"NULLIUS_DATASET": "", "NULLIUS_DATASET_ID": "websites98"}): + self.assertEqual(len(self.driver.units()["units"]), 98) + + +class DriverSites(Case): + """Overleaf down, or reset for one of its tasks, fails no other task's check.""" + + def setUp(self): + super().setUp() + self.driver = load_driver(self.tmp) + sites = self.driver.websites + for target, name, fake in ((sites, "PROXIES", self.w.PROXIES), (sites, "_resolves", self.w._resolves), + (sites, "LOCK", self.w.LOCK), (sites, "SITES_LOCK", self.w.SITES_LOCK), + (self.driver, "_task_file", lambda *a: None), (self.driver, "_nat", lambda: True), + (self.driver, "vm_image", lambda: "image"), (self.driver, "_provider_image", lambda: None), + (self.driver, "_write_limit", lambda: None), (self.driver, "ASSETS", HERE), + (self.driver._startup(), "network", lambda docker: "nullius-osworld2")): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + p = mock.patch.dict(os.environ, {sites.OPTION: SUFFIX}) + p.start() + self.addCleanup(p.stop) + + def test_the_reaper_keeps_the_unit_s_locks_also_when_its_time_ran_out(self): + given = [] + with mock.patch.object(self.driver.subprocess, "Popen", lambda *a, **kw: given.append(kw.get("pass_fds"))): + with self.driver.websites.held("005", SUFFIX, threading.Event()): + fds = self.driver.websites.held_fds() + self.driver._reaper(1, self.tmp, True) + with self.assertRaises(RuntimeError): + self.driver._stop(mock.Mock(pid=1), os.path.join(self.tmp, "x.log"), "the run", True) + self.driver._reaper(1, self.tmp, True) + self.assertEqual(len(fds), 1) + self.assertEqual(given, [fds, fds, ()]) + + def test_overleaf_down_fails_no_other_task(self): + Sites.down.add("overleaf") + self.assertEqual(self.driver._ready("tasks", "005"), "image") + + def test_no_site_s_state_is_read_for_a_unit_s_check(self): + Sites.stateless.update(self.w.STATEFUL) + self.assertEqual(self.driver._ready("tasks", "005"), "image") + with self.assertRaises(self.w.WebsiteError): + self.w.ready(SUFFIX) + + def test_overleaf_being_reset_for_its_task_fails_no_other_task(self): + inside, released = threading.Event(), threading.Event() + + def anew(value, closed=None): + Sites.down.add("overleaf") + inside.set() + released.wait(10) + Sites.down.discard("overleaf") + + with mock.patch.object(self.driver.websites, "reset_overleaf", anew): + def first(): + with self.driver.websites.held("057", SUFFIX, threading.Event()): + pass + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + self.assertEqual(self.driver._ready("tasks", "005"), "image") + self.assertEqual(self.driver._ready("tasks", "072"), "image") + released.set() + t.join(10) + + +def _web(): + stack = os.environ.get("NULLIUS_OSWORLD2_WEB", "") + return stack if stack and shutil.which("docker") and os.path.isfile(os.path.join(stack, "docker-compose.yml")) else "" + + +@unittest.skipUnless(_web(), "NULLIUS_OSWORLD2_WEB names no OSWorld-web, or no docker compose") +class Override(unittest.TestCase): + def test_the_project_as_compose_reads_it(self): + w = load() + tiers = w.allocation(8, 80) + settings = {"HOST_SUFFIX": SUFFIX, "CADDY_SCHEME": "http://", "OSWORLD2_WEB_ADDRESS": GATEWAY, + "OSWORLD2_WEB_DISK": "/dev/sda", + **{f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))}} + each, total = w.limits(settings) + self.assertLessEqual(total, (8 * w.MB, 80)) + self.assertGreater(total[0], 8 * w.MB - 1024) + said = subprocess.run(["docker", "compose", "-p", w.PROJECT, "-f", os.path.join(_web(), "docker-compose.yml"), + "-f", w.OVERRIDE, "config", "--format", "json"], capture_output=True, text=True, + env=dict(os.environ, **settings), timeout=120) + self.assertEqual(said.returncode, 0, said.stderr) + services = json.loads(said.stdout)["services"] + self.assertEqual(len(services), 40) + self.assertEqual(sorted(services), sorted(w.SERVICES)) + built = {n for n, s in services.items() if "build" in s} + self.assertEqual(built, {s for g, group, _ in w.GROUPS if g != "pulled" for s in group}) + for name, service in services.items(): + self.assertEqual(service.get("blkio_config"), { + "device_write_bps": [{"Path": "/dev/sda", "Rate": str(each[name][0])}], + "device_write_iops": [{"Path": "/dev/sda", "Rate": str(each[name][1])}]}, name) + self.assertEqual(service.get("logging"), {"driver": "json-file", "options": {"max-size": "10m", "max-file": "3"}}, name) + if name != "caddy": + self.assertNotIn("ports", service, name) + self.assertEqual([(p["host_ip"], p["published"]) for p in services["caddy"]["ports"]], [(GATEWAY, "80")]) + labelled = {v for s in services.values() for k, v in (s.get("labels") or {}).items() if k == "caddy"} + served = {h.strip()[len("http://"):-len(SUFFIX) - 1] for v in labelled for h in v.split(",")} + self.assertEqual(served, set(w.NAMES)) + self.assertEqual(served - {"studio.streamview", "interfacinglinux", "overleaf"}, set(w.STATEFUL)) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + """The tasks that import the website controller, and those that name Overleaf's site.""" + + def test_the_tasks_that_need_the_sites_and_overleaf_s(self): + w = load() + hosted, overleaf = set(), set() + for p in _task_classes(): + n = os.path.basename(p)[5:-3] + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [a.name for a in node.names] if isinstance(node, ast.Import) else []) + if any(m.endswith("controllers.website") for m in modules) or ( + isinstance(node, ast.ImportFrom) and (node.module or "").endswith("controllers") + and "website" in {a.name for a in node.names}): + hosted.add(n) + if isinstance(node, ast.Constant) and isinstance(node.value, str) and ( + node.value == "overleaf" or node.value.endswith("//overleaf.")): + overleaf.add(n) + self.assertEqual(sorted(hosted), sorted(w.TASKS)) + self.assertEqual(sorted(overleaf), sorted(w.OVERLEAF_TASKS)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_write_limit.py b/app/tests/test_osworld2_write_limit.py new file mode 100644 index 00000000..abf2d935 --- /dev/null +++ b/app/tests/test_osworld2_write_limit.py @@ -0,0 +1,555 @@ +"""OSWorld2's disk write limit: startup/sitecustomize.py on docker-py's create, and the driver's proof before a VM. +A fake docker-py, docker on PATH and sysfs stand in.""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +from nullius.standard import definitions, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +PROVIDER = "happysixd/osworld-docker@sha256:" + "0" * 64 +OFF = {"NULLIUS_OPTION_WRITE_MBPS": "none", "NULLIUS_OPTION_WRITE_IOPS": "none", "NULLIUS_OPTION_VM_PROVIDER": "docker"} +ON = {"NULLIUS_OPTION_WRITE_MBPS": "30", "NULLIUS_OPTION_WRITE_IOPS": "300"} +MB30 = 30 * 1024 * 1024 +CONTAINERS = """class Container: + drop = False + + def __init__(self, image, kwargs): + self.image, self.kwargs, self.attrs, self.removed, self.started = image, kwargs, {}, None, False + + def start(self): + self.started = True + + def reload(self): + kept = {} if Container.drop else self.kwargs + self.attrs = {"HostConfig": {"BlkioDeviceWriteBps": kept.get("device_write_bps"), + "BlkioDeviceWriteIOps": kept.get("device_write_iops")}} + + def remove(self, **kwargs): + self.removed = kwargs + + +class ContainerCollection: + made = [] + + def __init__(self, client=None): + self.client = client + + def create(self, image, command=None, **kwargs): + ContainerCollection.made.append(Container(image, kwargs)) + return ContainerCollection.made[-1] + + def run(self, image, command=None, **kwargs): + container = self.create(image=image, command=command, **kwargs) + container.start() + return container +""" +CHILD = """import json, multiprocessing, sys +import sibling +from docker.models.containers import ContainerCollection + + +def look(queue): + run = ContainerCollection.run + queue.put({"wrapped": hasattr(ContainerCollection.create, "__wrapped__"), + "other": run(ContainerCollection(), "ubuntu:24.04", x=1).kwargs}) + + +if __name__ == "__main__": + out = {"main": hasattr(ContainerCollection.create, "__wrapped__"), "path0": sys.path[0]} + for method in ("fork", "spawn"): + ctx = multiprocessing.get_context(method) + queue = ctx.Queue() + p = ctx.Process(target=look, args=(queue,)) + p.start() + out[method] = queue.get(timeout=60) + p.join() + print(json.dumps(out)) +""" +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + info) printf '%s\t%s\n' "$FAKE_ROOT" "${FAKE_CGROUP:-2}" ;; + ps) case "$*" in *name=*) printf '%s\n' $FAKE_PROBES ;; esac ;; + create) case "$FAKE_CREATE_HANG" in + all) exec sleep 5 ;; + once) [ -e "$FAKE_DOCKER_LOG.hung" ] || { : > "$FAKE_DOCKER_LOG.hung"; exec sleep 5; } ;; + esac + [ -z "$FAKE_CREATE_FAIL" ] || { echo "Error: refused" >&2; exit 1; } + echo "WARNING: write limit discarded" >&2; echo probe ;; + inspect) echo "$FAKE_HOST" ;; + network) echo "1|10.213.0.0/24 10.213.0.1" ;; + rm) [ -z "$FAKE_RM_FAIL" ] || { echo "Error: busy" >&2; exit 1; } ;; +esac +""" + + +def load_site(environ, name="osworld2_startup"): + spec = importlib.util.spec_from_file_location(name, os.path.join(STARTUP, "sitecustomize.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.dict(os.environ, environ): + spec.loader.exec_module(module) + return module + + +def load_driver(up, argv=()): + spec = importlib.util.spec_from_file_location("osworld2_driver_limit", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py", *argv]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def sysfs(case, site, root, kind="disk"): + """A fake sysfs and /dev for ``root``'s device: a disk, a partition, a device mapper volume, or no uevent.""" + st = os.stat(root).st_dev + tmp = tempfile.mkdtemp() + case.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + block, dev = os.path.join(tmp, "sys", "dev", "block"), os.path.join(tmp, "dev") + name = "dm-0" if kind == "dm" else "sda" + disk = os.path.join(tmp, "sys", "devices", "virtual" if kind == "dm" else "pci", name) + node = os.path.join(disk, "sda1") if kind == "partition" else disk + for d in (node, block, dev): + os.makedirs(d, exist_ok=True) + if kind != "no uevent": + with open(os.path.join(disk, "uevent"), "w") as f: + f.write(f"MAJOR=8\nMINOR=0\nDEVNAME={name}\nDEVTYPE=disk\n") + open(os.path.join(dev, name), "w").close() + if kind == "partition": + with open(os.path.join(node, "uevent"), "w") as f: + f.write("MAJOR=8\nMINOR=1\nDEVNAME=sda1\nDEVTYPE=partition\n") + with open(os.path.join(node, "partition"), "w") as f: + f.write("1\n") + open(os.path.join(dev, "sda1"), "w").close() + os.symlink(node, os.path.join(block, f"{os.major(st)}:{os.minor(st)}")) + for attr, value in (("SYS_BLOCK", block), ("DEV", dev)): + patched = mock.patch.object(site, attr, value) + patched.start() + case.addCleanup(patched.stop) + return os.path.join(dev, name) + + +class Client: + def __init__(self, root): + self.root, self.asked = root, 0 + + def info(self): + self.asked += 1 + return {"DockerRootDir": self.root} + + +class Intercept(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.root = os.path.join(self.tmp, "docker-root") + os.makedirs(self.root) + self.containers = types.ModuleType("docker.models.containers") + exec(CONTAINERS, self.containers.__dict__) + self.original = self.containers.ContainerCollection.create + modules = mock.patch.dict(sys.modules, {"docker": types.ModuleType("docker"), + "docker.models": types.ModuleType("docker.models"), + "docker.models.containers": self.containers}) + modules.start() + self.addCleanup(modules.stop) + self.site = load_site(dict(OFF, **ON)) + self.client = Client(self.root) + + def start(self, image, environ=ON, **kwargs): + self.site.wrap_docker_create(dict(OFF, **environ)) + return self.containers.ContainerCollection(self.client).run(image, **kwargs) + + def test_the_providers_container_is_limited_on_dockers_disk(self): + device = sysfs(self, self.site, self.root) + made = self.start(PROVIDER, detach=True, device_write_bps=[{"Path": "/dev/x", "Rate": 1}]) + self.assertEqual(made.image, PROVIDER) + self.assertTrue(made.kwargs["detach"]) + self.assertEqual(made.kwargs["device_write_bps"], [{"Path": "/dev/x", "Rate": 1}, {"Path": device, "Rate": MB30}]) + self.assertEqual(made.kwargs["device_write_iops"], [{"Path": device, "Rate": 300}]) + self.assertIsNone(made.removed) + self.assertTrue(made.started) + + def test_a_partition_is_limited_on_its_disk(self): + device = sysfs(self, self.site, self.root, "partition") + self.assertEqual(self.start(PROVIDER).kwargs["device_write_bps"][0]["Path"], device) + + def test_a_device_mapper_volume_is_limited_as_it_is(self): + device = sysfs(self, self.site, self.root, "dm") + self.assertEqual(device.rsplit(os.sep, 1)[1], "dm-0") + self.assertEqual(self.start(PROVIDER).kwargs["device_write_iops"], [{"Path": device, "Rate": 300}]) + + def test_one_limit_alone(self): + device = sysfs(self, self.site, self.root) + made = self.start(PROVIDER, {"NULLIUS_OPTION_WRITE_IOPS": "150"}) + made.kwargs.pop("labels") + self.assertEqual(made.kwargs, {"device_write_iops": [{"Path": device, "Rate": 150}], + "network": self.site.NETWORK}) + + def test_the_disk_is_found_once_a_process(self): + sysfs(self, self.site, self.root) + self.start(PROVIDER) + self.containers.ContainerCollection(self.client).run(PROVIDER) + self.assertEqual(self.client.asked, 1) + + def test_another_image_is_left_alone(self): + sysfs(self, self.site, self.root) + made = self.start("ubuntu:24.04", detach=True) + self.assertEqual((made.image, made.kwargs, self.client.asked), ("ubuntu:24.04", {"detach": True}, 0)) + + def test_a_container_kept_without_the_limit_is_removed_and_fails_the_start(self): + sysfs(self, self.site, self.root) + self.containers.Container.drop = True + with self.assertRaises(self.site.WriteLimitError) as raised: + self.start(PROVIDER) + made = self.containers.ContainerCollection.made[-1] + self.assertEqual((made.removed, made.started), ({"force": True, "v": True}, False)) + self.assertIn("Disk write MB/s and Disk write IOPS to none", str(raised.exception)) + + def test_no_device_fails_the_start(self): + empty = os.path.join(self.tmp, "empty") + os.makedirs(empty) + with mock.patch.object(self.site, "SYS_BLOCK", empty): + with self.assertRaises(self.site.WriteLimitError) as raised: + self.start(PROVIDER) + self.assertIn(self.root, str(raised.exception)) + self.assertEqual(self.containers.ContainerCollection.made, []) + + def test_no_uevent_fails_the_start(self): + sysfs(self, self.site, self.root, "no uevent") + with self.assertRaises(self.site.WriteLimitError): + self.start(PROVIDER) + self.assertEqual(self.containers.ContainerCollection.made, []) + + def test_no_docker_answer_fails_the_start(self): + sysfs(self, self.site, self.root) + self.client.info = lambda: {} + with self.assertRaises(self.site.WriteLimitError): + self.start(PROVIDER) + + def test_only_a_positive_whole_number_is_a_limit(self): + for value in ("30mb", "0", "007", "\u0663", "\u00b2", "-1", " 30"): + with self.subTest(value), self.assertRaises(self.site.WriteLimitError) as raised: + self.site.wanted({"NULLIUS_OPTION_WRITE_MBPS": value}) + self.assertIn("Disk write MB/s is", str(raised.exception)) + self.assertIn("Disk write IOPS to none", str(raised.exception)) + self.assertEqual(self.site.wanted({"NULLIUS_OPTION_WRITE_MBPS": "30"}), + [("device_write_bps", MB30, "BlkioDeviceWriteBps", "--device-write-bps")]) + + def test_the_driver_and_the_wrapper_name_one_image(self): + driver = load_driver(self.tmp) + self.assertEqual(self.site.PROVIDER, driver.PROVIDER_IMAGE.split("@")[0]) + + def test_no_limit_adds_none_and_asks_docker_nothing(self): + made = self.start(PROVIDER, {}, detach=True) + self.assertEqual(sorted(made.kwargs), ["detach", "labels", "network"]) + self.assertEqual(self.client.asked, 0) + + def test_no_docker_py_changes_nothing(self): + with mock.patch.dict(sys.modules, {"docker": None, "docker.models": None, "docker.models.containers": None}): + self.site.wrap_docker_create(dict(OFF, **ON)) + self.assertIs(self.containers.ContainerCollection.create, self.original) + + def test_only_python_startup_wraps_and_only_for_docker(self): + self.assertIs(self.containers.ContainerCollection.create, self.original) + load_site(dict(ON, NULLIUS_OPTION_VM_PROVIDER="aws"), "sitecustomize") + self.assertIs(self.containers.ContainerCollection.create, self.original) + load_site(dict(ON, NULLIUS_OPTION_VM_PROVIDER="docker"), "sitecustomize") + self.assertTrue(hasattr(self.containers.ContainerCollection.create, "__wrapped__")) + + +class Processes(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + package = os.path.join(self.tmp, "fake", "docker", "models") + os.makedirs(package) + for folder in (package, os.path.dirname(package)): + open(os.path.join(folder, "__init__.py"), "w").close() + with open(os.path.join(package, "containers.py"), "w") as f: + f.write(CONTAINERS) + self.scripts = os.path.join(self.tmp, "scripts") + os.makedirs(self.scripts) + open(os.path.join(self.scripts, "sibling.py"), "w").close() + with open(os.path.join(self.scripts, "runner.py"), "w") as f: + f.write(CHILD) + + def child(self, **environ): + env = load_driver(self.tmp)._env("001", **dict(OFF, **environ)) + self.assertEqual(env["PYTHONPATH"].split(os.pathsep), [STARTUP, self.tmp]) + env["PYTHONPATH"] += os.pathsep + os.path.join(self.tmp, "fake") + ran = subprocess.run([sys.executable, os.path.join(self.scripts, "runner.py")], cwd=self.tmp, env=env, + capture_output=True, text=True, timeout=120) + self.assertEqual(ran.returncode, 0, ran.stderr) + self.assertEqual(ran.stderr, "") + return json.loads(ran.stdout) + + def test_the_limit_holds_in_the_runner_and_its_workers(self): + out = self.child(**ON) + self.assertEqual(out["path0"], self.scripts) + self.assertTrue(out["main"]) + for method in ("fork", "spawn"): + self.assertEqual(out[method], {"wrapped": True, "other": {"x": 1}}) + + def test_with_no_limit_the_create_is_still_wrapped(self): + out = self.child() + self.assertTrue(out["main"]) + for method in ("fork", "spawn"): + self.assertEqual(out[method], {"wrapped": True, "other": {"x": 1}}) + + def test_another_vm_provider_leaves_every_process_alone(self): + out = self.child(NULLIUS_OPTION_VM_PROVIDER="aws", **ON) + self.assertFalse(out["main"]) + for method in ("fork", "spawn"): + self.assertFalse(out[method]["wrapped"]) + + def test_the_file_comes_from_the_drivers_command(self): + elsewhere = os.path.join(self.tmp, "snapshot", "startup", "sitecustomize.py") + driver = load_driver(self.tmp, [os.path.join(HERE, "agents.json"), os.path.join(HERE, "nop.py"), elsewhere]) + self.assertEqual(driver._env("001")["PYTHONPATH"].split(os.pathsep)[0], os.path.dirname(elsewhere)) + + +class Before(unittest.TestCase): + """The driver's proof, before a VM start, that the limit holds.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.root = os.path.join(self.tmp, "docker-root") + os.makedirs(self.root) + self.calls = os.path.join(self.tmp, "calls") + self.driver = load_driver(self.tmp) + self.device = sysfs(self, self.driver._startup(), self.root) + self.kept = json.dumps({"BlkioDeviceWriteBps": [{"Path": self.device, "Rate": MB30}], + "BlkioDeviceWriteIOps": [{"Path": self.device, "Rate": 300}]}) + find = importlib.util.find_spec + spec = mock.patch.object(importlib.util, "find_spec", lambda n, *a: object() if n == "docker" else find(n, *a)) + spec.start() + self.addCleanup(spec.stop) + self.env = dict(OFF, **ON, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_ROOT=self.root, FAKE_HOST=self.kept) + + def check(self, **environ): + with mock.patch.dict(os.environ, dict(self.env, **environ)): + self.driver._write_limit() + + def docker_calls(self): + try: + with open(self.calls) as f: + return [ln.split()[0] for ln in f] + except OSError: + return [] + + def test_a_limit_docker_keeps_passes_and_the_probe_is_removed(self): + self.check() + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "inspect", "rm"]) + with open(self.calls) as f: + create = f.read().splitlines()[2].split() + self.assertTrue(create[create.index("--name") + 1].startswith(f"nullius-osworld2-probe-{os.getuid()}-{os.getpid()}-")) + self.assertEqual(create[create.index("--device-write-bps") + 1], f"{self.device}:{MB30}") + self.assertEqual(create[create.index("--device-write-iops") + 1], f"{self.device}:300") + self.assertNotIn("start", self.docker_calls()) + + def test_a_limit_docker_drops_fails_with_what_to_do(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_HOST=json.dumps({"BlkioDeviceWriteBps": None, "BlkioDeviceWriteIOps": []})) + said = str(raised.exception) + self.assertIn("BlkioDeviceWriteBps, BlkioDeviceWriteIOps", said) + self.assertIn("write limit discarded", said) + self.assertIn("set Disk write MB/s and Disk write IOPS to none", said) + self.assertEqual(self.docker_calls()[-1], "rm") + + def test_a_failed_removal_does_not_hide_why_the_probe_failed(self): + real = self.driver._docker + + def docker(*args, **kw): + if args[0] == "rm": + raise RuntimeError("docker rm got no answer") + return real(*args, **kw) + + with mock.patch.object(self.driver, "_docker", docker), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_FAIL="1") + self.assertIn("a probe container was refused", str(raised.exception)) + + def test_a_refused_probe_fails(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_FAIL="1") + self.assertIn("refused", str(raised.exception)) + self.assertEqual(self.docker_calls()[-1], "rm") + + def test_no_disk_fails_before_any_container(self): + with mock.patch.object(self.driver._startup(), "SYS_BLOCK", os.path.join(self.tmp, "none")): + with self.assertRaises(RuntimeError): + self.check() + self.assertEqual(self.docker_calls(), ["info"]) + + def test_no_docker_py_fails(self): + with mock.patch.object(importlib.util, "find_spec", lambda n, *a: None): + with self.assertRaises(RuntimeError) as raised: + self.check() + self.assertIn("no docker-py", str(raised.exception)) + self.assertIn("install the driver's environment again", str(raised.exception)) + + def test_no_docker_root_asks_after_the_daemon(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_ROOT="") + self.assertIn("is Docker's daemon running", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info"]) + + def test_cgroup_v1_fails_before_any_container(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CGROUP="1") + self.assertIn("cgroup v1", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info"]) + + def names(self, verb): + with open(self.calls) as f: + lines = [ln.split() for ln in f if ln.startswith(verb + " ")] + return [ln[ln.index("--name") + 1] if verb == "create" else ln[-1] for ln in lines] + + def test_every_check_waits_on_docker_twice_the_cleanups_time(self): + waited, real = [], subprocess.run + + def ran(*a, **kw): + waited.append((a[0][1], kw.get("timeout"))) + return real(*a, **kw) + + with mock.patch.object(self.driver.subprocess, "run", ran), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES="")): + self.check() + self.driver._provider_image() + self.driver._old_probes() + self.assertEqual(self.driver.READY_SECONDS, 120) + self.assertEqual(self.driver.DOCKER_SECONDS, 60) + self.assertEqual(waited, [(c, 120) for c in ("info", "ps", "create", "inspect", "rm", "image")] + [("ps", 60)]) + + def test_a_create_with_no_answer_is_tried_again_under_a_new_name(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1): + self.check(FAKE_CREATE_HANG="once") + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "create", "inspect", "rm"]) + first, second = self.names("create") + self.assertNotEqual(first, second) + self.assertEqual(self.names("rm"), [first, second]) + + def test_two_creates_with_no_answer_fail_in_one_line_and_leave_no_probe(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_HANG="all") + self.assertEqual(str(raised.exception), + "Docker's daemon did not answer docker create in 1 s: it may be busy; " + "try again or run fewer units at once") + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "create", "rm"]) + self.assertEqual(self.names("rm"), self.names("create")) + + def test_no_second_create_while_the_first_may_stand(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_HANG="all", FAKE_RM_FAIL="1") + self.assertIn("did not answer docker create in 1 s", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "rm"]) + self.assertEqual(set(self.names("rm")), set(self.names("create"))) + + def test_only_this_users_gone_or_old_probes_are_removed(self): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + now, me = __import__("time").time_ns(), f"nullius-osworld2-probe-{os.getuid()}" + probes = {"dead": f"{me}-{gone.pid}-{now}", "old": f"{me}-{os.getpid()}-{now - 601 * 10 ** 9}", + "live": f"{me}-{os.getpid()}-{now}", "other": f"nullius-osworld2-probe-{os.getuid() + 1}-{gone.pid}-{now}", + "odd": f"{me}-x"} + with mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES=" ".join(probes.values()))): + self.driver._old_probes() + with open(self.calls) as f: + removed = [ln.split()[2:] for ln in f if ln.startswith("rm")] + self.assertEqual(removed, [[probes["dead"], probes["old"]]]) + + def test_no_startup_file_fails(self): + with mock.patch.object(self.driver, "SITE", os.path.join(self.tmp, "gone", "sitecustomize.py")): + with self.assertRaises(RuntimeError) as raised: + self.check() + self.assertIn("Disk write IOPS to none", str(raised.exception)) + + def test_no_limit_or_another_provider_asks_nothing(self): + self.check(**OFF) + self.check(NULLIUS_OPTION_VM_PROVIDER="aws") + self.assertEqual(self.docker_calls(), []) + + def test_the_reaper_removes_a_gone_drivers_probe(self): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + probe = f"nullius-osworld2-probe-{os.getuid()}-{gone.pid}-{__import__('time').time_ns()}" + with mock.patch.object(self.driver, "SWEEPS", (0,)), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES=probe)): + self.driver.reap(gone.pid, self.tmp, True) + with open(self.calls) as f: + self.assertIn(f"rm -f {probe}", f.read().splitlines()) + + def test_a_vm_start_is_refused_before_the_runner_and_the_null_control(self): + starts = {"original": lambda: self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", + "http://relay/v1", "k"), + "nop": lambda: self.driver.score("tasks__001", self.tmp)} + for name, start in starts.items(): + with self.subTest(name), mock.patch.object(self.driver, "vm_image", lambda: "image"), \ + mock.patch.object(self.driver, "_task_file", lambda *a: None), \ + mock.patch.object(self.driver, "ASSETS", self.tmp), \ + mock.patch.object(self.driver, "_provider_image", lambda: None), \ + mock.patch.object(self.driver, "_run", side_effect=AssertionError("no process started")), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_HOST="{}")): + with self.assertRaises(RuntimeError) as raised: + start() + self.assertIn("the VM's disk write limit cannot be applied", str(raised.exception)) + + +class Options(unittest.TestCase): + def test_the_options_reach_the_original_setup_and_the_null_control(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + given = {"write_mbps": "30", "write_iops": "300"} + for setup in ("original", "yours"): + values, bad = definitions.bench_options_of(bench, "offline36", given, setup) + self.assertEqual(bad, []) + env = run.option_env(values) + self.assertEqual((env["NULLIUS_OPTION_WRITE_MBPS"], env["NULLIUS_OPTION_WRITE_IOPS"]), ("30", "300"), setup) + values, _ = definitions.bench_options_of(bench, "offline36", None, "yours") + self.assertEqual((values["write_mbps"], values["write_iops"]), ("none", "none")) + self.assertTrue(definitions.bench_options_of(bench, "offline36", {"write_mbps": "30mb"}, "original")[1]) + + +class Turn(unittest.TestCase): + def test_the_original_setups_turn_holds_500_steps(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + original = json.load(f)["setups"]["original"] + self.assertEqual(original["turn_seconds"], 86400) + self.assertGreater(original["turn_seconds"], 2 * 500 * 32) + + +class Snapshot(unittest.TestCase): + def test_a_hosts_snapshot_carries_the_startup_file(self): + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + for name in ("driver.py", "nop.py", "agents.json", "startup/sitecustomize.py"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + manifest = json.loads((Path(root) / "manifest.json").read_text()) + self.assertIn("catalogue/benchmarks/osworld2/startup/sitecustomize.py", manifest["files"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_prerequisites.py b/app/tests/test_prerequisites.py index 3b2c90bf..d47e5be4 100644 --- a/app/tests/test_prerequisites.py +++ b/app/tests/test_prerequisites.py @@ -46,6 +46,9 @@ def test_uv_harbor_and_a_setup_s_programs_are_looked_for_on_path(self): self.assertEqual((rows["setup:sandbox"]["state"], rows["setup:sandbox"]["detail"]), ("missing", "bwrap not on PATH")) self.assertEqual(rows["setup:sandbox"]["install"], ["sudo apt-get install -y bubblewrap"]) self.assertFalse(prerequisites.ready([{"rows": list(rows.values())}])) + self.assertEqual(rows["setup:sandbox"]["label"], "sandbox") + named = {n: dict(x, label="Sandbox") for n, x in SETUP.items()} + self.assertEqual(by_name(prerequisites.check_here([], named, which=lambda p: p in have))["setup:sandbox"]["label"], "Sandbox") def test_the_docker_answer_is_kept_a_minute_unless_asked_fresh(self): calls = [] @@ -106,6 +109,8 @@ def test(host, programs=None): self.assertEqual(sorted(asked["programs"]), ["bwrap", "git", "python3", "srt"]) self.assertEqual((rows["python3"]["state"], rows["git"]["state"], rows["setup:sandbox"]["state"]), ("ok", "missing", "missing")) self.assertEqual(prerequisites.check_host(self.HOST, ["docker"], test=self.probe())[1]["state"], "ok") + named = {n: dict(x, label="Sandbox") for n, x in SETUP.items()} + self.assertEqual(by_name(prerequisites.check_host(self.HOST, [], named, test=self.probe()))["setup:sandbox"]["label"], "Sandbox") def test_a_boat_run_s_programs_are_asked_of_its_host(self): # a run on Boat starts and fills its VMs from the host: ssh, ssh-keygen, rsync and tar there diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 6d283043..94e4dfa4 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -285,6 +285,7 @@ def test_review_asks_for_docker_for_every_setup_and_the_null_control(self): setups = {n["name"]: n for n in out["host"]["needs"]} rows = prerequisites.check_here([], setups, which=lambda p: None) self.assertEqual([(x["name"], x["state"]) for x in rows], [("setup:docker", "missing")]) + self.assertEqual(rows[0]["label"], "docker") self.assertFalse(prerequisites.ready([{"rows": rows}])) self.assertTrue(prerequisites.ready([{"rows": prerequisites.check_here([], setups, which=lambda p: "/usr/bin/" + p)}])) diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py index 320fe3be..f5f35b8a 100644 --- a/app/tests/test_standard_gated.py +++ b/app/tests/test_standard_gated.py @@ -128,6 +128,7 @@ def test_gated_is_true_or_false(self): self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", definitions.problems(dict(self.b, gated="yes"), "benchmark")) + self.assertIs(definitions.get("benchmark", "osworld2").get("gated"), True) if __name__ == "__main__": diff --git a/app/tests/test_standard_options.py b/app/tests/test_standard_options.py index c35ed797..9c35789c 100644 --- a/app/tests/test_standard_options.py +++ b/app/tests/test_standard_options.py @@ -41,10 +41,26 @@ def test_values_or_a_pattern_and_plain_words_only(self): self.assertIn("options.effort: values or pattern, one of them", both) quote = definitions.problems(agent(options={"effort": dict(EFFORT, values=["low", 'x"; rm', "medium"])}), "agent") self.assertIn("options.effort.values: distinct plain words", quote) - pattern = {"summary": "s", "pattern": "^[0-9]{1,3}$", "default": "40"} + pattern = {"summary": "s", "pattern": "^[0-9]{1,3}$", "default": "40", "example": "16"} self.assertEqual(definitions.problems(agent(options={"effort": pattern}), "agent"), []) self.assertTrue(definitions.problems(agent(options={"effort": dict(pattern, default="x")}), "agent")) + def test_a_pattern_carries_an_example_it_allows(self): + pattern = {"summary": "s", "pattern": "^[0-9]{1,3}$", "default": "40"} + self.assertEqual(definitions.problems(agent(options={"effort": pattern}), "agent"), []) + for example in (16, "x", "1234", "40"): + o = dict(pattern, example=example) + self.assertIn("options.effort.example: a value its pattern allows besides its default, shown in its place", definitions.problems(agent(options={"effort": o}), "agent")) + + def test_every_bundled_pattern_carries_an_example(self): + for kind in ("benchmark", "agent", "capability"): + for d in definitions.load(kind, root=definitions.CATALOGUE, every=True)[0]: + for n, o in (d.get("options") or {}).items(): + self.assertTrue("pattern" not in o or "example" in o, f"{d['_ref']} {n}") + + def test_a_value_its_pattern_refuses_is_told_by_its_example(self): + self.assertEqual(definitions.option_problem({"pattern": "[0-9]{1,3}", "example": "16"}, "abc"), "'abc' is not a value it takes, e.g. 16") + def test_a_name_and_a_summary(self): bad = definitions.problems(agent(options={"Effort": dict(EFFORT), "effort": dict(EFFORT, summary="")}), "agent") self.assertIn("options: 'Effort' is not a name (lower case letters and _)", bad) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 16a00593..07017006 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -287,6 +287,17 @@ def versioned(self): self.stack.enter_context(patch.object(definitions, "load", lambda kind, root_=None, every=False: _LOAD(kind, str(root), every=every))) self.stack.enter_context(patch.object(definitions, "get", lambda kind, key, root_=None: _GET(kind, key, str(root)))) + def test_an_option_with_a_pattern_shows_its_example_not_the_pattern(self): + root = self.tmp / "catalogue" + shutil.copytree(ROOT, root) + path = root / "agents" / "fake" / "agent.json" + path.write_text(json.dumps(dict(json.loads(path.read_text()), options={"turns": {"summary": "Turns.", "pattern": "[1-9][0-9]{0,2}", "example": "40", "default": "10"}}))) + self.stack.enter_context(patch.object(definitions, "load", lambda kind, root_=None, every=False: _LOAD(kind, str(root), every=every))) + self.stack.enter_context(patch.object(definitions, "get", lambda kind, key, root_=None: _GET(kind, key, str(root)))) + html = self.get("/agents/fake").get_data(as_text=True) + self.assertIn("e.g. 40", html) + self.assertNotIn("[1-9]", html) + def test_an_agent_with_versions_is_one_card_and_a_page_per_version(self): self.versioned() html = self.get("/agents").get_data(as_text=True) diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 3b8d5797..28dac581 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -34,7 +34,7 @@ def test_every_standard_benchmark_draws_its_where_step_and_says_why_docker_is_no self.assertIn('title="Not on this machine: docker is not installed"', tile) self.assertEqual(html.count("Not on this machine"), 1) container = re.search(r'

    ", self.draw(None, "r.others=[];")["osworld1 local"]) + self.assertIn("Docker: Docker without sudo", self.draw(None)["osworld2 local"]) + + def test_an_option_shows_its_example_and_a_setup_its_label(self): + page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") + cut = lambda start, end: page[page.index(start):page.index(end, page.index(start))] + line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" + schema = dict(self.schema, hosts=[], standardHere={"docker": None, "licences": {}, "host": {"osworld2": {"docker": ["missing"]}}}) + program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" + + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + cut(" function benchOptions(", " // Only credentials") + + "const out={};for(const k of ['osworld2','gsm8k']){const r=C.createStandard(k);out[k]=benchOptions(r,C.standardBench(r),'shared')+benchOptions(r,C.standardBench(r),'setup')+benchOptions(r,C.standardBench(r));}" + + "const r=C.createStandard('osworld2'),b=C.standardBench(r);r.datasets=['offline36'];out.offline=benchOptions(r,b,'dataset');r.datasets=['test_v2'];out.all=benchOptions(r,b,'dataset');" + + "process.stdout.write(JSON.stringify(out));") + html = json.loads(subprocess.run([self.node, "-"], input=program, check=True, capture_output=True, text=True).stdout) + self.assertIn('title="Disk write limit for each VM. e.g. 16"', html["osworld2"]) + self.assertIn('placeholder="Provider default"', html["gsm8k"]) + self.assertEqual(("Website host suffix" in html["offline"], "Website host suffix" in html["all"]), (False, True)) if __name__ == "__main__": diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index c729c90b..874cebe7 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -12,6 +12,7 @@ No workspace is opened; the pages read the shipped catalogue only.""" from __future__ import annotations +import json import re import unittest from contextlib import ExitStack @@ -146,7 +147,7 @@ def test_every_listed_benchmark_is_a_card_opening_its_page(self): cat = catalog.load() keys = re.findall(r'', 1)[0] for b in cat.listed_benchmarks: @@ -218,6 +219,15 @@ def test_the_page_shows_its_folder(self): self.assertIn("benchmarks/swe-bench-verified/ ├─ benchmark.json ├─ driver.py ├─ images.json ├─ standard.json └─ upstream-settings.json", self.page("swe-bench-verified")) self.assertIn("benchmarks/terminal-bench/ ├─ benchmark.json ├─ driver.py ├─ standard.json └─ upstream-settings.json", self.page("terminal-bench")) + def test_an_option_with_a_pattern_shows_its_example_not_the_pattern(self): + for key in ("osworld2", "gsm8k", "humaneval", "gaia", "osworld1", "tau3-bench"): + html, opts = self.page_html(key), json.loads((Path(catalog.__file__).parent / "benchmarks" / key / "standard.json").read_text())["options"] + for o in opts.values(): + if "pattern" in o: + self.assertIn("e.g. %s" % o["example"], html) + self.assertNotIn(o["pattern"], unescape(html)) + self.assertNotIn("Provider default", self.page("osworld1")) + def test_a_machine_setup_says_what_checks_it_and_what_installs_it(self): osworld = self.page("osworld1") self.assertIn("Setup: docker", osworld) @@ -225,6 +235,14 @@ def test_a_machine_setup_says_what_checks_it_and_what_installs_it(self): self.assertNotIn("nothing installs it", osworld) self.assertIn("the builder's Where step checks it and can install it.", self.page("tau3-bench")) + def test_a_machine_setup_shows_its_label_and_its_key_without_one(self): + osworld2 = self.page("osworld2") + for label in ("Docker", "Scoring programs"): + self.assertIn("Setup: " + label, osworld2) + self.assertIn("needs " + label, osworld2) + self.assertNotIn("Setup: scoring_programs", osworld2) + self.assertIn("Setup: docker", self.page("osworld1")) + class DatasetsOnTheirPage(PageCase): """A benchmark's page carries its datasets: the task count, the official diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index acbb99d9..135ae26e 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -225,7 +225,8 @@ def row(name): return re.search(r']*id="%s".*?' % name, html, re.S).group(0) hosts_row, key = row("HOSTS"), row("MODEL_API_KEY") self.assertNotIn(">not set<", hosts_row); self.assertNotIn('data-entry=', hosts_row); self.assertIn('data-add-host aria-label="Add a host"', html) - page = re.sub(r'(?s)

    .*?
    ', '', html.split('
    ]*data-key-kind="benchmark">.*?', '', html, flags=re.S) + page = re.sub(r'(?s)
    .*?
    ', '', page.split('
    ', html.index('id="settings-form"'))] + form = re.sub(r'
    ]*data-key-kind="benchmark">.*?
    ', '', form, flags=re.S) self.assertEqual(form.count('type="submit"'), 2) # one Save for the page # re-posting the page unchanged changes nothing; an emptied field keeps the value; × removes it self.post(); self.assertIn("Nothing changed", self.page()) diff --git a/app/web/agents.py b/app/web/agents.py index 8784355a..608e595f 100644 --- a/app/web/agents.py +++ b/app/web/agents.py @@ -78,7 +78,7 @@ def _entries(kind: str, every: bool = False) -> dict: "install": _install_words(o), # its options, each with its default and what else it takes; its fidelity proof, when it has one "options": [{"name": n, "summary": x.get("summary", ""), "default": x.get("default"), - "values": list(x.get("values") or []), "pattern": x.get("pattern", "")} + "values": list(x.get("values") or []), "pattern": x.get("pattern", ""), "example": x.get("example", "")} for n, x in (o.get("options") or {}).items()], "fidelity": fid if isinstance(fid, dict) else None}) out.sort(key=lambda e: (not e["d"].get("example", False), definitions.title(e["d"]).casefold())) diff --git a/docs/agents/supported/README.md b/docs/agents/supported/README.md index 396b2815..1fcedad4 100644 --- a/docs/agents/supported/README.md +++ b/docs/agents/supported/README.md @@ -49,17 +49,17 @@ Every agent in the catalogue, the ports each version has proven, and the benchma ### Benchmarks each agent fits - + - - - - - - - - - + + + + + + + + +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    codex@2.2.2✓✓✓✗✓✓✓
    codex@0.16.0✓✓✓✗✓✓✓
    goose✓✓✓✗✓✓✓
    jiuwenswarm@0.2.7✗✓✓✗✓✓✓
    jiuwenswarm@0.2.8✓✓✓✗✓✓✓
    mini-swe-agent✗✗✗✗✓✗✗
    opencode✓✓✓✗✓✓✓
    reference✗✓✓✗✓✓✓
    terminus-2✗✗✗✗✗✗✓
    codex@2.2.2✓✓✓✗✗✓✓✓
    codex@0.16.0✓✓✓✗✗✓✓✓
    goose✓✓✓✗✗✓✓✓
    jiuwenswarm@0.2.7✗✓✓✗✗✓✓✓
    jiuwenswarm@0.2.8✓✓✓✗✗✓✓✓
    mini-swe-agent✗✗✗✗✗✓✗✗
    opencode✓✓✓✗✗✓✓✓
    reference✗✓✓✗✗✓✓✓
    terminus-2✗✗✗✗✗✗✗✓
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/codex.md b/docs/agents/supported/codex.md index f2e30756..3ef0213a 100644 --- a/docs/agents/supported/codex.md +++ b/docs/agents/supported/codex.md @@ -34,10 +34,10 @@ OpenAI's coding agent. ## Benchmarks it fits - + - - + +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    codex@2.2.2✓✓✓✗✓✓✓
    codex@0.16.0✓✓✓✗✓✓✓
    codex@2.2.2✓✓✓✗✗✓✓✓
    codex@0.16.0✓✓✓✗✗✓✓✓
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/goose.md b/docs/agents/supported/goose.md index 4ac61db7..47c858eb 100644 --- a/docs/agents/supported/goose.md +++ b/docs/agents/supported/goose.md @@ -30,9 +30,9 @@ The AAIF's open-source agent, formerly Block's. ## Benchmarks it fits - + - +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    goose✓✓✓✗✓✓✓
    goose✓✓✓✗✗✓✓✓
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/jiuwenswarm.md b/docs/agents/supported/jiuwenswarm.md index 0a536b16..3718dadb 100644 --- a/docs/agents/supported/jiuwenswarm.md +++ b/docs/agents/supported/jiuwenswarm.md @@ -34,10 +34,10 @@ openJiuwen's agent. ## Benchmarks it fits - + - - + +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    jiuwenswarm@0.2.7✗✓✓✗✓✓✓
    jiuwenswarm@0.2.8✓✓✓✗✓✓✓
    jiuwenswarm@0.2.7✗✓✓✗✗✓✓✓
    jiuwenswarm@0.2.8✓✓✓✗✗✓✓✓
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/mini-swe-agent.md b/docs/agents/supported/mini-swe-agent.md index dec313f7..1d514257 100644 --- a/docs/agents/supported/mini-swe-agent.md +++ b/docs/agents/supported/mini-swe-agent.md @@ -30,9 +30,9 @@ Princeton's 100-line agent, bash only. ## Benchmarks it fits - + - +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    mini-swe-agent✗✗✗✗✓✗✗
    mini-swe-agent✗✗✗✗✗✓✗✗
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/opencode.md b/docs/agents/supported/opencode.md index 34e1e610..d72342fe 100644 --- a/docs/agents/supported/opencode.md +++ b/docs/agents/supported/opencode.md @@ -30,9 +30,9 @@ The open-source terminal coding agent. ## Benchmarks it fits - + - +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    opencode✓✓✓✗✓✓✓
    opencode✓✓✓✗✗✓✓✓
    - ✗: the missing port, on hover. diff --git a/docs/agents/supported/terminus-2.md b/docs/agents/supported/terminus-2.md index 26469d36..e41d2acb 100644 --- a/docs/agents/supported/terminus-2.md +++ b/docs/agents/supported/terminus-2.md @@ -30,9 +30,9 @@ Harbor's terminal agent, used by Terminal-Bench. ## Benchmarks it fits - + - +
    GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
    GAIAGSM8KHumanEvalOSWorld1OSWorld2SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
    One task, Workspace files, SearchOne taskWorkspace filesComputer useComputer useWorkspace filesMCP tools, ConversationWorkspace files
    terminus-2✗✗✗✗✗✗✓
    terminus-2✗✗✗✗✗✗✗✓
    - ✗: the missing port, on hover. diff --git a/docs/benchmarks/osworld2-machine.md b/docs/benchmarks/osworld2-machine.md new file mode 100644 index 00000000..16b6ddb8 --- /dev/null +++ b/docs/benchmarks/osworld2-machine.md @@ -0,0 +1,192 @@ +# Prepare a machine for OSWorld2 + +OSWorld2's driver, its VMs, the mock websites, GitLab and the proxies run on one machine: this one, or the host chosen in the builder's **Where** step ([Hosts](../settings/run-hosts.md)). Run every step below there, as the user that runs Nullius, from the root of a checkout of this repository; a run on a host starts from the builder's Review, which sends the key to it; the last step's commands run it here with this machine's keys. How a run is scored: [Read an OSWorld2 run](osworld2-results.md). + +## What it needs + +The minimum is one task at a time: + +| Need | Amount, or how to check | +| --- | --- | +| System | Linux on x86_64 with apt; the commands below are Ubuntu's | +| Docker | `docker ps` answers without sudo; Docker Compose 2.24.4 or later and Buildx for the mock websites and GitLab | +| Kernel | `ip_tables` and `iptable_nat` loaded (`lsmod`): without them the VM's ports are not forwarded and it never becomes ready | +| Network | The subnet `10.213.0.0/24` free: Nullius makes the VMs' network `nullius-osworld2` on it, and the websites listen on its gateway, `10.213.0.1` | +| Libraries | `build-essential` to install the driver's environment, `libgl1` to run it | +| Scoring programs | ImageMagick (`identify`, `convert`), `ffmpeg`, `ffprobe` and `tesseract`: the evaluate() of tasks 025, 046, 051, 056, 085 and 095 runs them on this machine, and a run, the null control's too, is refused until they are on PATH | +| Programs | `git`, `python3`, `uv`, `curl` | +| CPU and memory | One VM, which the upstream's docker provider sets to 4 cores and 4 GB of memory, beside the driver, the agent's script and the services below | +| Disk | About 15 GB for the VM image (30 GB while its zip is unpacked), 5 GB for the gated assets, 2 GB for the driver's environment; under Docker's root, 2 GB for each running VM, about 20 GB for the mock websites' images (25 GB free while they build) and 7 GB for GitLab's; about 0.5 GB of output for 100 steps of M3. Measured: leave room above it | +| Network | GitHub, PyPI, Hugging Face and Docker Hub for the installs and the first GitLab task; OpenRouter for the agents and the LLM grader; the public internet for 29 tasks, 8 of them through a proxy of their own, which also install from the Ubuntu mirrors | +| KVM | Recommended: `/dev/kvm`; without it the upstream runs the VM unaccelerated, slower | + +More at once is optional: `-j N` runs N tasks side by side, each with a VM of its own, so N times its cores, memory and disk. Their VMs start one after another. A machine's constraints in Settings (CPUs, memory, processes) do not reach the VMs, the mock websites, GitLab or the proxies: Docker starts them outside the run's limits, so bound them with `-j`. + +One Unix user runs a machine's VMs: the upstream's docker provider keeps its port lock in `/tmp`, and another user's VMs then fail to start. Task 082 takes this machine's port 3000 on 127.0.0.1 for its VM, so its units run one at a time per machine, and fail before the VM starts if another program listens on that port. + +!!! warning "The machine's speed can change a task's outcome" + The upstream waits fixed times, not for the desktop: 60 s after a task is set up, 3 s after each action. Compare scores from machines of similar speed and load. + +## Which tasks it runs + +All 108 tasks of release `osworld-v2.1` run on the docker provider, but 103 and 104. + +| Tasks | Needs | +| --- | --- | +| 63, No hosted services | The VM and the gated assets; 27 of them also the LLM grader, the simulated user or the public internet | +| 35, Mock website tasks | The mock websites, step 8; Overleaf keeps one account for all, so its 2 tasks run one at a time, each on Overleaf started anew, a few minutes more | +| 2, tasks 026 and 041 | The mock websites, step 8, which serve the GitLab that Nullius makes on this machine before the first of them, about 8 minutes once; they run one at a time, each starting GitLab anew from its snapshot, about 5 minutes more | +| 8 that ask for a proxy | The proxy image, step 7, and for some the mock websites | + +103 and 104 are not supported: their evaluate() runs FreeCAD's Python on this machine, which Nullius neither checks nor installs. They stay in the datasets; without FreeCAD the null control scores them 0, and an agent's unit ends in an error. + +**Website host suffix** is a wildcard DNS name of the address the mock websites listen on, the VMs' gateway: `10.213.0.1.nip.io`. **VM provider** offers `docker` alone: the upstream's other providers need cloud accounts or a hypervisor of their own. + +## Set it up + +1. **See where it installs.** Nullius installs under `standard/` in its data directory (`STANDARD_DIR`; `~/.local/share/nullius/standard` on Linux): + + ```bash title="The driver's environment (ENV), OSWorld-V2's clone (UP) and the benchmark's folder (HERE)" + eval "$(PYTHONPATH=app python3 -c ' + import shlex + from nullius.standard import definitions, install + b = definitions.get("benchmark", "osworld2") + print("ENV=" + shlex.quote(install.driver_dir(b))) + print("UP=" + shlex.quote(install.upstream_dir(b))) + print("HERE=" + shlex.quote(b["_here"])) + ')" + echo "$ENV"; echo "$UP"; echo "$HERE" + ``` + + Where that disk lacks the space, make the data directory a link to a larger one before anything is installed (here `/data/nullius`, which must not exist yet): + + ```bash title="The data directory on /data" + if [ -e ~/.local/share/nullius ]; then mv ~/.local/share/nullius /data/nullius; else mkdir -p /data/nullius; fi + ln -s /data/nullius ~/.local/share/nullius + ``` + +2. **Install Docker, the libraries, the scoring programs and uv**, and load the kernel modules. + + ```bash title="System packages" + sudo apt-get update + sudo apt-get install -y docker.io docker-compose-v2 docker-buildx git curl unzip build-essential libgl1 imagemagick ffmpeg tesseract-ocr + sudo usermod -aG docker "$USER" # applies from the next login + sudo modprobe -a ip_tables iptable_nat + printf 'ip_tables\niptable_nat\n' | sudo tee /etc/modules-load.d/osworld2.conf + command -v uv || { curl -LsSf https://astral.sh/uv/install.sh | sh && . "$HOME/.local/bin/env"; } + ``` + + ```bash title="Check" + docker ps && docker compose version && docker buildx version + lsmod | grep -E '^(ip_tables|iptable_nat) ' + command -v identify convert ffmpeg ffprobe tesseract + test -e /dev/kvm && echo "KVM: yes" || echo "KVM: no" + ``` + +3. **Install OSWorld-V2 and its environment**, with Nullius's own installer, as a run would: OSWorld-V2 at its tested commit, then its `uv.lock`. A run that finds both installs nothing. + + ```bash title="Clone and install" + PYTHONPATH=app python3 -c ' + from nullius.standard import definitions, install + b = definitions.get("benchmark", "osworld2") + print(install.driver_packages(b)) + ' + ``` + +4. **Fetch the gated tasks and assets.** Ask for access to `xlangai/osworld_v2_tasks` and `xlangai/osworld_v2_assets_gated` on Hugging Face, log in with a token that has it, then run the upstream's own downloaders at the release: + + ```bash title="Task classes into evaluation_examples/task_class, assets into cache/osworld_v2_assets" + "$ENV/.venv/bin/hf" auth login + (cd "$UP" && "$ENV/.venv/bin/python" scripts/tools/download_osworld_v2_tasks.py --benchmark-release osworld-v2.1) + (cd "$UP" && "$ENV/.venv/bin/python" scripts/tools/download_osworld_v2_assets.py --benchmark-release osworld-v2.1) + ls "$UP"/evaluation_examples/task_class/task_*.py | wc -l # 108 + ``` + +5. **Fetch the VM image**, with the upstream's own downloader, at the revision it pins. Run again, it resumes: + + ```bash title="Download and unzip osworld-v2-ubuntu-x86.qcow2" + (cd "$UP" && PYTHONPATH="$UP" "$ENV/.venv/bin/python" -c "from desktop_env.providers.docker.manager import DockerVMManager; DockerVMManager().get_vm_path(os_type='Ubuntu', region=None)") + ls -l "$UP"/docker_vm_data/osworld-v2-ubuntu-x86.qcow2 + ``` + + Once the image is unpacked, the zip can go. Where Hugging Face is out of reach, `HF_ENDPOINT=https://hf-mirror.com` sends the upstream to its mirror. + +6. **Pull the VM's Docker image**, by the digest the driver runs: + + ```bash + docker pull happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9 + ``` + +7. **Build the proxy image**, for the 8 tasks that ask for a proxy (3 of Offline, none of No hosted services). Each such unit gets a tinyproxy of its own on this machine, given to the upstream in place of its DataImpulse entry, whose password is a placeholder. The upstream installs tinyproxy in the VM as each of the 8 is set up, through apt, which goes out directly: the 3 of Offline then need the Ubuntu mirrors too, and each is set up more slowly. + + ```bash + python3 "$HERE/proxy.py" --build + ``` + +8. **Build and start the mock websites**, for their 40 tasks: OSWorld-web at the tag `osworld-v2.1`, with its submodules, served under `10.213.0.1.nip.io` (no task of this release uses Moodle), here cloned into `/data`, a folder of this user's. The write limits are in MB/s and IOPS: the stack's share of the disk (below), and the builder's, here a whole budget of 120 and 1200 while nothing else runs. + + ```bash title="Clone, build, start, check" + git clone --recurse-submodules --branch osworld-v2.1 https://github.com/Task-Web/OSWorld-web.git /data/OSWorld-web + python3 "$HERE/websites.py" configure /data/OSWorld-web 10.213.0.1.nip.io --write-mbps 8 --write-iops 80 + python3 "$HERE/websites.py" builder /data/OSWorld-web up --write-mbps 120 --write-iops 1200 + python3 "$HERE/websites.py" build /data/OSWorld-web + python3 "$HERE/websites.py" builder /data/OSWorld-web down + python3 "$HERE/websites.py" reset /data/OSWorld-web + python3 "$HERE/websites.py" smoke /data/OSWorld-web + ``` + + The build stops before a group that would leave less than 25 GB free under Docker's root. Give runs the suffix as **Website host suffix** (`websites=10.213.0.1.nip.io`). + +9. **Add the keys** on the machine where Nullius runs, in Settings ([Model providers](../settings/model-providers.md)), never on a host: OpenRouter, for the agents and for the LLM grader, which the null control calls too on its 21 graded tasks. The builder sends a run on a host its key. Muse Spark's calls are priced only where OpenRouter says their cost; for an estimate and a spending cap, add its price: `nullius prices set meta/muse-spark-1.3 1.25 4.25 --cached 0.15`. + +10. **Run one task of the null control** on this machine, then no VM left: + + ```bash + PYTHONPATH=app python3 -m nullius standard run osworld2 nop --dataset offline36 --limit 1 --provider openrouter + docker ps --filter network=nullius-osworld2 + ``` + + The job is written under `jobs_standard/`; its task scores 0. Then one task of an agent, paid: `PYTHONPATH=app python3 -m nullius standard run osworld2 original --dataset offline36 --limit 1 --provider openrouter` asks M3 with MiniMax-M3 for up to 100 steps. + + Run the null control on the local backend: on the container backend, a run killed with SIGKILL leaves containers that fail the next run of that name. Never reuse a `--name`: the job would mix both runs, its `config.json` and `result.json` describing the last one only. + +## Write limits and load + +On a disk shared with others, limit what the VMs write with **Disk write MB/s** and **Disk write IOPS** (`write_mbps`, `write_iops`): each VM's container has a write limit of its own on Docker's disk, so with `-j N` give each the machine's budget divided by N, after the websites' share (at least 80 IOPS) and a GitLab task's, whose GitLab takes the VM's limits again, or 16 MB/s and 160 IOPS where they are none. Where Docker cannot apply a limit (btrfs, zfs, no io controller, cgroup v1), a unit with one fails before its VM starts. A tight limit slows the VM and can change a task's outcome. + +| Run | Time | Cost | +| --- | --- | --- | +| The null control, all 108 tasks | 7 to 10 hours of units; 1 h 23 min seven VMs at a time, a GitLab task counting as two, 2 h 18 min four at a time beside agent runs | About $0.15, the LLM grader | +| One task of M3, 100 steps | 52 min for task 010 | About $0.60 | +| One task of M3, 500 steps, Thinking adaptive | 5 min to 2 h 20 min on 7 tasks | $0.04 to $5.17 | +| One task of Muse Spark, 500 steps | About 30 min on 2 tasks | $4.08 to $5.34 | + +```bash title="The null control on all 108 tasks, six at a time" +PYTHONPATH=app python3 -m nullius standard run osworld2 nop --dataset test_v2 -j 6 --provider openrouter \ + --bench-option websites=10.213.0.1.nip.io --bench-option write_mbps=16 --bench-option write_iops=160 +``` + +## What is pinned + +| What | Pin | +| --- | --- | +| OSWorld-V2's code | Tag `osworld-v2.1`, commit `3d778a3c9a34a079316f70df023b166700445792`, its `uv.lock` | +| Task classes and assets | Hugging Face, tag `osworld-v2.1` of `xlangai/osworld_v2_tasks` and `xlangai/osworld_v2_assets_gated` | +| VM image | `xlangai/v2-image` at `6e16459a2feb5a8f1ed65babcfe7a2a6205d049d` | +| VM's Docker image | `happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9` | +| Mock websites | Task-Web/OSWorld-web, tag `osworld-v2.1`, commit `60c89fe6a8ed934668619d8d26132848239eb8ee` | +| GitLab | Task-Web/gitlab, commit `8655d651722f4254e59e813de9f68a6732ea525c` | +| Proxy | Alpine 3.24 by digest, its tinyproxy 1.11 (`proxy.py`) | + +## When it fails + +| Seen | Do | +| --- | --- | +| `VM failed to become ready` | Load `ip_tables` and `iptable_nat` (step 2) | +| `ImportError: libGL.so.1`, or `evdev` fails to build | Install `libgl1`, `build-essential` (step 2) | +| `permission denied` on the Docker socket | Log in again after `usermod`, in a new terminal session | +| A run refused for `scoring_programs` | Install ImageMagick, ffmpeg and tesseract (step 2) | +| `no task class`, `no assets` or `no VM image` | Steps 4 and 5 | +| A websites task refused before its VM starts | Step 8 and Website host suffix; `python3 "$HERE/websites.py" check 10.213.0.1.nip.io` | +| A GitLab task fails while Nullius makes its GitLab | Step 8; GitHub and Docker Hub reachable, and about 7 GB free under Docker's root | +| A Docker volume left after runs | `docker volume ls -f dangling=true`; remove your own | diff --git a/docs/benchmarks/osworld2-results.md b/docs/benchmarks/osworld2-results.md new file mode 100644 index 00000000..cd772e8d --- /dev/null +++ b/docs/benchmarks/osworld2-results.md @@ -0,0 +1,46 @@ +# Read an OSWorld2 run + +How OSWorld2's units are run and scored here, where that differs from the upstream or the paper. Preparing the machine is [OSWorld2 machine setup](osworld2-machine.md). + +## Against the paper + +The paper (Table 3) ran M3 on release osworld-v2-2026.06.24, on AWS with a residential proxy, 500 steps and thinking on. This entry runs release osworld-v2.1, whose task files differ for 80 of the 108 tasks, on the docker provider with no residential proxy, and by default 100 steps with thinking off. Steps per task 500 and Thinking adaptive restore the budget only. Nullius labels a run comparable with the paper by its model alone: check the release, dataset, steps, thinking and who served the model (Served by) before comparing scores. + +Muse Spark runs as published at this release, by default with its published budget of 500 steps. Its published score came from newer upstream code, Meta's own API and grader: scores here do not compare with it. + +Neither agent's history is shortened beyond the upstream's own screenshot trimming. In a run on OpenRouter pinned to MiniMax with Thinking adaptive, M3's history filled its window after about 220 steps: the call that overflows ends the episode, and the task's evaluate() scores the VM as it is. + +## The LLM grader and the simulated user + +The LLM grader (21 tasks) and the simulated user (6 tasks) are Claude Sonnet 4.6 through the run's provider, whichever agent runs; a null control pays for the grader too. The simulated user's calls count in the agent's tokens and cost; with M3 it is almost never asked. + +## Particular tasks + +- 29 tasks use the public internet: their traffic leaves from this machine, and the live sites can answer differently. +- Tasks 001, 006 and 010 start their mail client offline, unlike the upstream. +- Tasks 103 and 104 are not supported: see [Which tasks it runs](osworld2-machine.md#which-tasks-it-runs). +- The VM's clock runs at UTC+8: a task that reads the date, set up before 16:00 UTC and acted on after, can score 0: start runs clear of 16:00 UTC. + +## What a trial says + +| In the trial | What it means | What to do | +| --- | --- | --- | +| `setup_errors` in the score's record (`verifier/score.json`) | a setup step failed and the episode went on, as the upstream does | read the score as from a VM set up in part | +| `stopped`, `stopped_by` time, money, context window or other | a guard (its 10-hour limit, a spend guard or the model's window) stopped the episode, or came while its last reply was on its way; it was scored and stays in the mean | count it as a scored run that a guard marked | +| `grader_refused` | the grader refused an image that was not whole, of another type or over 5 MB, and the upstream's score was kept; the image can be the task's own | where the null control or several tasks carry it, read those tasks before counting their zeros against the agent | +| unscored, the LLM grader failed or its endpoint answered an error | an infrastructure error, outside the agent's rate | check that the provider serves Claude Sonnet 4.6, then run the unit again | +| unscored, an evaluation that gave up on the VM or a mock website | the upstream scored it without that read; its message gives that score | run the unit again | +| an error with no score: a screenshot the VM did not send, or Chrome DevTools timing out twice (told from the unit's log) | the upstream's own code failed | run the unit again | +| a setup error after the runner was killed by signal 9, 15 or 1 | the runner was terminated, for example when the machine ran out of memory | check the machine, then run the unit again | +| an error of no known cause | the runner died without its cleanup, of another signal with no traceback, or of another crash after a failed screenshot | read the log the message names, then run the unit again | +| rate-limited, or an error because the runner did not end an hour after its limit (a minute for the null control) | not run again unless retries are set | run the unit again | + +A time limit forced in the builder's Limits step or with `--timeout` ends a unit with no score where it is shorter than OSWorld2's own: leave them empty. + +## The null control + +OSWorld-V2 publishes no solutions, so there is no reference control. The null control acts DONE at once and scores the untouched VM, with 3600 s for each task's score step. It scores above 0 on 015, 076, 090 and 092, and on 077 in some runs. Its bound, a mean of at most 0.01, catches only errors: read it task by task. + +## What a trial keeps + +Each trial keeps the upstream's screenshots and API logs. The logs keep each image once, in `api_logs/images`: `python3 api_logs.py restore` on the `api_logs` folder, from this benchmark's folder, gives the files back byte for byte. A job copied from a host before packing ended can hold them partly packed: finish packing or restoring on the host, then copy it again. diff --git a/docs/benchmarks/supported/README.md b/docs/benchmarks/supported/README.md index 8401e40a..68b87061 100644 --- a/docs/benchmarks/supported/README.md +++ b/docs/benchmarks/supported/README.md @@ -18,6 +18,7 @@ Every benchmark in the catalogue: where it runs, what it needs, the ways it runs
    LoCoMo
    locomoVery long-term conversational memory over conversations of 19 to 32 sessions.Memory✓–10Official LongMemEval
    longmemevalLong-term chat memory: questions over timestamped chat histories.Memory✓–1500Official OSWorld1
    osworld1Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of its own.Computer use✓–369Official +OSWorld2
    osworld2Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a VM of its own.Computer use✓–108Official SWE-bench Verified
    swe-bench-verifiedSoftware engineering: fix a real GitHub issue so the failing tests pass.Coding✓✓500Official τ³-bench
    tau3-benchCustomer service with domain tools and a simulated user: airline, retail, telecom, banking.Customer service✓✓375Official Terminal-Bench 2.0
    terminal-benchReal terminal tasks in a container: build, configure, debug.Terminal✓✓89Official @@ -34,6 +35,7 @@ Every benchmark in the catalogue: where it runs, what it needs, the ways it runs LoCoMocap.memory/v1NulliusNullius✓✓✓ LongMemEvalcap.memory/v1NulliusNullius✓✓✓ OSWorld1agent.computer/v1Nullius–✓✓✓ +OSWorld2agent.computer/v1Nullius–✓✓– SWE-bench Verifiedagent.workspace/v1Nulliusthe authors✓✓✓ τ³-benchagent.tools/v1, agent.conversation/v1NulliusNullius✓✓✓ Terminal-Bench 2.0agent.workspace/v1Nulliusthe authors✓✓✓ diff --git a/docs/benchmarks/supported/gaia.md b/docs/benchmarks/supported/gaia.md index 76deed64..e80d708c 100644 --- a/docs/benchmarks/supported/gaia.md +++ b/docs/benchmarks/supported/gaia.md @@ -41,7 +41,7 @@ Real-world questions needing reasoning, web research and file understanding. | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| -| `temperature` | pattern `default\|(?:0\|[1-9][0-9]*)(?:\.[0-9]+)?`, default `default` | Provider default sends none; a non-negative decimal sets it. The paper does not state it. | `original` | +| `temperature` | e.g. `0.5`, default `default` | Provider default sends none; a non-negative decimal sets it. The paper does not state it. | `original` | ## Good to know diff --git a/docs/benchmarks/supported/gsm8k.md b/docs/benchmarks/supported/gsm8k.md index 73b16cf2..4fc8d07a 100644 --- a/docs/benchmarks/supported/gsm8k.md +++ b/docs/benchmarks/supported/gsm8k.md @@ -39,7 +39,7 @@ Grade-school math word problems that take two to eight steps of basic arithmetic | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| -| `temperature` | pattern `default\|(?:0\|[1-9][0-9]*)(?:\.[0-9]+)?`, default `default` | Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model. | `original` | +| `temperature` | e.g. `0.5`, default `default` | Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model. | `original` | ## Good to know diff --git a/docs/benchmarks/supported/humaneval.md b/docs/benchmarks/supported/humaneval.md index 9b58a16c..462b30c5 100644 --- a/docs/benchmarks/supported/humaneval.md +++ b/docs/benchmarks/supported/humaneval.md @@ -38,7 +38,7 @@ Python functions written from their signature and docstring, each checked by its | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| -| `temperature` | pattern `default\|(?:0\|[1-9][0-9]*)(?:\.[0-9]+)?`, default `default` | Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model. | `original` | +| `temperature` | e.g. `0.5`, default `default` | Sampling randomness. Provider default sends no temperature; enter a non-negative decimal to override it. Accepted values depend on the model. | `original` | ## Good to know diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 43d1e3a8..f8d53f05 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -38,7 +38,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| | `agent` | **`m3` (M3 Agent)**, `prompt` (PromptAgent), `claude` (Claude computer use), `gpt54` (GPT-5.4), `openai_cua` (OpenAI CUA), `qwen` (Qwen), `os_symphony` (OS-Symphony), `evocua` (EvoCUA), `autoglm` (AutoGLM), `autoglm_v` (AutoGLM-V), `mano` (Mano), `muse_spark` (Muse Spark) | Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit. | `original`, `model` | -| `max_steps` | pattern `default\|[1-9][0-9]{0,3}`, default `default` | The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it. | `original`, `model` | +| `max_steps` | e.g. `50`, default `default` | The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it. | `original`, `model` | | `thinking` | **`adaptive`**, `off` | M3_THINKING_MODE: adaptive, as run_multienv_m3.py's own documented invocation exports it (the model decides whether to think; temperature 1, no top_p), or off, the agent's code default when the variable is unset. | `original`, `model` | | `vm_provider` | **`docker`** | --provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own. | `original`, `model`, `yours` | diff --git a/docs/benchmarks/supported/osworld2.md b/docs/benchmarks/supported/osworld2.md new file mode 100644 index 00000000..9f86f5fa --- /dev/null +++ b/docs/benchmarks/supported/osworld2.md @@ -0,0 +1,82 @@ + + +# OSWorld2 + +Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a VM of its own. + + + + + + + + + + +
    Keyosworld2
    CategoryComputer use
    OriginOfficial
    Runs onNullius
    Tasks108
    PaperOSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks, arXiv · 2026
    Needs, ports provenagent.computer/v1
    + +## Datasets + +| Dataset | Key | Tasks | From | Verifier | +|---|---|---|---|---| +| OSWorld-V2, release osworld-v2.1 | `osworld2` | 108 | GitHub [xlang-ai/OSWorld-V2](https://github.com/xlang-ai/OSWorld-V2/tree/3d778a3c9a34a079316f70df023b166700445792) | Each task's own evaluate() on the VM's final state, 0 to 1. | + +## On the standard + +| Control | Written by | +|---|---| +| `nop` | Nullius | + +### Versions + +| Version | Setups as published, whose model | Options | +|---|---|---| +| `osworld2` | Original · OSWorld-V2 agents · Like the paper or Any of your choice | `agent`, `max_steps`, `thinking`, `vm_provider`, `websites`, `write_iops`, `write_mbps` | + +## Options + +| Option | Values, default in bold | What it sets | Setups | +|---|---|---|---| +| `agent` | **`m3` (MiniMax M3)**, `muse_spark` (Muse Spark) | Which of OSWorld-V2's own agents runs: each its script as published, with its own prompts, tools and history, asking the model it is published with. | `original` | +| `max_steps` | e.g. `500`, default `default` | The agent's step limit. default: each agent's own, as its wrapper runs it. | `original` | +| `thinking` | **`off`**, `adaptive` | Whether M3 thinks. The paper ran it adaptive. | `original` | +| `vm_provider` | **`docker`** | Where the VMs run. | `original`, `yours` | +| `websites` | e.g. `10.213.0.1.nip.io`, default `none` | Where the mock websites, and GitLab with them, are served. Needed by the tasks that use them. | `original`, `yours` | +| `write_iops` | e.g. `160`, default `none` | Disk write limit for each VM. | `original`, `yours` | +| `write_mbps` | e.g. `16`, default `none` | Disk write limit for each VM. | `original`, `yours` | + +| Option | Value | Refused without (`host`: the driver's machine; `embeddings`: a model the provider serves) | +|---|---|---| +| `vm_provider` | `docker` | host: docker, scoring_programs | + +## Good to know + +- Partial credit: each task scores 0 to 1; the score is the mean. + +## Nullius limitations + +- Not the paper's M3 setup: another release, no residential proxy, and 100 steps with thinking off by default; Steps per task 500 and Thinking adaptive restore the budget. +- Muse Spark's published score came from newer upstream code: scores here do not compare with it. +- The LLM grader (21 tasks) and the simulated user (6) are Claude Sonnet 4.6: the run's provider must serve it. +- 29 tasks use the public internet: their outcome can change with the live sites. +- Tasks 103 and 104 are not supported: they stay in the datasets, and an agent's unit ends in an error. +- A slow or loaded machine, or a run across 16:00 UTC (the VM's midnight), can change scores. +- The null control scores above 0 on 015, 076, 090 and 092, and on 077 in some runs. +- Only OSWorld-V2's own M3 and Muse Spark run it; no agent from the Agents page yet. +- Fetch the gated tasks and prepare the machine first: see OSWorld2 machine setup in the docs. + +## Cite + +??? citation "BibTeX" + + ```bibtex + @misc{yuan26osworld2, + title={OSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks}, + author={Mengqi Yuan and Zilong Zhou and Xinzhuang Xiong and Weiming Wu and Jiayang Sun and Jiamin Song and Kaiqian Cui and Bowen Wang and Haoyuan Wu and Yitong Li and Dunjie Lu and Haikong Lu and Qi Zhen and Xinyuan Wang and Jiaqi Deng and Yuhao Yang and Cheng Chen and Boyuan Zheng and Alex Su and Xiao Yu and Hao Zou and Saaket Agashe and Xing Han Lu and Manpreet Kaur and Zhengyang Qi and Vincent Sunn Chen and Frederic Sala and Dayiheng Liu and Junyang Lin and Zhou Yu and Yu Su and Siva Reddy and Xin Eric Wang and Peng Qi and Tianbao Xie and Tao Yu}, + year={2026}, + eprint={2606.29537}, + archivePrefix={arXiv}, + primaryClass={cs.AI}, + url={https://arxiv.org/abs/2606.29537}, + } + ``` diff --git a/docs/benchmarks/supported/tau3-bench.md b/docs/benchmarks/supported/tau3-bench.md index 712b2b9c..5d413d7b 100644 --- a/docs/benchmarks/supported/tau3-bench.md +++ b/docs/benchmarks/supported/tau3-bench.md @@ -39,8 +39,8 @@ Customer service with domain tools and a simulated user: airline, retail, teleco | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| | `agent_reasoning_effort` | **`default`**, `none`, `minimal`, `low`, `medium`, `high`, `xhigh` | How much reasoning the benchmark's own agent model uses, with `default` keeping the upstream settings. | `original` | -| `max_errors` | pattern `[1-9][0-9]{0,3}`, default `10` | The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's. | `original`, `yours` | -| `max_steps` | pattern `[1-9][0-9]?\|1[0-9]{2}\|200`, default `200` | The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met. | `original`, `yours` | +| `max_errors` | e.g. `5`, default `10` | The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's. | `original`, `yours` | +| `max_steps` | e.g. `100`, default `200` | The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met. | `original`, `yours` | | `retrieval` | **`bm25`**, `bm25_grep`, `grep_only`, `no_knowledge`, `full_kb`, `golden_retrieval`, `openai_embeddings`, `openai_embeddings_grep`, `alltools`, `terminal_use`, `terminal_use_write`, `bm25_reranker`, `bm25_reranker_grep`, `openai_embeddings_reranker`, `openai_embeddings_reranker_grep`, `qwen_embeddings`, `qwen_embeddings_grep`, `qwen_embeddings_reranker`, `qwen_embeddings_reranker_grep`, `alltools-qwen` | How the agent searches the bank's documents in banking_knowledge tasks. | `original`, `yours` | | `task_split` | **`base`**, `train`, `test`, `small`, `full` | Which group of tasks to run within the selected domain. | `original`, `yours` | | `upstream_agent` | **`llm_agent`**, `llm_agent_gt`, `llm_agent_solo` | Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools. | `original` | diff --git a/docs/standard.md b/docs/standard.md index 2573d2fb..abe23eed 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -477,7 +477,7 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | Key of an option | Meaning | | --- | --- | -| `summary`, `values` or `pattern`, `default` | What it changes, its allowed choices or input pattern, and its value when unset | +| `summary`, `values` or `pattern`, `example`, `default` | What it changes, its allowed choices or input pattern, and its value when unset. `example`, optional and checked when given, is a value the pattern allows besides the default; pages show it in the pattern's place, and a pattern without one shows nothing | | `datasets` | The datasets it applies to | | `about` | A line about each value | | `names` | A name for some values, shown in their place; a recipe and the CLI keep the value | @@ -493,7 +493,7 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is - **`embeddings`**: the value gets the run's own relay as the driver's `OPENAI_BASE_URL` (and its MCP servers'), the model pinned on its route. Bare OpenAI model names work on OpenAI or OpenRouter; qualified names such as `qwen/qwen3-embedding-8b` require OpenRouter. On OpenRouter, bare names acquire their maker's prefix in both the price inventory and the sent request. A judge route handed to litellm goes in its `api_base`. - **`reranker`**: the chat model used by the upstream search. The driver receives one `OPENAI_BASE_URL`: embeddings requests use the embedding pin when present, while chat and Responses requests use the reranker pin. A BM25 reranker needs no embedding model. Both models count toward the run's price requirements and spending cap. - **`files`**: a benchmark declares `{name: {url: "https://…", sha256: "…"}}`; an option value names them with `needs.files: [name]`. A needs block may add `when: {other_option: [values]}` to require its resources only under those selections. Only active options and needs fetch files, once per digest under `STANDARD_DATA_DIR` (the dataset cache by default), refusing a mismatched download. The driver receives each path as `NULLIUS_FILE_` (upper case). Names use lower case letters and underscores. -- **`host`**: `{summary, check: [programs on PATH], install: {apt, npm}}` (`standard/hostreq.py`), whatever the backend; hardware (a CUDA GPU, by `nvidia-smi`) has no `install`. Optional `gpu_memory_mb` is a positive whole number of MiB required on the largest GPU, read with `nvidia-smi` (total memory, not free memory or the sum across cards). The local builder and run startup refuse insufficient memory with the required and available amounts; an unavailable query names `nvidia-smi` instead. Remote GPU memory is checked when the run starts on the host. A run refuses missing programs and says the install commands when supplied. +- **`host`**: `{label, summary, check: [programs on PATH], install: {apt, npm}}`, `label` the name a page shows, the key without it (`standard/hostreq.py`), whatever the backend; hardware (a CUDA GPU, by `nvidia-smi`) has no `install`. Optional `gpu_memory_mb` is a positive whole number of MiB required on the largest GPU, read with `nvidia-smi` (total memory, not free memory or the sum across cards). The local builder and run startup refuse insufficient memory with the required and available amounts; an unavailable query names `nvidia-smi` instead. Remote GPU memory is checked when the run starts on the host. A run refuses missing programs and says the install commands when supplied. - **One integration per benchmark.** Add supported behavior as options on that entry, not numbered integration revisions. Record the selected options with each run and revalidate affected proofs. ### An agent @@ -533,7 +533,7 @@ An agent or a capability may keep several versions in its one file: an older rel | Rule | Detail | | --- | --- | -| Shape | A `summary`, `values` (or a `pattern`) and a `default`, the vendor's own; the summary says where it comes from | +| Shape | A `summary`, `values` (or a `pattern`, with an optional `example` it allows besides the default, shown in its place) and a `default`, the vendor's own; the summary says where it comes from | | Reaches the agent | As `{opt_}` in `model.env`, `env`, or a file `model.files` writes; `cli check` refuses a slot naming no option, and a default the option does not allow | | Set | `cli run` and `cli conform` take `--option name=value` (repeatable) | | Recorded | Every value under `standard.options` in `config.json`, defaults included | diff --git a/pyproject.toml b/pyproject.toml index 1b2a9d66..f4871800 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ _nullius_app = "app" dependencies = {file = ["app/requirements.txt"]} [tool.setuptools.package-data] -"nullius.catalogue" = ["agents/*/*.json", "agents/*/*.py", "agents/*/*.toml", "agents/*/*.yaml", "benchmarks/*/env/*", "benchmarks/*/*.json", "benchmarks/*/*.py", "capabilities/*/*.json", "capabilities/*/*.py"] +"nullius.catalogue" = ["agents/*/*.json", "agents/*/*.py", "agents/*/*.toml", "agents/*/*.yaml", "benchmarks/*/env/*", "benchmarks/*/*.json", "benchmarks/*/*.py", "benchmarks/*/*.yml", "benchmarks/*/startup/*.py", "capabilities/*/*.json", "capabilities/*/*.py"] "nullius.standard" = ["kits/*/*.json", "kits/*/*.py"] "nullius.judge" = ["floors.json"] _nullius_app = ["templates/**/*.html", "static/**/*.css", "static/**/*.js", "static/**/*.svg", "static/**/*.png", "static/**/*.json", "static/**/*.woff2", "static/**/*.ttf", "static/**/*.txt", "static/**/*.md", "requirements.txt", ".env.example"]