From 15513d00b9b4605e5f9ce7b3ea4062c9c536da26 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 15:56:32 +0000 Subject: [PATCH 01/35] Speak TRAPI 2.0 and build queries with translator_tom The harness now writes TRAPI 2.0 queries by default, built from translator_tom (TOM) models so each query is validated against the spec before it is sent. --trapi_version 1.6.x still works for services that have not moved: those queries are built from TOM's v1_6 models, and the SmartAPI registry is filtered to the same major.minor as before. Unsupported versions are refused at the CLI and by QueryRunner. Queries - MVP2 qualifiers go in constraints.qualifiers as one {qualifier_type_id: qualifier_value} mapping instead of 1.x's qualifier_constraints, which 2.0 servers such as Gandalf reject. - Unset qualifiers are left out rather than sent as "" (2.0 forbids an empty mapping); with none set, no constraints are sent. - Pathfinder queries are a plain QueryGraph with paths and no edges. - An absent Pathfinder category is omitted rather than sent as [null]. Responses 2.0 binds each QNode / QPath to one object with an ids list; 1.x to a list of objects each with an id. test_harness/trapi.py's binding_ids reads both, since the ARS can return either. Before this, every 2.0 response raised in the acceptance analysis and the agent was FAILED. The expected-answer lookup now checks membership in a binding's ids rather than only its last entry. Dependencies translator_tom needs pydantic>=2.12 and Python 3.10, so the pydantic pin is lifted and python_requires rises to 3.10 (CI and images are on 3.11). reasoner-validator was unused and conflicted with the new pydantic, so it is dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LyWodasv7FcfGMu7dGgvuR --- README.md | 18 ++ requirements.txt | 7 +- setup.py | 4 +- test_harness/acceptance_test_runner.py | 15 +- test_harness/main.py | 22 +- test_harness/pathfinder_test_runner.py | 6 +- test_harness/run.py | 8 +- test_harness/runner/generate_query.py | 285 +++++++++++--------- test_harness/runner/query_runner.py | 10 +- test_harness/runner/smart_api_registry.py | 4 +- test_harness/trapi.py | 64 +++++ tests/helpers/mock_responses.py | 29 +- tests/test_reporting.py | 10 +- tests/test_run.py | 4 +- tests/test_trapi.py | 308 ++++++++++++++++++++++ 15 files changed, 610 insertions(+), 184 deletions(-) create mode 100644 test_harness/trapi.py create mode 100644 tests/test_trapi.py diff --git a/README.md b/README.md index d4c7e47..0bea75a 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,24 @@ whenever `--query_type` is given. Saved result filenames are prefixed with the query type (eg `MVP1_test_report.json`) so runs of different slices stay distinguishable. +### TRAPI version +The harness speaks **TRAPI 2.0** by default. Queries are built with +[`translator_tom`](https://github.com/NCATSTranslator/TRAPIObjectModeling) +(TOM), the Translator-wide TRAPI object model, so each query is validated +against the spec before it is sent. `--trapi_version` picks the version the +queries are written in and which services are taken from the SmartAPI +registry (only those registered with the same `x-trapi` major.minor): +- `test-harness --trapi_version 1.6.0 download ` to test services still + on TRAPI 1.6. + +What 2.0 changed in the queries the harness sends: an MVP2 query's qualifiers +go in the query edge's `constraints.qualifiers` as one +`{qualifier_type_id: qualifier_value}` mapping (1.6's `qualifier_constraints` +is rejected by 2.0 services), and a Pathfinder query is an ordinary query graph +with `paths` and no `edges`. Responses are read in either shape — a 2.0 +binding is one object with an `ids` list, a 1.x binding a list of objects each +with an `id` — since the ARS can return either. + ### Result files The acceptance results CSV has a row per test asset, with the asset's name, its Information Radiator URL, the ARS pk, and the test case/asset ids. Every agent diff --git a/requirements.txt b/requirements.txt index f8dca51..8295600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ httpx==0.27.0 -pydantic==2.7.1 -# reasoner_pydantic==4.1.6 +# translator_tom needs pydantic>=2.12. +pydantic>=2.12.0 setproctitle==1.3.3 slack_sdk==3.27.2 tqdm==4.66.4 translator-testing-model==0.5.0 -reasoner-validator==4.2.5 +# TRAPI object model: builds the TRAPI 2.0 queries the harness sends. +translator_tom>=2.1.0 diff --git a/setup.py b/setup.py index 484e090..2a22a21 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="sri-test-harness", - version="0.7.1", + version="0.8.0", author="Max Wang", author_email="max@covar.com", url="https://github.com/TranslatorSRI/TestHarness", @@ -22,7 +22,7 @@ package_data={"test_harness": ["test_suites/*.json", "test_suites/README.md"]}, zip_safe=False, license="MIT", - python_requires=">=3.9", + python_requires=">=3.10", entry_points={ "console_scripts": [ "test-harness = test_harness.main:cli", diff --git a/test_harness/acceptance_test_runner.py b/test_harness/acceptance_test_runner.py index 09c14e1..6447fee 100644 --- a/test_harness/acceptance_test_runner.py +++ b/test_harness/acceptance_test_runner.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List +from test_harness.trapi import binding_ids from test_harness.utils import AgentReport, AgentStatus @@ -17,9 +18,8 @@ def run_acceptance_pass_fail_analysis( try: all_ids = [] for res in results: - for res_node, res_value in res["node_bindings"].items(): - for val in res_value: - ids = str(val["id"]) + for res_value in res["node_bindings"].values(): + for ids in binding_ids(res_value): if ids not in all_ids: all_ids.append(ids) if expect_output == "TopAnswer": @@ -38,8 +38,7 @@ def run_acceptance_pass_fail_analysis( n_perc_ids = [] for res in n_perc_res: for res_value in res["node_bindings"].values(): - for val in res_value: - ids = str(val["id"]) + for ids in binding_ids(res_value): if ids not in n_perc_ids: n_perc_ids.append(ids) # Record up front whether the expected answer came back at all. The @@ -57,11 +56,7 @@ def run_acceptance_pass_fail_analysis( for idx, res in enumerate(results): node_bindings = res.get("node_bindings", {}) for k in node_bindings.keys(): - nb = node_bindings[k] - the_id = None - for c in nb: - the_id = c.get("id") - if the_id == out_curie: + if out_curie in binding_ids(node_bindings[k]): ars_score = None ars_rank = None ara_score = None diff --git a/test_harness/main.py b/test_harness/main.py index 9266b7b..98d2ad7 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -8,7 +8,7 @@ import os import sys import time -from argparse import ArgumentParser +from argparse import ArgumentParser, ArgumentTypeError from urllib.parse import urlparse from uuid import uuid4 @@ -21,6 +21,7 @@ from test_harness.result_collector import ResultCollector from test_harness.run import run_tests from test_harness.slacker import LocalSlacker, Slacker +from test_harness.trapi import DEFAULT_TRAPI_VERSION, trapi_minor_version from test_harness.utils import QUERY_TYPE_PREDICATES, filter_tests_by_query_type setproctitle("TestHarness") @@ -176,6 +177,15 @@ def main(args): return EXIT_OK +def _trapi_version(value: str) -> str: + """Accept a TRAPI version only if the harness can write queries in it.""" + try: + trapi_minor_version(value) + except ValueError as e: + raise ArgumentTypeError(str(e)) + return value + + def cli(): """Parse args and run tests.""" parser = ArgumentParser(description="Translator SRI Automated Test Harness") @@ -304,9 +314,13 @@ def cli(): parser.add_argument( "--trapi_version", - type=str, - default="1.6.0", - help="TRAPI (SemVer) version assumed for testing (1.5.0, if not given)", + type=_trapi_version, + default=DEFAULT_TRAPI_VERSION, + help=( + "TRAPI (SemVer) version to test: queries are written in it and " + "only services registered for it are queried. 2.0.x or 1.6.x " + f"(default {DEFAULT_TRAPI_VERSION})." + ), ) parser.add_argument( diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index 8c6d921..4cb3212 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +from test_harness.trapi import binding_ids from test_harness.utils import AgentStatus, PathfinderReport @@ -13,9 +14,8 @@ def pathfinder_pass_fail_analysis( found_path_nodes = set() unmatched_paths = set() for analysis in message["results"][0]["analyses"]: - for path_bindings in analysis["path_bindings"].values(): - for path_binding in path_bindings: - path_id = path_binding["id"] + for path_binding in analysis["path_bindings"].values(): + for path_id in binding_ids(path_binding): matching_path_nodes = set() for edge_id in message["auxiliary_graphs"][path_id]["edges"]: edge = message["knowledge_graph"]["edges"][edge_id] diff --git a/test_harness/run.py b/test_harness/run.py index 6177b48..8a58146 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -28,6 +28,7 @@ from test_harness.reporter import Reporter from test_harness.result_collector import ResultCollector from test_harness.runner.query_runner import QueryRunner, env_map +from test_harness.trapi import DEFAULT_TRAPI_VERSION from test_harness.utils import ( AgentReport, AgentStatus, @@ -47,9 +48,12 @@ def run_tests( logger.info(f"Running {len(tests)} queries...") target_url = args.get("target_url") target = args.get("target") - query_runner = QueryRunner(logger, target_url=target_url, target=target) + trapi_version = args.get("trapi_version") or DEFAULT_TRAPI_VERSION + query_runner = QueryRunner( + logger, target_url=target_url, target=target, trapi_version=trapi_version + ) logger.info("Runner is getting service registry") - query_runner.retrieve_registry(trapi_version=args["trapi_version"]) + query_runner.retrieve_registry(trapi_version=trapi_version) # The overall test status is normally driven by the ARS. When the target # service specified in the tests is overridden (eg to run against a # locally running ARA), it is driven by the override target instead. diff --git a/test_harness/runner/generate_query.py b/test_harness/runner/generate_query.py index c9af93a..b84f62c 100644 --- a/test_harness/runner/generate_query.py +++ b/test_harness/runner/generate_query.py @@ -1,153 +1,180 @@ -"""Given a Test Asset, generate a TRAPI query.""" +"""Given a Test Asset, generate a TRAPI query. -import copy -from typing import Union +Queries are built from ``translator_tom`` (TOM) models for the TRAPI version +under test, so a malformed query fails here, against that version's schema, +rather than at the service it was sent to. ``to_dict()`` drops unset fields, +which is the JSON that goes on the wire. +""" +from typing import Dict, List, Optional, Union + +import translator_tom.v1_6 as trapi_1_6 +import translator_tom.v2_0 as trapi_2_0 from translator_testing_model.datamodel.pydanticmodel import ( PathfinderTestAsset, TestAsset, ) +from test_harness.trapi import DEFAULT_TRAPI_VERSION, is_trapi_2 from test_harness.utils import QUERY_TYPE_PREDICATES, get_qualifier_constraints -MVP1 = { - "message": { - "query_graph": { - "nodes": { - "ON": {"categories": ["biolink:Disease"]}, - "SN": {"categories": ["biolink:ChemicalEntity"]}, - }, - "edges": { - "t_edge": { - "object": "ON", - "subject": "SN", - "predicates": ["biolink:treats"], - } - }, - } - } +# Which query node a test asset's input pins, by query type and input +# category. SN is the subject (the chemical), ON the object. +INPUT_NODES = { + "MVP1": {"biolink:Disease": "ON"}, + "MVP2": {"biolink:ChemicalEntity": "SN", "biolink:Gene": "ON"}, } -MVP2 = { - "message": { - "query_graph": { - "nodes": { - "ON": {"categories": ["biolink:Gene"]}, - "SN": {"categories": ["biolink:ChemicalEntity"]}, - }, - "edges": { - "t_edge": { - "object": "ON", - "subject": "SN", - "predicates": ["biolink:affects"], - "qualifier_constraints": [ - { - "qualifier_set": [ - { - "qualifier_type_id": "biolink:object_aspect_qualifier", - "qualifier_value": "", - }, - { - "qualifier_type_id": "biolink:object_direction_qualifier", - "qualifier_value": "", - }, - ] - } - ], - } - }, - } - } +# The categories of the two query nodes, by query type. +NODE_CATEGORIES = { + "MVP1": {"SN": "biolink:ChemicalEntity", "ON": "biolink:Disease"}, + "MVP2": {"SN": "biolink:ChemicalEntity", "ON": "biolink:Gene"}, } -PATHFINDER = { - "message": { - "query_graph": { - "nodes": { - "SN": { - "set_interpretation": "BATCH", - "constraints": [], - "member_ids": [], - }, - "ON": { - "set_interpretation": "BATCH", - "constraints": [], - "member_ids": [], - }, - }, - "paths": {"p0": {"subject": "SN", "object": "ON"}}, - } + +def _query_type(test_asset: TestAsset) -> str: + for query_type, predicate in QUERY_TYPE_PREDICATES.items(): + if test_asset.predicate_id == predicate: + return query_type + raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") + + +def _qualifier_set(test_asset: TestAsset) -> Dict[str, str]: + """The asset's object aspect / direction qualifiers, leaving out unset ones.""" + aspect_qualifier, direction_qualifier = get_qualifier_constraints(test_asset) + qualifiers = { + "biolink:object_aspect_qualifier": aspect_qualifier, + "biolink:object_direction_qualifier": direction_qualifier, } -} + return {type_id: value for type_id, value in qualifiers.items() if value} -def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: - """Generate a TRAPI query.""" - query = {} - if isinstance(test_asset, PathfinderTestAsset): - source_id = test_asset.source_input_id - target_id = test_asset.target_input_id - query = copy.deepcopy(PATHFINDER) - query["message"]["query_graph"]["nodes"]["SN"] = { - "ids": [source_id], - "categories": [test_asset.source_input_category], - } - query["message"]["query_graph"]["nodes"]["ON"] = { - "ids": [target_id], - "categories": [test_asset.target_input_category], - } - elif test_asset.predicate_id == QUERY_TYPE_PREDICATES["MVP1"]: - # MVP1 - query = copy.deepcopy(MVP1) - # add id to node - if test_asset.input_category == "biolink:Disease": - query["message"]["query_graph"]["nodes"]["ON"]["ids"] = [ - test_asset.input_id - ] - else: - raise Exception( - f"Unsupported input category for MVP1: {test_asset.input_category}" - ) - # add knowledge_type - if "inferred" in test_asset.test_runner_settings: - query["message"]["query_graph"]["edges"]["t_edge"][ - "knowledge_type" - ] = "inferred" - elif test_asset.predicate_id == QUERY_TYPE_PREDICATES["MVP2"]: - # MVP2 - query = copy.deepcopy(MVP2) - # add id to corresponding node - if test_asset.input_category == "biolink:ChemicalEntity": - query["message"]["query_graph"]["nodes"]["SN"]["ids"] = [ - test_asset.input_id - ] - elif test_asset.input_category == "biolink:Gene": - query["message"]["query_graph"]["nodes"]["ON"]["ids"] = [ - test_asset.input_id +def _categories(category: Optional[str]) -> Optional[List[str]]: + return [category] if category else None + + +def _one_hop_query_2_0( + query_type: str, test_asset: TestAsset, input_node: str +) -> trapi_2_0.Query: + qualifier_set = _qualifier_set(test_asset) if query_type == "MVP2" else {} + edge = trapi_2_0.QEdge( + subject="SN", + object="ON", + predicates=[test_asset.predicate_id], + # TRAPI 2.0 gathers every edge constraint into one object, and a + # QualifierSetConstraint is a plain {qualifier_type_id: value} mapping. + constraints=( + trapi_2_0.QEdgeConstraints(qualifiers=[qualifier_set]) + if qualifier_set + else None + ), + knowledge_type=( + "inferred" if "inferred" in test_asset.test_runner_settings else None + ), + ) + nodes = { + node: trapi_2_0.QNode( + categories=[category], + ids=[test_asset.input_id] if node == input_node else None, + ) + for node, category in NODE_CATEGORIES[query_type].items() + } + return trapi_2_0.Query( + message=trapi_2_0.Message( + query_graph=trapi_2_0.QueryGraph(nodes=nodes, edges={"t_edge": edge}) + ) + ) + + +def _one_hop_query_1_6( + query_type: str, test_asset: TestAsset, input_node: str +) -> trapi_1_6.Query: + qualifier_set = _qualifier_set(test_asset) if query_type == "MVP2" else {} + edge = trapi_1_6.QEdge( + subject="SN", + object="ON", + predicates=[test_asset.predicate_id], + qualifier_constraints=( + [ + trapi_1_6.QualifierConstraint( + qualifier_set=[ + trapi_1_6.Qualifier( + qualifier_type_id=type_id, qualifier_value=value + ) + for type_id, value in qualifier_set.items() + ] + ) ] - else: - raise Exception(f"Unsupported input category: {test_asset.input_category}") - # add qualifier constraints - aspect_qualifier, direction_qualifier = get_qualifier_constraints(test_asset) - query["message"]["query_graph"]["edges"]["t_edge"]["qualifier_constraints"][0][ - "qualifier_set" - ][0]["qualifier_value"] = aspect_qualifier - query["message"]["query_graph"]["edges"]["t_edge"]["qualifier_constraints"][0][ - "qualifier_set" - ][1]["qualifier_value"] = direction_qualifier - # add knowledge_type - if "inferred" in test_asset.test_runner_settings: - query["message"]["query_graph"]["edges"]["t_edge"][ - "knowledge_type" - ] = "inferred" - else: - raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") + if qualifier_set + else None + ), + knowledge_type=( + "inferred" if "inferred" in test_asset.test_runner_settings else None + ), + ) + nodes = { + node: trapi_1_6.QNode( + categories=[category], + ids=[test_asset.input_id] if node == input_node else None, + ) + for node, category in NODE_CATEGORIES[query_type].items() + } + return trapi_1_6.Query( + message=trapi_1_6.Message( + query_graph=trapi_1_6.QueryGraph(nodes=nodes, edges={"t_edge": edge}) + ) + ) + + +def _pathfinder_query( + test_asset: PathfinderTestAsset, trapi_version: str +) -> Union[trapi_2_0.Query, trapi_1_6.Query]: + trapi = trapi_2_0 if is_trapi_2(trapi_version) else trapi_1_6 + nodes = { + "SN": trapi.QNode( + ids=[test_asset.source_input_id], + categories=_categories(test_asset.source_input_category), + ), + "ON": trapi.QNode( + ids=[test_asset.target_input_id], + categories=_categories(test_asset.target_input_category), + ), + } + paths = {"p0": trapi.QPath(subject="SN", object="ON")} + # TRAPI 2.0 makes QueryGraph.edges optional so one model covers Pathfinder + # queries; 1.6 has a separate PathfinderQueryGraph for them. + query_graph = ( + trapi_2_0.QueryGraph(nodes=nodes, paths=paths) + if trapi is trapi_2_0 + else trapi_1_6.PathfinderQueryGraph(nodes=nodes, paths=paths) + ) + return trapi.Query(message=trapi.Message(query_graph=query_graph)) + - return query +def generate_query( + test_asset: Union[TestAsset, PathfinderTestAsset], + trapi_version: str = DEFAULT_TRAPI_VERSION, +) -> dict: + """Generate a TRAPI query for ``trapi_version`` (2.0.x or 1.6.x).""" + if isinstance(test_asset, PathfinderTestAsset): + return _pathfinder_query(test_asset, trapi_version).to_dict() + + query_type = _query_type(test_asset) + input_node = INPUT_NODES[query_type].get(test_asset.input_category) + if input_node is None: + raise Exception( + f"Unsupported input category for {query_type}: " + f"{test_asset.input_category}" + ) + if is_trapi_2(trapi_version): + query = _one_hop_query_2_0(query_type, test_asset, input_node) + else: + query = _one_hop_query_1_6(query_type, test_asset, input_node) + return query.to_dict() if __name__ == "__main__": - test_asset = TestAsset.parse_obj( + test_asset = TestAsset.model_validate( { "id": "Asset_450", "name": "NeverShow: MMP3 increases activity or abundance of Potassium ion", diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 23c1e5c..827e18f 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -12,6 +12,7 @@ from test_harness.runner.generate_query import generate_query from test_harness.runner.smart_api_registry import retrieve_registry_from_smartapi +from test_harness.trapi import DEFAULT_TRAPI_VERSION, trapi_minor_version from test_harness.utils import hash_test_asset, normalize_curies MAX_QUERY_TIME = 600 @@ -33,9 +34,13 @@ def __init__( logger: logging.Logger, target_url: Optional[str] = None, target: Optional[str] = None, + trapi_version: str = DEFAULT_TRAPI_VERSION, ): """Initialize the Query Runner. + ``trapi_version`` is the TRAPI version queries are written in, and the + version of the services picked from the SmartAPI registry. + ``target_url`` and ``target`` override the target service specified in the tests themselves: when given, every query is sent to ``target_url`` instead of the services registered for the test's components, and the @@ -49,6 +54,9 @@ def __init__( if target is not None and not target.startswith("infores:"): target = f"infores:{target}" self.target_infores = target + # fail on an unsupported version now, not once per generated query + trapi_minor_version(trapi_version) + self.trapi_version = trapi_version def retrieve_registry(self, trapi_version: str): if self.target_url is not None: @@ -306,7 +314,7 @@ def run_queries( if asset_hash not in queries: # generate query try: - query = generate_query(test_asset) + query = generate_query(test_asset, self.trapi_version) queries[asset_hash] = { "query": query, "responses": {}, diff --git a/test_harness/runner/smart_api_registry.py b/test_harness/runner/smart_api_registry.py index 0a00c88..896a9c8 100644 --- a/test_harness/runner/smart_api_registry.py +++ b/test_harness/runner/smart_api_registry.py @@ -7,11 +7,13 @@ import httpx +from test_harness.trapi import DEFAULT_TRAPI_VERSION + LOGGER = logging.getLogger(__name__) def retrieve_registry_from_smartapi( - target_trapi_version="1.6.0", + target_trapi_version=DEFAULT_TRAPI_VERSION, ): """Returns a dict of smart api service endpoints defined with a dict like { diff --git a/test_harness/trapi.py b/test_harness/trapi.py new file mode 100644 index 0000000..add7193 --- /dev/null +++ b/test_harness/trapi.py @@ -0,0 +1,64 @@ +"""TRAPI version handling. + +The harness speaks TRAPI 2.0 by default and can still speak 1.6 for services +that have not moved yet. Queries are built with ``translator_tom`` (TOM), the +Translator-wide TRAPI object model, using the model package for the version +being tested, so a query is validated against that version's schema before it +is ever sent. + +Responses are read as plain dicts rather than parsed into TOM models: an ARS +merged message can be very large, and the harness only reads the bindings out +of it. What does differ between versions on that path is the shape of a +binding, which ``binding_ids`` reads either way. +""" + +import re +from typing import Any, List + +DEFAULT_TRAPI_VERSION = "2.0.0" + +# The TRAPI minor versions there is a TOM model package for, keyed by the +# "major.minor" prefix of a SemVer version. +SUPPORTED_TRAPI_VERSIONS = ("2.0", "1.6") + + +def trapi_minor_version(trapi_version: str) -> str: + """Return the ``major.minor`` of a TRAPI SemVer version, if it is supported. + + Examples: + >>> trapi_minor_version("2.0.0") + '2.0' + >>> trapi_minor_version("1.6.0-beta") + '1.6' + """ + match = re.match(r"(\d+)\.(\d+)", trapi_version or "") + minor = f"{match.group(1)}.{match.group(2)}" if match else None + if minor not in SUPPORTED_TRAPI_VERSIONS: + raise ValueError( + f"Unsupported TRAPI version '{trapi_version}'. " + f"Expected one of: {', '.join(v + '.x' for v in SUPPORTED_TRAPI_VERSIONS)}." + ) + return minor + + +def is_trapi_2(trapi_version: str) -> bool: + """Whether ``trapi_version`` is a TRAPI 2.x version.""" + return trapi_minor_version(trapi_version).startswith("2.") + + +def binding_ids(binding: Any) -> List[str]: + """Return the ids bound by one node, edge or path binding. + + TRAPI 2.0 binds each QNode / QEdge / QPath to a single object carrying an + ``ids`` list; TRAPI 1.x binds it to a list of objects each carrying an + ``id``. Both are read here, since the ARS can hand back either. + + Examples: + >>> binding_ids({"ids": ["MONDO:1", "MONDO:2"]}) + ['MONDO:1', 'MONDO:2'] + >>> binding_ids([{"id": "MONDO:1"}, {"id": "MONDO:2", "attributes": []}]) + ['MONDO:1', 'MONDO:2'] + """ + if isinstance(binding, dict): + return [str(binding_id) for binding_id in binding.get("ids") or []] + return [str(entry["id"]) for entry in binding or [] if "id" in entry] diff --git a/tests/helpers/mock_responses.py b/tests/helpers/mock_responses.py index 598de52..ed71715 100644 --- a/tests/helpers/mock_responses.py +++ b/tests/helpers/mock_responses.py @@ -1,4 +1,4 @@ -"""Mock Test Responses.""" +"""Mock Test Responses, in TRAPI 2.0.""" kp_response = { "message": { @@ -35,6 +35,8 @@ "subject": "MESH:D008687", "object": "MONDO:0005148", "predicate": "biolink:treats", + "knowledge_level": "knowledge_assertion", + "agent_type": "manual_agent", "sources": [ { "resource_id": "infores:kp0", @@ -48,30 +50,13 @@ "results": [ { "node_bindings": { - "n0": [ - { - "id": "MESH:D008687", - "attributes": [], - }, - ], - "n1": [ - { - "id": "MONDO:0005148", - "attributes": [], - }, - ], + "n0": {"ids": ["MESH:D008687"]}, + "n1": {"ids": ["MONDO:0005148"]}, }, "analyses": [ { - "resource_id": "kp0", - "edge_bindings": { - "n0n1": [ - { - "id": "n0n1", - "attributes": [], - }, - ], - }, + "resource_id": "infores:kp0", + "edge_bindings": {"n0n1": {"ids": ["n0n1"]}}, } ], }, diff --git a/tests/test_reporting.py b/tests/test_reporting.py index 26fb859..9e81480 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -523,7 +523,7 @@ def test_performance_test_is_finished_in_radiator(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished, "performance test was never finished in the radiator" @@ -552,7 +552,7 @@ def test_performance_checkpoint_failure_fails_the_radiator_test(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished[0][1] == AgentStatus.FAILED.value @@ -577,7 +577,7 @@ def test_performance_run_without_a_summary_fails_the_radiator_test(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished[0][1] == AgentStatus.FAILED.value @@ -601,7 +601,7 @@ def test_performance_test_finished_failed_on_error(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished @@ -633,7 +633,7 @@ def test_skipped_test_case_marks_all_assets_and_agents_skipped(mocker): reporter=reporter, collector=collector, logger=logger, - args={"suite": "acceptance", "trapi_version": "1.6.0"}, + args={"suite": "acceptance", "trapi_version": "2.0.0"}, ) # 3 assets total across the two acceptance cases in the fixture. diff --git a/tests/test_run.py b/tests/test_run.py index 3478834..298a402 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -53,7 +53,7 @@ def test_run_tests(mocker, httpx_mock: HTTPXMock): logger=logger, args={ "suite": "testing", - "trapi_version": "1.6.0", + "trapi_version": "2.0.0", }, ) @@ -96,7 +96,7 @@ def test_run_tests_with_target_override(httpx_mock: HTTPXMock): logger=logger, args={ "suite": "testing", - "trapi_version": "1.6.0", + "trapi_version": "2.0.0", "target_url": "http://localhost:8080", "target": "aragorn", }, diff --git a/tests/test_trapi.py b/tests/test_trapi.py new file mode 100644 index 0000000..76d5db6 --- /dev/null +++ b/tests/test_trapi.py @@ -0,0 +1,308 @@ +"""Tests for the TRAPI 2.0 surface: the queries sent and the responses read. + +Queries are checked by validating them against TOM, the TRAPI object model, +for the version they were written in. Responses are checked in both binding +shapes, since the ARS can return either. +""" + +import json + +import pytest +import translator_tom.v1_6 as trapi_1_6 +import translator_tom.v2_0 as trapi_2_0 +from pytest_httpx import HTTPXMock +from translator_testing_model.datamodel.pydanticmodel import ( + PathfinderTestAsset, + TestAsset, +) + +from test_harness.acceptance_test_runner import run_acceptance_pass_fail_analysis +from test_harness.main import _trapi_version +from test_harness.pathfinder_test_runner import pathfinder_pass_fail_analysis +from test_harness.run import run_tests +from test_harness.runner.generate_query import generate_query +from test_harness.runner.query_runner import QueryRunner +from test_harness.trapi import binding_ids, is_trapi_2, trapi_minor_version +from test_harness.utils import AgentReport, AgentStatus, PathfinderReport + +from .helpers.example_tests import example_test_cases +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response +from .helpers.mocks import MockReporter, MockResultCollector + +logger = setup_logger() + + +def _asset(predicate, input_id, input_category, qualifiers=(), inferred=True): + return TestAsset.model_validate( + { + "id": "Asset_1", + "test_runner_settings": ["inferred"] if inferred else [], + "input_id": input_id, + "input_category": input_category, + "predicate_id": predicate, + "output_id": "NCBIGene:1", + "qualifiers": [ + {"parameter": parameter, "value": value} + for parameter, value in qualifiers + ], + "expected_output": "TopAnswer", + } + ) + + +MVP2_ASSET = _asset( + "biolink:affects", + "CHEBI:2", + "biolink:ChemicalEntity", + qualifiers=[ + ("biolink_qualified_predicate", "biolink:causes"), + ("biolink_object_aspect_qualifier", "activity"), + ("biolink_object_direction_qualifier", "increased"), + ], +) + +PATHFINDER_ASSET = PathfinderTestAsset.model_validate( + { + "id": "PTFQ_1", + "source_input_id": "CHEBI:31690", + "source_input_category": "biolink:Drug", + "target_input_id": "MONDO:0004979", + "target_input_category": "biolink:Disease", + "predicate_id": "biolink:related_to", + "minimum_required_path_nodes": 1, + "path_nodes": [{"ids": ["NCBIGene:3815"], "name": "KIT"}], + "expected_output": "TopAnswer", + } +) + + +class TestVersions: + def test_minor_version(self): + assert trapi_minor_version("2.0.0") == "2.0" + assert trapi_minor_version("1.6.1") == "1.6" + assert is_trapi_2("2.0.0") + assert not is_trapi_2("1.6.0") + + @pytest.mark.parametrize("version", ["1.5.0", "3.0.0", "latest", ""]) + def test_unsupported_version_is_rejected(self, version): + with pytest.raises(ValueError, match="Unsupported TRAPI version"): + trapi_minor_version(version) + + def test_cli_rejects_unsupported_version(self): + from argparse import ArgumentTypeError + + assert _trapi_version("2.0.0") == "2.0.0" + with pytest.raises(ArgumentTypeError): + _trapi_version("1.5.0") + + def test_query_runner_rejects_unsupported_version(self): + with pytest.raises(ValueError): + QueryRunner(logger, trapi_version="1.5.0") + + +class TestQueries2_0: + """Queries default to TRAPI 2.0.""" + + def test_mvp1(self): + query = generate_query(_asset("biolink:treats", "MONDO:1", "biolink:Disease")) + trapi_2_0.Query.model_validate(query) + qgraph = query["message"]["query_graph"] + assert qgraph["nodes"] == { + "SN": {"categories": ["biolink:ChemicalEntity"]}, + "ON": {"ids": ["MONDO:1"], "categories": ["biolink:Disease"]}, + } + edge = qgraph["edges"]["t_edge"] + assert edge["predicates"] == ["biolink:treats"] + assert edge["knowledge_type"] == "inferred" + assert "constraints" not in edge + + def test_mvp2_qualifiers_are_one_constraint_mapping(self): + query = generate_query(MVP2_ASSET) + trapi_2_0.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["constraints"] == { + "qualifiers": [ + { + "biolink:object_aspect_qualifier": "activity", + "biolink:object_direction_qualifier": "increased", + } + ] + } + # the TRAPI 1.x spelling must not be sent: a 2.0 server rejects it + assert "qualifier_constraints" not in edge + + def test_mvp2_leaves_out_unset_qualifiers(self): + """2.0 forbids an empty qualifier mapping, so none is sent.""" + asset = _asset( + "biolink:affects", + "NCBIGene:1", + "biolink:Gene", + qualifiers=[("biolink_object_direction_qualifier", "decreased")], + inferred=False, + ) + query = generate_query(asset) + trapi_2_0.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["constraints"] == { + "qualifiers": [{"biolink:object_direction_qualifier": "decreased"}] + } + assert "knowledge_type" not in edge + assert query["message"]["query_graph"]["nodes"]["ON"]["ids"] == ["NCBIGene:1"] + + asset.qualifiers = [] + edge = generate_query(asset)["message"]["query_graph"]["edges"]["t_edge"] + assert "constraints" not in edge + + def test_pathfinder(self): + query = generate_query(PATHFINDER_ASSET) + trapi_2_0.Query.model_validate(query) + qgraph = query["message"]["query_graph"] + assert "edges" not in qgraph + assert qgraph["paths"] == {"p0": {"subject": "SN", "object": "ON"}} + assert qgraph["nodes"]["SN"] == { + "ids": ["CHEBI:31690"], + "categories": ["biolink:Drug"], + } + + def test_unsupported_input_category(self): + with pytest.raises(Exception, match="Unsupported input category for MVP1"): + generate_query(_asset("biolink:treats", "CHEBI:1", "biolink:Drug")) + + +class TestQueries1_6: + """A 1.6 query can still be asked for, for services that have not moved.""" + + def test_mvp2(self): + query = generate_query(MVP2_ASSET, "1.6.0") + trapi_1_6.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["qualifier_constraints"] == [ + { + "qualifier_set": [ + { + "qualifier_type_id": "biolink:object_aspect_qualifier", + "qualifier_value": "activity", + }, + { + "qualifier_type_id": "biolink:object_direction_qualifier", + "qualifier_value": "increased", + }, + ] + } + ] + assert "constraints" not in edge + + def test_pathfinder(self): + query = generate_query(PATHFINDER_ASSET, "1.6.0") + trapi_1_6.Query.model_validate(query) + assert query["message"]["query_graph"]["paths"] == { + "p0": {"subject": "SN", "object": "ON"} + } + + +class TestBindings: + def test_binding_ids_reads_both_shapes(self): + assert binding_ids({"ids": ["A", "B"]}) == ["A", "B"] + assert binding_ids([{"id": "A"}, {"id": "B"}]) == ["A", "B"] + assert binding_ids({}) == [] + assert binding_ids([]) == [] + + def _report(self, agent): + return {agent: AgentReport(AgentStatus.SKIPPED, None, None)} + + def test_acceptance_reads_2_0_bindings(self): + results = [ + { + "node_bindings": {"SN": {"ids": ["CHEBI:9"]}, "ON": {"ids": ["X:1"]}}, + "analyses": [{"resource_id": "infores:ara", "score": 0.9}], + }, + { + "node_bindings": { + "SN": {"ids": ["CHEBI:8", "CHEBI:2"]}, + "ON": {"ids": ["X:1"]}, + }, + "analyses": [{"resource_id": "infores:ara", "score": 0.4}], + }, + ] + report = run_acceptance_pass_fail_analysis( + self._report("ara"), "ara", results, "CHEBI:2", "TopAnswer" + ) + assert report["ara"].status == AgentStatus.PASSED + assert report["ara"].actual_output == { + "found": True, + "ars_score": None, + "ars_rank": None, + "ara_score": 0.4, + "ara_rank": 2, + } + + def test_acceptance_never_show_with_2_0_bindings(self): + results = [{"node_bindings": {"SN": {"ids": ["CHEBI:2"]}}, "analyses": []}] + report = run_acceptance_pass_fail_analysis( + self._report("ara"), "ara", results, "CHEBI:2", "NeverShow" + ) + assert report["ara"].status == AgentStatus.FAILED + assert report["ara"].message is None + + @pytest.mark.parametrize( + "path_binding", [{"ids": ["aux0"]}, [{"id": "aux0"}]], ids=["2.0", "1.x"] + ) + def test_pathfinder_reads_path_bindings(self, path_binding): + message = { + "results": [ + { + "node_bindings": {}, + "analyses": [ + { + "resource_id": "infores:ara", + "path_bindings": {"p0": path_binding}, + } + ], + } + ], + "auxiliary_graphs": {"aux0": {"edges": ["e0", "e1"]}}, + "knowledge_graph": { + "edges": { + "e0": {"subject": "CHEBI:31690", "object": "NCBIGene:3815"}, + "e1": {"subject": "NCBIGene:3815", "object": "MONDO:0004979"}, + } + }, + } + report = {"ara": PathfinderReport(AgentStatus.SKIPPED, None, None, "")} + pathfinder_pass_fail_analysis(report, "ara", message, [["NCBIGene:3815"]], 1) + assert report["ara"].status == AgentStatus.PASSED + assert report["ara"].expected_nodes_found == "NCBIGene:3815" + + +def test_run_sends_and_reads_trapi_2_0(httpx_mock: HTTPXMock): + """End to end: a 2.0 query goes out and a 2.0 response is scored.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={"MONDO:0010794": None, "DRUGBANK:DB00313": None, "MESH:D001463": None}, + ) + collector = MockResultCollector("ci", logger, target="gandalf") + tests = {"TestCase_1": example_test_cases["TestCase_1"]} + run_tests( + tests=tests, + reporter=MockReporter(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "2.0.0", + "target_url": "http://localhost:8080", + "target": "gandalf", + }, + ) + queries = [ + json.loads(request.content) + for request in httpx_mock.get_requests(url="http://localhost:8080/query") + ] + assert queries + for query in queries: + trapi_2_0.Query.model_validate(query) + # Both assets are NeverShow and kp_response binds neither expected answer, + # so both pass -- reading 2.0 bindings as 1.x lists would fail them instead. + assert collector.acceptance_stats["gandalf"]["NeverShow"]["PASSED"] == 2 From 23b7ffa18cc73ca5144a80a1955f82ccb0653f8b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 13:34:51 +0000 Subject: [PATCH 02/35] Report structured results to the new Information Radiator First phase of replacing the Zebrunner-based Information Radiator. The harness now dual-writes: Zebrunner is untouched, and a new client posts structured results to the replacement radiator in batches. - radiator_schema: the API contract shared by harness and radiator. Writes are upserts on client-chosen ids, so retries and replays are safe; results key on Tests-repo ids, which are stable across runs. - Per agent, the harness now records n_results, HTTP status, and response time (direct queries only; ARS fan-out is only seen through 10s polling, so timing it would mostly measure the polling). - An unreachable radiator never fails a run. Runs that aren't uploaded are saved to --output_dir and can be replayed with the new `test-harness-radiator push`. - docs/information-radiator.md records the agreed design and phases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WR6TD7jzBVdYPod4pKWcg --- README.md | 17 ++ compose.yml | 2 + deploy/cronjob.example.yaml | 10 + docs/information-radiator.md | 80 +++++++ radiator_schema/__init__.py | 166 ++++++++++++++ setup.py | 7 +- test_harness/main.py | 67 +++++- test_harness/radiator_client.py | 208 ++++++++++++++++++ test_harness/result_collector.py | 171 ++++++++++++++- test_harness/run.py | 22 ++ test_harness/runner/query_runner.py | 6 + test_harness/utils.py | 8 +- tests/conftest.py | 26 +++ tests/helpers/mocks.py | 9 +- tests/test_local.py | 6 +- tests/test_radiator.py | 328 ++++++++++++++++++++++++++++ 16 files changed, 1114 insertions(+), 19 deletions(-) create mode 100644 docs/information-radiator.md create mode 100644 radiator_schema/__init__.py create mode 100644 test_harness/radiator_client.py create mode 100644 tests/conftest.py create mode 100644 tests/test_radiator.py diff --git a/README.md b/README.md index d4c7e47..90e53f4 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,23 @@ saved to `--output_dir` automatically. Likewise, if the Information Radiator isn't configured (no `ZE_BASE_URL` / `ZE_REFRESH_TOKEN`), the harness falls back to a local reporter. +### The new Information Radiator +A replacement for the Zebrunner-based Information Radiator is being built in +this repo (see [`docs/information-radiator.md`](docs/information-radiator.md)). +Until it takes over, the harness reports to both. The new one gets structured +results instead of labels and log lines: for each test asset, every agent's +status, whether the expected answer was found and its rank/score, the number +of results, the HTTP status, and (for direct queries) the response time. + +It's configured with `RADIATOR_URL` and `RADIATOR_TOKEN` (or `--radiator_url` / +`--radiator_token`). Results are posted in batches; an unreachable radiator +never fails a run. Whenever a run isn't uploaded (`--local`, the radiator isn't +configured, or an upload failed) it's saved to `--output_dir` as +`radiator_.json` instead, which can be uploaded later: +- `test-harness-radiator push test_results/radiator_.json` + +Uploads are idempotent, so pushing a run that partly made it is safe. + ### Running local test files `test-harness download ` fetches the suites from [NCATSTranslator/Tests](https://github.com/NCATSTranslator/Tests) every run, diff --git a/compose.yml b/compose.yml index 3b4fa66..2d58242 100644 --- a/compose.yml +++ b/compose.yml @@ -15,3 +15,5 @@ services: SLACK_CHANNEL: ${SLACK_CHANNEL} ZE_BASE_URL: ${ZE_BASE_URL} ZE_REFRESH_TOKEN: ${ZE_REFRESH_TOKEN} + RADIATOR_URL: ${RADIATOR_URL} + RADIATOR_TOKEN: ${RADIATOR_TOKEN} diff --git a/deploy/cronjob.example.yaml b/deploy/cronjob.example.yaml index f3ad6e6..4629bfb 100644 --- a/deploy/cronjob.example.yaml +++ b/deploy/cronjob.example.yaml @@ -115,6 +115,16 @@ spec: - name: ZE_REFRESH_TOKEN valueFrom: secretKeyRef: {name: test-harness-secrets, key: ze-refresh-token} + # The new Information Radiator, reported to alongside + # Zebrunner until it takes over. Optional: without it each run + # is saved to --output_dir in the pod, which is lost with the + # pod unless that directory is a mounted volume. + - name: RADIATOR_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-url, optional: true} + - name: RADIATOR_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-token, optional: true} resources: # The load generator is the workload here: locust drives up to diff --git a/docs/information-radiator.md b/docs/information-radiator.md new file mode 100644 index 0000000..7076f99 --- /dev/null +++ b/docs/information-radiator.md @@ -0,0 +1,80 @@ +# Information Radiator replacement + +A custom replacement for the Zebrunner-based Information Radiator, built in this +repo alongside the harness. + +## Why + +Zebrunner's model forces the harness to put its results into labels and log +lines: per-agent status goes in as labels, and the full report (found, rank, +score, pks) as a JSON string in a log. None of it can be filtered or charted, +performance results never reach it at all, and each test costs about five API +calls. The replacement stores results as structured records, so it can show +what changed between runs and how assets and agents trend over time. + +## Decisions + +| | | +|---|---| +| Repo | Monorepo: the radiator lives here, in its own package and image. | +| Hosting | Same Kubernetes namespace as the harness CronJob. | +| UI login | One shared username/password for the consortium, from a k8s secret. Handled by the app (login form + signed session cookie); no external identity provider. | +| API auth | Ingest requires a bearer token (`RADIATOR_TOKEN`), separate from the UI login. Every API route requires authentication. | +| Raw responses | Not stored. Records keep the pks, which link to the ARS/ARAX UI. | +| History | Past runs are imported from Zebrunner. | + +## Layout + +``` +radiator_schema/ API contract (Pydantic), shared by harness and radiator +test_harness/ + radiator_client.py harness-side client + `test-harness-radiator push` +radiator/ (phase 2) FastAPI service, DB, importer, UI, Dockerfile +deploy/radiator.yaml (phase 5) Deployment, Service, Ingress +``` + +## API + +All writes go under `/api/ingest`, with `Authorization: Bearer `: + +| Method | Path | Body | +|---|---|---| +| `POST` | `/api/ingest/runs` | `RunCreate` | +| `POST` | `/api/ingest/runs/{run_id}/results` | `ResultBatch` | +| `PATCH` | `/api/ingest/runs/{run_id}` | `RunFinish` | + +Every write is an upsert: runs on the client-generated `run_id`, results on +`(run_id, test_case_id, asset_id)`. So retries and replays never duplicate. The +asset ids come from the Tests repo and are stable across runs, which is what +the trend views key on (`hash_test_asset` is not stable across processes). + +## Phases + +1. **Contract + harness (done on `claude/radiator-phase1`).** `radiator_schema`; + the harness records n_results, HTTP status, and response time (direct + queries only, since ARS fan-out is only seen through 10s polling) per agent; + it dual-writes to Zebrunner and the new radiator, and saves runs it couldn't + upload for `test-harness-radiator push`. +2. **Radiator backend.** FastAPI + Postgres (SQLAlchemy, Alembic); ingest + endpoints; read endpoints shaped for the views; shared-login session auth for + the UI and read API, bearer token for ingest; an artifact endpoint for + HelmsDeep's `report.html`. +3. **UI**, in priority order: run matrix (assets x agents with status, rank, + score, n_results, pk links); diff against the last comparable run (same + suite, env, and target); asset history; agent pass rate over time by + expected-output category; performance trends (knee, checkpoints per host). +4. **Zebrunner import.** Runs -> runs, tests -> assets via the + `TestCase`/`TestAsset`/`ExpectedOutput` labels, agent labels -> statuses, + found/rank/score from the report log where present. Older runs lack those, + and all lack n_results and response time; the UI shows them as not + recorded, not zero. Imported runs carry `origin: zebrunner_import`, and the + import is re-runnable. +5. **Deploy + cutover.** Deploy to the namespace; dual-write for 2-3 weeks; + point Slack at the new UI; remove `Reporter` and the `ZE_*` secrets. + +## Known gaps + +- ARA response times under the ARS aren't recorded. The ARS's own message + timestamps would give them; that needs confirming against the ARS API. +- A performance test whose HelmsDeep run raises (rather than returning an + error) isn't recorded in either radiator. This is existing behavior. diff --git a/radiator_schema/__init__.py b/radiator_schema/__init__.py new file mode 100644 index 0000000..06ec6e3 --- /dev/null +++ b/radiator_schema/__init__.py @@ -0,0 +1,166 @@ +"""The contract between the Test Harness and the Information Radiator. + +Both sides import these models: the harness builds them and posts them, the +radiator validates what it receives against them. Changing a model here changes +the API, so a change that is not purely additive needs ``SCHEMA_VERSION`` +bumped and the radiator taught to read both versions. + +Identity is chosen so that every write is an idempotent upsert, which is what +makes it safe to retry an upload or replay a saved payload: + +* a run is keyed by ``run_id``, a UUID the *client* generates, so re-posting a + run updates it instead of creating a second one; +* a result is keyed by ``(run_id, test_case_id, asset_id)``, the ids from the + Tests repo. They are stable across runs, which is also what lets the + radiator line an asset's results up over time. (``hash_test_asset`` is not: + it uses Python's per-process salted ``hash()``.) + +Fields the harness can't know are ``None``, never a stand-in like ``0`` or +``""``: "not recorded" and "zero results" have to stay distinguishable, +especially in runs imported from Zebrunner, which predate most of this. +""" + +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +SCHEMA_VERSION = 1 + +# Every write the harness makes goes under this prefix, authenticated with a +# bearer token. The rest of the radiator sits behind the consortium login. +INGEST_PREFIX = "/api/ingest" + + +class Status(str, Enum): + """Outcome of a test, or of a single agent's part in one. + + Mirrors ``test_harness.utils.AgentStatus``; redeclared so this package + stays importable without the harness. + """ + + PASSED = "PASSED" + FAILED = "FAILED" + NO_RESULTS = "NO_RESULTS" + SKIPPED = "SKIPPED" + ERROR = "ERROR" + + +class RunCreate(BaseModel): + """Opens a run. Posted once, before any results.""" + + run_id: UUID + suite: str + env: Optional[str] = None + # Set when the run overrode the services named in the tests, eg to test a + # locally running ARA. Runs with and without an override are different + # series and must not be compared as if they were the same thing. + target: Optional[str] = None + target_url: Optional[str] = None + query_type: Optional[str] = None + harness_version: Optional[str] = None + # Where the suite came from: the Tests repo archive URL, or a local dir. + tests_source: Optional[str] = None + started_at: datetime + # "harness" for live runs; the Zebrunner importer marks its runs so the UI + # can say why they are missing fields. + origin: Literal["harness", "zebrunner_import"] = "harness" + # The run's id in the system it came from, for imported runs. + origin_ref: Optional[str] = None + + +class AgentResult(BaseModel): + """One agent's part in one test asset.""" + + agent: str + status: Status + message: Optional[str] = None + # HTTP status, or the ARS's status code for an ARA it fanned out to. + http_status: Optional[int] = None + # Whether the expected output came back anywhere in this agent's results, + # and the rank/score it came back with (ARS sugeno for the ARS, the ARA's + # own analysis score for an ARA). + found: Optional[bool] = None + rank: Optional[int] = None + score: Optional[float] = None + n_results: Optional[int] = None + # Only recorded where the harness can measure it honestly (direct queries). + # ARS fan-out is polled every 10s, so timing it from the harness would + # mostly measure the polling. + response_time_s: Optional[float] = None + pk: Optional[str] = None + # Pathfinder only: which expected path nodes were found. + expected_nodes_found: Optional[str] = None + + +class AssetResult(BaseModel): + """The result of one test asset, across every agent it was sent to.""" + + test_case_id: str + asset_id: str + kind: Literal["acceptance", "pathfinder"] + name: Optional[str] = None + expected_output: Optional[str] = None + predicate: Optional[str] = None + # acceptance: input -> output; pathfinder: source -> target + input_curie: Optional[str] = None + output_curie: Optional[str] = None + # The overall status, driven by the ARS (or the override target). + status: Status + parent_pk: Optional[str] = None + agents: List[AgentResult] = Field(default_factory=list) + # Anything type-specific that doesn't merit a column yet, eg Pathfinder's + # expected path nodes. + details: Dict[str, Any] = Field(default_factory=dict) + + +class PerformanceResult(BaseModel): + """One HelmsDeep run against one host.""" + + test_case_id: str + asset_id: str + host: str + helmsdeep_target: Optional[str] = None + profile: Optional[str] = None + status: Status + # Set when the run never produced a summary at all. + error: Optional[str] = None + exit_code: Optional[int] = None + max_sustainable_concurrency: Optional[float] = None + knee_unsupported: Optional[bool] = None + # None when the run had no checkpoints to pass or fail. + checkpoints_passed: Optional[bool] = None + # HelmsDeep's summary.json, verbatim. The fields above are lifted out of it + # for querying; this stays the authoritative copy. + summary: Dict[str, Any] = Field(default_factory=dict) + + +class ResultBatch(BaseModel): + """A batch of results for an open run. Posted as many times as needed.""" + + results: List[AssetResult] = Field(default_factory=list) + performance: List[PerformanceResult] = Field(default_factory=list) + + +class RunFinish(BaseModel): + """Closes a run.""" + + ended_at: datetime + # Overall status counts, as reported in Slack. + counts: Dict[str, int] = Field(default_factory=dict) + + +class RunPayload(BaseModel): + """Everything about a run in one document. + + This is what the harness saves locally (``--local``, or when an upload + fails) and what ``test-harness-radiator push`` replays. + """ + + schema_version: int = SCHEMA_VERSION + run: RunCreate + results: List[AssetResult] = Field(default_factory=list) + performance: List[PerformanceResult] = Field(default_factory=list) + finish: Optional[RunFinish] = None diff --git a/setup.py b/setup.py index 484e090..aa6bf86 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,9 @@ description="Translator SRI Test Harness", long_description_content_type="text/markdown", long_description=readme, - packages=["test_harness"], + # radiator_schema is the API contract shared with the Information + # Radiator; it ships with the harness so the two can't drift apart. + packages=["test_harness", "radiator_schema"], include_package_data=True, # The editable local test suites ship with the package so `test-harness # load ` works from a plain (non-editable) install too, not just @@ -29,6 +31,9 @@ # Runs one suite against several services in turn, so a single # scheduled Job can sweep the ARS and each ARA sequentially. "test-harness-sweep = test_harness.sweep:cli", + # Uploads a run the harness saved instead of sending to the + # Information Radiator (--local, or the radiator was unreachable). + "test-harness-radiator = test_harness.radiator_client:cli", ], }, ) diff --git a/test_harness/main.py b/test_harness/main.py index 9266b7b..96a0621 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -9,14 +9,18 @@ import sys import time from argparse import ArgumentParser +from datetime import datetime +from importlib.metadata import PackageNotFoundError, version from urllib.parse import urlparse from uuid import uuid4 from setproctitle import setproctitle +from radiator_schema import RunCreate from test_harness.download import download_tests, load_tests from test_harness.logger import get_logger, setup_logger from test_harness.performance_test_runner import PROFILES +from test_harness.radiator_client import RadiatorClient from test_harness.reporter import LocalReporter, Reporter from test_harness.result_collector import ResultCollector from test_harness.run import run_tests @@ -45,6 +49,13 @@ def url_type(arg): raise TypeError("Invalid URL") +def harness_version(): + try: + return version("sri-test-harness") + except PackageNotFoundError: + return None + + def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] @@ -116,13 +127,43 @@ def main(args): slacker = LocalSlacker(output_dir=output_dir, logger=logger) else: slacker = Slacker() - collector = ResultCollector(test_env, logger, target=args.get("target")) + # The new Information Radiator runs alongside Zebrunner until it takes + # over. It is additive: without RADIATOR_URL/RADIATOR_TOKEN, or with + # --local, it only records, and the run is saved to --output_dir instead. + radiator = RadiatorClient( + base_url=args.get("radiator_url"), + token=args.get("radiator_token"), + enabled=not local, + logger=logger, + ) + radiator.start_run( + RunCreate( + run_id=uuid4(), + suite=args["suite"], + env=test_env, + target=args.get("target"), + target_url=args.get("target_url"), + query_type=query_type, + harness_version=harness_version(), + tests_source=args.get("tests_url") or args.get("tests_dir"), + started_at=datetime.now().astimezone(), + ) + ) + radiator_link = ( + f"\n<{radiator.run_url}|View in the new Information Radiator>" + if radiator.run_url + else "" + ) + + collector = ResultCollector( + test_env, logger, target=args.get("target"), radiator=radiator + ) queried_envs = set() for test in tests.values(): queried_envs.add(test.test_env) slacker.post_notification( messages=[ - f"Running {args['suite']} ({sum([len(test.test_assets) for test in tests.values()])} tests, {len(tests.values())} queries)...\n<{reporter.base_path}/test-runs/{reporter.test_run_id}|View in the Information Radiator>" + f"Running {args['suite']} ({sum([len(test.test_assets) for test in tests.values()])} tests, {len(tests.values())} queries)...\n<{reporter.base_path}/test-runs/{reporter.test_run_id}|View in the Information Radiator>{radiator_link}" ] ) start_time = time.time() @@ -130,11 +171,12 @@ def main(args): slacker.post_notification( messages=[ - """Test Suite: {test_suite}\nDuration: {duration} | Environment(s): {envs}\n<{ir_url}|View in the Information Radiator>\n{result_summary}""".format( + """Test Suite: {test_suite}\nDuration: {duration} | Environment(s): {envs}\n<{ir_url}|View in the Information Radiator>{radiator_link}\n{result_summary}""".format( test_suite=args["suite"], duration=round(time.time() - start_time, 2), envs=(",").join(list(queried_envs)), ir_url=f"{reporter.base_path}/test-runs/{reporter.test_run_id}", + radiator_link=radiator_link, result_summary=collector.dump_result_summary(), ) ] @@ -164,6 +206,13 @@ def main(args): logger.info("Finishing up test run...") reporter.finish_test_run() + radiator.finish_run(counts=collector.acceptance_report) + if not radiator.enabled or radiator.upload_failed: + path = radiator.save(output_dir, prefix=target_prefix) + logger.info( + f"Saved the run for the Information Radiator to {path}; upload it " + "with `test-harness-radiator push`." + ) if args["json_output"]: os.makedirs(output_dir, exist_ok=True) @@ -247,6 +296,18 @@ def cli(): help="Access token for authentication with the Testing Dashboard", ) + parser.add_argument( + "--radiator_url", + type=url_type, + help="URL of the new Information Radiator (defaults to $RADIATOR_URL)", + ) + + parser.add_argument( + "--radiator_token", + type=str, + help="Ingest token for the new Information Radiator (defaults to $RADIATOR_TOKEN)", + ) + parser.add_argument( "--save_to_dashboard", action="store_true", diff --git a/test_harness/radiator_client.py b/test_harness/radiator_client.py new file mode 100644 index 0000000..a414522 --- /dev/null +++ b/test_harness/radiator_client.py @@ -0,0 +1,208 @@ +"""Client for the Information Radiator's ingest API. + +Runs alongside the Zebrunner ``Reporter`` while the new radiator is brought up. +Unlike the Reporter, it never makes a call per test: results are buffered and +posted in batches as structured records (see ``radiator_schema``). + +An unreachable radiator must never cost a run its results, so nothing here +raises. Every record is kept in ``payload`` whether or not it was uploaded; +when an upload fails the harness saves that payload to disk, and +``test-harness-radiator push`` replays it later. Every write is an upsert on +ids the client chose, so replaying a run that partly made it is safe. +""" + +import json +import logging +import os +import sys +from argparse import ArgumentParser +from datetime import datetime +from typing import Optional + +import httpx + +from radiator_schema import ( + INGEST_PREFIX, + AssetResult, + PerformanceResult, + ResultBatch, + RunCreate, + RunFinish, + RunPayload, +) + + +class RadiatorClient: + """Posts a run's results to the Information Radiator.""" + + def __init__( + self, + base_url: Optional[str] = None, + token: Optional[str] = None, + enabled: bool = True, + batch_size: int = 50, + logger: logging.Logger = logging.getLogger(__name__), + ): + """Initialize the client. + + With ``enabled=False``, or without a URL and token, nothing is sent: + the client only records, which is what ``--local`` runs want. + """ + self.base_url = (base_url or os.getenv("RADIATOR_URL") or "").rstrip("/") + self.token = token or os.getenv("RADIATOR_TOKEN") + self.enabled = enabled and bool(self.base_url and self.token) + self.batch_size = batch_size + self.logger = logger + self.payload: Optional[RunPayload] = None + # Set by the first failed upload. Nothing is sent after that: the run + # is replayed from the saved payload instead, which avoids a half-run + # in the radiator that looks complete. + self.upload_failed = False + self._pending = ResultBatch() + self._client: Optional[httpx.Client] = None + + @staticmethod + def is_configured(base_url=None, token=None) -> bool: + """Return True if enough config exists to talk to the radiator.""" + return bool( + (base_url or os.getenv("RADIATOR_URL")) + and (token or os.getenv("RADIATOR_TOKEN")) + ) + + @property + def run_url(self) -> Optional[str]: + """Link to the run in the radiator UI, once a run is open.""" + if not self.enabled or self.payload is None: + return None + return f"{self.base_url}/runs/{self.payload.run.run_id}" + + def start_run(self, run: RunCreate): + """Open a run.""" + self.payload = RunPayload(run=run) + self._send("POST", "/runs", run.model_dump(mode="json")) + + def add_result(self, result: AssetResult): + """Record one asset's result, uploading once a batch has built up.""" + self.payload.results.append(result) + self._pending.results.append(result) + if len(self._pending.results) >= self.batch_size: + self.flush() + + def add_performance(self, result: PerformanceResult): + """Record one performance run and upload it straight away. + + These come hours apart in a sweep, so they aren't held for a batch. + """ + self.payload.performance.append(result) + self._pending.performance.append(result) + self.flush() + + def flush(self): + """Upload whatever results are buffered.""" + if not (self._pending.results or self._pending.performance): + return + batch, self._pending = self._pending, ResultBatch() + self._send( + "POST", + f"/runs/{self.payload.run.run_id}/results", + batch.model_dump(mode="json"), + ) + + def finish_run( + self, counts: Optional[dict] = None, finish: Optional[RunFinish] = None + ): + """Upload any remaining results and close the run. + + ``finish`` closes it with a given end time instead of now, for replays. + """ + self.flush() + self.payload.finish = finish or RunFinish( + ended_at=datetime.now().astimezone(), counts=counts or {} + ) + self._send( + "PATCH", + f"/runs/{self.payload.run.run_id}", + self.payload.finish.model_dump(mode="json"), + ) + if self._client is not None: + self._client.close() + self._client = None + + def save(self, output_dir: str, prefix: str = "") -> str: + """Write the whole run to ``output_dir`` for replay; return the path.""" + os.makedirs(output_dir, exist_ok=True) + path = os.path.join( + output_dir, f"{prefix}radiator_{self.payload.run.run_id}.json" + ) + with open(path, "w") as f: + f.write(self.payload.model_dump_json(indent=2)) + return path + + def _send(self, method: str, path: str, body: dict): + if not self.enabled or self.upload_failed: + return + if self._client is None: + self._client = httpx.Client( + base_url=f"{self.base_url}{INGEST_PREFIX}", + headers={"Authorization": f"Bearer {self.token}"}, + timeout=30, + # retries connection failures only; an HTTP error is an answer + transport=httpx.HTTPTransport(retries=2), + ) + try: + res = self._client.request(method, path, json=body) + res.raise_for_status() + except Exception as e: + self.upload_failed = True + self.logger.error( + f"Upload to the Information Radiator failed ({method} {path}): " + f"{e}. Not sending the rest of this run; it will be saved " + "locally for `test-harness-radiator push`." + ) + + +def push(payload: RunPayload, client: RadiatorClient) -> bool: + """Replay a saved run into the radiator. Returns True on success.""" + client.start_run(payload.run) + for result in payload.results: + client.add_result(result) + for result in payload.performance: + client.add_performance(result) + if payload.finish is not None: + # keep the original end time and counts, not the time of the replay + client.finish_run(finish=payload.finish) + else: + # the run never finished (eg the harness crashed); leave it open + client.flush() + return not client.upload_failed + + +def cli(): + """Entrypoint for ``test-harness-radiator``.""" + parser = ArgumentParser(description="Information Radiator tools") + subparsers = parser.add_subparsers(dest="command", required=True) + push_parser = subparsers.add_parser( + "push", + help="Upload a run saved by the harness (radiator_.json)", + ) + push_parser.add_argument("files", nargs="+", help="Saved run payload(s)") + parser.add_argument("--radiator_url", help="Defaults to $RADIATOR_URL") + parser.add_argument("--radiator_token", help="Defaults to $RADIATOR_TOKEN") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger("test-harness-radiator") + if not RadiatorClient.is_configured(args.radiator_url, args.radiator_token): + logger.error("Set RADIATOR_URL and RADIATOR_TOKEN (or pass the flags).") + sys.exit(1) + + failed = 0 + for path in args.files: + with open(path) as f: + payload = RunPayload.model_validate(json.load(f)) + client = RadiatorClient(args.radiator_url, args.radiator_token, logger=logger) + if push(payload, client): + logger.info(f"Pushed {path} -> {client.run_url}") + else: + failed += 1 + sys.exit(1 if failed else 0) diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index e7d8df3..b32e736 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -14,7 +14,9 @@ TestEnvEnum, ) +from radiator_schema import AgentResult, AssetResult, PerformanceResult from test_harness.performance_test_runner import checkpoint_verdicts +from test_harness.radiator_client import RadiatorClient from test_harness.utils import AgentReport, AgentStatus, TestReport @@ -39,6 +41,7 @@ def __init__( test_env: Optional[TestEnvEnum], logger: logging.Logger, target: Optional[str] = None, + radiator: Optional[RadiatorClient] = None, ): """Initialize the Collector. @@ -47,8 +50,12 @@ def __init__( service directly, so results are collected for that one agent instead of the ARS + ARA roster; an ARS override still fans out to ARAs and keeps the usual roster. + + ``radiator``, when given, is handed a structured record of every result + as it is collected. """ self.logger = logger + self.radiator = radiator self.has_acceptance_results = False self.has_performance_results = False agents = [ @@ -114,6 +121,7 @@ def collect_acceptance_result( parent_pk: Union[str, None], url: str, force_skipped: bool = False, + status: Optional[AgentStatus] = None, ): """Add a single report to the total output. @@ -121,6 +129,10 @@ def collect_acceptance_result( in ``report``. It is used when the test as a whole was skipped (eg the query never ran) so the per-agent stats/CSV agree with the skipped test-level status instead of reporting incidental per-ARA errors. + + ``status`` is the test's overall status, for the radiator. Without it, + it is worked out the way ``run_tests`` does: from the ARS, or from the + single agent when an override target narrowed the roster to one. """ self.has_acceptance_results = True # add result to stats @@ -155,23 +167,131 @@ def collect_acceptance_result( f""""{asset.name}",{url},{pk_url},{test.id},{asset.id},{agent_results}\n""" ) - @staticmethod - def _expected_answer_cells(agent_report: AgentReport) -> List[str]: + if self.radiator is not None: + if status is None: + status_agent = "ars" if "ars" in self.agents else self.agents[0] + agent_report = report.result.get(status_agent) + status = ( + agent_report.status + if agent_report is not None and not force_skipped + else AgentStatus.SKIPPED + ) + # A record the schema rejects costs the radiator one asset, never + # the run its results. + try: + record = self._asset_result( + test, asset, report, parent_pk, status, force_skipped + ) + except Exception as e: + self.logger.error( + f"[{test.id}] couldn't record asset {asset.id} for the " + f"Information Radiator: {e}" + ) + else: + self.radiator.add_result(record) + + def _asset_result( + self, + test: Union[TestCase, PathfinderTestCase], + asset: Union[TestAsset, PathfinderTestAsset], + report: TestReport, + parent_pk: Optional[str], + status: AgentStatus, + force_skipped: bool, + ) -> AssetResult: + """The radiator's record of one asset, with a row for every agent. + + Agents are the same roster as the CSV's columns, and an agent with no + response is SKIPPED here too, so the two always agree. + """ + agents = [] + for agent in self.agents: + pk = report.pks.get(agent) + # the query runner writes the string "None" when the ARS never + # produced a merged message + pk = None if pk in (None, "", "None") else pk + agent_report = None if force_skipped else report.result.get(agent) + if agent_report is None: + agents.append( + AgentResult(agent=agent, status=AgentStatus.SKIPPED.value, pk=pk) + ) + continue + found, rank, score = self._expected_answer(agent_report) + agents.append( + AgentResult( + agent=agent, + status=agent_report.status.value, + message=agent_report.message, + http_status=agent_report.http_status, + found=found, + rank=rank, + score=score, + n_results=agent_report.n_results, + response_time_s=agent_report.response_time_s, + pk=pk, + expected_nodes_found=getattr( + agent_report, "expected_nodes_found", None + ), + ) + ) + + if isinstance(asset, PathfinderTestAsset): + kind = "pathfinder" + input_curie = asset.source_input_id + output_curie = asset.target_input_id + else: + kind = "acceptance" + input_curie = asset.input_id + output_curie = asset.output_id + return AssetResult( + test_case_id=test.id, + asset_id=asset.id, + kind=kind, + name=asset.name or asset.description, + expected_output=asset.expected_output, + predicate=asset.predicate_id, + input_curie=input_curie, + output_curie=output_curie, + status=status.value, + parent_pk=parent_pk or None, + agents=agents, + details=report.test_details or {}, + ) + + @classmethod + def _expected_answer_cells(cls, agent_report: AgentReport) -> List[str]: """CSV cells for where the expected answer landed for a single agent. + Returns (found, rank, score) as strings; see ``_expected_answer``. + Cells are left blank when the agent never got far enough for the + question to have an answer (eg it errored out). + """ + found, rank, score = cls._expected_answer(agent_report) + return [ + "" if found is None else ("true" if found else "false"), + "" if rank is None else str(rank), + "" if score is None else str(score), + ] + + @staticmethod + def _expected_answer( + agent_report: AgentReport, + ) -> Tuple[Optional[bool], Optional[int], Optional[float]]: + """Where the expected answer landed for a single agent. + Returns (found, rank, score), pulled from the same ``actual_output`` that goes into the report JSON uploaded to the radiator. ARS results are scored/ranked by the ARS itself (sugeno), ARA results by their own analyses, so whichever of the two the analysis filled in is used. - Cells are left blank when the agent never got far enough for the - question to have an answer (eg it errored out). + Each is None when the agent never got far enough for the question to + have an answer (eg it errored out). """ actual_output = agent_report.actual_output or {} if not actual_output: # No results at all means the expected answer definitely wasn't in # the response; anything else (an error, a timeout) is unknown. - found = "false" if agent_report.status == AgentStatus.NO_RESULTS else "" - return [found, "", ""] + found = False if agent_report.status == AgentStatus.NO_RESULTS else None + return found, None, None rank = actual_output.get("ars_rank") if rank is None: @@ -185,11 +305,7 @@ def _expected_answer_cells(agent_report: AgentReport) -> List[str]: # answer that was in the response. found = rank is not None or score is not None - return [ - "true" if found else "false", - "" if rank is None else str(rank), - "" if score is None else str(score), - ] + return bool(found), rank, score def collect_performance_result( self, @@ -260,6 +376,39 @@ def collect_performance_result( "summary": summary, } + if self.radiator is not None: + # same verdict run_tests gives the test: a run with no summary, or + # one that missed a checkpoint, failed + failed = bool(results.get("error")) or ( + summary.get("checkpoints_passed") is False + ) + try: + record = PerformanceResult( + test_case_id=test.id, + asset_id=asset.id, + host=host_url, + helmsdeep_target=results.get("helmsdeep_target"), + profile=results.get("profile"), + status=( + AgentStatus.FAILED.value if failed else AgentStatus.PASSED.value + ), + error=results.get("error"), + exit_code=results.get("exit_code"), + max_sustainable_concurrency=summary.get( + "max_sustainable_concurrency" + ), + knee_unsupported=summary.get("knee_unsupported"), + checkpoints_passed=summary.get("checkpoints_passed"), + summary=summary, + ) + except Exception as e: + self.logger.error( + f"[{test.id}] couldn't record the performance run against " + f"{host_url} for the Information Radiator: {e}" + ) + else: + self.radiator.add_performance(record) + @property def performance_checkpoints_passed(self) -> Optional[bool]: """Overall checkpoint verdict across every performance run. diff --git a/test_harness/run.py b/test_harness/run.py index 6177b48..f1d8330 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -36,6 +36,25 @@ ) +def record_response_meta(agent_report: AgentReport, response: Any) -> None: + """Record what the response itself says, before any analysis of it. + + ``n_results`` is only recorded for a successful response: the query runner + stands in an empty result list when an agent errors or times out, and that + is not an agent answering with zero results. + """ + if not isinstance(response, dict): + return + agent_report.http_status = response.get("status_code") + agent_report.response_time_s = response.get("elapsed_s") + body = response.get("response") + message = body.get("message") if isinstance(body, dict) else None + results = message.get("results") if isinstance(message, dict) else None + status_code = agent_report.http_status + if isinstance(results, list) and isinstance(status_code, int) and status_code < 300: + agent_report.n_results = len(results) + + def run_tests( tests: Dict[str, Union[TestCase, PathfinderTestCase]], reporter: Reporter, @@ -125,6 +144,7 @@ def run_tests( actual_output=None, ) agent_report = report.result[agent] + record_response_meta(agent_report, response) try: if response["status_code"] > 299: agent_report.status = AgentStatus.FAILED @@ -214,6 +234,7 @@ def run_tests( test_query["pks"].get("parent_pk"), f"{reporter.base_path}/test-runs/{reporter.test_run_id}/tests/{test_id}", force_skipped=force_skipped, + status=status, ) try: @@ -252,6 +273,7 @@ def run_tests( None, f"{reporter.base_path}/test-runs/{reporter.test_run_id}/tests/{test_id}", force_skipped=True, + status=status, ) try: reporter.upload_labels( diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 23c1e5c..6590823 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -82,9 +82,12 @@ def run_query( # send message response = {} status_code = 418 + elapsed_s = None with httpx.Client(timeout=600) as client: try: + start_time = time.monotonic() res = client.post(url, json=message) + elapsed_s = time.monotonic() - start_time status_code = res.status_code res.raise_for_status() response = res.json() @@ -102,6 +105,9 @@ def run_query( responses[single_infores] = { "response": response, "status_code": status_code, + # Only a direct query is timed: the ARS's own submit returns + # straight away, and its ARAs are only seen through polling. + "elapsed_s": elapsed_s, } return query_hash, responses, pks diff --git a/test_harness/utils.py b/test_harness/utils.py index 1abefc2..98a3f5d 100644 --- a/test_harness/utils.py +++ b/test_harness/utils.py @@ -47,13 +47,19 @@ class AgentReport: # Where the expected answer landed in this agent's response: a "found" # flag plus the ARS/ARA rank (int) and score (float) when it was found. actual_output: Optional[dict[str, Union[bool, int, float, None]]] + # About the response itself rather than the analysis of it. None when it + # isn't known: no response, or (for response_time_s) one fanned out by the + # ARS, which the harness only sees through its 10s polling. + http_status: Optional[int] = None + n_results: Optional[int] = None + response_time_s: Optional[float] = None @dataclass class PathfinderReport(AgentReport): """Dictionary for single Pathfinder agent report.""" - expected_nodes_found: str + expected_nodes_found: Optional[str] = None @dataclass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..35a93ac --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,26 @@ +"""Fixtures shared by every test.""" + +import pytest + +from test_harness.radiator_client import RadiatorClient + + +@pytest.fixture(autouse=True) +def isolate_radiator(monkeypatch, tmp_path): + """Keep the Information Radiator client out of the developer's environment. + + ``main`` saves the run for the radiator whenever it isn't uploaded, which + in tests is always. Saves aimed at the default ``test_results/`` land in + the test's tmp dir instead of the checkout, and a radiator configured in + the developer's shell is never used. + """ + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + save = RadiatorClient.save + + def save_to_tmp(self, output_dir, prefix=""): + if output_dir == "test_results": + output_dir = str(tmp_path) + return save(self, output_dir, prefix) + + monkeypatch.setattr(RadiatorClient, "save", save_to_tmp) diff --git a/tests/helpers/mocks.py b/tests/helpers/mocks.py index 2d7ba95..ad73399 100644 --- a/tests/helpers/mocks.py +++ b/tests/helpers/mocks.py @@ -86,9 +86,16 @@ def collect_acceptance_result( parent_pk: str | None, url: str, force_skipped: bool = False, + status=None, ): return super().collect_acceptance_result( - test, asset, report, parent_pk, url, force_skipped=force_skipped + test, + asset, + report, + parent_pk, + url, + force_skipped=force_skipped, + status=status, ) def collect_performance_result( diff --git a/tests/test_local.py b/tests/test_local.py index 2f5eeb8..31cf6b4 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -144,10 +144,12 @@ def fake_run_tests(tests, reporter, collector, logger, args): ) saved = os.listdir(tmp_path) - # acceptance json + csv and the json report were all saved with the prefix - assert len(saved) == 3 + # acceptance json + csv, the json report, and the run saved for the + # Information Radiator were all saved with the prefix + assert len(saved) == 4 assert all(name.startswith("aragorn_") for name in saved) assert "aragorn_test_report.json" in saved + assert any(name.startswith("aragorn_radiator_") for name in saved) def test_target_override_requires_both_flags(mocker, tmp_path): diff --git a/tests/test_radiator.py b/tests/test_radiator.py new file mode 100644 index 0000000..c8c77bd --- /dev/null +++ b/tests/test_radiator.py @@ -0,0 +1,328 @@ +"""Tests for reporting to the new Information Radiator. + +Covers the structured records the collector builds (including the response +metadata Zebrunner never got: n_results, HTTP status, response time), and the +client's batching, auth, failure handling, and save/replay. +""" + +import json +from uuid import uuid4 + +from pytest_httpx import HTTPXMock + +from radiator_schema import AssetResult, RunCreate, RunPayload +from test_harness.radiator_client import RadiatorClient, push +from test_harness.result_collector import ResultCollector +from test_harness.run import record_response_meta, run_tests +from test_harness.utils import AgentReport, AgentStatus, TestReport + +from .helpers.example_tests import example_test_cases +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response +from .helpers.mocks import MockReporter, MockResultCollector + +logger = setup_logger() + +BASE = "http://radiator" +INGEST = f"{BASE}/api/ingest" + + +class _Asset: + name = "asset-name" + description = None + id = "asset-1" + expected_output = "TopAnswer" + predicate_id = "biolink:treats" + input_id = "MONDO:1" + output_id = "CHEBI:2" + + +class _Case: + id = "case-1" + + +def _run(**kwargs): + return RunCreate( + run_id=uuid4(), suite="suite", started_at="2026-10-06T00:00:00Z", **kwargs + ) + + +def _recording_client(): + """A client that records but sends nothing, with a run open.""" + client = RadiatorClient(enabled=False) + client.start_run(_run()) + return client + + +def _report(status, **kwargs): + return AgentReport(status=status, message=None, actual_output=None, **kwargs) + + +def test_record_response_meta_counts_results(): + report = _report(AgentStatus.SKIPPED) + record_response_meta( + report, + { + "response": {"message": {"results": [{}, {}, {}]}}, + "status_code": 200, + "elapsed_s": 1.5, + }, + ) + assert report.http_status == 200 + assert report.n_results == 3 + assert report.response_time_s == 1.5 + + +def test_record_response_meta_ignores_stand_in_results_on_error(): + """An errored agent gets an empty stand-in result list from the query + runner; that must not read as the agent answering with zero results.""" + report = _report(AgentStatus.SKIPPED) + record_response_meta( + report, {"response": {"message": {"results": []}}, "status_code": 598} + ) + assert report.http_status == 598 + assert report.n_results is None + assert report.response_time_s is None + + +def test_collector_builds_structured_asset_result(): + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={"parent_pk": "parent", "ars": "merged", "shepherd-arax": "arax-pk"}, + result={ + "ars": _report(AgentStatus.PASSED, http_status=200, n_results=120), + "shepherd-arax": _report( + AgentStatus.NO_RESULTS, http_status=200, n_results=0 + ), + }, + test_details=None, + ) + report.result["ars"].actual_output = { + "found": True, + "ars_rank": 4, + "ars_score": 0.91, + } + collector.collect_acceptance_result(_Case(), _Asset(), report, "parent", "url") + + [result] = client.payload.results + assert result.test_case_id == "case-1" + assert result.asset_id == "asset-1" + assert result.kind == "acceptance" + assert result.input_curie == "MONDO:1" + assert result.output_curie == "CHEBI:2" + # derived from the ARS, as run_tests does + assert result.status == "PASSED" + assert result.parent_pk == "parent" + + agents = {agent.agent: agent for agent in result.agents} + # one row per agent on the roster, like the CSV + assert list(agents) == collector.agents + ars = agents["ars"] + assert (ars.found, ars.rank, ars.score) == (True, 4, 0.91) + assert ars.n_results == 120 + assert ars.pk == "merged" + arax = agents["shepherd-arax"] + assert arax.status == "NO_RESULTS" + assert arax.found is False + assert arax.n_results == 0 + # no response at all: skipped, and nothing is claimed about it + bte = agents["shepherd-bte"] + assert bte.status == "SKIPPED" + assert bte.found is None and bte.n_results is None + + +def test_collector_survives_a_record_the_schema_rejects(): + """A malformed value costs the radiator that asset, not the run.""" + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={}, + result={"ars": _report(AgentStatus.PASSED)}, + test_details=None, + ) + report.result["ars"].actual_output = {"found": True, "ars_rank": "first"} + collector.collect_acceptance_result(_Case(), _Asset(), report, None, "url") + + assert client.payload.results == [] + # the existing outputs are unaffected + assert collector.acceptance_stats["ars"]["TopAnswer"]["PASSED"] == 1 + + +def test_collector_force_skipped_skips_every_agent(): + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={"ars": "None"}, + result={"ars": _report(AgentStatus.FAILED, http_status=500)}, + test_details=None, + ) + collector.collect_acceptance_result( + _Case(), _Asset(), report, None, "url", force_skipped=True + ) + + [result] = client.payload.results + assert result.status == "SKIPPED" + assert {agent.status for agent in result.agents} == {"SKIPPED"} + # the query runner's "None" placeholder is not a pk + assert all(agent.pk is None for agent in result.agents) + + +def test_collector_records_failed_performance_checkpoint(): + client = _recording_client() + collector = ResultCollector("ci", logger, radiator=client) + collector.collect_performance_result( + _Case(), + _Asset(), + "url", + "https://ars.ci.transltr.io", + { + "helmsdeep_target": "ars", + "exit_code": 0, + "summary": { + "checkpoints_passed": False, + "max_sustainable_concurrency": 12.5, + }, + }, + ) + + [perf] = client.payload.performance + assert perf.status == "FAILED" + assert perf.checkpoints_passed is False + assert perf.max_sustainable_concurrency == 12.5 + assert perf.summary["checkpoints_passed"] is False + + +def test_run_tests_records_response_metadata(httpx_mock: HTTPXMock): + """End to end through run_tests: a direct query's result count and + response time reach the radiator record.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + # nothing normalizes, so every curie is kept as is + json={ + curie: None + for curie in [ + "MONDO:0010794", + "DRUGBANK:DB00313", + "MESH:D001463", + "CHEBI:18295", + "CHEBI:31690", + "CL:0000097", + "MONDO:0004979", + "NCBIGene:3815", + "NCBIGene:4254", + "PR:000049994", + ] + }, + ) + client = _recording_client() + collector = MockResultCollector("ci", logger, target="aragorn", radiator=client) + run_tests( + tests=example_test_cases, + reporter=MockReporter(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + + assert client.payload.results + for result in client.payload.results: + [aragorn] = result.agents + assert aragorn.agent == "aragorn" + assert aragorn.http_status == 200 + assert aragorn.n_results == len(kp_response["message"]["results"]) + assert aragorn.response_time_s is not None + + +def test_client_needs_url_and_token(): + assert not RadiatorClient.is_configured() + assert not RadiatorClient.is_configured(base_url=BASE) + assert RadiatorClient.is_configured(base_url=BASE, token="tok") + assert not RadiatorClient(base_url=BASE).enabled + assert not RadiatorClient(base_url=BASE, token="tok", enabled=False).enabled + + +def test_disabled_client_sends_nothing(httpx_mock: HTTPXMock): + client = RadiatorClient(base_url=BASE, token="tok", enabled=False) + client.start_run(_run()) + client.add_result(_asset_result()) + client.finish_run() + assert httpx_mock.get_requests() == [] + assert client.run_url is None + assert len(client.payload.results) == 1 + + +def _asset_result(asset_id="asset-1"): + return AssetResult( + test_case_id="case-1", asset_id=asset_id, kind="acceptance", status="PASSED" + ) + + +def test_client_batches_results(httpx_mock: HTTPXMock): + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="PATCH", url=f"{INGEST}/runs/{run.run_id}") + + client = RadiatorClient(base_url=BASE + "/", token="tok", batch_size=2) + client.start_run(run) + for i in range(3): + client.add_result(_asset_result(f"asset-{i}")) + client.finish_run(counts={"PASSED": 3}) + + requests = httpx_mock.get_requests() + assert [(r.method, r.url.path) for r in requests] == [ + ("POST", "/api/ingest/runs"), + ("POST", f"/api/ingest/runs/{run.run_id}/results"), + ("POST", f"/api/ingest/runs/{run.run_id}/results"), + ("PATCH", f"/api/ingest/runs/{run.run_id}"), + ] + assert all(r.headers["Authorization"] == "Bearer tok" for r in requests) + # a full batch of two, then the remainder on finish + batches = [json.loads(r.content)["results"] for r in requests[1:3]] + assert [len(batch) for batch in batches] == [2, 1] + assert json.loads(requests[3].content)["counts"] == {"PASSED": 3} + assert not client.upload_failed + assert client.run_url == f"{BASE}/runs/{run.run_id}" + + +def test_client_failure_stops_uploads_and_replays(httpx_mock: HTTPXMock, tmp_path): + """A failed upload never raises, stops further uploads, and the saved run + replays in full, keeping its original end time.""" + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs", status_code=503) + + client = RadiatorClient(base_url=BASE, token="tok", batch_size=1) + client.start_run(run) + client.add_result(_asset_result("asset-1")) + client.add_result(_asset_result("asset-2")) + client.finish_run(counts={"PASSED": 2}) + + assert client.upload_failed + assert len(httpx_mock.get_requests()) == 1 + path = client.save(str(tmp_path), prefix="aragorn_") + assert path.endswith(f"aragorn_radiator_{run.run_id}.json") + + with open(path) as f: + payload = RunPayload.model_validate(json.load(f)) + assert len(payload.results) == 2 + + httpx_mock.reset(assert_all_responses_were_requested=False) + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="PATCH", url=f"{INGEST}/runs/{run.run_id}") + replay = RadiatorClient(base_url=BASE, token="tok") + assert push(payload, replay) + + requests = httpx_mock.get_requests() + assert len(json.loads(requests[1].content)["results"]) == 2 + finish = json.loads(requests[2].content) + assert finish["ended_at"] == payload.finish.model_dump(mode="json")["ended_at"] + assert finish["counts"] == {"PASSED": 2} From f77bb5a21154553f1ac36e9dde02e16b5424ad67 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 13:34:51 +0000 Subject: [PATCH 03/35] Bump minor version Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WR6TD7jzBVdYPod4pKWcg --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa6bf86..3a6fcc0 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="sri-test-harness", - version="0.7.1", + version="0.8.0", author="Max Wang", author_email="max@covar.com", url="https://github.com/TranslatorSRI/TestHarness", From 3b7551f8543afe694e7a8ff602bc191a7e62ebc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 14:09:21 +0000 Subject: [PATCH 04/35] Add the Information Radiator service and dashboard The replacement for the Zebrunner-based Information Radiator, as its own package and (later) image in this repo. - FastAPI + Postgres (SQLAlchemy, Alembic). Ingest endpoints take the radiator_schema contract with a bearer token; writes are upserts on the contract's ids, so retries and replays never duplicate. - One shared consortium login (signed session cookie, throttled), and every API route requires the token or a login. - Pages: runs list; a run's results grid (asset x agent, with rank) and per-agent summary; what changed since the previous run in the same series; an asset's status/rank/result-count history; agent pass-rate trends by expected output; performance trends. Charts are server-side SVG with a small hover layer, each paired with a table. - `python -m radiator.demo` seeds made-up history for development. - Tests run against real Postgres, including the harness's own client posting to the app. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WR6TD7jzBVdYPod4pKWcg --- pytest.ini | 5 + radiator/__init__.py | 1 + radiator/alembic.ini | 39 ++ radiator/api.py | 80 +++ radiator/app.py | 120 +++++ radiator/auth.py | 91 ++++ radiator/charts.py | 222 ++++++++ radiator/config.py | 40 ++ radiator/db.py | 9 + radiator/demo.py | 298 +++++++++++ radiator/ingest.py | 107 ++++ radiator/migrations/env.py | 49 ++ radiator/migrations/script.py.mako | 25 + .../versions/0001_initial_schema.py | 136 +++++ radiator/models.py | 165 ++++++ radiator/queries.py | 474 ++++++++++++++++ radiator/requirements-test.txt | 5 + radiator/requirements.txt | 12 + radiator/static/charts.js | 100 ++++ radiator/static/style.css | 313 +++++++++++ radiator/templates/_macros.html | 21 + radiator/templates/asset.html | 97 ++++ radiator/templates/base.html | 25 + radiator/templates/diff.html | 66 +++ radiator/templates/login.html | 23 + radiator/templates/performance.html | 48 ++ radiator/templates/run.html | 143 +++++ radiator/templates/runs.html | 68 +++ radiator/templates/trends.html | 63 +++ radiator/tests/__init__.py | 0 radiator/tests/conftest.py | 84 +++ radiator/tests/factories.py | 53 ++ radiator/tests/test_auth.py | 83 +++ radiator/tests/test_ingest.py | 179 +++++++ radiator/tests/test_pages.py | 199 +++++++ radiator/web.py | 504 ++++++++++++++++++ 36 files changed, 3947 insertions(+) create mode 100644 pytest.ini create mode 100644 radiator/__init__.py create mode 100644 radiator/alembic.ini create mode 100644 radiator/api.py create mode 100644 radiator/app.py create mode 100644 radiator/auth.py create mode 100644 radiator/charts.py create mode 100644 radiator/config.py create mode 100644 radiator/db.py create mode 100644 radiator/demo.py create mode 100644 radiator/ingest.py create mode 100644 radiator/migrations/env.py create mode 100644 radiator/migrations/script.py.mako create mode 100644 radiator/migrations/versions/0001_initial_schema.py create mode 100644 radiator/models.py create mode 100644 radiator/queries.py create mode 100644 radiator/requirements-test.txt create mode 100644 radiator/requirements.txt create mode 100644 radiator/static/charts.js create mode 100644 radiator/static/style.css create mode 100644 radiator/templates/_macros.html create mode 100644 radiator/templates/asset.html create mode 100644 radiator/templates/base.html create mode 100644 radiator/templates/diff.html create mode 100644 radiator/templates/login.html create mode 100644 radiator/templates/performance.html create mode 100644 radiator/templates/run.html create mode 100644 radiator/templates/runs.html create mode 100644 radiator/templates/trends.html create mode 100644 radiator/tests/__init__.py create mode 100644 radiator/tests/conftest.py create mode 100644 radiator/tests/factories.py create mode 100644 radiator/tests/test_auth.py create mode 100644 radiator/tests/test_ingest.py create mode 100644 radiator/tests/test_pages.py create mode 100644 radiator/web.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5383816 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +# The harness's tests. The radiator's run separately, in their own +# environment and against Postgres: pytest radiator/tests +testpaths = tests +pythonpath = . diff --git a/radiator/__init__.py b/radiator/__init__.py new file mode 100644 index 0000000..a38b76e --- /dev/null +++ b/radiator/__init__.py @@ -0,0 +1 @@ +"""The Information Radiator: an always-on dashboard of Test Harness results.""" diff --git a/radiator/alembic.ini b/radiator/alembic.ini new file mode 100644 index 0000000..3ad1bae --- /dev/null +++ b/radiator/alembic.ini @@ -0,0 +1,39 @@ +# Migrations for the Information Radiator. Run from the repo root: +# alembic -c radiator/alembic.ini upgrade head +# The database URL comes from RADIATOR_DATABASE_URL. +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = %(here)s/.. +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/radiator/api.py b/radiator/api.py new file mode 100644 index 0000000..1b87828 --- /dev/null +++ b/radiator/api.py @@ -0,0 +1,80 @@ +"""The JSON API: ingest for the harness, and a small read API for scripts.""" + +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from sqlalchemy.orm import Session + +import radiator_schema as schema +from radiator import ingest, queries +from radiator.auth import require_token, require_token_or_login + +ingest_router = APIRouter( + prefix=schema.INGEST_PREFIX, dependencies=[Depends(require_token)] +) +read_router = APIRouter(prefix="/api", dependencies=[Depends(require_token_or_login)]) + + +def get_session(request: Request): + with request.app.state.sessionmaker() as session: + yield session + + +def _commit_or_404(session: Session, run_id: uuid.UUID, write): + try: + write() + except ingest.RunNotFound: + session.rollback() + raise HTTPException( + status.HTTP_404_NOT_FOUND, + f"No run {run_id}: open it with POST {schema.INGEST_PREFIX}/runs first", + ) + session.commit() + + +@ingest_router.post("/runs", status_code=status.HTTP_201_CREATED) +def create_run(run: schema.RunCreate, session: Session = Depends(get_session)): + ingest.upsert_run(session, run) + session.commit() + return {"run_id": str(run.run_id)} + + +@ingest_router.post("/runs/{run_id}/results") +def add_results( + run_id: uuid.UUID, + batch: schema.ResultBatch, + session: Session = Depends(get_session), +): + _commit_or_404( + session, run_id, lambda: ingest.upsert_results(session, run_id, batch) + ) + return {"results": len(batch.results), "performance": len(batch.performance)} + + +@ingest_router.patch("/runs/{run_id}") +def finish_run( + run_id: uuid.UUID, + finish: schema.RunFinish, + session: Session = Depends(get_session), +): + _commit_or_404(session, run_id, lambda: ingest.finish_run(session, run_id, finish)) + return {"run_id": str(run_id)} + + +@read_router.get("/runs") +def list_runs( + suite: str | None = None, + env: str | None = None, + limit: int = 50, + session: Session = Depends(get_session), +): + runs = queries.list_runs(session, suite=suite, env=env, limit=min(limit, 500)) + return [queries.run_json(run) for run in runs] + + +@read_router.get("/runs/{run_id}") +def get_run(run_id: uuid.UUID, session: Session = Depends(get_session)): + run = queries.get_run(session, run_id) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No such run") + return queries.run_json(run, with_results=True) diff --git a/radiator/app.py b/radiator/app.py new file mode 100644 index 0000000..531c3c2 --- /dev/null +++ b/radiator/app.py @@ -0,0 +1,120 @@ +"""The Information Radiator web app. + +Run with ``uvicorn radiator.app:create_app --factory``. Behind the cluster +ingress, pass ``--proxy-headers --forwarded-allow-ips='*'`` so the login +throttle sees each client's address rather than the ingress's. +""" + +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +from fastapi import FastAPI, Form, Request +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from sqlalchemy import text +from starlette.middleware.sessions import SessionMiddleware + +from radiator import api, web +from radiator.auth import LoginThrottle, check_credentials, is_logged_in +from radiator.config import Settings +from radiator.db import make_sessionmaker + +HERE = Path(__file__).parent +SESSION_MAX_AGE_S = 30 * 24 * 3600 + + +def _safe_next(next_url: Optional[str]) -> str: + """Only redirect back to a path on this site after login.""" + if not next_url: + return "/" + parsed = urlparse(next_url) + if parsed.scheme or parsed.netloc or not next_url.startswith("/"): + return "/" + if next_url.startswith("//"): + return "/" + return next_url + + +def create_app(settings: Optional[Settings] = None) -> FastAPI: + settings = settings or Settings.from_env() + app = FastAPI(title="Information Radiator", docs_url=None, redoc_url=None) + app.state.settings = settings + app.state.sessionmaker = make_sessionmaker(settings.database_url) + app.state.login_throttle = LoginThrottle() + templates = Jinja2Templates(directory=HERE / "templates") + web.register_template_helpers(templates) + app.state.templates = templates + + app.add_middleware( + SessionMiddleware, + secret_key=settings.session_secret, + session_cookie="radiator_session", + max_age=SESSION_MAX_AGE_S, + same_site="lax", + https_only=settings.secure_cookies, + ) + app.mount("/static", StaticFiles(directory=HERE / "static"), name="static") + app.include_router(api.ingest_router) + app.include_router(api.read_router) + app.include_router(web.router) + + @app.exception_handler(web.LoginRequired) + async def login_required(request: Request, exc: web.LoginRequired): + target = request.url.path + if request.url.query: + target += f"?{request.url.query}" + return RedirectResponse(f"/login?next={target}", status_code=303) + + @app.get("/healthz", include_in_schema=False) + def healthz(): + """Liveness/readiness: the app is up and can reach its database.""" + try: + with app.state.sessionmaker() as session: + session.execute(text("SELECT 1")) + except Exception: + return JSONResponse({"ok": False}, status_code=503) + return {"ok": True} + + @app.get("/login", response_class=HTMLResponse, include_in_schema=False) + def login_page(request: Request, next: Optional[str] = None): + if is_logged_in(request): + return RedirectResponse(_safe_next(next), status_code=303) + return templates.TemplateResponse( + request, "login.html", {"next": _safe_next(next), "error": None} + ) + + @app.post("/login", include_in_schema=False) + def login( + request: Request, + username: str = Form(...), + password: str = Form(...), + next: Optional[str] = Form(None), + ): + throttle: LoginThrottle = app.state.login_throttle + client = request.client.host if request.client else "unknown" + error = None + if throttle.blocked(client): + error = "Too many failed attempts. Try again in a few minutes." + elif check_credentials(request, username, password): + throttle.succeeded(client) + request.session.clear() + request.session["user"] = settings.username + return RedirectResponse(_safe_next(next), status_code=303) + else: + throttle.failed(client) + error = "That username and password didn't match." + return templates.TemplateResponse( + request, + "login.html", + {"next": _safe_next(next), "error": error}, + status_code=401, + ) + + @app.post("/logout", include_in_schema=False) + def logout(request: Request): + request.session.clear() + return RedirectResponse("/login", status_code=303) + + return app diff --git a/radiator/auth.py b/radiator/auth.py new file mode 100644 index 0000000..d54fcfc --- /dev/null +++ b/radiator/auth.py @@ -0,0 +1,91 @@ +"""Authentication. + +Two ways in, and nothing is reachable without one of them (bar the health +check and the login page itself): + +* the UI: one shared username/password for the consortium, which sets a + signed session cookie; +* the API: a bearer token, for the harness to upload with and for scripts. + +The JSON read API accepts either, so the UI's own pages could use it too. +""" + +import hmac +import time +from collections import defaultdict, deque +from typing import Optional + +from fastapi import HTTPException, Request, status + + +def _equal(a: Optional[str], b: str) -> bool: + return a is not None and hmac.compare_digest(a.encode(), b.encode()) + + +def has_valid_token(request: Request) -> bool: + header = request.headers.get("authorization", "") + scheme, _, token = header.partition(" ") + return scheme.lower() == "bearer" and _equal( + token.strip(), request.app.state.settings.api_token + ) + + +def is_logged_in(request: Request) -> bool: + return request.session.get("user") == request.app.state.settings.username + + +def require_token(request: Request) -> None: + """Dependency for the ingest API: bearer token only.""" + if not has_valid_token(request): + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, + "A valid bearer token is required", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def require_token_or_login(request: Request) -> None: + """Dependency for the read API.""" + if not (has_valid_token(request) or is_logged_in(request)): + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, + "Log in, or send a bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def check_credentials(request: Request, username: str, password: str) -> bool: + settings = request.app.state.settings + # evaluate both, so a wrong username takes as long as a wrong password + user_ok = _equal(username, settings.username) + password_ok = _equal(password, settings.password) + return user_ok and password_ok + + +class LoginThrottle: + """Slow down password guessing against the one shared password. + + Counts failed logins per client address in a sliding window, in memory. + That is enough for the single replica this runs as; it resets on restart. + """ + + def __init__(self, max_failures: int = 10, window_s: float = 15 * 60): + self.max_failures = max_failures + self.window_s = window_s + self._failures: dict[str, deque] = defaultdict(deque) + + def _prune(self, key: str, now: float) -> deque: + failures = self._failures[key] + while failures and now - failures[0] > self.window_s: + failures.popleft() + return failures + + def blocked(self, key: str) -> bool: + return len(self._prune(key, time.monotonic())) >= self.max_failures + + def failed(self, key: str) -> None: + now = time.monotonic() + self._prune(key, now).append(now) + + def succeeded(self, key: str) -> None: + self._failures.pop(key, None) diff --git a/radiator/charts.py b/radiator/charts.py new file mode 100644 index 0000000..061185a --- /dev/null +++ b/radiator/charts.py @@ -0,0 +1,222 @@ +"""Server-rendered SVG charts. + +Charts are drawn here rather than by a JS library, so pages work without a +frontend build and render the same for everyone. ``static/charts.js`` adds the +hover layer (a crosshair that snaps to the nearest run and a tooltip listing +every series there) from the data embedded in each chart. + +Specs follow the house rules: 2px lines, >=8px end dots with a 2px surface +ring, hairline solid gridlines, one y-axis, a legend for 2+ series, and every +chart paired with a table view in its template. +""" + +import json +import math +from dataclasses import dataclass, field +from datetime import datetime +from typing import Callable, Optional + +from markupsafe import Markup, escape + +AGENT_COLOR_SLOTS = { + "ars": 1, + "aragorn": 2, + "shepherd-aragorn": 2, + "arax": 3, + "shepherd-arax": 3, + "biothings-explorer": 4, + "shepherd-bte": 4, + "improving-agent": 5, + "unsecret-agent": 6, + "cqs": 7, +} + + +# Approximate on-screen widths of the page layouts, so text renders near 1:1. +WIDTH_FULL = 1180 +WIDTH_HALF = 570 +WIDTH_THIRD = 360 + + +def agent_color(agent: str) -> str: + """The CSS color for an agent. Follows the agent, never its position.""" + return f"var(--agent-{AGENT_COLOR_SLOTS.get(agent, 8)})" + + +@dataclass +class Series: + name: str + color: str + # (x, y) with y None for a gap; x is when the run started + points: list[tuple[datetime, Optional[float]]] = field(default_factory=list) + + +def nice_ticks(lo: float, hi: float, count: int = 4) -> list[float]: + if hi == lo: + hi = lo + 1 + raw = (hi - lo) / count + magnitude = 10 ** math.floor(math.log10(raw)) + step = next(m * magnitude for m in (1, 2, 2.5, 5, 10) if m * magnitude >= raw) + start = math.floor(lo / step) * step + ticks = [] + value = start + while value <= hi + step * 1e-9: + ticks.append(round(value, 10)) + value += step + if ticks[-1] < hi: + ticks.append(round(value, 10)) + return ticks + + +def _date_ticks(x0: float, x1: float, count: int = 5) -> list[float]: + if x1 == x0: + return [x0] + return [x0 + (x1 - x0) * i / (count - 1) for i in range(count)] + + +def line_chart( + series: list[Series], + *, + y_format: Callable[[float], str] = lambda v: f"{v:g}", + y_min: Optional[float] = None, + y_max: Optional[float] = None, + invert: bool = False, + width: int = 720, + height: int = 220, + label: str = "", +) -> Markup: + """A time-series line chart, one line per series, on a single y-axis. + + ``invert`` puts small values at the top, for ranks (1 is best). + + The SVG scales to its container, text included, so ``width`` should be + about the width it is shown at; the ``WIDTH_*`` constants match the + layouts the pages use. + """ + values = [y for s in series for _, y in s.points if y is not None] + xs = sorted({x for s in series for x, _ in s.points}) + if not values or not xs: + return Markup('

No data for this range.

') + + lo = min(values) if y_min is None else y_min + hi = max(values) if y_max is None else y_max + ticks = nice_ticks(lo, hi) + if y_min is not None and ticks[0] < y_min: + # eg ranks start at 1: label the floor itself, not the 0 below it + ticks[0] = y_min + lo, hi = min(ticks[0], lo), max(ticks[-1], hi) + + left, right, top, bottom = 48, 16, 10, 26 + plot_w, plot_h = width - left - right, height - top - bottom + t0, t1 = xs[0].timestamp(), xs[-1].timestamp() + + def sx(x: datetime) -> float: + if t1 == t0: + return left + plot_w / 2 + return left + (x.timestamp() - t0) / (t1 - t0) * plot_w + + def sy(y: float) -> float: + frac = (y - lo) / (hi - lo) if hi != lo else 0.5 + if invert: + frac = 1 - frac + return top + plot_h - frac * plot_h + + parts = [ + f'' + ] + for tick in ticks: + y = sy(tick) + parts.append( + f'' + f'{escape(y_format(tick))}' + ) + base_y = top + plot_h + parts.append( + f'' + ) + for i, t in enumerate(_date_ticks(t0, t1, 5 if t1 > t0 else 1)): + x = left + (t - t0) / (t1 - t0) * plot_w if t1 > t0 else left + plot_w / 2 + anchor = ( + "start" + if i == 0 and t1 > t0 + else ("end" if x >= width - right - 1 else "middle") + ) + when = datetime.fromtimestamp(t, tz=xs[0].tzinfo).strftime("%b %-d") + parts.append( + f'{when}' + ) + + for s in series: + runs: list[list[tuple[float, float]]] = [[]] + for x, y in sorted(s.points, key=lambda p: p[0]): + if y is None: + runs.append([]) + else: + runs[-1].append((sx(x), sy(y))) + for run in runs: + if len(run) > 1: + d = " ".join(f"{px:.1f},{py:.1f}" for px, py in run) + parts.append( + f'' + ) + elif len(run) == 1: + px, py = run[0] + parts.append( + f'' + ) + last = next( + ( + (x, y) + for x, y in sorted(s.points, key=lambda p: p[0], reverse=True) + if y is not None + ), + None, + ) + if last is not None: + parts.append( + f'' + ) + + parts.append( + f'' + ) + parts.append( + f'' + ) + parts.append("") + + # what the hover layer needs: x positions (in viewBox units) and, at each, + # every series' formatted value + hover = { + "width": width, + "xs": [round(sx(x), 1) for x in xs], + "titles": [x.strftime("%b %-d %Y, %H:%M") for x in xs], + "series": [ + { + "name": s.name, + "color": s.color, + "values": [ + (lambda y: None if y is None else y_format(y))( + dict(s.points).get(x) + ) + for x in xs + ], + } + for s in series + ], + } + legend = "" + if len(series) > 1: + legend = ( + '
' + + "".join( + f'{escape(s.name)}' + for s in series + ) + + "
" + ) + data = escape(json.dumps(hover)) + return Markup( + f'
{"".join(parts)}
{legend}' + ) diff --git a/radiator/config.py b/radiator/config.py new file mode 100644 index 0000000..ef4869f --- /dev/null +++ b/radiator/config.py @@ -0,0 +1,40 @@ +"""Settings, all from the environment so they can come from a k8s secret.""" + +import os +from dataclasses import dataclass + + +class ConfigError(RuntimeError): + """A required setting is missing.""" + + +@dataclass(frozen=True) +class Settings: + database_url: str + # Bearer token for the API, which the harness uses to upload. + api_token: str + # The one shared login for the consortium. + username: str + password: str + # Signs the session cookie. Changing it logs everyone out. + session_secret: str + # Only send the session cookie over HTTPS. Off just for local development. + secure_cookies: bool = True + + @classmethod + def from_env(cls) -> "Settings": + def required(name): + value = os.getenv(name) + if not value: + raise ConfigError(f"{name} must be set") + return value + + return cls( + database_url=required("RADIATOR_DATABASE_URL"), + api_token=required("RADIATOR_API_TOKEN"), + username=required("RADIATOR_USERNAME"), + password=required("RADIATOR_PASSWORD"), + session_secret=required("RADIATOR_SESSION_SECRET"), + secure_cookies=os.getenv("RADIATOR_SECURE_COOKIES", "true").lower() + not in ("0", "false", "no"), + ) diff --git a/radiator/db.py b/radiator/db.py new file mode 100644 index 0000000..d272661 --- /dev/null +++ b/radiator/db.py @@ -0,0 +1,9 @@ +"""Database engine and sessions.""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + + +def make_sessionmaker(database_url: str) -> sessionmaker[Session]: + engine = create_engine(database_url, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) diff --git a/radiator/demo.py b/radiator/demo.py new file mode 100644 index 0000000..50f6bc5 --- /dev/null +++ b/radiator/demo.py @@ -0,0 +1,298 @@ +"""Fill a development database with made-up runs. + + RADIATOR_DATABASE_URL=... python -m radiator.demo + +Everything here is fake: it exists to develop and screenshot the dashboard +against something shaped like real history (regressions, fixes, ranks that +drift, a flaky agent, performance sweeps). Never point this at production. +""" + +import random +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator.config import Settings # noqa: F401 (documents the env var) +from radiator.db import make_sessionmaker +from radiator.ingest import ingest_payload + +AGENTS = ["ars", "shepherd-aragorn", "shepherd-arax", "shepherd-bte"] + +PAIRS = [ + ("Metformin", "type 2 diabetes mellitus", "CHEBI:6801", "MONDO:0005148"), + ("Imatinib", "chronic myelogenous leukemia", "CHEBI:45783", "MONDO:0011996"), + ("Albuterol", "asthma", "CHEBI:2549", "MONDO:0004979"), + ("Levothyroxine", "hypothyroidism", "CHEBI:18332", "MONDO:0005420"), + ("Sumatriptan", "migraine disorder", "CHEBI:10650", "MONDO:0005277"), + ("Lisinopril", "hypertension", "CHEBI:43755", "MONDO:0005044"), + ("Methotrexate", "rheumatoid arthritis", "CHEBI:44185", "MONDO:0008383"), + ("Donepezil", "Alzheimer disease", "CHEBI:53289", "MONDO:0004975"), + ("Omeprazole", "gastroesophageal reflux", "CHEBI:7772", "MONDO:0007186"), + ("Sertraline", "major depressive disorder", "CHEBI:9123", "MONDO:0002009"), + ("Allopurinol", "gout", "CHEBI:40279", "MONDO:0005393"), + ("Tamoxifen", "breast carcinoma", "CHEBI:41774", "MONDO:0004989"), + ("Warfarin", "venous thromboembolism", "CHEBI:10033", "MONDO:0005147"), + ("Hydroxychloroquine", "lupus erythematosus", "CHEBI:5801", "MONDO:0004670"), + ("Isoniazid", "tuberculosis", "CHEBI:6030", "MONDO:0018076"), + ("Riluzole", "amyotrophic lateral sclerosis", "CHEBI:8863", "MONDO:0004976"), + ("Ivermectin", "onchocerciasis", "CHEBI:6078", "MONDO:0005301"), + ("Montelukast", "allergic rhinitis", "CHEBI:50730", "MONDO:0011786"), + ("Pyridostigmine", "myasthenia gravis", "CHEBI:8665", "MONDO:0009688"), + ("Dapsone", "leprosy", "CHEBI:4325", "MONDO:0005124"), + ("Insulin glargine", "type 1 diabetes mellitus", "CHEBI:5931", "MONDO:0005147"), + ("Penicillamine", "Wilson disease", "CHEBI:7959", "MONDO:0010200"), + ("Ursodiol", "primary biliary cholangitis", "CHEBI:9907", "MONDO:0005388"), + ("Nitisinone", "tyrosinemia type I", "CHEBI:50378", "MONDO:0010161"), +] +GENES = [ + ("Bortezomib", "PSMB5", "CHEBI:52717", "NCBIGene:5693"), + ("Erlotinib", "EGFR", "CHEBI:114785", "NCBIGene:1956"), + ("Simvastatin", "HMGCR", "CHEBI:9150", "NCBIGene:3156"), + ("Celecoxib", "PTGS2", "CHEBI:41423", "NCBIGene:5743"), + ("Rapamycin", "MTOR", "CHEBI:9168", "NCBIGene:2475"), + ("Vemurafenib", "BRAF", "CHEBI:63637", "NCBIGene:673"), + ("Tofacitinib", "JAK3", "CHEBI:71200", "NCBIGene:3718"), + ("Olaparib", "PARP1", "CHEBI:83766", "NCBIGene:142"), +] +NEVER = [ + ("Ethanol", "type 2 diabetes mellitus", "CHEBI:16236", "MONDO:0005148"), + ("Caffeine", "chronic myelogenous leukemia", "CHEBI:27732", "MONDO:0011996"), + ("Sodium chloride", "hypertension", "CHEBI:26710", "MONDO:0005044"), + ("Glucose", "migraine disorder", "CHEBI:17234", "MONDO:0005277"), +] + + +def build_assets(): + assets = [] + expectations = [ + "TopAnswer", + "TopAnswer", + "Acceptable", + "Acceptable", + "BadButForgivable", + ] + for i, (drug, disease, chem, mondo) in enumerate(PAIRS): + assets.append( + dict( + test_case_id=f"TestCase_{i // 4 + 1}", + asset_id=f"Asset_{i + 1}", + name=f"{drug} treats {disease}", + expected_output=expectations[i % 5], + predicate="biolink:treats", + input_curie=mondo, + output_curie=chem, + difficulty=random.random(), + ) + ) + for j, (drug, gene, chem, ncbi) in enumerate(GENES): + assets.append( + dict( + test_case_id=f"TestCase_{20 + j // 4}", + asset_id=f"Asset_{100 + j}", + name=f"{drug} affects {gene}", + expected_output="TopAnswer" if j % 3 else "Acceptable", + predicate="biolink:affects", + input_curie=ncbi, + output_curie=chem, + difficulty=random.random(), + ) + ) + for k, (chem_name, disease, chem, mondo) in enumerate(NEVER): + assets.append( + dict( + test_case_id="TestCase_30", + asset_id=f"Asset_{200 + k}", + name=f"{chem_name} treats {disease}", + expected_output="NeverShow", + predicate="biolink:treats", + input_curie=mondo, + output_curie=chem, + difficulty=0.2, + ) + ) + return assets + + +AGENT_SKILL = { + "ars": 0.78, + "shepherd-aragorn": 0.7, + "shepherd-arax": 0.66, + "shepherd-bte": 0.55, +} + + +def agent_result(agent, spec, run_index, n_runs, rng): + skill = AGENT_SKILL[agent] + # aragorn regresses for a stretch mid-month, then a fix lands + if agent == "shepherd-aragorn" and n_runs * 0.45 < run_index < n_runs * 0.7: + skill -= 0.25 + # bte gets steadily better + if agent == "shepherd-bte": + skill += 0.25 * run_index / n_runs + roll = rng.random() + if roll < 0.03: + return schema.AgentResult( + agent=agent, status="FAILED", message="Timed out", http_status=598 + ) + if roll < 0.06 and agent != "ars": + return schema.AgentResult( + agent=agent, status="ERROR", message="Status code: 500", http_status=500 + ) + n_results = int(rng.lognormvariate(5.2, 0.7)) + if rng.random() < 0.05: + return schema.AgentResult( + agent=agent, + status="NO_RESULTS", + message="No results", + http_status=200, + n_results=0, + found=False, + ) + expected = spec["expected_output"] + good = rng.random() < skill + 0.25 - spec["difficulty"] * 0.4 + if expected == "NeverShow": + found = not good + rank = rng.randint(1, n_results) if found else None + status = "PASSED" if not found else "FAILED" + else: + found = good or rng.random() < 0.5 + if found: + base = 1 + spec["difficulty"] * 40 + rank = max(1, int(rng.gauss(base if good else base * 4 + 30, 5))) + rank = min(rank, n_results) + else: + rank = None + limit = { + "TopAnswer": 30, + "Acceptable": n_results // 2, + "BadButForgivable": n_results, + }[expected] + status = "PASSED" if found and rank is not None and rank <= limit else "FAILED" + score = ( + None + if rank is None + else round(max(0.05, 1 - rank / max(n_results, 1)) * rng.uniform(0.85, 1.0), 3) + ) + return schema.AgentResult( + agent=agent, + status=status, + http_status=200, + found=found, + rank=rank, + score=score, + n_results=n_results, + pk=str(uuid.UUID(int=rng.getrandbits(128))), + ) + + +def demo_runs(days=30, env="ci", suite="sprint_4_tests", seed=7): + rng = random.Random(seed) + random.seed(seed) + assets = build_assets() + now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + n_runs = days + payloads = [] + for i in range(n_runs): + started = ( + now + - timedelta(days=n_runs - 1 - i, hours=-6 if i == n_runs - 1 else 0) + - timedelta(hours=8) + ) + run = schema.RunCreate( + run_id=uuid.uuid5(uuid.NAMESPACE_URL, f"demo:{suite}:{env}:{i}"), + suite=suite, + env=env, + harness_version="0.8.0", + tests_source="https://github.com/NCATSTranslator/Tests/archive/refs/heads/main.zip", + started_at=started, + ) + results = [] + counts = {} + for spec in assets: + agents = [agent_result(a, spec, i, n_runs, rng) for a in AGENTS] + overall = agents[0].status + counts[overall] = counts.get(overall, 0) + 1 + fields = {k: v for k, v in spec.items() if k != "difficulty"} + results.append( + schema.AssetResult( + kind="acceptance", + status=overall, + agents=agents, + parent_pk=str(uuid.UUID(int=rng.getrandbits(128))), + **fields, + ) + ) + payloads.append( + schema.RunPayload( + run=run, + results=results, + finish=schema.RunFinish( + ended_at=started + timedelta(minutes=rng.randint(38, 75)), + counts=counts, + ), + ) + ) + + # weekly performance sweeps + hosts = [ + ("https://ars.ci.transltr.io", "ars", 14.0), + ("https://aragorn.ci.transltr.io", "aragorn", 9.0), + ("https://arax.ci.transltr.io", "arax", 6.5), + ("https://bte.ci.transltr.io", "bte", 11.0), + ] + for week in range(max(days // 7, 1) + 1): + started = now - timedelta(days=7 * week, hours=30) + run = schema.RunCreate( + run_id=uuid.uuid5(uuid.NAMESPACE_URL, f"demo:perf:{week}"), + suite="performance_tests", + env=env, + harness_version="0.8.0", + started_at=started, + ) + perf = [] + for host, target, base in hosts: + msc = round(max(1.0, rng.gauss(base - week * 0.4, 1.2)), 1) + passed = msc >= base * 0.8 + perf.append( + schema.PerformanceResult( + test_case_id="Perf_1", + asset_id=f"Perf_{target}", + host=host, + helmsdeep_target=target, + profile="mixed", + status="PASSED" if passed else "FAILED", + exit_code=0, + max_sustainable_concurrency=msc, + checkpoints_passed=passed, + summary={ + "max_sustainable_concurrency": msc, + "checkpoints_passed": passed, + }, + ) + ) + payloads.append( + schema.RunPayload( + run=run, + performance=perf, + finish=schema.RunFinish( + ended_at=started + timedelta(hours=4, minutes=12) + ), + ) + ) + return payloads + + +def main(): + import os + + sessionmaker = make_sessionmaker(os.environ["RADIATOR_DATABASE_URL"]) + payloads = demo_runs() + with sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + print(f"Loaded {len(payloads)} demo runs.") + + +if __name__ == "__main__": + main() diff --git a/radiator/ingest.py b/radiator/ingest.py new file mode 100644 index 0000000..0d18d7a --- /dev/null +++ b/radiator/ingest.py @@ -0,0 +1,107 @@ +"""Writing results: what the ingest API and the Zebrunner importer share. + +Every write is an upsert on the identity ``radiator_schema`` defines, so a +retried batch or a replayed run updates what's there instead of duplicating it. +An asset's agent rows are replaced wholesale on each upsert: the newest upload +is the whole truth about that asset in that run. +""" + +import uuid + +from sqlalchemy import delete, insert, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +import radiator_schema as schema +from radiator.models import AgentResult, AssetResult, PerformanceResult, Run + + +class RunNotFound(LookupError): + pass + + +def upsert_run(session: Session, run: schema.RunCreate) -> None: + values = run.model_dump(exclude={"run_id"}) + stmt = pg_insert(Run).values(id=run.run_id, counts={}, **values) + session.execute(stmt.on_conflict_do_update(index_elements=[Run.id], set_=values)) + + +def _require_run(session: Session, run_id: uuid.UUID) -> None: + if session.get(Run, run_id) is None: + raise RunNotFound(run_id) + + +def upsert_results( + session: Session, run_id: uuid.UUID, batch: schema.ResultBatch +) -> None: + _require_run(session, run_id) + for result in batch.results: + values = result.model_dump(exclude={"agents"}) + values["status"] = result.status.value + stmt = pg_insert(AssetResult).values(run_id=run_id, **values) + asset_result_id = session.execute( + stmt.on_conflict_do_update( + index_elements=[ + AssetResult.run_id, + AssetResult.test_case_id, + AssetResult.asset_id, + ], + set_=values, + ).returning(AssetResult.id) + ).scalar_one() + session.execute( + delete(AgentResult).where(AgentResult.asset_result_id == asset_result_id) + ) + if result.agents: + session.execute( + insert(AgentResult), + [ + { + **agent.model_dump(), + "status": agent.status.value, + "asset_result_id": asset_result_id, + } + for agent in result.agents + ], + ) + + for perf in batch.performance: + values = perf.model_dump() + values["status"] = perf.status.value + stmt = pg_insert(PerformanceResult).values(run_id=run_id, **values) + session.execute( + stmt.on_conflict_do_update( + index_elements=[ + PerformanceResult.run_id, + PerformanceResult.test_case_id, + PerformanceResult.asset_id, + PerformanceResult.host, + ], + set_=values, + ) + ) + + +def finish_run(session: Session, run_id: uuid.UUID, finish: schema.RunFinish) -> None: + _require_run(session, run_id) + session.execute( + update(Run) + .where(Run.id == run_id) + .values(ended_at=finish.ended_at, counts=finish.counts) + ) + + +def ingest_payload(session: Session, payload: schema.RunPayload) -> None: + """Write a whole run at once (the importer's path).""" + upsert_run(session, payload.run) + upsert_results( + session, + payload.run.run_id, + schema.ResultBatch(results=payload.results, performance=payload.performance), + ) + if payload.finish is not None: + finish_run(session, payload.run.run_id, payload.finish) + + +def run_exists(session: Session, run_id: uuid.UUID) -> bool: + return session.scalar(select(Run.id).where(Run.id == run_id)) is not None diff --git a/radiator/migrations/env.py b/radiator/migrations/env.py new file mode 100644 index 0000000..b3ad1fd --- /dev/null +++ b/radiator/migrations/env.py @@ -0,0 +1,49 @@ +"""Alembic environment for the Information Radiator.""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine + +from radiator.models import Base + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + + +def database_url() -> str: + url = config.attributes.get("database_url") or os.getenv("RADIATOR_DATABASE_URL") + if not url: + raise RuntimeError("RADIATOR_DATABASE_URL must be set") + return url + + +def run_migrations_offline(): + context.configure( + url=database_url(), + target_metadata=Base.metadata, + literal_binds=True, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + engine = create_engine(database_url()) + with engine.connect() as connection: + context.configure( + connection=connection, + target_metadata=Base.metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/radiator/migrations/script.py.mako b/radiator/migrations/script.py.mako new file mode 100644 index 0000000..fe3b097 --- /dev/null +++ b/radiator/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/radiator/migrations/versions/0001_initial_schema.py b/radiator/migrations/versions/0001_initial_schema.py new file mode 100644 index 0000000..4540f80 --- /dev/null +++ b/radiator/migrations/versions/0001_initial_schema.py @@ -0,0 +1,136 @@ +"""Initial schema + +Revision ID: 0001 +Revises: +Create Date: 2026-10-06 13:59:11.385508 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "runs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("suite", sa.String(length=255), nullable=False), + sa.Column("env", sa.String(length=32), nullable=True), + sa.Column("target", sa.String(length=255), nullable=True), + sa.Column("target_url", sa.Text(), nullable=True), + sa.Column("query_type", sa.String(length=32), nullable=True), + sa.Column("harness_version", sa.String(length=64), nullable=True), + sa.Column("tests_source", sa.Text(), nullable=True), + sa.Column("origin", sa.String(length=32), nullable=False), + sa.Column("origin_ref", sa.String(length=255), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("counts", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_runs_series", "runs", ["suite", "env", "started_at"], unique=False + ) + op.create_index("ix_runs_started_at", "runs", ["started_at"], unique=False) + op.create_index( + "uq_runs_origin_ref", + "runs", + ["origin", "origin_ref"], + unique=True, + postgresql_where="origin_ref IS NOT NULL", + ) + op.create_table( + "asset_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("run_id", sa.Uuid(), nullable=False), + sa.Column("test_case_id", sa.String(length=255), nullable=False), + sa.Column("asset_id", sa.String(length=255), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("expected_output", sa.String(length=64), nullable=True), + sa.Column("predicate", sa.String(length=255), nullable=True), + sa.Column("input_curie", sa.String(length=255), nullable=True), + sa.Column("output_curie", sa.String(length=255), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("parent_pk", sa.String(length=255), nullable=True), + sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("run_id", "test_case_id", "asset_id"), + ) + op.create_index( + "ix_asset_results_asset", + "asset_results", + ["test_case_id", "asset_id"], + unique=False, + ) + op.create_table( + "performance_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("run_id", sa.Uuid(), nullable=False), + sa.Column("test_case_id", sa.String(length=255), nullable=False), + sa.Column("asset_id", sa.String(length=255), nullable=False), + sa.Column("host", sa.String(length=512), nullable=False), + sa.Column("helmsdeep_target", sa.String(length=64), nullable=True), + sa.Column("profile", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("exit_code", sa.Integer(), nullable=True), + sa.Column("max_sustainable_concurrency", sa.Float(), nullable=True), + sa.Column("knee_unsupported", sa.Boolean(), nullable=True), + sa.Column("checkpoints_passed", sa.Boolean(), nullable=True), + sa.Column("summary", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("run_id", "test_case_id", "asset_id", "host"), + ) + op.create_index( + "ix_performance_results_host", "performance_results", ["host"], unique=False + ) + op.create_table( + "agent_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("asset_result_id", sa.Integer(), nullable=False), + sa.Column("agent", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("message", sa.Text(), nullable=True), + sa.Column("http_status", sa.Integer(), nullable=True), + sa.Column("found", sa.Boolean(), nullable=True), + sa.Column("rank", sa.Integer(), nullable=True), + sa.Column("score", sa.Float(), nullable=True), + sa.Column("n_results", sa.Integer(), nullable=True), + sa.Column("response_time_s", sa.Float(), nullable=True), + sa.Column("pk", sa.String(length=255), nullable=True), + sa.Column("expected_nodes_found", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["asset_result_id"], ["asset_results.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("asset_result_id", "agent"), + ) + op.create_index("ix_agent_results_agent", "agent_results", ["agent"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_agent_results_agent", table_name="agent_results") + op.drop_table("agent_results") + op.drop_index("ix_performance_results_host", table_name="performance_results") + op.drop_table("performance_results") + op.drop_index("ix_asset_results_asset", table_name="asset_results") + op.drop_table("asset_results") + op.drop_index( + "uq_runs_origin_ref", + table_name="runs", + postgresql_where="origin_ref IS NOT NULL", + ) + op.drop_index("ix_runs_started_at", table_name="runs") + op.drop_index("ix_runs_series", table_name="runs") + op.drop_table("runs") diff --git a/radiator/models.py b/radiator/models.py new file mode 100644 index 0000000..ab9484a --- /dev/null +++ b/radiator/models.py @@ -0,0 +1,165 @@ +"""Database tables. + +The shape follows ``radiator_schema``: a run has asset results, each with a row +per agent, plus any performance results. The unique constraints are the +upsert keys the ingest API relies on, so a retried or replayed upload updates +rows rather than duplicating them. +""" + +import uuid +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy import ( + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +# Postgres only: the ingest upserts use INSERT ... ON CONFLICT. +JsonType = JSONB + + +def utcnow(): + return datetime.now(timezone.utc) + + +class Base(DeclarativeBase): + pass + + +class Run(Base): + __tablename__ = "runs" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True) + suite: Mapped[str] = mapped_column(String(255)) + env: Mapped[Optional[str]] = mapped_column(String(32)) + target: Mapped[Optional[str]] = mapped_column(String(255)) + target_url: Mapped[Optional[str]] = mapped_column(Text) + query_type: Mapped[Optional[str]] = mapped_column(String(32)) + harness_version: Mapped[Optional[str]] = mapped_column(String(64)) + tests_source: Mapped[Optional[str]] = mapped_column(Text) + origin: Mapped[str] = mapped_column(String(32), default="harness") + origin_ref: Mapped[Optional[str]] = mapped_column(String(255)) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + ended_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + counts: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow + ) + + assets: Mapped[list["AssetResult"]] = relationship( + back_populates="run", cascade="all, delete-orphan", passive_deletes=True + ) + performance: Mapped[list["PerformanceResult"]] = relationship( + back_populates="run", cascade="all, delete-orphan", passive_deletes=True + ) + + __table_args__ = ( + # "the runs of this series", newest first: the runs list, and finding + # the previous comparable run for a diff + Index("ix_runs_series", "suite", "env", "started_at"), + Index("ix_runs_started_at", "started_at"), + # an import is re-runnable: one row per source run + Index( + "uq_runs_origin_ref", + "origin", + "origin_ref", + unique=True, + postgresql_where="origin_ref IS NOT NULL", + ), + ) + + +class AssetResult(Base): + __tablename__ = "asset_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) + test_case_id: Mapped[str] = mapped_column(String(255)) + asset_id: Mapped[str] = mapped_column(String(255)) + kind: Mapped[str] = mapped_column(String(32)) + name: Mapped[Optional[str]] = mapped_column(Text) + expected_output: Mapped[Optional[str]] = mapped_column(String(64)) + predicate: Mapped[Optional[str]] = mapped_column(String(255)) + input_curie: Mapped[Optional[str]] = mapped_column(String(255)) + output_curie: Mapped[Optional[str]] = mapped_column(String(255)) + status: Mapped[str] = mapped_column(String(32)) + parent_pk: Mapped[Optional[str]] = mapped_column(String(255)) + details: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + + run: Mapped[Run] = relationship(back_populates="assets") + agents: Mapped[list["AgentResult"]] = relationship( + back_populates="asset", + cascade="all, delete-orphan", + passive_deletes=True, + order_by="AgentResult.id", + ) + + __table_args__ = ( + UniqueConstraint("run_id", "test_case_id", "asset_id"), + # an asset's history across runs + Index("ix_asset_results_asset", "test_case_id", "asset_id"), + ) + + +class AgentResult(Base): + __tablename__ = "agent_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + asset_result_id: Mapped[int] = mapped_column( + ForeignKey("asset_results.id", ondelete="CASCADE") + ) + agent: Mapped[str] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(32)) + message: Mapped[Optional[str]] = mapped_column(Text) + http_status: Mapped[Optional[int]] = mapped_column(Integer) + found: Mapped[Optional[bool]] = mapped_column(Boolean) + rank: Mapped[Optional[int]] = mapped_column(Integer) + score: Mapped[Optional[float]] = mapped_column(Float) + n_results: Mapped[Optional[int]] = mapped_column(Integer) + response_time_s: Mapped[Optional[float]] = mapped_column(Float) + pk: Mapped[Optional[str]] = mapped_column(String(255)) + expected_nodes_found: Mapped[Optional[str]] = mapped_column(Text) + + asset: Mapped[AssetResult] = relationship(back_populates="agents") + + __table_args__ = ( + UniqueConstraint("asset_result_id", "agent"), + Index("ix_agent_results_agent", "agent"), + ) + + +class PerformanceResult(Base): + __tablename__ = "performance_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) + test_case_id: Mapped[str] = mapped_column(String(255)) + asset_id: Mapped[str] = mapped_column(String(255)) + host: Mapped[str] = mapped_column(String(512)) + helmsdeep_target: Mapped[Optional[str]] = mapped_column(String(64)) + profile: Mapped[Optional[str]] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(32)) + error: Mapped[Optional[str]] = mapped_column(Text) + exit_code: Mapped[Optional[int]] = mapped_column(Integer) + max_sustainable_concurrency: Mapped[Optional[float]] = mapped_column(Float) + knee_unsupported: Mapped[Optional[bool]] = mapped_column(Boolean) + checkpoints_passed: Mapped[Optional[bool]] = mapped_column(Boolean) + summary: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + + run: Mapped[Run] = relationship(back_populates="performance") + + __table_args__ = ( + UniqueConstraint("run_id", "test_case_id", "asset_id", "host"), + Index("ix_performance_results_host", "host"), + ) diff --git a/radiator/queries.py b/radiator/queries.py new file mode 100644 index 0000000..4d05ad6 --- /dev/null +++ b/radiator/queries.py @@ -0,0 +1,474 @@ +"""Read queries behind the dashboard's views and the read API.""" + +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Optional + +from sqlalchemy import func, select +from sqlalchemy.orm import Session, selectinload + +from radiator.models import AgentResult, AssetResult, PerformanceResult, Run + +STATUSES = ["PASSED", "FAILED", "NO_RESULTS", "ERROR", "SKIPPED"] +# Statuses that mean the agent answered and got it wrong, as opposed to the +# test not happening (SKIPPED). +NOT_PASSING = {"FAILED", "NO_RESULTS", "ERROR"} + +# Fixed display order, so the ARS leads and an agent's column (and color) never +# moves between runs. Agents not listed sort after these, alphabetically. +AGENT_ORDER = [ + "ars", + "aragorn", + "arax", + "biothings-explorer", + "improving-agent", + "unsecret-agent", + "cqs", + "shepherd-aragorn", + "shepherd-arax", + "shepherd-bte", +] + + +def agent_sort_key(agent: str): + try: + return (0, AGENT_ORDER.index(agent), agent) + except ValueError: + return (1, 0, agent) + + +def pass_rate(counts: dict) -> Optional[float]: + """Passed over everything that actually ran (skips excluded).""" + ran = sum(counts.get(status, 0) for status in STATUSES if status != "SKIPPED") + if ran == 0: + return None + return counts.get("PASSED", 0) / ran + + +# --- runs ------------------------------------------------------------------ + + +def list_runs( + session: Session, + suite: Optional[str] = None, + env: Optional[str] = None, + limit: int = 50, + offset: int = 0, +) -> list[Run]: + stmt = select(Run).order_by(Run.started_at.desc()).limit(limit).offset(offset) + if suite: + stmt = stmt.where(Run.suite == suite) + if env: + stmt = stmt.where(Run.env == env) + return list(session.scalars(stmt)) + + +def count_runs(session: Session, suite=None, env=None) -> int: + stmt = select(func.count()).select_from(Run) + if suite: + stmt = stmt.where(Run.suite == suite) + if env: + stmt = stmt.where(Run.env == env) + return session.scalar(stmt) + + +def filter_options(session: Session) -> dict: + return { + "suites": list( + session.scalars(select(Run.suite).distinct().order_by(Run.suite)) + ), + "envs": [ + env + for env in session.scalars(select(Run.env).distinct().order_by(Run.env)) + if env + ], + } + + +def status_counts(session: Session, run_ids: list[uuid.UUID]) -> dict: + """Overall (asset-level) status counts per run, from the stored results. + + Computed rather than read from ``Run.counts``: imported and unfinished runs + have no counts, and these always agree with what the run page shows. + """ + counts: dict = {run_id: {} for run_id in run_ids} + if not run_ids: + return counts + rows = session.execute( + select(AssetResult.run_id, AssetResult.status, func.count()) + .where(AssetResult.run_id.in_(run_ids)) + .group_by(AssetResult.run_id, AssetResult.status) + ) + for run_id, status, n in rows: + counts[run_id][status] = n + return counts + + +def performance_counts(session: Session, run_ids: list[uuid.UUID]) -> dict: + """Performance verdict counts per run, for runs with performance results.""" + counts: dict = defaultdict(dict) + if not run_ids: + return counts + rows = session.execute( + select(PerformanceResult.run_id, PerformanceResult.status, func.count()) + .where(PerformanceResult.run_id.in_(run_ids)) + .group_by(PerformanceResult.run_id, PerformanceResult.status) + ) + for run_id, status, n in rows: + counts[run_id][status] = n + return counts + + +def get_run(session: Session, run_id: uuid.UUID) -> Optional[Run]: + return session.scalar( + select(Run) + .where(Run.id == run_id) + .options( + selectinload(Run.assets).selectinload(AssetResult.agents), + selectinload(Run.performance), + ) + ) + + +def previous_run(session: Session, run: Run) -> Optional[Run]: + """The run before this one in the same series. + + A series is the same suite, env, target override, and query type: a run + against a locally running ARA, or of only the MVP1 queries, isn't + comparable with a full run against the deployed services. + """ + return session.scalar( + select(Run) + .where( + Run.suite == run.suite, + Run.env.is_not_distinct_from(run.env), + Run.target.is_not_distinct_from(run.target), + Run.query_type.is_not_distinct_from(run.query_type), + Run.started_at < run.started_at, + ) + .order_by(Run.started_at.desc()) + .limit(1) + ) + + +def run_agents(run: Run) -> list[str]: + agents = {agent.agent for asset in run.assets for agent in asset.agents} + return sorted(agents, key=agent_sort_key) + + +def agent_counts(run: Run) -> dict[str, dict[str, int]]: + counts: dict = defaultdict(lambda: defaultdict(int)) + for asset in run.assets: + for agent in asset.agents: + counts[agent.agent][agent.status] += 1 + return counts + + +# --- diff ------------------------------------------------------------------ + +DIFF_KINDS = [ + ("regression", "Regressions", "Passed last run, not passing now"), + ("fixed", "Fixed", "Not passing last run, passing now"), + ("changed", "Other status changes", "Changed between non-passing statuses"), + ("skipped", "Skips", "Started or stopped being skipped"), + ("rank", "Rank moves", "Same status, expected answer moved 5 or more places"), + ("added", "New assets", "Not in the previous run"), + ("removed", "Dropped assets", "In the previous run, not in this one"), +] + + +@dataclass +class Change: + kind: str + asset: AssetResult + agent: Optional[str] = None + before: Optional[AgentResult] = None + after: Optional[AgentResult] = None + + @property + def rank_delta(self) -> Optional[int]: + """Positive when the expected answer moved up (a smaller rank).""" + if self.before is None or self.after is None: + return None + if self.before.rank is None or self.after.rank is None: + return None + return self.before.rank - self.after.rank + + +@dataclass +class Diff: + base: Optional[Run] + changes: dict[str, list[Change]] = field(default_factory=dict) + + def count(self, kind: str) -> int: + return len(self.changes.get(kind, [])) + + +# Smaller rank moves are mostly scoring noise between runs. +RANK_MOVE_THRESHOLD = 5 + + +def _classify(before: AgentResult, after: AgentResult) -> Optional[str]: + if before.status != after.status: + if "SKIPPED" in (before.status, after.status): + return "skipped" + if before.status == "PASSED": + return "regression" + if after.status == "PASSED": + return "fixed" + return "changed" + if ( + before.rank is not None + and after.rank is not None + and abs(before.rank - after.rank) >= RANK_MOVE_THRESHOLD + ): + return "rank" + return None + + +def diff_runs(base: Optional[Run], run: Run) -> Diff: + diff = Diff(base=base, changes={kind: [] for kind, _, _ in DIFF_KINDS}) + if base is None: + return diff + before_assets = {(a.test_case_id, a.asset_id): a for a in base.assets} + after_assets = {(a.test_case_id, a.asset_id): a for a in run.assets} + + for key, asset in after_assets.items(): + previous = before_assets.get(key) + if previous is None: + diff.changes["added"].append(Change("added", asset)) + continue + before_agents = {a.agent: a for a in previous.agents} + for after in asset.agents: + before = before_agents.get(after.agent) + if before is None: + continue + kind = _classify(before, after) + if kind: + diff.changes[kind].append( + Change(kind, asset, after.agent, before, after) + ) + for key, asset in before_assets.items(): + if key not in after_assets: + diff.changes["removed"].append(Change("removed", asset)) + + for changes in diff.changes.values(): + changes.sort( + key=lambda c: ( + agent_sort_key(c.agent or ""), + -abs(c.rank_delta or 0), + c.asset.name or "", + ) + ) + return diff + + +# --- asset history ----------------------------------------------------------- + + +@dataclass +class AssetHistory: + test_case_id: str + asset_id: str + latest: Optional[AssetResult] + # oldest first + points: list[tuple[Run, AssetResult]] + agents: list[str] + + def agent_result(self, asset: AssetResult, agent: str) -> Optional[AgentResult]: + for result in asset.agents: + if result.agent == agent: + return result + return None + + +def asset_history( + session: Session, + test_case_id: str, + asset_id: str, + env: Optional[str] = None, + limit: int = 60, +) -> AssetHistory: + stmt = ( + select(Run, AssetResult) + .join(AssetResult, AssetResult.run_id == Run.id) + .where( + AssetResult.test_case_id == test_case_id, + AssetResult.asset_id == asset_id, + ) + .options(selectinload(AssetResult.agents)) + .order_by(Run.started_at.desc()) + .limit(limit) + ) + if env: + stmt = stmt.where(Run.env == env) + points = list(reversed(session.execute(stmt).all())) + agents = sorted( + {agent.agent for _, asset in points for agent in asset.agents}, + key=agent_sort_key, + ) + return AssetHistory( + test_case_id=test_case_id, + asset_id=asset_id, + latest=points[-1][1] if points else None, + points=points, + agents=agents, + ) + + +def asset_envs(session: Session, test_case_id: str, asset_id: str) -> list[str]: + return [ + env + for env in session.scalars( + select(Run.env) + .join(AssetResult, AssetResult.run_id == Run.id) + .where( + AssetResult.test_case_id == test_case_id, + AssetResult.asset_id == asset_id, + ) + .distinct() + ) + if env + ] + + +# --- trends ---------------------------------------------------------------- + + +@dataclass +class TrendPoint: + run: Run + # agent -> expected_output -> status -> count + counts: dict + + +def agent_trends( + session: Session, suite: str, env: Optional[str], days: int = 90 +) -> list[TrendPoint]: + """Per-run, per-agent status counts by expected output, oldest first. + + Only full runs of the suite: target overrides and single-query-type runs + measure something else and would make the line jump. + """ + since = datetime.now(timezone.utc) - timedelta(days=days) + run_filter = [ + Run.suite == suite, + Run.started_at >= since, + Run.target.is_(None), + Run.query_type.is_(None), + ] + if env: + run_filter.append(Run.env == env) + runs = list( + session.scalars(select(Run).where(*run_filter).order_by(Run.started_at)) + ) + if not runs: + return [] + rows = session.execute( + select( + AssetResult.run_id, + AgentResult.agent, + AssetResult.expected_output, + AgentResult.status, + func.count(), + ) + .join(AgentResult, AgentResult.asset_result_id == AssetResult.id) + .where(AssetResult.run_id.in_([run.id for run in runs])) + .group_by( + AssetResult.run_id, + AgentResult.agent, + AssetResult.expected_output, + AgentResult.status, + ) + ) + by_run: dict = { + run.id: defaultdict(lambda: defaultdict(lambda: defaultdict(int))) + for run in runs + } + for run_id, agent, expected, status, n in rows: + by_run[run_id][agent][expected or "Unspecified"][status] += n + return [TrendPoint(run=run, counts=by_run[run.id]) for run in runs] + + +# --- performance ------------------------------------------------------------- + + +def performance_history( + session: Session, days: int = 180 +) -> dict[str, list[tuple[Run, PerformanceResult]]]: + """Performance results per host (and HelmsDeep target), oldest first.""" + since = datetime.now(timezone.utc) - timedelta(days=days) + rows = session.execute( + select(Run, PerformanceResult) + .join(PerformanceResult, PerformanceResult.run_id == Run.id) + .where(Run.started_at >= since) + .order_by(Run.started_at) + ) + series: dict = defaultdict(list) + for run, perf in rows: + key = perf.host + if perf.helmsdeep_target: + key = f"{perf.host} ({perf.helmsdeep_target})" + series[key].append((run, perf)) + return dict(sorted(series.items())) + + +# --- JSON for the read API --------------------------------------------------- + + +def run_json(run: Run, with_results: bool = False) -> dict: + data = { + "run_id": str(run.id), + "suite": run.suite, + "env": run.env, + "target": run.target, + "query_type": run.query_type, + "harness_version": run.harness_version, + "origin": run.origin, + "started_at": run.started_at.isoformat(), + "ended_at": run.ended_at.isoformat() if run.ended_at else None, + "counts": run.counts, + } + if with_results: + data["results"] = [ + { + "test_case_id": asset.test_case_id, + "asset_id": asset.asset_id, + "name": asset.name, + "kind": asset.kind, + "expected_output": asset.expected_output, + "input_curie": asset.input_curie, + "output_curie": asset.output_curie, + "status": asset.status, + "agents": [ + { + "agent": agent.agent, + "status": agent.status, + "found": agent.found, + "rank": agent.rank, + "score": agent.score, + "n_results": agent.n_results, + "http_status": agent.http_status, + "response_time_s": agent.response_time_s, + "pk": agent.pk, + } + for agent in asset.agents + ], + } + for asset in run.assets + ] + data["performance"] = [ + { + "host": perf.host, + "helmsdeep_target": perf.helmsdeep_target, + "profile": perf.profile, + "status": perf.status, + "max_sustainable_concurrency": perf.max_sustainable_concurrency, + "checkpoints_passed": perf.checkpoints_passed, + "error": perf.error, + } + for perf in run.performance + ] + return data diff --git a/radiator/requirements-test.txt b/radiator/requirements-test.txt new file mode 100644 index 0000000..333c276 --- /dev/null +++ b/radiator/requirements-test.txt @@ -0,0 +1,5 @@ +-r requirements.txt +pytest==9.1.1 +httpx==0.28.1 +# Starlette's TestClient +httpx2==2.13.1 diff --git a/radiator/requirements.txt b/radiator/requirements.txt new file mode 100644 index 0000000..735e144 --- /dev/null +++ b/radiator/requirements.txt @@ -0,0 +1,12 @@ +# The Information Radiator's own dependencies. Kept apart from the harness's +# requirements: the two ship as separate images and pin independently. The +# shared contract, radiator_schema, needs only pydantic 2 and works with both. +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +sqlalchemy==2.1.3 +psycopg[binary]==3.3.6 +alembic==1.20.0 +jinja2==3.1.6 +itsdangerous==2.2.0 +python-multipart==0.0.32 +pydantic==2.13.5 diff --git a/radiator/static/charts.js b/radiator/static/charts.js new file mode 100644 index 0000000..55c4593 --- /dev/null +++ b/radiator/static/charts.js @@ -0,0 +1,100 @@ +// Hover layer for the server-rendered charts and status cells. Tooltips only +// enhance: every value is also in the page's tables. All text is inserted with +// textContent, since names and messages come from uploaded data. +(function () { + const tip = document.createElement("div"); + tip.className = "tooltip"; + document.body.appendChild(tip); + + function place(event) { + const pad = 14; + let x = event.clientX + pad; + let y = event.clientY + pad; + const rect = tip.getBoundingClientRect(); + if (x + rect.width > window.innerWidth - 8) x = event.clientX - rect.width - pad; + if (y + rect.height > window.innerHeight - 8) y = event.clientY - rect.height - pad; + tip.style.left = x + "px"; + tip.style.top = y + "px"; + } + + function show(title, rows, event) { + tip.replaceChildren(); + const head = document.createElement("div"); + head.className = "tt-title"; + head.textContent = title; + tip.appendChild(head); + for (const row of rows) { + const line = document.createElement("div"); + line.className = "tt-row"; + if (row.color) { + const key = document.createElement("span"); + key.className = "tt-key"; + key.style.background = row.color; + line.appendChild(key); + } + const value = document.createElement("span"); + value.className = "tt-value"; + value.textContent = row.value; + const name = document.createElement("span"); + name.className = "tt-name"; + name.textContent = row.name; + line.append(value, name); + tip.appendChild(line); + } + tip.style.display = "block"; + place(event); + } + + function hide() { + tip.style.display = "none"; + } + + // Line charts: a crosshair that snaps to the nearest run, listing every series. + for (const wrap of document.querySelectorAll(".chart-wrap[data-hover]")) { + const data = JSON.parse(wrap.dataset.hover); + const svg = wrap.querySelector("svg"); + const cross = svg.querySelector(".crosshair"); + svg.addEventListener("pointermove", (event) => { + const box = svg.getBoundingClientRect(); + const x = ((event.clientX - box.left) / box.width) * data.width; + let best = 0; + for (let i = 1; i < data.xs.length; i++) { + if (Math.abs(data.xs[i] - x) < Math.abs(data.xs[best] - x)) best = i; + } + cross.setAttribute("x1", data.xs[best]); + cross.setAttribute("x2", data.xs[best]); + cross.setAttribute("visibility", "visible"); + const rows = data.series + .filter((s) => s.values[best] !== null) + .map((s) => ({ color: s.color, value: s.values[best], name: s.name })); + if (!rows.length) rows.push({ value: "no data", name: "" }); + show(data.titles[best], rows, event); + }); + svg.addEventListener("pointerleave", () => { + cross.setAttribute("visibility", "hidden"); + hide(); + }); + } + + // Cells and bar segments: each carries its own tooltip. + function cellTip(event) { + const el = event.target.closest("[data-tip]"); + if (!el) return; + const lines = el.dataset.tip.split("\n"); + show(lines[0], lines.slice(1).map((l) => ({ value: l, name: "" })), event); + } + document.addEventListener("pointerover", cellTip); + document.addEventListener("pointermove", (event) => { + if (event.target.closest("[data-tip]")) place(event); + }); + document.addEventListener("pointerout", (event) => { + if (event.target.closest("[data-tip]")) hide(); + }); + document.addEventListener("focusin", (event) => { + const el = event.target.closest("[data-tip]"); + if (!el) return; + const box = el.getBoundingClientRect(); + cellTip({ target: el, clientX: box.right, clientY: box.bottom }); + }); + document.addEventListener("focusout", hide); +})(); diff --git a/radiator/static/style.css b/radiator/static/style.css new file mode 100644 index 0000000..b7e9383 --- /dev/null +++ b/radiator/static/style.css @@ -0,0 +1,313 @@ +/* Information Radiator. Colors are tokens: light by default, dark from the OS + setting. Status colors are fixed and always paired with an icon and label; + agent colors are fixed per agent so an agent looks the same everywhere. */ + +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --surface-2: #f3f2ef; + --ink: #0b0b0b; + --ink-2: #52514e; + --ink-muted: #6f6d68; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.1); + --link: #1c5cab; + --accent: #2a78d6; + --up: #006300; + --down: #b12f2f; + + --status-passed: #0ca30c; + --status-no-results: #fab219; + --status-error: #ec835a; + --status-failed: #d03b3b; + --status-skipped: #d6d5ce; + + --agent-1: #2a78d6; + --agent-2: #eb6834; + --agent-3: #1baf7a; + --agent-4: #eda100; + --agent-5: #e87ba4; + --agent-6: #008300; + --agent-7: #4a3aa7; + --agent-8: #e34948; +} + +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --surface-2: #232321; + --ink: #ffffff; + --ink-2: #c3c2b7; + --ink-muted: #9a9890; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.1); + --link: #86b6ef; + --accent: #3987e5; + --up: #0ca30c; + --down: #e66767; + --status-skipped: #4a4a46; + + --agent-1: #3987e5; + --agent-2: #d95926; + --agent-3: #199e70; + --agent-4: #c98500; + --agent-5: #d55181; + --agent-6: #008300; + --agent-7: #9085e9; + --agent-8: #e66767; + } +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} + +a { color: var(--link); text-decoration: none; } +a:hover { text-decoration: underline; } + +code, .mono { + font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + font-size: 12px; +} + +/* --- frame ------------------------------------------------------------- */ + +.topbar { + display: flex; + align-items: center; + gap: 24px; + padding: 0 24px; + height: 52px; + background: var(--surface); + border-bottom: 1px solid var(--border); +} +.brand { + font-weight: 650; + color: var(--ink); + display: flex; + align-items: center; + gap: 8px; +} +.brand:hover { text-decoration: none; } +.brand-mark { + width: 18px; height: 18px; border-radius: 5px; + background: conic-gradient(var(--status-passed) 0 60%, var(--status-failed) 0 80%, var(--status-no-results) 0); +} +.nav { display: flex; gap: 4px; flex: 1; } +.nav a { + color: var(--ink-2); + padding: 6px 10px; + border-radius: 6px; +} +.nav a:hover { background: var(--surface-2); text-decoration: none; } +.nav a.active { color: var(--ink); background: var(--surface-2); font-weight: 550; } +.logout button { + background: none; border: 0; color: var(--ink-2); cursor: pointer; font: inherit; +} + +main { max-width: 1280px; margin: 0 auto; padding: 24px; } + +.page-head { margin-bottom: 20px; } +.page-head h1 { font-size: 22px; font-weight: 650; margin: 0 0 4px; } +.crumbs { color: var(--ink-muted); font-size: 13px; margin-bottom: 6px; } +.meta { color: var(--ink-2); display: flex; flex-wrap: wrap; gap: 6px 16px; } + +.card { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 10px; + padding: 16px 20px; + margin-bottom: 20px; +} +.card h2 { font-size: 15px; font-weight: 650; margin: 0 0 2px; } +.card .sub { color: var(--ink-muted); font-size: 13px; margin: 0 0 12px; } +.card-head { display: flex; justify-content: space-between; align-items: baseline; gap: 16px; } + +.empty { color: var(--ink-muted); padding: 24px 0; text-align: center; } + +/* --- filters: one row above what they scope ------------------------------ */ + +.filters { + display: flex; flex-wrap: wrap; gap: 8px; align-items: end; margin-bottom: 16px; +} +.filters label { display: flex; flex-direction: column; font-size: 12px; color: var(--ink-muted); gap: 2px; } +.filters select, .filters input { + font: inherit; color: var(--ink); background: var(--surface); + border: 1px solid var(--axis); border-radius: 6px; padding: 5px 8px; min-width: 120px; +} +.filters button, .button { + font: inherit; border: 1px solid var(--axis); background: var(--surface); + color: var(--ink); border-radius: 6px; padding: 5px 12px; cursor: pointer; +} +.filters .clear { align-self: center; font-size: 13px; } + +/* --- stat tiles ---------------------------------------------------------- */ + +.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 12px; margin-bottom: 20px; } +.tile { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 14px 16px; +} +.tile .label { color: var(--ink-2); font-size: 13px; } +.tile .value { font-size: 26px; font-weight: 600; margin-top: 2px; } +.tile.hero .value { font-size: 48px; line-height: 1.1; } +.tile .delta { font-size: 13px; color: var(--ink-muted); } +.delta.up { color: var(--up); } +.delta.down { color: var(--down); } + +/* --- tables -------------------------------------------------------------- */ + +table.data { width: 100%; border-collapse: collapse; } +table.data th { + text-align: left; font-weight: 550; color: var(--ink-2); font-size: 12px; + padding: 6px 8px; border-bottom: 1px solid var(--axis); white-space: nowrap; + position: sticky; top: 0; background: var(--surface); +} +table.data td { padding: 6px 8px; border-bottom: 1px solid var(--grid); vertical-align: middle; } +table.data td.num, table.data th.num { text-align: right; font-variant-numeric: tabular-nums; } +table.data tr:hover td { background: var(--surface-2); } +.scroll { overflow-x: auto; } +.muted { color: var(--ink-muted); } +.nowrap { white-space: nowrap; } + +.badge { + display: inline-block; font-size: 11px; padding: 1px 7px; border-radius: 999px; + border: 1px solid var(--axis); color: var(--ink-2); white-space: nowrap; +} + +/* --- status: color + icon + label, never color alone -------------------- */ + +.status { + display: inline-flex; align-items: center; gap: 5px; white-space: nowrap; font-size: 13px; +} +.status .icon { + display: inline-grid; place-items: center; width: 18px; height: 18px; border-radius: 4px; + font-size: 11px; font-weight: 700; color: #0b0b0b; +} +.st-PASSED .icon, .cell.st-PASSED { background: var(--status-passed); } +/* In the matrix and history most cells pass: a tint keeps the page calm, so + what isn't passing is what stands out. */ +.matrix .cell.st-PASSED, .strip .cell.st-PASSED { + background: color-mix(in srgb, var(--status-passed) 16%, var(--surface)); + color: var(--up); + box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--status-passed) 35%, transparent); +} +.st-NO_RESULTS .icon, .cell.st-NO_RESULTS { background: var(--status-no-results); } +.st-ERROR .icon, .cell.st-ERROR { background: var(--status-error); } +.st-FAILED .icon, .cell.st-FAILED { background: var(--status-failed); color: #fff; } +.st-SKIPPED .icon, .cell.st-SKIPPED { background: var(--status-skipped); color: var(--ink-2); } + +.legend { display: flex; flex-wrap: wrap; gap: 6px 14px; font-size: 12px; color: var(--ink-2); margin: 8px 0 0; } +.legend .key { display: inline-flex; align-items: center; gap: 6px; } +.legend .swatch { width: 12px; height: 12px; border-radius: 3px; } +.legend .line { width: 16px; height: 2px; border-radius: 1px; } + +/* stacked status bar: part-to-whole, 2px surface gaps between segments */ +.stackbar { display: flex; gap: 2px; height: 10px; min-width: 120px; } +.stackbar span { display: block; height: 100%; min-width: 2px; } +.stackbar span:first-child { border-radius: 4px 0 0 4px; } +.stackbar span:last-child { border-radius: 0 4px 4px 0; } +.stackbar span:only-child { border-radius: 4px; } +.seg-PASSED { background: var(--status-passed); } +.seg-NO_RESULTS { background: var(--status-no-results); } +.seg-ERROR { background: var(--status-error); } +.seg-FAILED { background: var(--status-failed); } +.seg-SKIPPED { background: var(--status-skipped); } + +/* run matrix: one cell per asset x agent */ +.matrix td.agent-cell { padding: 3px 4px; } +.cell { + display: flex; align-items: center; justify-content: center; gap: 4px; + min-width: 64px; height: 26px; border-radius: 5px; font-size: 12px; + font-weight: 600; color: #0b0b0b; font-variant-numeric: tabular-nums; +} +.cell:hover { text-decoration: none; outline: 2px solid var(--ink); outline-offset: 1px; } +.cell .rank { font-weight: 500; opacity: 0.85; } +th.agent-head { text-align: center; } +.agent-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 5px; vertical-align: 1px; } + +/* history strip: rows are agents, columns are runs, oldest -> newest */ +.strip { display: grid; gap: 2px; align-items: center; } +.strip .row-label { font-size: 12px; color: var(--ink-2); padding-right: 10px; white-space: nowrap; } +.strip .cell { min-width: 0; height: 22px; font-size: 10px; border-radius: 3px; } +.strip .axis-label { font-size: 11px; color: var(--ink-muted); } + +/* diff */ +.diff-summary { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 16px; } +.diff-summary a { + border: 1px solid var(--border); background: var(--surface); border-radius: 8px; + padding: 8px 12px; color: var(--ink); min-width: 120px; +} +.diff-summary a:hover { text-decoration: none; border-color: var(--axis); } +.diff-summary .n { font-size: 20px; font-weight: 600; display: block; } +.diff-summary .k { font-size: 12px; color: var(--ink-2); } +.arrow { color: var(--ink-muted); margin: 0 4px; } + +/* charts */ +.chart { width: 100%; height: auto; display: block; overflow: visible; } +.chart text { fill: var(--ink-muted); font-size: 11px; font-variant-numeric: tabular-nums; } +.chart .grid { stroke: var(--grid); stroke-width: 1; } +.chart .baseline { stroke: var(--axis); stroke-width: 1; } +.chart .series { fill: none; stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart .dot { stroke: var(--surface); stroke-width: 2; } +.chart .crosshair { stroke: var(--ink-muted); stroke-width: 1; } +.chart-wrap { position: relative; } +.multiples { display: grid; grid-template-columns: repeat(auto-fit, minmax(380px, 1fr)); gap: 12px 24px; } +.multiples.two { grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); } +.multiples h3 { font-size: 13px; font-weight: 600; margin: 4px 0; } + +details.table-view { margin-top: 8px; } +details.table-view summary { cursor: pointer; color: var(--ink-2); font-size: 12px; } +details.table-view[open] summary { margin-bottom: 8px; } + +/* tooltip (one for the page; filled with textContent by charts.js) */ +.tooltip { + position: fixed; pointer-events: none; z-index: 10; display: none; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); border-radius: 8px; padding: 8px 10px; + font-size: 12px; max-width: 320px; +} +.tooltip .tt-title { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; align-items: center; gap: 8px; } +.tooltip .tt-key { width: 12px; height: 2px; border-radius: 1px; flex: none; } +.tooltip .tt-value { font-weight: 650; font-variant-numeric: tabular-nums; } +.tooltip .tt-name { color: var(--ink-2); } + +/* login */ +.login-page { min-height: 100vh; display: grid; place-items: center; padding: 16px; } +.login { + width: 100%; max-width: 360px; background: var(--surface); border: 1px solid var(--border); + border-radius: 12px; padding: 28px; +} +.login h1 { font-size: 20px; margin: 0 0 4px; } +.login p { color: var(--ink-2); margin: 0 0 20px; } +.login label { display: block; font-size: 13px; color: var(--ink-2); margin-bottom: 12px; } +.login input { + display: block; width: 100%; margin-top: 4px; font: inherit; color: var(--ink); + background: var(--page); border: 1px solid var(--axis); border-radius: 6px; padding: 8px 10px; +} +.login button { + width: 100%; margin-top: 8px; font: inherit; font-weight: 600; color: #fff; + background: var(--accent); border: 0; border-radius: 6px; padding: 9px; cursor: pointer; +} +.error { + background: color-mix(in srgb, var(--status-failed) 12%, transparent); + border: 1px solid color-mix(in srgb, var(--status-failed) 40%, transparent); + border-radius: 6px; padding: 8px 10px; margin-bottom: 14px; font-size: 13px; +} +.pager { display: flex; gap: 12px; justify-content: flex-end; margin-top: 12px; } + +@media (max-width: 640px) { + main { padding: 16px; } + .topbar { padding: 0 16px; gap: 12px; } + .tile.hero .value { font-size: 40px; } +} diff --git a/radiator/templates/_macros.html b/radiator/templates/_macros.html new file mode 100644 index 0000000..26524d4 --- /dev/null +++ b/radiator/templates/_macros.html @@ -0,0 +1,21 @@ +{% macro agent_head(agent) -%} +{{ agent }} +{%- endmacro %} + +{% macro cell(asset, result, env, href=None) -%} +{% if result is none -%} +– +{%- else -%} + + + {%- if result.rank is not none %}#{{ result.rank }}{% endif -%} + +{%- endif %} +{%- endmacro %} + +{% macro status_legend() -%} +
+ {% for s in STATUSES %}{{ status_badge(s) }}{% endfor %} +
+{%- endmacro %} diff --git a/radiator/templates/asset.html b/radiator/templates/asset.html new file mode 100644 index 0000000..9ce8269 --- /dev/null +++ b/radiator/templates/asset.html @@ -0,0 +1,97 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% set a = history.latest %} +{% block title %}{{ a.name or a.asset_id }}{% endblock %} +{% block content %} +
+
Runs / {{ history.test_case_id }} / {{ history.asset_id }}
+

{{ a.name or a.asset_id }}

+
+ {{ a.input_curie or '' }}{% if a.output_curie %} → {{ a.output_curie }}{% endif %} + {% if a.predicate %}{{ a.predicate }}{% endif %} + {% if a.expected_output %}expected: {{ a.expected_output }}{% endif %} + {{ a.kind }} +
+
+ +{% if envs|length > 1 %} +
+ + +
+{% endif %} + +
+

Status history

+

The last {{ history.points|length }} runs in {{ env or 'every environment' }}, oldest to newest. Click a cell to open that run.

+ {% set n = history.points|length %} +
+
+ {% for agent in history.agents %} +
{{ m.agent_head(agent) }}
+ {% for run, asset in history.points %}{% set r = history.agent_result(asset, agent) %} + {% if r %} + {{ STATUS_LABELS[r.status][0] }} + {% else %} + · + {% endif %} + {% endfor %} + {% endfor %} +
+ {% for run, asset in history.points %} +
{% if loop.first or loop.last %}{{ run.started_at.strftime('%b %-d') }}{% endif %}
+ {% endfor %} +
+
+ {{ m.status_legend() }} +
+ +
+
+

Rank of the expected answer

+

Lower is better; runs where it wasn't found are gaps

+ {{ rank_chart if rank_chart else '

The expected answer has no recorded rank in this history.

'|safe }} +
+
+

Number of results

+

How many results each agent returned

+ {{ count_chart if count_chart else '

Result counts weren\'t recorded for these runs.

'|safe }} +
+
+ +
+
+

{{ 'Every run' if show_all or history.points|length <= 10 else 'Latest 10 runs' }}

+ {% if history.points|length > 10 %} + {% if show_all %}Show the latest 10 + {% else %}Show all {{ history.points|length }}{% endif %} + {% endif %} +
+
+ + + + {% for run, asset in table_runs %} + {% for r in asset.agents %} + + + + + + + + + + + + {% endfor %} + {% endfor %} + +
RunAgentStatusFoundRankScoreResultsTimeResponse
{% if loop.first %}{{ when(run.started_at) }}{% endif %}{{ m.agent_head(r.agent) }}{{ status_badge(r.status) }}{% if r.message %} {{ r.message }}{% endif %}{{ '–' if r.found is none else ('yes' if r.found else 'no') }}{{ r.rank if r.rank is not none else '–' }}{{ '%.3g'|format(r.score) if r.score is not none else '–' }}{{ '{:,}'.format(r.n_results) if r.n_results is not none else '–' }}{{ '%.1fs'|format(r.response_time_s) if r.response_time_s is not none else '–' }}{% if r.pk %}{{ r.pk[:8] }}…{% endif %}
+
+
+{% endblock %} diff --git a/radiator/templates/base.html b/radiator/templates/base.html new file mode 100644 index 0000000..113ddda --- /dev/null +++ b/radiator/templates/base.html @@ -0,0 +1,25 @@ + + + + + + {% block title %}Information Radiator{% endblock %} · Translator + + + + +
+ Information Radiator + +
+
+
+ {% block content %}{% endblock %} +
+ + + diff --git a/radiator/templates/diff.html b/radiator/templates/diff.html new file mode 100644 index 0000000..38f9cfc --- /dev/null +++ b/radiator/templates/diff.html @@ -0,0 +1,66 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Changes · {{ run.suite }}{% endblock %} +{% block content %} +
+
Runs / {{ run.suite }} · {{ when(run.started_at) }} / changes
+

What changed

+
+ {% if base %} + {{ when(base.started_at) }} → {{ when(run.started_at) }} + {{ run.suite }} · {{ run.env }} + {% else %} + There's no earlier run of this suite in this environment to compare with. + {% endif %} +
+
+{% if base %} +
+ {% for kind, title, _ in kinds %} + {{ diff.count(kind) }}{{ title }} + {% endfor %} +
+ +{% for kind, title, description in kinds %}{% set changes = diff.changes[kind] %} +
+

{{ title }} ({{ changes|length }})

+

{{ description }}

+ {% if changes %} +
+ + + + {% if kind not in ('added', 'removed') %}{% else %}{% endif %} + + + {% for c in changes[:300] %} + + + + {% if kind not in ('added', 'removed') %} + + + + + + {% else %} + + {% endif %} + + {% endfor %} + +
AssetExpectedAgentBeforeAfterRankStatus
{{ c.asset.name or c.asset.asset_id }}{{ c.asset.expected_output or '–' }}{{ m.agent_head(c.agent) }}{{ status_badge(c.before.status) }}→{{ status_badge(c.after.status) }} + {% if c.before.rank is not none or c.after.rank is not none %} + {{ c.before.rank if c.before.rank is not none else '–' }} → {{ c.after.rank if c.after.rank is not none else '–' }} + {% if c.rank_delta %}({{ '%+d'|format(c.rank_delta) }}){% endif %} + {% endif %} + {{ status_badge(c.asset.status) }}
+
+ {% if changes|length > 300 %}

Showing the first 300 of {{ changes|length }}.

{% endif %} + {% else %} +

None.

+ {% endif %} +
+{% endfor %} +{% endif %} +{% endblock %} diff --git a/radiator/templates/login.html b/radiator/templates/login.html new file mode 100644 index 0000000..71d4774 --- /dev/null +++ b/radiator/templates/login.html @@ -0,0 +1,23 @@ + + + + + + Log in · Information Radiator + + + + + + + diff --git a/radiator/templates/performance.html b/radiator/templates/performance.html new file mode 100644 index 0000000..f515f95 --- /dev/null +++ b/radiator/templates/performance.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% block title %}Performance{% endblock %} +{% block content %} +
+

Performance

+
HelmsDeep load tests: the highest concurrency each service sustained within its SLO, and checkpoint verdicts.
+
+ +
+ + +
+ +{% if not panels %} +

No performance runs in the last {{ days }} days.

+{% endif %} +
+{% for key, rows, chart in panels %} +
+

{{ key }}

+ {% set last = rows[-1][1] %} +

Latest: {{ '%.1f'|format(last.max_sustainable_concurrency) if last.max_sustainable_concurrency is not none else 'no stage met the SLO' }} + {% if last.checkpoints_passed is not none %} · checkpoints {{ status_badge('PASSED' if last.checkpoints_passed else 'FAILED') }}{% endif %}

+ {{ chart }} +
+ Table view + + + + {% for run, perf in rows|reverse %} + + + + + + + {% endfor %} + +
RunProfileMax concurrencyCheckpoints
{{ when(run.started_at) }}{{ perf.profile or 'default' }}{{ '%.1f'|format(perf.max_sustainable_concurrency) if perf.max_sustainable_concurrency is not none else '–' }}{% if perf.error %}{{ status_badge('ERROR') }}{% elif perf.checkpoints_passed is none %}none{% else %}{{ status_badge('PASSED' if perf.checkpoints_passed else 'FAILED') }}{% endif %}
+
+
+{% endfor %} +
+{% endblock %} diff --git a/radiator/templates/run.html b/radiator/templates/run.html new file mode 100644 index 0000000..ee0fc8f --- /dev/null +++ b/radiator/templates/run.html @@ -0,0 +1,143 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}{{ run.suite }} · {{ when(run.started_at) }}{% endblock %} +{% block content %} +
+
Runs / {{ run.suite }}
+

{{ run.suite }} · {{ run.env or 'no env' }}

+
+ {{ when(run.started_at) }} + Duration {{ duration(run) }}{% if not run.ended_at %} running{% endif %} + {% if run.target %}target: {{ run.target }}{% endif %} + {% if run.query_type %}{{ run.query_type }} only{% endif %} + {% if run.harness_version %}harness {{ run.harness_version }}{% endif %} + {% if run.origin == 'zebrunner_import' %}imported from Zebrunner{% endif %} +
+
+ +
+
+
Pass rate
+
{{ pct(rate) }}
+ {% if previous_rate is not none and rate is not none %} + {% set d = (rate - previous_rate) * 100 %} +
+ {{ '%+.1f'|format(d) }} pts vs previous run +
+ {% endif %} +
+
+
Assets tested
+
{{ overall.values()|sum }}
+
{{ overall.get('PASSED', 0) }} passed · {{ overall.get('SKIPPED', 0) }} skipped
+
+
+
Regressions
+
{% if previous %}{{ diff.count('regression') }}{% else %}–{% endif %}
+
agent results that stopped passing
+
+
+
Fixed
+
{% if previous %}{{ diff.count('fixed') }}{% else %}–{% endif %}
+
{% if previous %}vs {{ when(previous.started_at) }}{% else %}no earlier run to compare{% endif %}
+
+
+ +
+
+

By agent

Each agent's results across every asset in this run

+ {% if previous %}What changed since the last run →{% endif %} +
+ + + {% for s in STATUSES %}{% endfor %} + + {% for agent in agents %}{% set c = agent_counts[agent] %} + + + + + {% for s in STATUSES %}{% endfor %} + + {% endfor %} + +
AgentResultsPass rate{{ STATUS_LABELS[s][1] }}
{{ m.agent_head(agent) }}{{ status_bar(c) }}{{ pct(pass_rate(c)) }}{% if c.get(s) %}{{ c.get(s) }}{% else %}0{% endif %}
+ {{ m.status_legend() }} +
+ +{% if run.performance %} +
+

Performance

+

HelmsDeep runs in this test run

+ + + + {% for perf in run.performance %} + + + + + + + {% endfor %} + +
HostRun typeProfileResultMax sustainable concurrencyCheckpoints
{{ perf.host }}{{ perf.helmsdeep_target or '–' }}{{ perf.profile or 'default' }}{{ status_badge(perf.status) }}{% if perf.error %} {{ perf.error }}{% endif %}{{ '%.1f'|format(perf.max_sustainable_concurrency) if perf.max_sustainable_concurrency is not none else '–' }}{% if perf.knee_unsupported %} unsupported{% endif %}{% if perf.checkpoints_passed is none %}none configured{% else %}{{ status_badge('PASSED' if perf.checkpoints_passed else 'FAILED') }}{% endif %}
+
+{% endif %} + +
+
+

Results

Every asset against every agent. The number is the rank the expected answer came back at.

+
+
+ + + + + + {% if filtered %}Clear{% endif %} +
+ {% if assets %} +
+ + + + {% for a in agents %}{% endfor %} + + + {% for asset in assets %} + {% set by_agent = {} %}{% for r in asset.agents %}{% set _ = by_agent.update({r.agent: r}) %}{% endfor %} + + + + + {% for a in agents %}{% set r = by_agent.get(a) %} + + {% endfor %} + + {% endfor %} + +
AssetExpectedOverall{{ m.agent_head(a) }}
{{ asset.name or asset.asset_id }} +
{{ asset.input_curie or '' }}{% if asset.output_curie %} → {{ asset.output_curie }}{% endif %}
{{ asset.expected_output or '–' }}{{ status_badge(asset.status) }}{{ m.cell(asset, r, run.env, pk_url(r.pk, run.env) if r else None) }}
+
+

{{ assets|length }} of {{ run.assets|length }} assets. Hover a cell for details; click it to open the response.

+ {% else %} +

No assets match these filters.

+ {% endif %} +
+{% endblock %} diff --git a/radiator/templates/runs.html b/radiator/templates/runs.html new file mode 100644 index 0000000..e546bc8 --- /dev/null +++ b/radiator/templates/runs.html @@ -0,0 +1,68 @@ +{% extends "base.html" %} +{% block title %}Runs{% endblock %} +{% block content %} +
+

Test runs

+
{{ total }} run{{ '' if total == 1 else 's' }}{% if suite or env %} matching{% endif %}
+
+ +
+ + + + {% if suite or env %}Clear{% endif %} +
+ +
+ {% if runs %} +
+ + + + + + + {% for run in runs %} + {% set c = counts[run.id] %}{% set pc = perf_counts.get(run.id) %} + + + + + {% if pc and not c %} + + + + {% else %} + + + + {% endif %} + + + + {% endfor %} + +
StartedSuiteEnvResultsPass rateAssetsDuration
{{ when(run.started_at) }}
{{ ago(run.started_at) }}		{{ run.suite }} + {% if run.target %}target: {{ run.target }}{% endif %} + {% if run.query_type %}{{ run.query_type }} only{% endif %} + {{ run.env or '–' }}{{ status_bar(pc) }}performance · {{ pc.values()|sum }} host{{ '' if pc.values()|sum == 1 else 's' }}{{ pct(pass_rate(pc)) }}–{{ status_bar(c) }}{{ pct(pass_rate(c)) }}{{ c.values()|sum }}{% if run.ended_at %}{{ duration(run) }}{% else %}running{% endif %}{% if run.origin == 'zebrunner_import' %}imported{% endif %}
+
+
+ {% if page > 1 %}← Newer{% endif %} + {% if has_next %}Older →{% endif %} +
+ {% else %} +

No runs yet. They appear here as soon as the harness starts one.

+ {% endif %} +
+{% endblock %} diff --git a/radiator/templates/trends.html b/radiator/templates/trends.html new file mode 100644 index 0000000..3ad0349 --- /dev/null +++ b/radiator/templates/trends.html @@ -0,0 +1,63 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Trends{% endblock %} +{% block content %} +
+

Trends

+
Pass rate per agent over full runs of a suite: skips are left out, and runs with a target override or a single query type aren't included.
+
+ +
+ + + + +
+ +{% if not points %} +

No full runs of {{ suite or 'this suite' }} in the last {{ days }} days.

+{% else %} +
+

Pass rate by agent

+

{{ points|length }} runs · all expected outputs

+ {{ overall_chart }} +
+ Table view +
+ + {% for a in agents %}{% endfor %} + + {% for p in points|reverse %} + + {% for a in agents %}{% endfor %} + {% endfor %} + +
Run{{ m.agent_head(a) }}
{{ when(p.run.started_at) }}{{ pct(rate(p, a), 1) }}
+
+
+
+ +
+

By expected output

+

The same pass rate, split by what the test expects of the answer

+
+ {% for expected, chart in multiples %} +

{{ expected }}

{{ chart }}
+ {% endfor %} +
+
+{% endif %} +{% endblock %} diff --git a/radiator/tests/__init__.py b/radiator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/radiator/tests/conftest.py b/radiator/tests/conftest.py new file mode 100644 index 0000000..1e4a536 --- /dev/null +++ b/radiator/tests/conftest.py @@ -0,0 +1,84 @@ +"""Fixtures: a migrated Postgres database and an app pointed at it. + +Needs a Postgres the tests may wipe, given as RADIATOR_TEST_DATABASE_URL, eg +postgresql+psycopg://postgres@localhost:5432/radiator_test +""" + +import os + +import pytest +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, text + +from radiator.app import create_app +from radiator.config import Settings + +HERE = os.path.dirname(__file__) +TOKEN = "test-token" +USERNAME = "translator" +PASSWORD = "correct horse battery staple" + + +@pytest.fixture(scope="session") +def database_url(): + url = os.getenv("RADIATOR_TEST_DATABASE_URL") + if not url: + pytest.skip("RADIATOR_TEST_DATABASE_URL is not set") + engine = create_engine(url) + with engine.begin() as conn: + conn.execute(text("DROP SCHEMA public CASCADE; CREATE SCHEMA public")) + engine.dispose() + config = Config(os.path.join(HERE, "..", "alembic.ini")) + config.attributes["database_url"] = url + command.upgrade(config, "head") + return url + + +@pytest.fixture +def settings(database_url): + return Settings( + database_url=database_url, + api_token=TOKEN, + username=USERNAME, + password=PASSWORD, + session_secret="test-secret", + secure_cookies=False, + ) + + +@pytest.fixture +def app(settings): + app = create_app(settings) + yield app + with app.state.sessionmaker() as session: + session.execute( + text("TRUNCATE runs, asset_results, agent_results, performance_results") + ) + session.commit() + + +@pytest.fixture +def client(app): + """Unauthenticated.""" + return TestClient(app) + + +@pytest.fixture +def api(app): + """Authenticated with the API token.""" + return TestClient(app, headers={"Authorization": f"Bearer {TOKEN}"}) + + +@pytest.fixture +def browser(app): + """Logged in through the login form, like a person.""" + client = TestClient(app) + res = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD}, + follow_redirects=False, + ) + assert res.status_code == 303 + return client diff --git a/radiator/tests/factories.py b/radiator/tests/factories.py new file mode 100644 index 0000000..24fdf7e --- /dev/null +++ b/radiator/tests/factories.py @@ -0,0 +1,53 @@ +"""Builders for schema payloads in tests and the demo seed.""" + +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema + + +def run_create(started_at=None, **kwargs): + defaults = dict( + run_id=uuid.uuid4(), + suite="sprint_4_tests", + env="ci", + started_at=started_at or datetime.now(timezone.utc), + ) + defaults.update(kwargs) + return schema.RunCreate(**defaults) + + +def agent(name, status="PASSED", **kwargs): + return schema.AgentResult(agent=name, status=status, **kwargs) + + +def asset( + asset_id="Asset_1", + test_case_id="TestCase_1", + status="PASSED", + agents=None, + **kwargs, +): + defaults = dict( + kind="acceptance", + name=f"{asset_id} name", + expected_output="TopAnswer", + predicate="biolink:treats", + input_curie="MONDO:0004979", + output_curie="CHEBI:31690", + ) + defaults.update(kwargs) + return schema.AssetResult( + test_case_id=test_case_id, + asset_id=asset_id, + status=status, + agents=agents if agents is not None else [agent("ars", status)], + **defaults, + ) + + +def finish(started_at=None, minutes=42, counts=None): + started_at = started_at or datetime.now(timezone.utc) + return schema.RunFinish( + ended_at=started_at + timedelta(minutes=minutes), counts=counts or {} + ) diff --git a/radiator/tests/test_auth.py b/radiator/tests/test_auth.py new file mode 100644 index 0000000..ff101e6 --- /dev/null +++ b/radiator/tests/test_auth.py @@ -0,0 +1,83 @@ +"""Nothing is reachable without logging in or a token.""" + +import pytest + +from radiator.app import _safe_next + +from .conftest import PASSWORD, USERNAME + + +def test_health_check_needs_no_auth(client): + assert client.get("/healthz").json() == {"ok": True} + + +@pytest.mark.parametrize("path", ["/", "/trends", "/performance"]) +def test_pages_redirect_to_login(client, path): + res = client.get(path, follow_redirects=False) + assert res.status_code == 303 + assert res.headers["location"] == f"/login?next={path}" + + +def test_read_api_needs_token_or_login(client, api, browser): + assert client.get("/api/runs").status_code == 401 + assert api.get("/api/runs").status_code == 200 + assert browser.get("/api/runs").status_code == 200 + + +def test_ingest_needs_the_token(client, browser): + body = { + "run_id": "6e3c1d8e-5f0e-4bb8-9d7e-1f1e0b6f0a11", + "suite": "s", + "started_at": "2026-10-06T00:00:00Z", + } + assert client.post("/api/ingest/runs", json=body).status_code == 401 + bad = client.post( + "/api/ingest/runs", json=body, headers={"Authorization": "Bearer nope"} + ) + assert bad.status_code == 401 + # a logged-in browser still can't write: ingest is token-only + assert browser.post("/api/ingest/runs", json=body).status_code == 401 + + +def test_login_and_logout(client): + res = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD, "next": "/trends"}, + follow_redirects=False, + ) + assert res.status_code == 303 + assert res.headers["location"] == "/trends" + assert client.get("/", follow_redirects=False).status_code == 200 + + client.post("/logout", follow_redirects=False) + assert client.get("/", follow_redirects=False).status_code == 303 + + +def test_wrong_password(client): + res = client.post("/login", data={"username": USERNAME, "password": "nope"}) + assert res.status_code == 401 + assert "didn" in res.text + assert client.get("/", follow_redirects=False).status_code == 303 + + +def test_login_is_throttled(client): + for _ in range(10): + client.post("/login", data={"username": USERNAME, "password": "nope"}) + # even the right password is refused once the address is throttled + res = client.post("/login", data={"username": USERNAME, "password": PASSWORD}) + assert res.status_code == 401 + assert "Too many" in res.text + + +@pytest.mark.parametrize( + "next_url, expected", + [ + (None, "/"), + ("/runs/abc?x=1", "/runs/abc?x=1"), + ("https://evil.example", "/"), + ("//evil.example", "/"), + ("javascript:alert(1)", "/"), + ], +) +def test_login_only_redirects_on_site(next_url, expected): + assert _safe_next(next_url) == expected diff --git a/radiator/tests/test_ingest.py b/radiator/tests/test_ingest.py new file mode 100644 index 0000000..4d6c5ca --- /dev/null +++ b/radiator/tests/test_ingest.py @@ -0,0 +1,179 @@ +"""The ingest API, including against the harness's own client.""" + +import uuid + +from fastapi.testclient import TestClient + +import radiator_schema as schema +from radiator import queries +from test_harness.radiator_client import RadiatorClient, push + +from . import factories as f +from .conftest import TOKEN + + +def _get(app, run_id): + with app.state.sessionmaker() as session: + return queries.get_run(session, run_id) + + +def test_full_run(app, api): + run = f.run_create() + assert ( + api.post("/api/ingest/runs", json=run.model_dump(mode="json")).status_code + == 201 + ) + batch = schema.ResultBatch( + results=[ + f.asset( + "Asset_1", + agents=[ + f.agent( + "ars", found=True, rank=3, score=0.9, n_results=120, pk="pk1" + ), + f.agent("aragorn", "NO_RESULTS", n_results=0, http_status=200), + ], + ) + ], + performance=[ + schema.PerformanceResult( + test_case_id="Perf_1", + asset_id="Asset_9", + host="https://ars", + status="PASSED", + max_sustainable_concurrency=12.5, + summary={"knee": {"users": 13}}, + ) + ], + ) + res = api.post( + f"/api/ingest/runs/{run.run_id}/results", json=batch.model_dump(mode="json") + ) + assert res.json() == {"results": 1, "performance": 1} + finish = f.finish(run.started_at, counts={"PASSED": 1}) + assert ( + api.patch( + f"/api/ingest/runs/{run.run_id}", json=finish.model_dump(mode="json") + ).status_code + == 200 + ) + + stored = _get(app, run.run_id) + assert stored.ended_at == finish.ended_at + assert stored.counts == {"PASSED": 1} + [asset] = stored.assets + ars, aragorn = asset.agents + assert (ars.found, ars.rank, ars.score, ars.n_results, ars.pk) == ( + True, + 3, + 0.9, + 120, + "pk1", + ) + assert (aragorn.status, aragorn.n_results) == ("NO_RESULTS", 0) + [perf] = stored.performance + assert perf.max_sustainable_concurrency == 12.5 + assert perf.summary == {"knee": {"users": 13}} + + +def test_writes_are_idempotent(app, api): + """Re-posting a run, or a result, updates it instead of duplicating it, + and an asset's agent rows are replaced by the newest upload.""" + run = f.run_create() + for _ in range(2): + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + first = schema.ResultBatch( + results=[f.asset("Asset_1", agents=[f.agent("ars"), f.agent("arax")])] + ) + second = schema.ResultBatch( + results=[f.asset("Asset_1", status="FAILED", agents=[f.agent("ars", "FAILED")])] + ) + for batch in (first, second): + api.post( + f"/api/ingest/runs/{run.run_id}/results", json=batch.model_dump(mode="json") + ) + + with app.state.sessionmaker() as session: + assert len(queries.list_runs(session)) == 1 + [asset] = _get(app, run.run_id).assets + assert asset.status == "FAILED" + assert [(a.agent, a.status) for a in asset.agents] == [("ars", "FAILED")] + + +def test_results_for_an_unknown_run_are_refused(api): + run_id = uuid.uuid4() + res = api.post(f"/api/ingest/runs/{run_id}/results", json={"results": []}) + assert res.status_code == 404 + res = api.patch( + f"/api/ingest/runs/{run_id}", + json=f.finish().model_dump(mode="json"), + ) + assert res.status_code == 404 + + +def test_invalid_body_is_rejected(api): + run = f.run_create() + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + res = api.post( + f"/api/ingest/runs/{run.run_id}/results", + json={ + "results": [ + { + "test_case_id": "x", + "asset_id": "y", + "kind": "acceptance", + "status": "MAYBE", + } + ] + }, + ) + assert res.status_code == 422 + + +def _harness_client(app, batch_size=2): + """The harness's RadiatorClient, talking to this app in-process.""" + client = RadiatorClient( + base_url="http://radiator", token=TOKEN, batch_size=batch_size + ) + client._client = TestClient( + app, + base_url=f"http://radiator{schema.INGEST_PREFIX}", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + return client + + +def test_harness_client_round_trip(app): + """The contract test: what the harness sends is what the radiator stores.""" + client = _harness_client(app) + run = f.run_create() + client.start_run(run) + for i in range(5): + client.add_result(f.asset(f"Asset_{i}")) + client.finish_run(counts={"PASSED": 5}) + assert not client.upload_failed + + stored = _get(app, run.run_id) + assert sorted(a.asset_id for a in stored.assets) == [f"Asset_{i}" for i in range(5)] + assert stored.counts == {"PASSED": 5} + + +def test_replaying_a_partly_uploaded_run(app): + """A run that half made it, replayed from the saved payload, ends up whole + and without duplicates.""" + run = f.run_create() + partial = _harness_client(app) + partial.start_run(run) + partial.add_result(f.asset("Asset_0")) + partial.add_result(f.asset("Asset_1")) # flushed: batch of 2 + + payload = schema.RunPayload( + run=run, + results=[f.asset(f"Asset_{i}") for i in range(4)], + finish=f.finish(run.started_at), + ) + assert push(payload, _harness_client(app)) + + stored = _get(app, run.run_id) + assert len(stored.assets) == 4 + assert stored.ended_at == payload.finish.ended_at diff --git a/radiator/tests/test_pages.py b/radiator/tests/test_pages.py new file mode 100644 index 0000000..35922ef --- /dev/null +++ b/radiator/tests/test_pages.py @@ -0,0 +1,199 @@ +"""The dashboard's pages and the queries behind them.""" + +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator import queries +from radiator.ingest import ingest_payload + +from . import factories as f + +T0 = datetime(2026, 9, 1, 6, tzinfo=timezone.utc) + + +def _load(app, *payloads): + with app.state.sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + + +def _payload(started_at, results, **run_kwargs): + run = f.run_create(started_at=started_at, **run_kwargs) + return schema.RunPayload(run=run, results=results, finish=f.finish(started_at)) + + +def test_pages_render(app, browser): + perf = schema.RunPayload( + run=f.run_create(started_at=T0, suite="performance_tests"), + performance=[ + schema.PerformanceResult( + test_case_id="P", + asset_id="A", + host="https://ars.ci.transltr.io", + helmsdeep_target="ars", + status="PASSED", + max_sustainable_concurrency=12.0, + ) + ], + ) + first = _payload(T0, [f.asset("Asset_1", agents=[f.agent("ars", rank=4)])]) + second = _payload( + T0 + timedelta(days=1), + [ + f.asset( + "Asset_1", status="FAILED", agents=[f.agent("ars", "FAILED", rank=90)] + ) + ], + ) + _load(app, perf, first, second) + run_id = second.run.run_id + + runs = browser.get("/") + assert runs.status_code == 200 + assert "sprint_4_tests" in runs.text and "performance_tests" in runs.text + + page = browser.get(f"/runs/{run_id}") + assert page.status_code == 200 + assert "Asset_1 name" in page.text + assert 'href="/runs/{}/diff#regression">1<'.format(run_id) in page.text + + diff = browser.get(f"/runs/{run_id}/diff") + assert diff.status_code == 200 + assert "4 → 90" in diff.text + + asset = browser.get("/assets/TestCase_1/Asset_1?env=ci") + assert asset.status_code == 200 + assert "Rank of the expected answer" in asset.text + + assert browser.get("/trends").status_code == 200 + performance = browser.get("/performance") + assert performance.status_code == 200 + assert "https://ars.ci.transltr.io (ars)" in performance.text + + +def test_empty_database(browser): + assert "No runs yet" in browser.get("/").text + assert browser.get("/trends").status_code == 200 + assert browser.get("/performance").status_code == 200 + + +def test_missing_things_404(browser): + assert browser.get("/runs/6e3c1d8e-5f0e-4bb8-9d7e-1f1e0b6f0a11").status_code == 404 + assert browser.get("/assets/nope/nope").status_code == 404 + + +def test_uploaded_text_is_escaped(app, browser): + """Names and messages come from uploads; they must never become markup.""" + evil = '' + payload = _payload( + T0, + [ + f.asset( + "Asset_1", name=evil, agents=[f.agent("ars", "FAILED", message=evil)] + ) + ], + ) + _load(app, payload) + page = browser.get(f"/runs/{payload.run.run_id}").text + assert evil not in page + assert "<script>" in page + + +def test_matrix_filters(app, browser): + payload = _payload( + T0, + [ + f.asset("Asset_1", agents=[f.agent("ars"), f.agent("arax", "FAILED")]), + f.asset("Asset_2", status="FAILED", agents=[f.agent("ars", "FAILED")]), + ], + ) + _load(app, payload) + url = f"/runs/{payload.run.run_id}" + failed_overall = browser.get(url, params={"status": "FAILED"}).text + assert "Asset_2 name" in failed_overall and "Asset_1 name" not in failed_overall + failed_arax = browser.get(url, params={"status": "FAILED", "agent": "arax"}).text + assert "Asset_1 name" in failed_arax and "Asset_2 name" not in failed_arax + search = browser.get(url, params={"q": "asset_2"}).text + assert "Asset_2 name" in search and "Asset_1 name" not in search + + +def _run(app, payload): + with app.state.sessionmaker() as session: + return queries.get_run(session, payload.run.run_id) + + +def test_diff_classification(app): + before = _payload( + T0, + [ + f.asset("regressed", agents=[f.agent("ars", rank=3)]), + f.asset("fixed", agents=[f.agent("ars", "FAILED")]), + f.asset("changed", agents=[f.agent("ars", "FAILED")]), + f.asset("skipped", agents=[f.agent("ars")]), + f.asset("moved", agents=[f.agent("ars", rank=40)]), + f.asset("jittered", agents=[f.agent("ars", rank=10)]), + f.asset("dropped"), + ], + ) + after = _payload( + T0 + timedelta(days=1), + [ + f.asset("regressed", agents=[f.agent("ars", "FAILED", rank=80)]), + f.asset("fixed", agents=[f.agent("ars")]), + f.asset("changed", agents=[f.agent("ars", "NO_RESULTS")]), + f.asset("skipped", agents=[f.agent("ars", "SKIPPED")]), + f.asset("moved", agents=[f.agent("ars", rank=4)]), + f.asset("jittered", agents=[f.agent("ars", rank=12)]), + f.asset("new"), + ], + ) + _load(app, before, after) + diff = queries.diff_runs(_run(app, before), _run(app, after)) + found = { + kind: [c.asset.asset_id for c in changes] + for kind, changes in diff.changes.items() + } + assert found == { + "regression": ["regressed"], + "fixed": ["fixed"], + "changed": ["changed"], + "skipped": ["skipped"], + # a 2-place wobble isn't a move + "rank": ["moved"], + "added": ["new"], + "removed": ["dropped"], + } + [moved] = diff.changes["rank"] + assert moved.rank_delta == 36 + + +def test_previous_run_stays_in_its_series(app): + """A run with a target override or another query type isn't the + 'previous run' of a full run, and doesn't show up in its trends.""" + full_1 = _payload(T0, [f.asset()]) + override = _payload( + T0 + timedelta(hours=1), + [f.asset()], + target="aragorn", + target_url="http://localhost:8080", + ) + mvp1 = _payload(T0 + timedelta(hours=2), [f.asset()], query_type="MVP1") + other_env = _payload(T0 + timedelta(hours=3), [f.asset()], env="test") + full_2 = _payload(T0 + timedelta(hours=4), [f.asset()]) + _load(app, full_1, override, mvp1, other_env, full_2) + + with app.state.sessionmaker() as session: + latest = queries.get_run(session, full_2.run.run_id) + assert queries.previous_run(session, latest).id == full_1.run.run_id + assert ( + queries.previous_run(session, queries.get_run(session, full_1.run.run_id)) + is None + ) + points = queries.agent_trends(session, "sprint_4_tests", "ci", days=100000) + assert [p.run.id for p in points] == [full_1.run.run_id, full_2.run.run_id] + + +def test_pass_rate_leaves_out_skips(): + assert queries.pass_rate({"PASSED": 3, "FAILED": 1, "SKIPPED": 10}) == 0.75 + assert queries.pass_rate({"SKIPPED": 4}) is None diff --git a/radiator/web.py b/radiator/web.py new file mode 100644 index 0000000..8b06648 --- /dev/null +++ b/radiator/web.py @@ -0,0 +1,504 @@ +"""The dashboard's pages. Everything here requires the shared login.""" + +import uuid +from datetime import datetime, timezone +from typing import Optional +from urllib.parse import urlencode + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import HTMLResponse +from markupsafe import Markup, escape +from sqlalchemy.orm import Session + +from radiator import charts, queries +from radiator.api import get_session +from radiator.auth import is_logged_in + + +class LoginRequired(Exception): + """Raised for a page request without a session; redirects to /login.""" + + +def require_login(request: Request) -> None: + if not is_logged_in(request): + raise LoginRequired() + + +router = APIRouter(dependencies=[Depends(require_login)]) + +STATUS_LABELS = { + "PASSED": ("✓", "Passed"), + "FAILED": ("✕", "Failed"), + "NO_RESULTS": ("∅", "No results"), + "ERROR": ("!", "Error"), + "SKIPPED": ("–", "Skipped"), +} + +# Where a pk can be looked at. The ARAX UI shows any ARS message for the +# environment it is deployed in. +PK_VIEWERS = { + "dev": "https://arax.ci.transltr.io/?r={pk}", + "ci": "https://arax.ci.transltr.io/?r={pk}", + "test": "https://arax.test.transltr.io/?r={pk}", + "prod": "https://arax.transltr.io/?r={pk}", +} + +TREND_WINDOWS = [30, 90, 180, 365] + + +# --- template helpers ---------------------------------------------------------- + + +def status_badge(status: str) -> Markup: + icon, label = STATUS_LABELS.get(status, ("?", status)) + return Markup( + f'{escape(label)}" + ) + + +def status_bar(counts: dict) -> Markup: + total = sum(counts.values()) + if not total: + return Markup('no results') + segments = [] + for status in queries.STATUSES: + n = counts.get(status, 0) + if not n: + continue + label = STATUS_LABELS[status][1] + tip = f"{label}\n{n} of {total} ({n / total:.0%})" + segments.append( + f'' + ) + return Markup( + f'' + ) + + +def _counts_text(counts: dict) -> str: + return ", ".join( + f"{counts[s]} {STATUS_LABELS[s][1].lower()}" + for s in queries.STATUSES + if counts.get(s) + ) + + +def pct(value: Optional[float], digits: int = 0) -> str: + return "–" if value is None else f"{value * 100:.{digits}f}%" + + +def when(value: Optional[datetime]) -> str: + if value is None: + return "–" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def ago(value: Optional[datetime]) -> str: + if value is None: + return "" + seconds = (datetime.now(timezone.utc) - value).total_seconds() + for unit, size in (("d", 86400), ("h", 3600), ("m", 60)): + if seconds >= size: + return f"{int(seconds // size)}{unit} ago" + return "just now" + + +def duration(run) -> str: + if run.ended_at is None: + return "–" + minutes = int((run.ended_at - run.started_at).total_seconds() // 60) + return f"{minutes // 60}h {minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m" + + +def pk_url(pk: Optional[str], env: Optional[str]) -> Optional[str]: + if not pk: + return None + return PK_VIEWERS.get(env or "ci", PK_VIEWERS["ci"]).format(pk=pk) + + +def cell_tip(asset, agent) -> str: + lines = [ + f"{agent.agent} · {STATUS_LABELS.get(agent.status, ('', agent.status))[1]}" + ] + if agent.found is not None: + lines.append( + "expected answer found" if agent.found else "expected answer not found" + ) + if agent.rank is not None: + lines.append(f"rank {agent.rank}") + if agent.score is not None: + lines.append(f"score {agent.score:.3g}") + if agent.n_results is not None: + lines.append(f"{agent.n_results:,} results") + if agent.response_time_s is not None: + lines.append(f"{agent.response_time_s:.1f}s") + if agent.message: + lines.append(agent.message) + return "\n".join(lines) + + +def query_string(request: Request, **changes) -> str: + params = dict(request.query_params) + for key, value in changes.items(): + if value is None: + params.pop(key, None) + else: + params[key] = value + return "?" + urlencode(params) if params else "?" + + +def register_template_helpers(templates) -> None: + env = templates.env + env.globals.update( + status_badge=status_badge, + status_bar=status_bar, + pct=pct, + when=when, + ago=ago, + duration=duration, + pk_url=pk_url, + cell_tip=cell_tip, + agent_color=charts.agent_color, + STATUS_LABELS=STATUS_LABELS, + STATUSES=queries.STATUSES, + query_string=query_string, + ) + + +def render(request: Request, template: str, **context) -> HTMLResponse: + return request.app.state.templates.TemplateResponse(request, template, context) + + +# --- pages ------------------------------------------------------------------- + + +@router.get("/", response_class=HTMLResponse) +def runs_page( + request: Request, + suite: Optional[str] = None, + env: Optional[str] = None, + page: int = 1, + session: Session = Depends(get_session), +): + per_page = 30 + page = max(page, 1) + runs = queries.list_runs( + session, suite=suite, env=env, limit=per_page, offset=(page - 1) * per_page + ) + total = queries.count_runs(session, suite=suite, env=env) + run_ids = [run.id for run in runs] + return render( + request, + "runs.html", + nav="runs", + runs=runs, + counts=queries.status_counts(session, run_ids), + perf_counts=queries.performance_counts(session, run_ids), + pass_rate=queries.pass_rate, + options=queries.filter_options(session), + suite=suite, + env=env, + page=page, + has_next=page * per_page < total, + total=total, + ) + + +def _load_run(session: Session, run_id: uuid.UUID): + run = queries.get_run(session, run_id) + if run is None: + raise HTTPException(404, "No such run") + return run + + +@router.get("/runs/{run_id}", response_class=HTMLResponse) +def run_page( + request: Request, + run_id: uuid.UUID, + status: Optional[str] = None, + agent: Optional[str] = None, + expected: Optional[str] = None, + q: Optional[str] = None, + session: Session = Depends(get_session), +): + run = _load_run(session, run_id) + previous = queries.previous_run(session, run) + if previous is not None: + previous = queries.get_run(session, previous.id) + diff = queries.diff_runs(previous, run) + agents = queries.run_agents(run) + + overall = queries.status_counts(session, [run.id])[run.id] + previous_rate = None + if previous is not None: + previous_rate = queries.pass_rate( + queries.status_counts(session, [previous.id])[previous.id] + ) + + assets = sorted(run.assets, key=lambda a: (a.test_case_id, a.name or a.asset_id)) + if status: + if agent: + assets = [ + a + for a in assets + if any(r.agent == agent and r.status == status for r in a.agents) + ] + else: + assets = [a for a in assets if a.status == status] + if expected: + assets = [a for a in assets if a.expected_output == expected] + if q: + needle = q.lower() + assets = [ + a + for a in assets + if needle + in " ".join( + filter( + None, + [a.name, a.asset_id, a.test_case_id, a.input_curie, a.output_curie], + ) + ).lower() + ] + + return render( + request, + "run.html", + nav="runs", + run=run, + previous=previous, + diff=diff, + agents=agents, + agent_counts=queries.agent_counts(run), + overall=overall, + rate=queries.pass_rate(overall), + previous_rate=previous_rate, + assets=assets, + expected_outputs=sorted( + {a.expected_output for a in run.assets if a.expected_output} + ), + filters=dict(status=status, agent=agent, expected=expected, q=q), + filtered=any([status, expected, q]), + pass_rate=queries.pass_rate, + ) + + +@router.get("/runs/{run_id}/diff", response_class=HTMLResponse) +def diff_page( + request: Request, + run_id: uuid.UUID, + base: Optional[uuid.UUID] = None, + session: Session = Depends(get_session), +): + run = _load_run(session, run_id) + if base is not None: + base_run = _load_run(session, base) + else: + previous = queries.previous_run(session, run) + base_run = queries.get_run(session, previous.id) if previous else None + return render( + request, + "diff.html", + nav="runs", + run=run, + base=base_run, + diff=queries.diff_runs(base_run, run), + kinds=queries.DIFF_KINDS, + ) + + +@router.get("/assets/{test_case_id}/{asset_id}", response_class=HTMLResponse) +def asset_page( + request: Request, + test_case_id: str, + asset_id: str, + env: Optional[str] = None, + all: bool = False, + session: Session = Depends(get_session), +): + envs = queries.asset_envs(session, test_case_id, asset_id) + if env is None and envs: + # default to the env with the most history rather than mixing them + env = "ci" if "ci" in envs else envs[0] + history = queries.asset_history(session, test_case_id, asset_id, env=env) + if history.latest is None: + raise HTTPException(404, "No results for this asset") + + def series_for(attr): + return [ + charts.Series( + name=agent, + color=charts.agent_color(agent), + points=[ + ( + run.started_at, + getattr(history.agent_result(asset, agent), attr, None), + ) + for run, asset in history.points + ], + ) + for agent in history.agents + ] + + rank_series = [ + s for s in series_for("rank") if any(y is not None for _, y in s.points) + ] + count_series = [ + s for s in series_for("n_results") if any(y is not None for _, y in s.points) + ] + return render( + request, + "asset.html", + nav="runs", + history=history, + envs=envs, + env=env, + table_runs=list(reversed(history.points))[: None if all else 10], + show_all=all, + rank_chart=( + charts.line_chart( + rank_series, + invert=True, + y_min=1, + y_format=lambda v: f"{v:,.0f}", + width=charts.WIDTH_HALF, + label="Rank of the expected answer over time", + ) + if rank_series + else None + ), + count_chart=( + charts.line_chart( + count_series, + y_min=0, + y_format=lambda v: f"{v:,.0f}", + width=charts.WIDTH_HALF, + label="Number of results over time", + ) + if count_series + else None + ), + ) + + +@router.get("/trends", response_class=HTMLResponse) +def trends_page( + request: Request, + suite: Optional[str] = None, + env: Optional[str] = None, + days: int = 90, + session: Session = Depends(get_session), +): + options = queries.filter_options(session) + if suite is None and options["suites"]: + latest = queries.list_runs(session, limit=1) + suite = latest[0].suite if latest else options["suites"][0] + env = env or (latest[0].env if latest else None) + days = days if days in TREND_WINDOWS else 90 + points = queries.agent_trends(session, suite, env, days) if suite else [] + + agents = sorted( + {agent for p in points for agent in p.counts}, key=queries.agent_sort_key + ) + expected_outputs = [ + e + for e in ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] + if any(e in p.counts.get(a, {}) for p in points for a in agents) + ] + sorted( + {e for p in points for a in agents for e in p.counts.get(a, {})} + - {"TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"} + ) + + def rate(point, agent, expected=None): + by_expected = point.counts.get(agent, {}) + if expected is None: + merged: dict = {} + for counts in by_expected.values(): + for status, n in counts.items(): + merged[status] = merged.get(status, 0) + n + return queries.pass_rate(merged) + return queries.pass_rate(by_expected.get(expected, {})) + + def chart(expected=None, height=240, width=charts.WIDTH_FULL): + return charts.line_chart( + [ + charts.Series( + agent, + charts.agent_color(agent), + [(p.run.started_at, rate(p, agent, expected)) for p in points], + ) + for agent in agents + ], + y_min=0, + y_max=1, + y_format=lambda v: f"{v * 100:.0f}%", + height=height, + width=width, + label=f"Pass rate by agent{f' for {expected}' if expected else ''}", + ) + + return render( + request, + "trends.html", + nav="trends", + options=options, + suite=suite, + env=env, + days=days, + windows=TREND_WINDOWS, + points=points, + agents=agents, + overall_chart=chart() if points else None, + multiples=( + [ + (e, chart(e, height=200, width=charts.WIDTH_HALF)) + for e in expected_outputs + ] + if points + else [] + ), + rate=rate, + ) + + +@router.get("/performance", response_class=HTMLResponse) +def performance_page( + request: Request, days: int = 180, session: Session = Depends(get_session) +): + days = days if days in TREND_WINDOWS else 180 + history = queries.performance_history(session, days) + panels = [] + for key, rows in history.items(): + series = [ + charts.Series( + "max sustainable concurrency", + "var(--agent-1)", + [ + (run.started_at, perf.max_sustainable_concurrency) + for run, perf in rows + ], + ) + ] + panels.append( + ( + key, + rows, + charts.line_chart( + series, + y_min=0, + y_format=lambda v: f"{v:g}", + height=170, + width=charts.WIDTH_THIRD, + label=f"Max sustainable concurrency for {key}", + ), + ) + ) + return render( + request, + "performance.html", + nav="performance", + panels=panels, + days=days, + windows=TREND_WINDOWS, + ) From 25c63f70088c90270a9ae9cfb0d3bc78c7e66a5f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 19:06:59 +0000 Subject: [PATCH 05/35] Import past runs from Zebrunner `python -m radiator.zebrunner_import` reads Zebrunner's own read API (launches, their tests with labels, and per-test logs) and posts each run to the radiator's ingest API. Statuses come from the per-agent labels the harness set; found, rank, score and pks from the TestReport the harness logged (--no-logs skips those for speed). Zebrunner never had result counts, HTTP status, response times or the environment, so those stay unknown; --env sets the environment, otherwise runs with prod-only agents are marked prod. Run ids derive from the launch id, so a re-run skips (or with --force updates) what's already there. Performance tests are skipped: they predate HelmsDeep and have no summary. All pk links now open the ARAX UI on ci, as the harness always did. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WR6TD7jzBVdYPod4pKWcg --- radiator/tests/test_zebrunner_import.py | 277 ++++++++++++ radiator/web.py | 17 +- radiator/zebrunner_import.py | 558 ++++++++++++++++++++++++ 3 files changed, 840 insertions(+), 12 deletions(-) create mode 100644 radiator/tests/test_zebrunner_import.py create mode 100644 radiator/zebrunner_import.py diff --git a/radiator/tests/test_zebrunner_import.py b/radiator/tests/test_zebrunner_import.py new file mode 100644 index 0000000..a87d18b --- /dev/null +++ b/radiator/tests/test_zebrunner_import.py @@ -0,0 +1,277 @@ +"""The Zebrunner import, against a fake Zebrunner speaking its read API.""" + +import json + +import httpx +from fastapi.testclient import TestClient + +from radiator import queries +from radiator.zebrunner_import import ( + Radiator, + Zebrunner, + convert_test, + infer_env, + import_runs, + run_id_for, + suite_from_name, +) + +from .conftest import TOKEN + +ZE = "http://zebrunner.test" + + +def label(key, value): + return {"key": key, "value": value} + + +def acceptance_test(test_id, asset, agents, status="PASSED", expected="TopAnswer"): + return { + "id": test_id, + "name": f"{asset} name", + "status": status, + "testClass": "TestCase_1", + "labels": [ + label("TestCase", "TestCase_1"), + label("TestAsset", asset), + label("ExpectedOutput", expected), + label("InputCurie", "MONDO:0004979"), + label("OutputCurie", "CHEBI:31690"), + *[label(agent, s) for agent, s in agents.items()], + ], + } + + +REPORT = { + "pks": {"parent_pk": "parent-1", "ars": "merged-1", "aragorn": "child-1"}, + "result": { + "ars": { + "status": "PASSED", + "message": None, + "actual_output": {"found": True, "ars_rank": 3, "ars_score": 0.97}, + }, + "aragorn": { + "status": "FAILED", + "message": None, + "actual_output": {"found": True, "ara_rank": 88, "ara_score": 0.2}, + }, + }, + "test_details": None, +} + +LAUNCHES = [ + { + "id": 101, + "name": "sprint_4_tests: 2025_03_02_06_00", + "startedAt": "2025-03-02T06:00:00Z", + "endedAt": "2025-03-02T06:50:00Z", + "build": "v0.3.3", + }, + { + "id": 100, + "name": "sprint_4_tests: 2025_03_01_06_00", + "startedAt": "2025-03-01T06:00:00Z", + "endedAt": "2025-03-01T06:45:00Z", + "build": "v0.3.3", + }, + { + "id": 99, + "name": "performance_tests: 2025_02_28_06_00", + "startedAt": "2025-02-28T06:00:00Z", + "endedAt": "2025-02-28T07:00:00Z", + }, +] +TESTS = { + 101: [ + acceptance_test(1, "Asset_1", {"ars": "PASSED", "aragorn": "FAILED"}), + # Zebrunner refused the NO_RESULTS finish, so the test is IN_PROGRESS; + # the agent label still has the harness's status + acceptance_test( + 2, + "Asset_2", + {"ars": "NO_RESULTS", "aragorn": "SKIPPED"}, + status="IN_PROGRESS", + ), + ], + 100: [ + acceptance_test( + 3, "Asset_1", {"ars": "FAILED", "aragorn": "FAILED"}, status="FAILED" + ) + ], + 99: [ + { + "id": 4, + "name": "perf", + "status": "PASSED", + "labels": [ + label("TestCase", "Perf_1"), + label("TestAsset", "Asset_9"), + label("TestRunTime", "60"), + label("UserSpawnRate", "1"), + ], + } + ], +} +LOGS = { + 1: [ + {"kind": "LOG", "message": json.dumps({"message": {"query_graph": {}}})}, + {"kind": "LOG", "message": json.dumps(REPORT, indent=4)}, + ], +} + + +def fake_zebrunner(calls): + """Zebrunner's read API, with an access token that expires once.""" + state = {"expired_once": False} + + def handler(request: httpx.Request): + calls.append((request.method, request.url.path, dict(request.url.params))) + path = request.url.path + if path == "/api/iam/v1/auth/refresh": + assert json.loads(request.content) == {"refreshToken": "refresh"} + return httpx.Response(200, json={"authToken": "access"}) + assert request.headers["Authorization"] == "Bearer access" + if path == "/api/reporting/v1/launches": + if not state["expired_once"]: + state["expired_once"] = True + return httpx.Response(401) + page, size = int(request.url.params["page"]), int( + request.url.params["pageSize"] + ) + items = LAUNCHES[(page - 1) * size : page * size] + return httpx.Response(200, json={"items": items, "_meta": {}}) + if path.startswith("/api/reporting/v1/launches/") and path.endswith("/tests"): + launch_id = int(path.split("/")[5]) + return httpx.Response(200, json={"items": TESTS[launch_id]}) + if path.endswith("/logs"): + test_id = int(path.split("/")[7]) + logs = LOGS.get(test_id, []) + # one log per page, to exercise the page token + token = int(request.url.params.get("pageToken", 0)) + meta = {"nextPageToken": str(token + 1)} if token + 1 < len(logs) else {} + return httpx.Response( + 200, json={"items": logs[token : token + 1], "_meta": meta} + ) + return httpx.Response(404) + + return httpx.MockTransport(handler) + + +def _radiator(app): + radiator = Radiator.__new__(Radiator) + radiator.client = TestClient(app, headers={"Authorization": f"Bearer {TOKEN}"}) + return radiator + + +def _zebrunner(calls): + zebrunner = Zebrunner(ZE, "refresh", project_id=7, transport=fake_zebrunner(calls)) + return zebrunner + + +def test_import(app): + calls = [] + stats = import_runs(_zebrunner(calls), _radiator(app), env="ci") + + assert stats.imported == 2 + assert stats.skipped_empty == 1 # the performance-only run + assert stats.performance_tests_skipped == 1 + assert not stats.failed + # every Zebrunner call carried the project id, paging worked through the 401 + assert all( + params.get("projectId") == "7" + for _, path, params in calls + if "auth" not in path + ) + + with app.state.sessionmaker() as session: + latest = queries.get_run(session, run_id_for(ZE, 101)) + assert latest.origin == "zebrunner_import" + assert latest.origin_ref == "101" + assert latest.suite == "sprint_4_tests" + assert latest.env == "ci" + assert latest.harness_version == "v0.3.3" + assert latest.counts == {"PASSED": 1, "NO_RESULTS": 1} + + assets = {a.asset_id: a for a in latest.assets} + one = assets["Asset_1"] + assert one.status == "PASSED" + assert one.parent_pk == "parent-1" + agents = {a.agent: a for a in one.agents} + assert ( + agents["ars"].found, + agents["ars"].rank, + agents["ars"].score, + agents["ars"].pk, + ) == (True, 3, 0.97, "merged-1") + assert (agents["aragorn"].status, agents["aragorn"].rank) == ("FAILED", 88) + # what Zebrunner never had stays unknown, not zero + assert agents["ars"].n_results is None and agents["ars"].http_status is None + + two = assets["Asset_2"] + assert two.status == "NO_RESULTS" + assert {a.agent: a.status for a in two.agents} == { + "ars": "NO_RESULTS", + "aragorn": "SKIPPED", + } + + # the two imported runs are one series, so they diff against each other + previous = queries.previous_run(session, latest) + assert previous.id == run_id_for(ZE, 100) + + +def test_reimport_skips_then_force_updates(app): + import_runs(_zebrunner([]), _radiator(app), env="ci") + again = import_runs(_zebrunner([]), _radiator(app), env="ci") + assert again.imported == 0 and again.skipped_existing == 2 + + forced = import_runs(_zebrunner([]), _radiator(app), env="ci", force=True) + assert forced.imported == 2 + with app.state.sessionmaker() as session: + assert len(queries.list_runs(session)) == 2 + + +def test_dry_run_writes_nothing(app): + stats = import_runs(_zebrunner([]), None, since=None, fetch_logs=False) + assert stats.imported == 2 + # no env given: the prod-only aragorn agent marks these as prod runs + assert stats.unknown_env == 0 + with app.state.sessionmaker() as session: + assert queries.list_runs(session) == [] + + +def test_filters(app): + from datetime import datetime, timezone + + stats = import_runs( + _zebrunner([]), + _radiator(app), + env="ci", + since=datetime(2025, 3, 2, tzinfo=timezone.utc), + ) + assert stats.launches == 1 and stats.imported == 1 + + +def test_conversion_details(): + assert suite_from_name("sprint_4_tests: 2025_03_02_06_00") == "sprint_4_tests" + assert suite_from_name("odd name") == "odd name" + # an asset from a target-override run: the single agent drives the status + result = convert_test(acceptance_test(5, "Asset_5", {"aragorn": "FAILED"}), None) + assert result.status == "FAILED" + # malformed report values are dropped, not fatal + report = { + "result": {"ars": {"status": "PASSED", "actual_output": {"ars_rank": "n/a"}}} + } + result = convert_test(acceptance_test(6, "Asset_6", {"ars": "PASSED"}), report) + assert result.agents[0].rank is None + + +def test_env_inference(): + prod = convert_test( + acceptance_test(7, "A", {"ars": "PASSED", "arax": "PASSED"}), None + ) + shepherd = convert_test( + acceptance_test(8, "A", {"ars": "PASSED", "shepherd-arax": "PASSED"}), None + ) + assert infer_env([prod]) == "prod" + # dev, ci and test all use the shepherd agents: can't tell which + assert infer_env([shepherd]) is None diff --git a/radiator/web.py b/radiator/web.py index 8b06648..c58c3fd 100644 --- a/radiator/web.py +++ b/radiator/web.py @@ -34,14 +34,9 @@ def require_login(request: Request) -> None: "SKIPPED": ("–", "Skipped"), } -# Where a pk can be looked at. The ARAX UI shows any ARS message for the -# environment it is deployed in. -PK_VIEWERS = { - "dev": "https://arax.ci.transltr.io/?r={pk}", - "ci": "https://arax.ci.transltr.io/?r={pk}", - "test": "https://arax.test.transltr.io/?r={pk}", - "prod": "https://arax.transltr.io/?r={pk}", -} +# Where a pk is opened: the ARAX UI on ci, for every environment, as the +# harness has always linked them. +PK_VIEWER = "https://arax.ci.transltr.io/?r={pk}" TREND_WINDOWS = [30, 90, 180, 365] @@ -111,10 +106,8 @@ def duration(run) -> str: return f"{minutes // 60}h {minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m" -def pk_url(pk: Optional[str], env: Optional[str]) -> Optional[str]: - if not pk: - return None - return PK_VIEWERS.get(env or "ci", PK_VIEWERS["ci"]).format(pk=pk) +def pk_url(pk: Optional[str], env: Optional[str] = None) -> Optional[str]: + return PK_VIEWER.format(pk=pk) if pk else None def cell_tip(asset, agent) -> str: diff --git a/radiator/zebrunner_import.py b/radiator/zebrunner_import.py new file mode 100644 index 0000000..4a51311 --- /dev/null +++ b/radiator/zebrunner_import.py @@ -0,0 +1,558 @@ +"""Import past runs from Zebrunner into the Information Radiator. + + python -m radiator.zebrunner_import \\ + --zebrunner-url https://zebrunner.example.org --refresh-token ... \\ + --radiator-url https://radiator.example.org --radiator-token ... \\ + [--project-id 1] [--env ci] [--since 2025-01-01] [--dry-run] + +Reads Zebrunner's own read API (the same service the harness reports to): + +* ``GET /api/reporting/v1/launches`` - runs ("launches"), paged +* ``GET /api/reporting/v1/launches/{id}/tests`` - a run's tests, with labels +* ``GET /api/reporting/v1/test-runs/{id}/tests/{testId}/logs`` - a test's logs + +and posts each run to the radiator's ingest API, exactly as the harness does. + +What comes across, and from where: + +* From the test labels the harness set: the test case and asset ids, the + expected output, the CURIEs, and each agent's status. +* From the per-test report the harness logged (the JSON ``TestReport``): each + agent's message, whether the expected answer was found, its rank and score, + and the pks. ``--no-logs`` skips fetching these, which is much faster. +* Not available at all: the number of results, HTTP status and response time + (never recorded), and the environment (never sent to Zebrunner; see + ``--env``). + +Imported runs are marked ``origin=zebrunner_import`` and get a run id derived +from the Zebrunner launch id, so re-running the import updates them instead of +duplicating them. Already-imported launches are skipped unless ``--force``. +Performance tests are skipped: they predate HelmsDeep and carry no summary. +""" + +import argparse +import json +import logging +import sys +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Iterator, Optional + +import httpx + +import radiator_schema as schema + +logger = logging.getLogger("zebrunner-import") + +# Labels the harness set on every test; any other label key is an agent. +META_LABELS = { + "TestCase", + "TestAsset", + "ExpectedOutput", + "InputCurie", + "OutputCurie", + "SourceInputCurie", + "TargetInputCurie", + "TestRunTime", + "UserSpawnRate", +} +PERFORMANCE_LABELS = {"TestRunTime", "UserSpawnRate"} +STATUSES = {s.value for s in schema.Status} +# Zebrunner's own test statuses, for tests without an agent label to go by. +# It has no NO_RESULTS or ERROR, so the harness's attempts to finish a test +# with those were refused and the test stayed IN_PROGRESS. +ZEBRUNNER_STATUS = { + "PASSED": "PASSED", + "FAILED": "FAILED", + "SKIPPED": "SKIPPED", + "ABORTED": "ERROR", +} +# Agents that only exist in prod; the shepherd-* agents are dev/ci/test. +PROD_AGENTS = { + "aragorn", + "arax", + "biothings-explorer", + "improving-agent", + "unsecret-agent", + "cqs", +} +IMPORT_NAMESPACE = uuid.UUID("6f1c8a52-1d7e-4a8f-9a55-0b7c3f6d2e11") + + +def run_id_for(zebrunner_url: str, launch_id) -> uuid.UUID: + """Stable per launch, so re-importing updates rather than duplicates.""" + return uuid.uuid5(IMPORT_NAMESPACE, f"{zebrunner_url.rstrip('/')}/{launch_id}") + + +def parse_time(value) -> Optional[datetime]: + if value in (None, ""): + return None + if isinstance(value, (int, float)): + # epoch millis + return datetime.fromtimestamp(value / 1000, tz=timezone.utc) + text = str(value).replace("Z", "+00:00") + parsed = datetime.fromisoformat(text) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def suite_from_name(name: str) -> str: + """The harness named runs ``": "``.""" + suite, sep, _ = (name or "").rpartition(":") + return suite.strip() if sep and suite.strip() else (name or "unknown").strip() + + +# --- Zebrunner client -------------------------------------------------------- + + +class Zebrunner: + """The handful of Zebrunner read calls the import needs.""" + + def __init__( + self, + base_url: str, + refresh_token: str, + project_id: Optional[int] = None, + transport: Optional[httpx.BaseTransport] = None, + ): + self.base_url = base_url.rstrip("/") + self.refresh_token = refresh_token + self.project_id = project_id + self.client = httpx.Client( + base_url=self.base_url, timeout=60, transport=transport + ) + self._authenticate() + + def _authenticate(self): + res = self.client.post( + "/api/iam/v1/auth/refresh", json={"refreshToken": self.refresh_token} + ) + res.raise_for_status() + self.client.headers["Authorization"] = f"Bearer {res.json()['authToken']}" + + def _get(self, path: str, **params) -> dict: + params = {k: v for k, v in params.items() if v is not None} + if self.project_id is not None: + params.setdefault("projectId", self.project_id) + res = self.client.get(path, params=params) + if res.status_code == 401: + # access tokens are short-lived; an import can outlast one + self._authenticate() + res = self.client.get(path, params=params) + res.raise_for_status() + return res.json() + + def launches(self, page_size: int = 50) -> Iterator[dict]: + page = 1 + seen = set() + while True: + items = self._get( + "/api/reporting/v1/launches", page=page, pageSize=page_size + ).get("items", []) + fresh = [item for item in items if item.get("id") not in seen] + if not fresh: + return + for item in fresh: + seen.add(item.get("id")) + yield item + if len(items) < page_size: + return + page += 1 + + def tests(self, launch_id) -> list[dict]: + return self._get(f"/api/reporting/v1/launches/{launch_id}/tests").get( + "items", [] + ) + + def logs(self, launch_id, test_id, page_size: int = 100) -> list[dict]: + logs, token = [], None + while True: + payload = self._get( + f"/api/reporting/v1/test-runs/{launch_id}/tests/{test_id}/logs", + maxPageSize=page_size, + pageToken=token, + ) + logs.extend(payload.get("items", [])) + token = (payload.get("_meta") or {}).get("nextPageToken") + if not token or not payload.get("items"): + return logs + + +# --- conversion ---------------------------------------------------------------- + + +@dataclass +class ImportStats: + launches: int = 0 + imported: int = 0 + skipped_existing: int = 0 + skipped_empty: int = 0 + assets: int = 0 + with_report: int = 0 + performance_tests_skipped: int = 0 + unknown_env: int = 0 + failed: list = field(default_factory=list) + + +def labels_of(test: dict) -> dict[str, str]: + labels = {} + for label in test.get("labels") or []: + if isinstance(label, dict) and label.get("key") is not None: + labels[str(label["key"])] = ( + None if label.get("value") is None else str(label["value"]) + ) + return labels + + +def find_report(logs: list[dict]) -> Optional[dict]: + """The harness's TestReport: the last log line that parses as one.""" + for log in reversed(logs): + try: + data = json.loads(log.get("message") or "") + except (TypeError, ValueError): + continue + if isinstance(data, dict) and isinstance(data.get("result"), dict): + return data + return None + + +def _number(value, kind): + try: + return None if value is None else kind(value) + except (TypeError, ValueError): + return None + + +def expected_answer(agent_report: dict) -> tuple: + """(found, rank, score), as the harness's collector reads them.""" + actual = agent_report.get("actual_output") + if not isinstance(actual, dict) or not actual: + status = agent_report.get("status") + return (False if status == "NO_RESULTS" else None), None, None + rank = actual.get("ars_rank") + if rank is None: + rank = actual.get("ara_rank") + score = actual.get("ars_score") + if score is None: + score = actual.get("ara_score") + found = actual.get("found") + if found is None: + found = rank is not None or score is not None + return bool(found), _number(rank, int), _number(score, float) + + +def convert_test(test: dict, report: Optional[dict]) -> Optional[schema.AssetResult]: + labels = labels_of(test) + if PERFORMANCE_LABELS & labels.keys(): + return None + agents_status = { + key: value + for key, value in labels.items() + if key not in META_LABELS and value in STATUSES + } + report_agents = (report or {}).get("result") or {} + pks = (report or {}).get("pks") or {} + + agents = [] + for agent in sorted(set(agents_status) | set(report_agents)): + detail = report_agents.get(agent) or {} + status = agents_status.get(agent) or detail.get("status") + if status not in STATUSES: + continue + found, rank, score = expected_answer({**detail, "status": status}) + pk = pks.get(agent) + agents.append( + schema.AgentResult( + agent=agent, + status=status, + message=detail.get("message"), + found=found, + rank=rank, + score=score, + pk=None if pk in (None, "", "None") else str(pk), + expected_nodes_found=detail.get("expected_nodes_found"), + ) + ) + + # the overall status the harness gave the test: the ARS's, or the single + # override agent's; Zebrunner's own status only as a fallback + by_agent = {a.agent: a.status for a in agents} + if "ars" in by_agent: + status = by_agent["ars"] + elif len(by_agent) == 1: + status = next(iter(by_agent.values())) + else: + status = ZEBRUNNER_STATUS.get(test.get("status"), "SKIPPED") + + pathfinder = "SourceInputCurie" in labels + parent_pk = pks.get("parent_pk") + return schema.AssetResult( + test_case_id=labels.get("TestCase") or test.get("testClass") or "unknown", + asset_id=labels.get("TestAsset") or f"zebrunner-test-{test.get('id')}", + kind="pathfinder" if pathfinder else "acceptance", + name=test.get("name"), + expected_output=labels.get("ExpectedOutput"), + input_curie=labels.get("SourceInputCurie" if pathfinder else "InputCurie"), + output_curie=labels.get("TargetInputCurie" if pathfinder else "OutputCurie"), + status=status, + parent_pk=None if parent_pk in (None, "", "None") else str(parent_pk), + agents=agents, + details=(report or {}).get("test_details") or {}, + ) + + +def infer_env(results: list[schema.AssetResult]) -> Optional[str]: + agents = {a.agent for r in results for a in r.agents} + if agents & PROD_AGENTS: + return "prod" + return None + + +def convert_launch( + zebrunner_url: str, + launch: dict, + tests: list[dict], + reports: dict, + env: Optional[str], + stats: ImportStats, +) -> Optional[schema.RunPayload]: + results = [] + for test in tests: + result = convert_test(test, reports.get(test.get("id"))) + if result is None: + stats.performance_tests_skipped += 1 + continue + results.append(result) + if not results: + return None + + # the same asset can appear twice if a test was retried; keep the last + deduped = {(r.test_case_id, r.asset_id): r for r in results} + results = list(deduped.values()) + + started_at = parse_time(launch.get("startedAt")) or datetime.now(timezone.utc) + ended_at = parse_time(launch.get("endedAt")) + run_env = env or infer_env(results) + if run_env is None: + stats.unknown_env += 1 + counts: dict = {} + for result in results: + counts[result.status.value] = counts.get(result.status.value, 0) + 1 + + return schema.RunPayload( + run=schema.RunCreate( + run_id=run_id_for(zebrunner_url, launch["id"]), + suite=suite_from_name(launch.get("name")), + env=run_env, + harness_version=launch.get("build"), + started_at=started_at, + origin="zebrunner_import", + origin_ref=str(launch["id"]), + ), + results=results, + finish=( + schema.RunFinish(ended_at=ended_at, counts=counts) if ended_at else None + ), + ) + + +# --- radiator side --------------------------------------------------------------- + + +class Radiator: + def __init__(self, base_url: str, token: str, transport=None): + self.client = httpx.Client( + base_url=base_url.rstrip("/"), + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + transport=transport, + ) + + def has_run(self, run_id: uuid.UUID) -> bool: + res = self.client.get(f"/api/runs/{run_id}") + if res.status_code == 404: + return False + res.raise_for_status() + return True + + def push(self, payload: schema.RunPayload, batch_size: int = 200) -> None: + prefix = schema.INGEST_PREFIX + run_id = payload.run.run_id + self.client.post( + f"{prefix}/runs", json=payload.run.model_dump(mode="json") + ).raise_for_status() + for i in range(0, len(payload.results), batch_size): + batch = schema.ResultBatch(results=payload.results[i : i + batch_size]) + self.client.post( + f"{prefix}/runs/{run_id}/results", json=batch.model_dump(mode="json") + ).raise_for_status() + if payload.finish is not None: + self.client.patch( + f"{prefix}/runs/{run_id}", json=payload.finish.model_dump(mode="json") + ).raise_for_status() + + +# --- the import ------------------------------------------------------------------ + + +def import_runs( + zebrunner: Zebrunner, + radiator: Optional[Radiator], + *, + env: Optional[str] = None, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + suite: Optional[str] = None, + fetch_logs: bool = True, + force: bool = False, + workers: int = 8, + limit: Optional[int] = None, +) -> ImportStats: + """Import every matching launch. ``radiator=None`` is a dry run.""" + stats = ImportStats() + for launch in zebrunner.launches(): + started = parse_time(launch.get("startedAt")) + if since and started and started < since: + continue + if until and started and started >= until: + continue + if suite and suite_from_name(launch.get("name")) != suite: + continue + if limit is not None and stats.launches >= limit: + break + stats.launches += 1 + name = f"launch {launch.get('id')} ({launch.get('name')})" + run_id = run_id_for(zebrunner.base_url, launch["id"]) + try: + if radiator is not None and not force and radiator.has_run(run_id): + stats.skipped_existing += 1 + logger.info(f"{name}: already imported, skipping") + continue + tests = zebrunner.tests(launch["id"]) + reports = {} + if fetch_logs and tests: + with ThreadPoolExecutor(max_workers=workers) as pool: + found = pool.map( + lambda t: ( + t.get("id"), + find_report(zebrunner.logs(launch["id"], t.get("id"))), + ), + tests, + ) + reports = {test_id: report for test_id, report in found if report} + payload = convert_launch( + zebrunner.base_url, launch, tests, reports, env, stats + ) + if payload is None: + stats.skipped_empty += 1 + logger.info(f"{name}: no acceptance results, skipping") + continue + stats.assets += len(payload.results) + stats.with_report += sum(1 for t in tests if t.get("id") in reports) + if radiator is not None: + radiator.push(payload) + stats.imported += 1 + logger.info( + f"{name}: {'would import' if radiator is None else 'imported'} " + f"{len(payload.results)} assets ({payload.run.suite}, env " + f"{payload.run.env or 'unknown'})" + ) + except Exception as e: + stats.failed.append((launch.get("id"), str(e))) + logger.error(f"{name}: failed: {e}") + return stats + + +def _date(text: str) -> datetime: + return parse_time(text if "T" in text else f"{text}T00:00:00+00:00") + + +def cli(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Import past runs from Zebrunner into the Information Radiator." + ) + parser.add_argument("--zebrunner-url", required=True) + parser.add_argument( + "--refresh-token", + required=True, + help="Zebrunner API refresh token (ZE_REFRESH_TOKEN)", + ) + parser.add_argument( + "--project-id", + type=int, + help="Zebrunner project id, if your instance needs one", + ) + parser.add_argument( + "--radiator-url", help="Where to import to (omit with --dry-run)" + ) + parser.add_argument("--radiator-token", help="The radiator's API token") + parser.add_argument( + "--env", + help=( + "Environment to record the imported runs under. Zebrunner never " + "stored it; without this, runs with prod agents are marked prod " + "and the rest have no environment." + ), + ) + parser.add_argument("--suite", help="Only import runs of this suite") + parser.add_argument( + "--since", type=_date, help="Only runs started on/after this date (YYYY-MM-DD)" + ) + parser.add_argument( + "--until", type=_date, help="Only runs started before this date" + ) + parser.add_argument("--limit", type=int, help="Stop after this many runs") + parser.add_argument( + "--no-logs", + action="store_true", + help="Skip per-test logs: no ranks, scores or pks, but much faster", + ) + parser.add_argument( + "--force", action="store_true", help="Re-import runs that were already imported" + ) + parser.add_argument( + "--workers", type=int, default=8, help="Parallel log requests per run" + ) + parser.add_argument( + "--dry-run", action="store_true", help="Read and convert, but write nothing" + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + if not args.dry_run and not (args.radiator_url and args.radiator_token): + parser.error( + "--radiator-url and --radiator-token are required unless --dry-run" + ) + + zebrunner = Zebrunner(args.zebrunner_url, args.refresh_token, args.project_id) + radiator = ( + None if args.dry_run else Radiator(args.radiator_url, args.radiator_token) + ) + stats = import_runs( + zebrunner, + radiator, + env=args.env, + since=args.since, + until=args.until, + suite=args.suite, + fetch_logs=not args.no_logs, + force=args.force, + workers=args.workers, + limit=args.limit, + ) + logger.info( + f"Done: {stats.imported} of {stats.launches} runs " + f"{'would be ' if args.dry_run else ''}imported, {stats.assets} assets " + f"({stats.with_report} with a logged report). Skipped: " + f"{stats.skipped_existing} already imported, {stats.skipped_empty} " + f"without acceptance results, {stats.performance_tests_skipped} " + f"performance tests. {stats.unknown_env} runs have no environment." + ) + for launch_id, error in stats.failed: + logger.error(f"Failed: launch {launch_id}: {error}") + return 1 if stats.failed else 0 + + +if __name__ == "__main__": + sys.exit(cli()) From 4e27d1e407564c1620fd9f29e2953bd448651b80 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 19:10:53 +0000 Subject: [PATCH 06/35] Deploy the Information Radiator, and fit it to phones - radiator/Dockerfile (same base image as the harness, non-root). - deploy/: the radiator (Deployment with a migration init container, Service, Ingress), an optional in-namespace Postgres, and a one-off Job for the Zebrunner import. - CI runs the radiator's tests against a Postgres service and checks the migrations match the models; releases push a second image, ghcr.io/translatorsri/testharness-radiator. - httpx is a runtime dependency (the import uses it). - Pages no longer overflow at phone width, and the wide trend chart has a narrow twin so its labels stay legible there. - docs/information-radiator.md covers deploying, importing, and developing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WR6TD7jzBVdYPod4pKWcg --- .github/workflows/dev.yml | 41 ++++++ .github/workflows/release.yml | 10 ++ deploy/radiator-import.example.yaml | 37 +++++ deploy/radiator-postgres.example.yaml | 59 ++++++++ deploy/radiator.example.yaml | 108 +++++++++++++++ docs/information-radiator.md | 189 +++++++++++++++++++------- radiator/Dockerfile | 28 ++++ radiator/requirements-test.txt | 1 - radiator/requirements.txt | 2 + radiator/static/style.css | 18 ++- radiator/templates/base.html | 2 +- radiator/templates/performance.html | 4 +- radiator/templates/run.html | 4 + radiator/templates/trends.html | 3 +- radiator/web.py | 5 + 15 files changed, 452 insertions(+), 59 deletions(-) create mode 100644 deploy/radiator-import.example.yaml create mode 100644 deploy/radiator-postgres.example.yaml create mode 100644 deploy/radiator.example.yaml create mode 100644 radiator/Dockerfile diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 229a8c2..084df1a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -26,6 +26,47 @@ jobs: - name: Run tests and get output run: pytest + radiator-test: + name: Run the Information Radiator's tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: radiator_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + name: Setup python + with: + python-version: '3.11' + cache: 'pip' + + - name: Install requirements + run: pip install -r radiator/requirements-test.txt + + - name: Check the migrations match the models + env: + RADIATOR_DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/radiator_test + run: | + alembic -c radiator/alembic.ini upgrade head + alembic -c radiator/alembic.ini check + + - name: Run tests + env: + RADIATOR_TEST_DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/radiator_test + run: pytest radiator/tests + check-format: name: Check that code matches Black formatter runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15e87a0..e754514 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,3 +23,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io tags: latest,${{ steps.get_version.outputs.VERSION }} + - name: Push the Information Radiator to GitHub Packages + uses: docker/build-push-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + repository: translatorsri/testharness-radiator + dockerfile: radiator/Dockerfile + path: . + tags: latest,${{ steps.get_version.outputs.VERSION }} diff --git a/deploy/radiator-import.example.yaml b/deploy/radiator-import.example.yaml new file mode 100644 index 0000000..962dae0 --- /dev/null +++ b/deploy/radiator-import.example.yaml @@ -0,0 +1,37 @@ +# One-off Job: import past runs from Zebrunner into the Information Radiator. +# Run it once the radiator is up; it's safe to re-run (already-imported runs +# are skipped). Try `--dry-run` and `--limit 5` first to check what comes +# across. +# +# Zebrunner never recorded the environment. Pass --env for the environment +# these runs were against; to import runs from several environments, run the +# Job once per environment with --since/--until/--suite narrowing each. +apiVersion: batch/v1 +kind: Job +metadata: + name: radiator-zebrunner-import +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: import + image: ghcr.io/translatorsri/testharness-radiator:latest + command: ["python", "-m", "radiator.zebrunner_import"] + args: + - --zebrunner-url=$(ZE_BASE_URL) + - --refresh-token=$(ZE_REFRESH_TOKEN) + - --radiator-url=http://radiator + - --radiator-token=$(RADIATOR_TOKEN) + - --env=ci + env: + - name: ZE_BASE_URL + valueFrom: {secretKeyRef: {name: test-harness-secrets, key: ze-base-url}} + - name: ZE_REFRESH_TOKEN + valueFrom: {secretKeyRef: {name: test-harness-secrets, key: ze-refresh-token}} + - name: RADIATOR_TOKEN + valueFrom: {secretKeyRef: {name: radiator-secrets, key: api-token}} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} diff --git a/deploy/radiator-postgres.example.yaml b/deploy/radiator-postgres.example.yaml new file mode 100644 index 0000000..b8df052 --- /dev/null +++ b/deploy/radiator-postgres.example.yaml @@ -0,0 +1,59 @@ +# A small Postgres for the Information Radiator, for when the namespace has no +# database to use. Skip this if you have a managed one. +# +# kubectl create secret generic radiator-postgres \ +# --from-literal=password="$(openssl rand -base64 24)" +# +# and use that password in radiator-secrets' database-url: +# postgresql+psycopg://radiator:@radiator-postgres:5432/radiator +# +# Back the volume up: it is the only copy of the results history. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: radiator-postgres + labels: {app: radiator-postgres} +spec: + serviceName: radiator-postgres + replicas: 1 + selector: + matchLabels: {app: radiator-postgres} + template: + metadata: + labels: {app: radiator-postgres} + spec: + containers: + - name: postgres + image: postgres:16 + ports: + - {name: postgres, containerPort: 5432} + env: + - {name: POSTGRES_USER, value: radiator} + - {name: POSTGRES_DB, value: radiator} + - name: POSTGRES_PASSWORD + valueFrom: {secretKeyRef: {name: radiator-postgres, key: password}} + - {name: PGDATA, value: /var/lib/postgresql/data/pgdata} + readinessProbe: + exec: {command: ["pg_isready", "-U", "radiator"]} + periodSeconds: 10 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} + volumeMounts: + - {name: data, mountPath: /var/lib/postgresql/data} + volumeClaimTemplates: + - metadata: {name: data} + spec: + accessModes: [ReadWriteOnce] + resources: + requests: {storage: 10Gi} +--- +apiVersion: v1 +kind: Service +metadata: + name: radiator-postgres + labels: {app: radiator-postgres} +spec: + selector: {app: radiator-postgres} + ports: + - {name: postgres, port: 5432, targetPort: postgres} diff --git a/deploy/radiator.example.yaml b/deploy/radiator.example.yaml new file mode 100644 index 0000000..845521b --- /dev/null +++ b/deploy/radiator.example.yaml @@ -0,0 +1,108 @@ +# Example manifests for the Information Radiator, in the same namespace as the +# harness CronJob. Adapt before applying: the image tag, hostname, TLS secret, +# ingress class, and every value in the Secret are placeholders. +# +# The radiator needs a Postgres database. Point RADIATOR_DATABASE_URL at an +# existing one, or apply deploy/radiator-postgres.example.yaml for a small +# in-namespace instance. +# +# Generate the secrets with eg `openssl rand -base64 32`: +# kubectl create secret generic radiator-secrets \ +# --from-literal=database-url='postgresql+psycopg://radiator:PASSWORD@radiator-postgres:5432/radiator' \ +# --from-literal=api-token="$(openssl rand -base64 32)" \ +# --from-literal=username=translator \ +# --from-literal=password="$(openssl rand -base64 18)" \ +# --from-literal=session-secret="$(openssl rand -base64 32)" +# +# Then give the harness the same api-token as RADIATOR_TOKEN (see +# deploy/cronjob.example.yaml), and share the username/password with the +# consortium. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: radiator + labels: {app: radiator} +spec: + # One replica: the login throttle is in memory, and the load is a handful + # of people looking at a dashboard. Recreate so a migration never runs + # against a database an old pod is still using. + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: radiator} + template: + metadata: + labels: {app: radiator} + spec: + securityContext: + runAsNonRoot: true + initContainers: + # Bring the schema up to date before the app starts. Alembic is a + # no-op when it already is. + - name: migrate + image: ghcr.io/translatorsri/testharness-radiator:latest + command: ["alembic", "-c", "radiator/alembic.ini", "upgrade", "head"] + env: + - name: RADIATOR_DATABASE_URL + valueFrom: {secretKeyRef: {name: radiator-secrets, key: database-url}} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {memory: 256Mi} + containers: + - name: radiator + image: ghcr.io/translatorsri/testharness-radiator:latest + ports: + - {name: http, containerPort: 8000} + env: + - name: RADIATOR_DATABASE_URL + valueFrom: {secretKeyRef: {name: radiator-secrets, key: database-url}} + - name: RADIATOR_API_TOKEN + valueFrom: {secretKeyRef: {name: radiator-secrets, key: api-token}} + - name: RADIATOR_USERNAME + valueFrom: {secretKeyRef: {name: radiator-secrets, key: username}} + - name: RADIATOR_PASSWORD + valueFrom: {secretKeyRef: {name: radiator-secrets, key: password}} + - name: RADIATOR_SESSION_SECRET + valueFrom: {secretKeyRef: {name: radiator-secrets, key: session-secret}} + readinessProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 10 + livenessProbe: + # generous: a database blip shouldn't restart the app + httpGet: {path: /healthz, port: http} + periodSeconds: 30 + failureThreshold: 6 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 512Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: radiator + labels: {app: radiator} +spec: + selector: {app: radiator} + ports: + - {name: http, port: 80, targetPort: http} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: radiator + annotations: + # uploads from the harness come in batches; leave room for big ones + nginx.ingress.kubernetes.io/proxy-body-size: "16m" +spec: + ingressClassName: nginx + tls: + - hosts: [radiator.example.org] + secretName: radiator-tls + rules: + - host: radiator.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: {name: radiator, port: {name: http}} diff --git a/docs/information-radiator.md b/docs/information-radiator.md index 7076f99..0450f5a 100644 --- a/docs/information-radiator.md +++ b/docs/information-radiator.md @@ -1,41 +1,73 @@ -# Information Radiator replacement +# Information Radiator -A custom replacement for the Zebrunner-based Information Radiator, built in this -repo alongside the harness. +A replacement for the Zebrunner-based Information Radiator, built in this repo +alongside the harness. The harness posts structured results to it; it shows +them to the consortium behind one shared login. ## Why -Zebrunner's model forces the harness to put its results into labels and log -lines: per-agent status goes in as labels, and the full report (found, rank, -score, pks) as a JSON string in a log. None of it can be filtered or charted, -performance results never reach it at all, and each test costs about five API -calls. The replacement stores results as structured records, so it can show -what changed between runs and how assets and agents trend over time. +Zebrunner's model forced the harness to put its results into labels and log +lines: per-agent status went in as labels, and the full report (found, rank, +score, pks) as a JSON string in a log. None of it could be filtered or charted, +performance results never reached it, and each test cost about five API calls. +The radiator stores results as records, so it can show what changed between +runs and how assets and agents trend over time. ## Decisions | | | |---|---| -| Repo | Monorepo: the radiator lives here, in its own package and image. | +| Repo | Monorepo: `radiator/` here, shipped as its own image (`ghcr.io/translatorsri/testharness-radiator`). | | Hosting | Same Kubernetes namespace as the harness CronJob. | -| UI login | One shared username/password for the consortium, from a k8s secret. Handled by the app (login form + signed session cookie); no external identity provider. | -| API auth | Ingest requires a bearer token (`RADIATOR_TOKEN`), separate from the UI login. Every API route requires authentication. | -| Raw responses | Not stored. Records keep the pks, which link to the ARS/ARAX UI. | -| History | Past runs are imported from Zebrunner. | +| UI login | One shared username/password for the consortium (a k8s secret). Signed session cookie; failed logins are throttled per client. | +| API auth | Every API route needs the bearer token (`RADIATOR_API_TOKEN`); the read API also accepts a logged-in session. Uploads need the token. | +| Raw responses | Not stored. Results keep their pks, which open in the ARAX UI on ci. | +| History | Imported from Zebrunner's API (`radiator.zebrunner_import`). | -## Layout +## What's where ``` -radiator_schema/ API contract (Pydantic), shared by harness and radiator +radiator_schema/ the API contract (Pydantic), shared by both sides test_harness/ - radiator_client.py harness-side client + `test-harness-radiator push` -radiator/ (phase 2) FastAPI service, DB, importer, UI, Dockerfile -deploy/radiator.yaml (phase 5) Deployment, Service, Ingress + radiator_client.py harness side: batched uploads, save + `test-harness-radiator push` +radiator/ + app.py, auth.py FastAPI app, login, token auth + api.py, ingest.py ingest + read API; upserts + models.py, migrations/ Postgres schema (SQLAlchemy, Alembic) + queries.py, web.py the dashboard's queries and pages + templates/, static/ server-rendered HTML, CSS, a small hover script + charts.py server-side SVG charts + zebrunner_import.py the history import + demo.py made-up data for development +deploy/ + radiator.example.yaml Deployment (+ migration init container), Service, Ingress + radiator-postgres.example.yaml a small in-namespace Postgres, if you need one + radiator-import.example.yaml one-off Job for the Zebrunner import ``` +## The dashboard + +- **Runs**: every run, newest first, with its status breakdown, pass rate, and + duration; filter by suite and environment. +- **A run**: pass rate against the previous run, regressions and fixes, a + per-agent breakdown, and the results grid: every asset against every agent, + with the rank the expected answer came back at. Filter by status (overall or + per agent), expected output, or search by name/id/CURIE. Each cell opens its + pk in ARAX. +- **What changed**: against the previous run *of the same series* (same suite, + env, target override, and query type): regressions, fixes, other status + changes, skips, rank moves of 5+ places, and assets added or dropped. +- **An asset**: its status per agent across the last 60 runs, the rank of the + expected answer and the number of results over time, and a table of runs. +- **Trends**: each agent's pass rate over full runs of a suite, overall and by + expected output. Skips are left out; runs with a target override or a single + query type aren't included. +- **Performance**: HelmsDeep's max sustainable concurrency per host over time, + with checkpoint verdicts. + ## API -All writes go under `/api/ingest`, with `Authorization: Bearer `: +Uploads go under `/api/ingest`, with `Authorization: Bearer `: | Method | Path | Body | |---|---|---| @@ -44,37 +76,96 @@ All writes go under `/api/ingest`, with `Authorization: Bearer `: | `PATCH` | `/api/ingest/runs/{run_id}` | `RunFinish` | Every write is an upsert: runs on the client-generated `run_id`, results on -`(run_id, test_case_id, asset_id)`. So retries and replays never duplicate. The -asset ids come from the Tests repo and are stable across runs, which is what -the trend views key on (`hash_test_asset` is not stable across processes). - -## Phases - -1. **Contract + harness (done on `claude/radiator-phase1`).** `radiator_schema`; - the harness records n_results, HTTP status, and response time (direct - queries only, since ARS fan-out is only seen through 10s polling) per agent; - it dual-writes to Zebrunner and the new radiator, and saves runs it couldn't - upload for `test-harness-radiator push`. -2. **Radiator backend.** FastAPI + Postgres (SQLAlchemy, Alembic); ingest - endpoints; read endpoints shaped for the views; shared-login session auth for - the UI and read API, bearer token for ingest; an artifact endpoint for - HelmsDeep's `report.html`. -3. **UI**, in priority order: run matrix (assets x agents with status, rank, - score, n_results, pk links); diff against the last comparable run (same - suite, env, and target); asset history; agent pass rate over time by - expected-output category; performance trends (knee, checkpoints per host). -4. **Zebrunner import.** Runs -> runs, tests -> assets via the - `TestCase`/`TestAsset`/`ExpectedOutput` labels, agent labels -> statuses, - found/rank/score from the report log where present. Older runs lack those, - and all lack n_results and response time; the UI shows them as not - recorded, not zero. Imported runs carry `origin: zebrunner_import`, and the - import is re-runnable. -5. **Deploy + cutover.** Deploy to the namespace; dual-write for 2-3 weeks; - point Slack at the new UI; remove `Reporter` and the `ZE_*` secrets. +`(run_id, test_case_id, asset_id)`, and an asset's agent rows are replaced by +the newest upload. Retries and replays never duplicate. The asset ids come from +the Tests repo and are stable across runs, which is what history and trends key +on. + +Reading, with the token or a session: `GET /api/runs?suite=&env=&limit=` and +`GET /api/runs/{run_id}` (with every result). + +## Deploying + +1. **Database.** Use an existing Postgres, or apply + `deploy/radiator-postgres.example.yaml`. Back it up: it's the only copy of + the history. +2. **Secrets.** Create `radiator-secrets` as described at the top of + `deploy/radiator.example.yaml`. +3. **The radiator.** Adapt and apply `deploy/radiator.example.yaml` (image tag, + hostname, TLS secret, ingress class). Migrations run in an init container on + every start; they're a no-op when the schema is current. +4. **The harness.** Add `RADIATOR_URL` (the radiator's **public** URL: it's + also the link posted to Slack) and `RADIATOR_TOKEN` (the same api-token) to + `test-harness-secrets`; `deploy/cronjob.example.yaml` already reads them. + The harness then reports to both Zebrunner and the radiator. +5. **History.** Run the Zebrunner import (below). +6. **Cutover**, after a couple of weeks of both: point people at the new UI, + then remove the Zebrunner `Reporter` and the `ZE_*` secrets. + +The image is built and pushed by the release workflow alongside the harness's. +To build it by hand, from the repo root: +`docker build -f radiator/Dockerfile -t testharness-radiator .` + +## Importing from Zebrunner + +``` +python -m radiator.zebrunner_import \ + --zebrunner-url "$ZE_BASE_URL" --refresh-token "$ZE_REFRESH_TOKEN" \ + --radiator-url https://radiator.example.org --radiator-token "$RADIATOR_TOKEN" \ + --env ci --dry-run --limit 5 +``` + +Drop `--dry-run`/`--limit` once that looks right; `deploy/radiator-import.example.yaml` +runs it in the cluster. It reads Zebrunner's API (launches, their tests' +labels, and each test's logs) and posts runs like the harness does. + +- **What comes across:** test case/asset ids, expected output, CURIEs, every + agent's status (from the labels the harness set), and from the logged report, + each agent's found/rank/score and the pks. `--no-logs` skips the logs: much + faster, but no ranks, scores or pks. +- **What can't:** result counts, HTTP status and response times were never + recorded, so those show as "not recorded", never zero. The environment was + never sent to Zebrunner: pass `--env`, or runs with prod-only agents are + marked prod and the rest get no environment. To import several environments, + run once per environment, narrowing with `--suite`, `--since`, `--until`. +- **Skipped:** performance tests (pre-HelmsDeep, no summary). +- **Re-runnable:** imported runs get ids derived from the Zebrunner launch, so + a second run skips them, or updates them with `--force`. +- **If your Zebrunner needs it**, pass `--project-id`. + +The endpoints were read from Zebrunner's published images +(`zebrunner/reporting-service` 1.54.1, `zebrunner/test-execution-logs-service` +1.6.1). An older instance may differ; `--dry-run --limit 5` shows quickly +whether it works against yours. + +## Developing + +``` +# a Postgres to develop against, then: +export RADIATOR_DATABASE_URL=postgresql+psycopg://postgres@localhost:5432/radiator +export RADIATOR_API_TOKEN=dev-token RADIATOR_USERNAME=dev RADIATOR_PASSWORD=dev \ + RADIATOR_SESSION_SECRET=dev RADIATOR_SECURE_COOKIES=false +pip install -r radiator/requirements-test.txt +alembic -c radiator/alembic.ini upgrade head +python -m radiator.demo # made-up history to look at +uvicorn radiator.app:create_app --factory --reload +``` + +Tests need a database they may wipe: +`RADIATOR_TEST_DATABASE_URL=... pytest radiator/tests`. After changing +`models.py`, add a migration with +`alembic -c radiator/alembic.ini revision --autogenerate -m "..."`, review it, +and check `alembic -c radiator/alembic.ini check` is clean (CI runs it). + +To point a local harness run at it: `RADIATOR_URL=http://localhost:8000 +RADIATOR_TOKEN=dev-token test-harness load local_acceptance`. ## Known gaps -- ARA response times under the ARS aren't recorded. The ARS's own message - timestamps would give them; that needs confirming against the ARS API. +- ARA response times under the ARS aren't recorded: the harness only sees + ARS fan-out through 10s polling. The ARS's own message timestamps could + give them. - A performance test whose HelmsDeep run raises (rather than returning an error) isn't recorded in either radiator. This is existing behavior. +- HelmsDeep's `report.html` isn't uploaded yet (it still goes to Slack); the + radiator stores the `summary.json` it is drawn from. diff --git a/radiator/Dockerfile b/radiator/Dockerfile new file mode 100644 index 0000000..28604ea --- /dev/null +++ b/radiator/Dockerfile @@ -0,0 +1,28 @@ +# The Information Radiator. Build from the repo root, since it needs the +# shared radiator_schema package: +# docker build -f radiator/Dockerfile -t testharness-radiator . +FROM ghcr.io/translatorsri/renci-python-image:3.11.5 + +LABEL org.opencontainers.image.source /github.com/TranslatorSRI/TestHarness + +WORKDIR /app + +COPY radiator/requirements.txt radiator/requirements.txt +RUN pip install --no-cache-dir -r radiator/requirements.txt + +COPY radiator_schema radiator_schema +COPY radiator radiator +RUN rm -rf radiator/tests + +ENV PYTHONPATH=/app \ + PYTHONUNBUFFERED=1 + +# the non-root user defined in the base image +USER nru + +EXPOSE 8000 + +# --proxy-headers so the login throttle sees each client, not the ingress. +# Migrations run separately (an init container in deploy/radiator.example.yaml): +# alembic -c radiator/alembic.ini upgrade head +CMD ["uvicorn", "radiator.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", "--forwarded-allow-ips", "*"] diff --git a/radiator/requirements-test.txt b/radiator/requirements-test.txt index 333c276..29d64a6 100644 --- a/radiator/requirements-test.txt +++ b/radiator/requirements-test.txt @@ -1,5 +1,4 @@ -r requirements.txt pytest==9.1.1 -httpx==0.28.1 # Starlette's TestClient httpx2==2.13.1 diff --git a/radiator/requirements.txt b/radiator/requirements.txt index 735e144..2a22e6f 100644 --- a/radiator/requirements.txt +++ b/radiator/requirements.txt @@ -10,3 +10,5 @@ jinja2==3.1.6 itsdangerous==2.2.0 python-multipart==0.0.32 pydantic==2.13.5 +# for the Zebrunner import +httpx==0.28.1 diff --git a/radiator/static/style.css b/radiator/static/style.css index b7e9383..7048234 100644 --- a/radiator/static/style.css +++ b/radiator/static/style.css @@ -103,15 +103,17 @@ code, .mono { width: 18px; height: 18px; border-radius: 5px; background: conic-gradient(var(--status-passed) 0 60%, var(--status-failed) 0 80%, var(--status-no-results) 0); } -.nav { display: flex; gap: 4px; flex: 1; } +.nav { display: flex; gap: 4px; flex: 1; min-width: 0; overflow-x: auto; } .nav a { + white-space: nowrap; color: var(--ink-2); padding: 6px 10px; border-radius: 6px; } .nav a:hover { background: var(--surface-2); text-decoration: none; } .nav a.active { color: var(--ink); background: var(--surface-2); font-weight: 550; } -.logout button { +.brand { white-space: nowrap; } +.logout button { white-space: nowrap; background: none; border: 0; color: var(--ink-2); cursor: pointer; font: inherit; } @@ -261,8 +263,8 @@ th.agent-head { text-align: center; } .chart .dot { stroke: var(--surface); stroke-width: 2; } .chart .crosshair { stroke: var(--ink-muted); stroke-width: 1; } .chart-wrap { position: relative; } -.multiples { display: grid; grid-template-columns: repeat(auto-fit, minmax(380px, 1fr)); gap: 12px 24px; } -.multiples.two { grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); } +.multiples { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(380px, 100%), 1fr)); gap: 12px 24px; } +.multiples.two { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); } .multiples h3 { font-size: 13px; font-weight: 600; margin: 4px 0; } details.table-view { margin-top: 8px; } @@ -306,8 +308,14 @@ details.table-view[open] summary { margin-bottom: 8px; } } .pager { display: flex; gap: 12px; justify-content: flex-end; margin-top: 12px; } +.narrow-only { display: none; } @media (max-width: 640px) { + .wide-only { display: none; } + .narrow-only { display: block; } main { padding: 16px; } - .topbar { padding: 0 16px; gap: 12px; } + .topbar { padding: 0 16px; gap: 8px; } + .brand-text { display: none; } + .card { padding: 14px; } + .card-head { flex-direction: column; gap: 4px; } .tile.hero .value { font-size: 40px; } } diff --git a/radiator/templates/base.html b/radiator/templates/base.html index 113ddda..205f253 100644 --- a/radiator/templates/base.html +++ b/radiator/templates/base.html @@ -9,7 +9,7 @@
- Information Radiator + Information Radiator