From 9a36783132e2ed930c21186cad6b475ee7917224 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 8 Jul 2026 11:39:16 +0100 Subject: [PATCH 001/344] feat: Support RHOAI RC images Signed-off-by: Alberto Perdomo --- .../toolbox/deploy_custom_catalog/main.py | 3 + .../templates/catalogsource.yaml.j2 | 3 + projects/llm_d/orchestration/ci.py | 29 +++- .../orchestration/config.d/platform.yaml | 8 +- .../manifests/quay-registry-icsp.yaml | 9 ++ projects/llm_d/orchestration/prepare_phase.py | 134 ++++++++++++++++++ projects/llm_d/tests/test_profiles.py | 110 +++++++++++++- vaults/psap-rhoai-rc.yaml | 8 ++ 8 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml create mode 100644 vaults/psap-rhoai-rc.yaml diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index eea38593d..641184cb8 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -17,6 +17,7 @@ def run( catalog_image: str, *, display_name: str = "", + publisher: str = "", ) -> int: """ Deploy a CatalogSource from a custom index image and wait for it to become READY. @@ -26,6 +27,7 @@ def run( catalog_namespace: Namespace where the CatalogSource will be deployed catalog_image: Index image backing the CatalogSource display_name: Optional human-friendly label for logs + publisher: Optional publisher label for the CatalogSource """ execute_tasks(locals()) @@ -39,6 +41,7 @@ def setup_directories(args, ctx): shell.mkdir("src") shell.mkdir("artifacts") ctx.display_name = args.display_name or args.catalog_source_name + ctx.publisher = args.publisher or "" return f"Prepared directories for {ctx.display_name}" diff --git a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 index c3244fca4..8f00bdc1a 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 +++ b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 @@ -7,3 +7,6 @@ spec: sourceType: grpc image: {{ args.catalog_image }} displayName: {{ ctx.display_name }} +{% if args.publisher %} + publisher: {{ args.publisher }} +{% endif %} diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index d08376451..d604f8b02 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -36,6 +36,32 @@ def init(): run.init() config.init(Path(__file__).parent) +def list_vaults() -> list[str]: + """List all vaults (includes both mandatory and optional).""" + all_vaults = vault.phase_vault_list_all() + if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): + return [*all_vaults, "psap-rhoai-rc"] + + return all_vaults + + +def init_vaults_for_phase(phase: str) -> None: + mandatory_vaults = [ + *config.project.get_config("vaults.all", []), + *config.project.get_config(f"vaults.{phase}", []), + ] + optional_vaults = [ + *config.project.get_config("vaults.all-optional", []), + *config.project.get_config(f"vaults.{phase}-optional", []), + ] + + if phase == "prepare" and config.project.get_config( + "platform.rhoai.custom_catalog.enabled", False + ): + optional_vaults = [*optional_vaults, "psap-rhoai-rc"] + + vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) + @click.group(cls=ci_lib.HelpfulGroup) @click.pass_context @@ -49,7 +75,7 @@ def main(ctx): logger.info("No need to initialize the vaults for the resolve step") return - vault.phase_vault_init(ctx.invoked_subcommand) + init_vaults_for_phase(ctx.invoked_subcommand) @main.command() @@ -113,6 +139,7 @@ def post_cleanup(ctx) -> int: main.add_command( create_fournos_resolve_entrypoint( vault_list_funcs=[ + list_vaults, vault.phase_vault_list_all, caliper_export_list_vaults, caliper_export_list_optional_vaults, diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 44a941275..1d1f16985 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -50,10 +50,14 @@ operators: rhoai: custom_catalog: enabled: false - name: rhoai-custom + name: rhoai-catalog-dev namespace: openshift-marketplace image: null - display_name: Red Hat OpenShift AI Custom Catalog + display_name: Red Hat OpenShift AI + publisher: RHOAI Development Catalog + pull_secret: + vault: + name: psap-rhoai-rc namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml new file mode 100644 index 000000000..d18702599 --- /dev/null +++ b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml @@ -0,0 +1,9 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: quay-registry +spec: + repositoryDigestMirrors: + - source: registry.redhat.io/rhoai + mirrors: + - quay.io/rhoai diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 138c24509..7b9f31e36 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -1,8 +1,12 @@ from __future__ import annotations +import base64 import hashlib import json import logging +import tempfile +import time +from pathlib import Path from typing import Any from projects.cluster.toolbox.bootstrap_lws_operator import main as bootstrap_lws_operator @@ -35,6 +39,11 @@ ) logger = logging.getLogger(__name__) +RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" +RHOAI_PULL_SECRET_NAME = "pull-secret" +RHOAI_REGISTRY = "quay.io/rhoai" +RHOAI_PULL_SECRET_CONTENT = "rhoai_rc.secret" +RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: @@ -102,9 +111,130 @@ def deploy_rhoai_custom_catalog(*, rhoai: dict) -> int: catalog_namespace=custom_catalog["namespace"], catalog_image=custom_catalog["image"], display_name=custom_catalog.get("display_name", custom_catalog["name"]), + publisher=custom_catalog.get("publisher", ""), ) +def _custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: + pull_secret = custom_catalog.get("pull_secret", {}) + vault_spec = pull_secret.get("vault", {}) + vault_name = vault_spec.get("name") + content_name = vault_spec.get("content", RHOAI_PULL_SECRET_CONTENT) + + if not vault_name: + raise RuntimeError("RHOAI custom catalog pull_secret.vault.name is required") + + secret_path = vault.get_vault_content_path(vault_name, content_name) + if secret_path is None: + raise RuntimeError( + f"RHOAI pull secret content '{content_name}' was not found in vault '{vault_name}'" + ) + + return secret_path + + +def _decode_pull_secret(secret_data: dict[str, Any]) -> str: + encoded = secret_data.get("data", {}).get(".dockerconfigjson") + if not encoded: + raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") + + return base64.b64decode(encoded).decode("utf-8") + + +def wait_for_rhoai_pull_secret_ready( + *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 +) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + try: + secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded = _decode_pull_secret(secret) + if RHOAI_REGISTRY not in decoded: + raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + + mcp_status = oc( + "get", + "mcp", + "-o", + r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', + check=False, + log_stdout=False, + log_stderr=False, + ).stdout.strip() + if "False" in mcp_status: + raise RuntimeError("machine config pools are still updating") + + logger.info( + "RHOAI pull secret now contains %s and machine config pools are updated", + RHOAI_REGISTRY, + ) + return + except RuntimeError as exc: + last_error = str(exc) + logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) + time.sleep(poll_interval_seconds) + + raise RuntimeError( + f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + ) + + +def prepare_rhoai_pull_secret(custom_catalog: dict[str, Any]) -> None: + pull_secret_path = _custom_catalog_pull_secret_path(custom_catalog) + auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() + if not auth_basic: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + + if RHOAI_REGISTRY in decoded_secret: + logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + return + + logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: + temp_file.write(decoded_secret) + temp_path = Path(temp_file.name) + + try: + oc( + "registry", + "login", + f"--registry={RHOAI_REGISTRY}", + f"--auth-basic={auth_basic}", + "--to", + str(temp_path), + ) + oc( + "set", + "data", + f"secret/{RHOAI_PULL_SECRET_NAME}", + "-n", + RHOAI_PULL_SECRET_NAMESPACE, + f"--from-file=.dockerconfigjson={temp_path}", + ) + finally: + temp_path.unlink(missing_ok=True) + + wait_for_rhoai_pull_secret_ready() + + +def apply_rhoai_icsp() -> None: + logger.info("Applying RHOAI ICSP manifest: %s", RHOAI_ICSP_MANIFEST) + oc("apply", "-f", str(RHOAI_ICSP_MANIFEST)) + + def rhoai_operator_spec( *, rhoai: dict, @@ -179,7 +309,11 @@ def prepare_gpu_operator() -> None: def prepare_rhoai_operator() -> None: platform = runtime_config.get_platform_config() + custom_catalog = platform["rhoai"]["custom_catalog"] prepare_rhcl_operator() + if custom_catalog["enabled"]: + prepare_rhoai_pull_secret(custom_catalog) + apply_rhoai_icsp() deploy_rhoai_custom_catalog(rhoai=platform["rhoai"]) operator_spec = operator_spec_by_package(platform, "rhods-operator") operator_spec = rhoai_operator_spec(rhoai=platform["rhoai"], operator_spec=operator_spec) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index a798fb8e2..e2cc81df6 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -8,7 +8,7 @@ from projects.core.library import config as core_config from projects.core.library import env from projects.llm_d.orchestration import ci as llmd_ci -from projects.llm_d.orchestration import runtime_config, test_phase +from projects.llm_d.orchestration import prepare_phase, runtime_config, test_phase from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) @@ -198,6 +198,114 @@ def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided assert runtime_config.get_benchmark_keys() == ["short"] +def test_list_vaults_only_includes_rhoai_rc_vault_for_custom_catalog_runs() -> None: + _init_project_config() + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + assert "psap-rhoai-rc" not in llmd_ci.list_vaults() + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + assert "psap-rhoai-rc" in llmd_ci.list_vaults() + + +def test_prepare_phase_adds_rhoai_rc_vault_only_for_custom_catalog_runs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + calls: list[dict[str, object]] = [] + + def _fake_init(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(llmd_ci.vault, "init", _fake_init) + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + llmd_ci.init_vaults_for_phase("prepare") + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + llmd_ci.init_vaults_for_phase("prepare") + + assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] + assert "psap-rhoai-rc" in calls[1]["optional_vaults"] + + +def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + platform = { + "operators": { + "rhcl-operator": { + "package": "rhcl-operator", + "namespace": "openshift-operators", + "source": "redhat-operators", + "channel": "stable", + }, + "rhods-operator": { + "package": "rhods-operator", + "namespace": "redhat-ods-operator", + "source": "redhat-operators", + "channel": "stable-3.x", + }, + }, + "rhoai": { + "custom_catalog": { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": {"vault": {"name": "psap-rhoai-rc"}}, + }, + "namespace": "redhat-ods-applications", + "datasciencecluster_name": "default-dsc", + "components": ["kserve"], + "required_crds_before_dsc": ["datascienceclusters.datasciencecluster.opendatahub.io"], + "required_crds_after_dsc": ["llminferenceservices.serving.kserve.io"], + }, + } + + calls: list[object] = [] + + monkeypatch.setattr(prepare_phase.runtime_config, "get_platform_config", lambda: platform) + monkeypatch.setattr(prepare_phase, "prepare_rhcl_operator", lambda: calls.append("rhcl")) + monkeypatch.setattr( + prepare_phase, + "prepare_rhoai_pull_secret", + lambda custom_catalog: calls.append(("pull", custom_catalog["image"])), + ) + monkeypatch.setattr(prepare_phase, "apply_rhoai_icsp", lambda: calls.append("icsp")) + monkeypatch.setattr( + prepare_phase, + "deploy_rhoai_custom_catalog", + lambda rhoai: calls.append(("catalog", rhoai["custom_catalog"]["publisher"])), + ) + monkeypatch.setattr( + prepare_phase, + "ensure_operator_subscription", + lambda operator_spec: calls.append(("sub", operator_spec["package"])), + ) + monkeypatch.setattr( + prepare_phase, + "ensure_required_crds_before_dsc", + lambda: calls.append("crds"), + ) + + prepare_phase.prepare_rhoai_operator() + + assert calls == [ + "rhcl", + ("pull", "quay.io/rhoai/rhoai-fbc-fragment@sha256:test"), + "icsp", + ("catalog", "RHOAI Development Catalog"), + ("sub", "rhods-operator"), + "crds", + ] + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/vaults/psap-rhoai-rc.yaml b/vaults/psap-rhoai-rc.yaml new file mode 100644 index 000000000..f9bbc10c4 --- /dev/null +++ b/vaults/psap-rhoai-rc.yaml @@ -0,0 +1,8 @@ +env_key: PSAP_RHOAI_RC_SECRET_PATH +description: | + Vault containing credentials for the RHOAI release-candidate registry + +content: + rhoai_rc.secret: + description: | + Basic auth string used to extend the cluster pull secret for quay.io/rhoai From 27f0b0686074bd4307479243fc28ad2bf2712001 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 9 Jul 2026 10:55:31 +0100 Subject: [PATCH 002/344] fix: Address review comments Signed-off-by: Alberto Perdomo --- .../orchestration/config.d/platform.yaml | 1 + projects/llm_d/orchestration/prepare_phase.py | 218 +------------- projects/llm_d/tests/test_profiles.py | 53 +++- projects/rhoai/library/deploy.py | 276 ++++++++++++++++++ vaults/psap-rhoai-rc.yaml | 2 + 5 files changed, 330 insertions(+), 220 deletions(-) create mode 100644 projects/rhoai/library/deploy.py diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 1d1f16985..f9b2975e2 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -58,6 +58,7 @@ rhoai: pull_secret: vault: name: psap-rhoai-rc + content: rhoai_rc.secret namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 7b9f31e36..3e3b63047 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -1,17 +1,11 @@ from __future__ import annotations -import base64 import hashlib import json import logging -import tempfile -import time from pathlib import Path -from typing import Any from projects.cluster.toolbox.bootstrap_lws_operator import main as bootstrap_lws_operator -from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator -from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.dsl.utils.k8s import ( @@ -33,32 +27,22 @@ run as capture_prepare_state_toolbox_run, ) from projects.llm_d.toolbox.ensure_gateway import main as ensure_gateway_command +from projects.rhoai.library.deploy import ( + ensure_operator_subscription, + operator_spec_by_package, +) +from projects.rhoai.library.deploy import ( + prepare_rhoai_operator as rhoai_prepare_rhoai_operator, +) from projects.rhoai.toolbox.apply_datasciencecluster import main as apply_datasciencecluster_command from projects.rhoai.toolbox.wait_datasciencecluster_ready import ( main as wait_datasciencecluster_ready_command, ) logger = logging.getLogger(__name__) -RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" -RHOAI_PULL_SECRET_NAME = "pull-secret" -RHOAI_REGISTRY = "quay.io/rhoai" -RHOAI_PULL_SECRET_CONTENT = "rhoai_rc.secret" RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" -def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: - operators = platform["operators"] - if isinstance(operators, dict): - if package in operators: - return {"package": package, **operators[package]} - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") - - for operator_spec in operators: - if operator_spec["package"] == package: - return operator_spec - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") - - def verify_oc_access() -> None: oc("whoami") @@ -85,177 +69,11 @@ def verify_cluster_version() -> None: ) -def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: - return cluster_deploy_operator.run( - package_name=operator_spec["package"], - target_namespace=operator_spec["namespace"], - source_name=operator_spec["source"], - channel=operator_spec["channel"], - source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), - display_name=operator_spec.get("display_name", operator_spec["package"]), - artifact_dirname_suffix=f"_{operator_spec['package']}", - ) - - -def deploy_rhoai_custom_catalog(*, rhoai: dict) -> int: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - logger.info("RHOAI custom catalog disabled; using default catalog source") - return 0 - - if not custom_catalog.get("image"): - raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") - - return deploy_custom_catalog.run( - catalog_source_name=custom_catalog["name"], - catalog_namespace=custom_catalog["namespace"], - catalog_image=custom_catalog["image"], - display_name=custom_catalog.get("display_name", custom_catalog["name"]), - publisher=custom_catalog.get("publisher", ""), - ) - - -def _custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: - pull_secret = custom_catalog.get("pull_secret", {}) - vault_spec = pull_secret.get("vault", {}) - vault_name = vault_spec.get("name") - content_name = vault_spec.get("content", RHOAI_PULL_SECRET_CONTENT) - - if not vault_name: - raise RuntimeError("RHOAI custom catalog pull_secret.vault.name is required") - - secret_path = vault.get_vault_content_path(vault_name, content_name) - if secret_path is None: - raise RuntimeError( - f"RHOAI pull secret content '{content_name}' was not found in vault '{vault_name}'" - ) - - return secret_path - - -def _decode_pull_secret(secret_data: dict[str, Any]) -> str: - encoded = secret_data.get("data", {}).get(".dockerconfigjson") - if not encoded: - raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") - - return base64.b64decode(encoded).decode("utf-8") - - -def wait_for_rhoai_pull_secret_ready( - *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 -) -> None: - deadline = time.monotonic() + timeout_seconds - last_error: str | None = None - - while time.monotonic() < deadline: - try: - secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded = _decode_pull_secret(secret) - if RHOAI_REGISTRY not in decoded: - raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") - - mcp_status = oc( - "get", - "mcp", - "-o", - r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', - check=False, - log_stdout=False, - log_stderr=False, - ).stdout.strip() - if "False" in mcp_status: - raise RuntimeError("machine config pools are still updating") - - logger.info( - "RHOAI pull secret now contains %s and machine config pools are updated", - RHOAI_REGISTRY, - ) - return - except RuntimeError as exc: - last_error = str(exc) - logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) - time.sleep(poll_interval_seconds) - - raise RuntimeError( - f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" - ) - - -def prepare_rhoai_pull_secret(custom_catalog: dict[str, Any]) -> None: - pull_secret_path = _custom_catalog_pull_secret_path(custom_catalog) - auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() - if not auth_basic: - raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") - - current_secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded_secret = _decode_pull_secret(current_secret) - - if RHOAI_REGISTRY in decoded_secret: - logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) - return - - logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: - temp_file.write(decoded_secret) - temp_path = Path(temp_file.name) - - try: - oc( - "registry", - "login", - f"--registry={RHOAI_REGISTRY}", - f"--auth-basic={auth_basic}", - "--to", - str(temp_path), - ) - oc( - "set", - "data", - f"secret/{RHOAI_PULL_SECRET_NAME}", - "-n", - RHOAI_PULL_SECRET_NAMESPACE, - f"--from-file=.dockerconfigjson={temp_path}", - ) - finally: - temp_path.unlink(missing_ok=True) - - wait_for_rhoai_pull_secret_ready() - - def apply_rhoai_icsp() -> None: logger.info("Applying RHOAI ICSP manifest: %s", RHOAI_ICSP_MANIFEST) oc("apply", "-f", str(RHOAI_ICSP_MANIFEST)) -def rhoai_operator_spec( - *, - rhoai: dict, - operator_spec: dict[str, str], -) -> dict[str, str]: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - return operator_spec - - updated_spec = dict(operator_spec) - updated_spec["source"] = custom_catalog["name"] - updated_spec["source_namespace"] = custom_catalog["namespace"] - return updated_spec - - -def prepare_rhcl_operator() -> None: - platform = runtime_config.get_platform_config() - operator_spec = operator_spec_by_package(platform, "rhcl-operator") - ensure_operator_subscription(operator_spec) - - def prepare_cert_manager() -> None: platform = runtime_config.get_platform_config() operator_spec = operator_spec_by_package(platform, "openshift-cert-manager-operator") @@ -309,24 +127,10 @@ def prepare_gpu_operator() -> None: def prepare_rhoai_operator() -> None: platform = runtime_config.get_platform_config() - custom_catalog = platform["rhoai"]["custom_catalog"] - prepare_rhcl_operator() - if custom_catalog["enabled"]: - prepare_rhoai_pull_secret(custom_catalog) - apply_rhoai_icsp() - deploy_rhoai_custom_catalog(rhoai=platform["rhoai"]) - operator_spec = operator_spec_by_package(platform, "rhods-operator") - operator_spec = rhoai_operator_spec(rhoai=platform["rhoai"], operator_spec=operator_spec) - ensure_operator_subscription(operator_spec) - ensure_required_crds_before_dsc() - - -def ensure_required_crds_before_dsc() -> None: - platform = runtime_config.get_platform_config() - rhoai = platform["rhoai"] - wait_for_crds_command.run( - crd_names=rhoai["required_crds_before_dsc"], - display_name="RHOAI pre-DSC CRDs", + rhoai_prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=apply_rhoai_icsp, ) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index e2cc81df6..5b4148816 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -8,10 +8,11 @@ from projects.core.library import config as core_config from projects.core.library import env from projects.llm_d.orchestration import ci as llmd_ci -from projects.llm_d.orchestration import prepare_phase, runtime_config, test_phase +from projects.llm_d.orchestration import runtime_config, test_phase from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) +from projects.rhoai.library import deploy as rhoai_deploy PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" @@ -258,7 +259,7 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", "display_name": "Red Hat OpenShift AI", "publisher": "RHOAI Development Catalog", - "pull_secret": {"vault": {"name": "psap-rhoai-rc"}}, + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, }, "namespace": "redhat-ods-applications", "datasciencecluster_name": "default-dsc", @@ -270,31 +271,35 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy calls: list[object] = [] - monkeypatch.setattr(prepare_phase.runtime_config, "get_platform_config", lambda: platform) - monkeypatch.setattr(prepare_phase, "prepare_rhcl_operator", lambda: calls.append("rhcl")) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "prepare_rhcl_operator", lambda platform: calls.append("rhcl") + ) + monkeypatch.setattr( + rhoai_deploy, "prepare_rhoai_pull_secret", - lambda custom_catalog: calls.append(("pull", custom_catalog["image"])), + lambda custom_catalog: calls.append(("pull", custom_catalog.image)), ) - monkeypatch.setattr(prepare_phase, "apply_rhoai_icsp", lambda: calls.append("icsp")) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "deploy_rhoai_custom_catalog", - lambda rhoai: calls.append(("catalog", rhoai["custom_catalog"]["publisher"])), + lambda custom_catalog: calls.append(("catalog", custom_catalog.publisher)), ) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "ensure_operator_subscription", lambda operator_spec: calls.append(("sub", operator_spec["package"])), ) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "ensure_required_crds_before_dsc", - lambda: calls.append("crds"), + lambda rhoai: calls.append("crds"), ) - prepare_phase.prepare_rhoai_operator() + rhoai_deploy.prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=lambda: calls.append("icsp"), + ) assert calls == [ "rhcl", @@ -306,6 +311,28 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy ] +def test_custom_catalog_pull_secret_path_uses_explicit_vault_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, str] = {} + + def _fake_get_vault_content_path(vault_name: str, content_name: str): + captured["vault_name"] = vault_name + captured["content_name"] = content_name + return Path("/tmp/rhoai_rc.secret") + + monkeypatch.setattr(rhoai_deploy.vault, "get_vault_content_path", _fake_get_vault_content_path) + + path = rhoai_deploy.custom_catalog_pull_secret_path( + { + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + } + ) + + assert path == Path("/tmp/rhoai_rc.secret") + assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py new file mode 100644 index 000000000..220ec8854 --- /dev/null +++ b/projects/rhoai/library/deploy.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import base64 +import logging +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator +from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog +from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command +from projects.core.dsl.utils.k8s import oc, oc_get_json +from projects.core.library import vault + +logger = logging.getLogger(__name__) + +RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" +RHOAI_PULL_SECRET_NAME = "pull-secret" +RHOAI_REGISTRY = "quay.io/rhoai" + + +class _RhoaiCustomCatalogPullSecretVaultConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + content: str + + +class _RhoaiCustomCatalogPullSecretConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + vault: _RhoaiCustomCatalogPullSecretVaultConfig + + +class _RhoaiCustomCatalogPullSecretInput(BaseModel): + model_config = ConfigDict(extra="ignore") + + pull_secret: _RhoaiCustomCatalogPullSecretConfig + + +class RhoaiCustomCatalogConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + enabled: bool = False + name: str + namespace: str + image: str | None = None + display_name: str | None = None + publisher: str | None = None + pull_secret: _RhoaiCustomCatalogPullSecretConfig + + +class RhoaiOperatorConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + custom_catalog: RhoaiCustomCatalogConfig + namespace: str + datasciencecluster_name: str + components: list[str] = Field(default_factory=list) + required_crds_before_dsc: list[str] = Field(default_factory=list) + required_crds_after_dsc: list[str] = Field(default_factory=list) + + +def custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: + catalog = _RhoaiCustomCatalogPullSecretInput.model_validate(custom_catalog) + secret_path = vault.get_vault_content_path( + catalog.pull_secret.vault.name, + catalog.pull_secret.vault.content, + ) + if secret_path is None: + raise RuntimeError( + "RHOAI pull secret content " + f"'{catalog.pull_secret.vault.content}' was not found in vault " + f"'{catalog.pull_secret.vault.name}'" + ) + return secret_path + + +def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: + operators = platform["operators"] + if isinstance(operators, dict): + if package in operators: + return {"package": package, **operators[package]} + raise KeyError(f"Unknown operator package: {package}") + + for operator_spec in operators: + if operator_spec["package"] == package: + return operator_spec + raise KeyError(f"Unknown operator package: {package}") + + +def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: + return cluster_deploy_operator.run( + package_name=operator_spec["package"], + target_namespace=operator_spec["namespace"], + source_name=operator_spec["source"], + channel=operator_spec["channel"], + source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), + display_name=operator_spec.get("display_name", operator_spec["package"]), + artifact_dirname_suffix=f"_{operator_spec['package']}", + ) + + +def deploy_rhoai_custom_catalog(*, custom_catalog: RhoaiCustomCatalogConfig) -> int: + if not custom_catalog.enabled: + logger.info("RHOAI custom catalog disabled; using default catalog source") + return 0 + + if not custom_catalog.image: + raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") + + return deploy_custom_catalog.run( + catalog_source_name=custom_catalog.name, + catalog_namespace=custom_catalog.namespace, + catalog_image=custom_catalog.image, + display_name=custom_catalog.display_name or custom_catalog.name, + publisher=custom_catalog.publisher or "", + ) + + +def rhoai_operator_spec( + *, + custom_catalog: RhoaiCustomCatalogConfig, + operator_spec: dict[str, str], +) -> dict[str, str]: + if not custom_catalog.enabled: + return operator_spec + + updated_spec = dict(operator_spec) + updated_spec["source"] = custom_catalog.name + updated_spec["source_namespace"] = custom_catalog.namespace + return updated_spec + + +def prepare_rhcl_operator(platform: dict[str, Any]) -> None: + operator_spec = operator_spec_by_package(platform, "rhcl-operator") + ensure_operator_subscription(operator_spec) + + +def _decode_pull_secret(secret_data: dict[str, Any]) -> str: + encoded = secret_data.get("data", {}).get(".dockerconfigjson") + if not encoded: + raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") + + return base64.b64decode(encoded).decode("utf-8") + + +def wait_for_rhoai_pull_secret_ready( + *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 +) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + try: + secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded = _decode_pull_secret(secret) + if RHOAI_REGISTRY not in decoded: + raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + + mcp_status = oc( + "get", + "mcp", + "-o", + r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', + check=False, + log_stdout=False, + log_stderr=False, + ).stdout.strip() + if "False" in mcp_status: + raise RuntimeError("machine config pools are still updating") + + logger.info( + "RHOAI pull secret now contains %s and machine config pools are updated", + RHOAI_REGISTRY, + ) + return + except RuntimeError as exc: + last_error = str(exc) + logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) + time.sleep(poll_interval_seconds) + + raise RuntimeError( + f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + ) + + +def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: + pull_secret_path = vault.get_vault_content_path( + custom_catalog.pull_secret.vault.name, + custom_catalog.pull_secret.vault.content, + ) + if pull_secret_path is None: + raise RuntimeError( + "RHOAI pull secret content " + f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " + f"'{custom_catalog.pull_secret.vault.name}'" + ) + auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() + if not auth_basic: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + + if RHOAI_REGISTRY in decoded_secret: + logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + return + + logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: + temp_file.write(decoded_secret) + temp_path = Path(temp_file.name) + + try: + oc( + "registry", + "login", + f"--registry={RHOAI_REGISTRY}", + f"--auth-basic={auth_basic}", + "--to", + str(temp_path), + ) + oc( + "set", + "data", + f"secret/{RHOAI_PULL_SECRET_NAME}", + "-n", + RHOAI_PULL_SECRET_NAMESPACE, + f"--from-file=.dockerconfigjson={temp_path}", + ) + finally: + temp_path.unlink(missing_ok=True) + + wait_for_rhoai_pull_secret_ready() + + +def ensure_required_crds_before_dsc(rhoai: RhoaiOperatorConfig) -> None: + wait_for_crds_command.run( + crd_names=rhoai.required_crds_before_dsc, + display_name="RHOAI pre-DSC CRDs", + ) + + +def prepare_rhoai_operator( + *, + platform: dict[str, Any], + rhoai: dict[str, Any], + icsp_applier: Callable[[], None], +) -> None: + rhoai_config = RhoaiOperatorConfig.model_validate(rhoai) + + prepare_rhcl_operator(platform) + if rhoai_config.custom_catalog.enabled: + prepare_rhoai_pull_secret(rhoai_config.custom_catalog) + icsp_applier() + deploy_rhoai_custom_catalog(custom_catalog=rhoai_config.custom_catalog) + operator_spec = operator_spec_by_package(platform, "rhods-operator") + operator_spec = rhoai_operator_spec( + custom_catalog=rhoai_config.custom_catalog, + operator_spec=operator_spec, + ) + ensure_operator_subscription(operator_spec) + ensure_required_crds_before_dsc(rhoai_config) diff --git a/vaults/psap-rhoai-rc.yaml b/vaults/psap-rhoai-rc.yaml index f9bbc10c4..e35cd4560 100644 --- a/vaults/psap-rhoai-rc.yaml +++ b/vaults/psap-rhoai-rc.yaml @@ -6,3 +6,5 @@ content: rhoai_rc.secret: description: | Basic auth string used to extend the cluster pull secret for quay.io/rhoai + sample: | + rhoai+rhoai_external_readonly_bot:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From 87b0035a1cd3809ad1ec65e5f69d2736f9f5bf8f Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 9 Jul 2026 11:12:47 +0100 Subject: [PATCH 003/344] fix: MCP status check passing on empty output Signed-off-by: Alberto Perdomo --- projects/llm_d/tests/test_profiles.py | 31 +++++++++++++++++++++++++++ projects/rhoai/library/deploy.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 5b4148816..97a03cdfb 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -333,6 +333,37 @@ def _fake_get_vault_content_path(vault_name: str, content_name: str): assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} +def test_wait_for_rhoai_pull_secret_ready_treats_empty_mcp_status_as_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + mcp_outputs = iter(["", "True"]) + clock = {"value": 0.0} + + def _fake_monotonic() -> float: + value = clock["value"] + clock["value"] += 0.1 + return value + + def _fake_sleep(_seconds: float) -> None: + clock["value"] += 0.1 + + def _fake_oc_get_json(*args, **kwargs): + return {"data": {".dockerconfigjson": "unused"}} + + def _fake_oc(*args, **kwargs): + if args[:2] == ("get", "mcp"): + return type("Result", (), {"stdout": next(mcp_outputs), "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy, "_decode_pull_secret", lambda _secret: "quay.io/rhoai") + monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) + monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) + + rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 220ec8854..fa4c6203c 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -175,7 +175,7 @@ def wait_for_rhoai_pull_secret_ready( log_stdout=False, log_stderr=False, ).stdout.strip() - if "False" in mcp_status: + if not mcp_status or "False" in mcp_status: raise RuntimeError("machine config pools are still updating") logger.info( From c4854cfc2e3a5d8bd390d3145ec953c43f986907 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 9 Jul 2026 15:43:27 -0400 Subject: [PATCH 004/344] fix: update caliper postprocess config fields after PR 110 schema change PR 110 renamed caliper.postprocess.kpi fields: - kpi.generate -> kpi.artifacts_to_kpis - kpi.export -> removed (no longer needed) - analyze.baseline -> analyze.historical_kpis These three project configs were not updated, causing pydantic ValidationError (extra_forbidden) during postprocessing. Co-authored-by: Cursor --- projects/llm_d_legacy/testing/config.yaml | 6 ++---- projects/mcp_gateway/orchestration/config.yaml | 6 ++---- projects/rhaiis/orchestration/config.yaml | 4 +--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/projects/llm_d_legacy/testing/config.yaml b/projects/llm_d_legacy/testing/config.yaml index b16b0008f..dd4050ccf 100644 --- a/projects/llm_d_legacy/testing/config.yaml +++ b/projects/llm_d_legacy/testing/config.yaml @@ -364,13 +364,11 @@ caliper: kpi: enabled: false - generate: + artifacts_to_kpis: enabled: false output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 34f67a0a9..b73f7da9d 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -34,14 +34,12 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json export: diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 045227198..6c932e4bb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -31,11 +31,9 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false export: from: null From 18eab4e4e6e54dd4a974b6a0a4db3016af253423 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 9 Jul 2026 16:33:14 -0400 Subject: [PATCH 005/344] docs: document available pipelines and spec.env variables Add pipeline reference table and spec.env documentation (PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_SHA) to the gitops README. Co-authored-by: Cursor --- fournos/gitops/README.md | 52 +++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/fournos/gitops/README.md b/fournos/gitops/README.md index 5f1821b3e..4eae41404 100644 --- a/fournos/gitops/README.md +++ b/fournos/gitops/README.md @@ -37,12 +37,52 @@ gitops/ - **Build**: Shipwright build configuration for building forge image from source - **BuildRun**: Triggers the build process -### Workflows -- **Task**: `forge-step` - Reusable Tekton task for executing forge commands -- **Pipelines**: Multiple pipeline variants: - - `forge-full`: Complete pipeline with pre-cleanup, prepare, test, export-artifacts, and post-cleanup - - `pipeline-test-only`: Test execution only - - `pipeline-replot`: Replotting functionality +### Workflows +- **Task**: `forge-step` — reusable Tekton task for executing forge commands +- **Pipelines**: + +| Pipeline | Steps | Finally | Use case | +|---|---|---|---| +| `forge-full` | pre-cleanup → prepare → preflight → test | post-cleanup → export-artifacts | Full lifecycle with cleanup on both ends | +| `forge-prepare-test` | prepare → preflight → test | export-artifacts | Prepare + test without cleanup | +| `forge-test-only` | preflight → test | export-artifacts | Skip preparation, just run the test | +| `forge-prepare-only` | prepare → preflight | export-artifacts | Set up the cluster without benchmarks | +| `forge-replot` | replot | export-artifacts | Re-run visualization/postprocessing | + +Select a pipeline via `spec.pipeline` in the FournosJob CR (default: `fournos-full`). + +### FournosJob `spec.env` variables + +The `forge-step` task bootstrap script reads certain well-known variables +from `spec.env` and acts on them before launching the execution engine. + +#### Git checkout + +| Variable | Description | +|---|---| +| `PULL_PULL_SHA` | Checkout a specific git commit SHA. The task runs `git fetch origin ` and `git reset --hard FETCH_HEAD` before executing. | +| `PULL_NUMBER` | Checkout the HEAD of a GitHub PR (`refs/pull//head`). Also enables GitHub PR notifications — test results are posted as a comment on the PR. | +| `PULL_BASE_SHA` | Base branch commit SHA (informational context for the execution engine). | + +If neither `PULL_PULL_SHA` nor `PULL_NUMBER` is set, the execution engine +runs with the code baked into the container image. + +`PULL_NUMBER` is the most convenient option for testing a PR: it checks +out the latest code from that PR **and** posts results back to GitHub. + +#### Example + +```yaml +spec: + env: + PULL_NUMBER: "118" # checkout PR #118 HEAD + post results to PR +``` + +```yaml +spec: + env: + PULL_PULL_SHA: "abc123" # checkout exact commit, no PR notification +``` ## Environments From 2a9f27f0e93972e8ecca7ecf4745c76124a721e9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:17 +0200 Subject: [PATCH 006/344] [caliper] s3: remove the import/export prefix config field --- .../caliper/cli/orchestration_entrypoints.py | 8 +++---- projects/caliper/cli/s3_export.py | 21 ++++--------------- .../orchestration/postprocess_config.py | 8 ------- .../caliper/orchestration/step_logging.py | 13 +++++++++++- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 41203348e..5e8fba7d4 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -75,12 +75,10 @@ def s3_import_entrypoint( s3_parent_config = postprocess_config.s3 s3_config = postprocess_config.s3.import_ - # Build import prefix using instance + s3.prefix + directory + # Build import prefix using instance + directory import_prefix_parts = [] if s3_parent_config.instance: import_prefix_parts.append(s3_parent_config.instance) - if s3_parent_config.prefix: - import_prefix_parts.append(s3_parent_config.prefix.rstrip("/")) if s3_parent_config.directory: import_prefix_parts.append(s3_parent_config.directory) @@ -89,7 +87,8 @@ def s3_import_entrypoint( log_s3_import_command( bucket=s3_parent_config.bucket, - prefix=import_prefix, + instance=s3_parent_config.instance, + directory=s3_parent_config.directory, output_dir=import_dir, include_kpis_json=s3_config.include_kpis_json, include_kpis_csv=s3_config.include_kpis_csv, @@ -139,7 +138,6 @@ def s3_export_entrypoint( ai_data_dir=ai_data_dir, analysis_file=analysis_file, bucket=s3_parent_config.bucket, - prefix=s3_parent_config.prefix, instance=s3_parent_config.instance, directory=s3_parent_config.directory, upload_id=s3_config.upload_id, diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index c60bd9d36..299305f17 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -224,7 +224,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path | None = None, analysis_file: Path | None = None, bucket: str, - prefix: str = "", instance: str | None = None, directory: str | None = None, upload_id: str | None = None, @@ -240,7 +239,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path to AI data directory to upload analysis_file: Path to analysis file to upload bucket: S3 bucket name - prefix: S3 object prefix/path instance: Instance identifier for S3 organization directory: Directory identifier for S3 organization upload_id: Custom upload identifier (uses timestamp if not provided) @@ -288,12 +286,10 @@ def run_s3_export_with_explicit_paths( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ s3_path_components = [] if instance: s3_path_components.append(instance) - if prefix: - s3_path_components.append(prefix.rstrip("/")) if directory: s3_path_components.append(directory) s3_path_components.append(upload_id) @@ -302,9 +298,7 @@ def run_s3_export_with_explicit_paths( logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - logger.info( - f"Path structure: {instance}/{prefix.rstrip('/') if prefix else 'no-prefix'}/{directory}/{upload_id}/" - ) + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") @@ -601,12 +595,10 @@ def run_s3_export( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ s3_path_components = [] if instance: s3_path_components.append(instance) - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix: - s3_path_components.append(s3_parent_config.prefix.rstrip("/")) if directory: s3_path_components.append(directory) s3_path_components.append(upload_id) @@ -615,12 +607,7 @@ def run_s3_export( logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - prefix_component = ( - s3_parent_config.prefix.rstrip("/") - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix - else "no-prefix" - ) - logger.info(f"Path structure: {instance}/{prefix_component}/{directory}/{upload_id}/") + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index 4d1ca6815..b30ccb222 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -167,10 +167,6 @@ class CaliperOrchestrationS3ExportSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False - prefix: str | None = Field( - default=None, - description="S3 key prefix/folder path (optional).", - ) upload_id: str | None = Field( default=None, description="Custom upload identifier; uses timestamp (YY-MM-DD_HHMMSS) if null.", @@ -225,10 +221,6 @@ class CaliperOrchestrationS3Section(BaseModel): default=None, description="Directory identifier for S3 organization (required when import or export is enabled).", ) - prefix: str | None = Field( - default=None, - description="S3 prefix for organizing uploads and imports.", - ) vault: CaliperOrchestrationS3VaultSection = Field( default_factory=CaliperOrchestrationS3VaultSection, description="Vault configuration for AWS credentials.", diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 3aeae431b..474ed72c4 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -390,7 +390,8 @@ def log_ai_data_command( def log_s3_import_command( bucket: str, - prefix: str, + instance: str | None, + directory: str | None, output_dir: Path, include_kpis_json: bool = False, include_kpis_csv: bool = False, @@ -398,6 +399,14 @@ def log_s3_import_command( max_downloads: int = 50, ) -> None: """Log the CLI command to reproduce the S3 import step.""" + # Build prefix from instance and directory + prefix_parts = [] + if instance: + prefix_parts.append(instance) + if directory: + prefix_parts.append(directory) + prefix = "/".join(prefix_parts) if prefix_parts else "" + command_parts = [ f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' ] @@ -416,6 +425,8 @@ def log_s3_import_command( step_args = { "bucket": bucket, + "instance": instance, + "directory": directory, "prefix": prefix, "output_dir": str(output_dir), "include_kpis_json": include_kpis_json, From d3031b414a0c758a525e034aadc73380c88e0e1b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:30 +0200 Subject: [PATCH 007/344] [llm_d] orchestration: config: remove the s3 config field --- projects/llm_d/orchestration/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 56265eb55..36f12e78f 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -78,7 +78,6 @@ caliper: vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials - prefix: llm_d/ import: enabled: true output_dir: historical_data From f8f29963adb93b8595f577420d143b1f20a4e492 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:48 +0200 Subject: [PATCH 008/344] [skeleton] orchestration: config: remove the prefix config field --- projects/skeleton/orchestration/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 4c15f6591..91f2a4761 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -86,7 +86,6 @@ caliper: bucket: psap-dashboard-data instance: smoke directory: skeleton-dashboard - prefix: skeleton/ vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials From beb882acb9c9b34253f55dc14548d17ba76a7ed4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:57:01 +0200 Subject: [PATCH 009/344] .github: workflows: update workflows to actions/checkout@v5 --- .github/steps/prepare_minimal_ubuntu_image/action.yml | 1 + .github/workflows/check_consistency.yml | 3 ++- .github/workflows/check_python_syntax.yml | 3 ++- .github/workflows/image-push.yaml | 4 +++- .github/workflows/ruff.yaml | 4 +++- .github/workflows/test_toolbox_dsl.yml | 4 +++- 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/steps/prepare_minimal_ubuntu_image/action.yml b/.github/steps/prepare_minimal_ubuntu_image/action.yml index ebbedd53c..4570b25fc 100644 --- a/.github/steps/prepare_minimal_ubuntu_image/action.yml +++ b/.github/steps/prepare_minimal_ubuntu_image/action.yml @@ -1,4 +1,5 @@ name: "Prepare FORGE minimal ubuntu image" +description: "Install minimal Python dependencies required for FORGE repository operations" runs: using: "composite" # <-- this is the important part diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index 04556928e..0fd25b2a5 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -18,9 +18,10 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/steps/prepare_minimal_ubuntu_image diff --git a/.github/workflows/check_python_syntax.yml b/.github/workflows/check_python_syntax.yml index d13fd5ca4..41e9f3c40 100644 --- a/.github/workflows/check_python_syntax.yml +++ b/.github/workflows/check_python_syntax.yml @@ -10,9 +10,10 @@ jobs: max-parallel: 4 steps: - name: Use checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Check that all the files can be compiled run: | diff --git a/.github/workflows/image-push.yaml b/.github/workflows/image-push.yaml index 4cd5141f2..d4a880269 100644 --- a/.github/workflows/image-push.yaml +++ b/.github/workflows/image-push.yaml @@ -10,7 +10,9 @@ jobs: image-push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - name: Remove GitHub credentials from git config run: | diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index 3f4200572..6fea10352 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -16,7 +16,9 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/test_toolbox_dsl.yml b/.github/workflows/test_toolbox_dsl.yml index 669fdafc6..66ac3c418 100644 --- a/.github/workflows/test_toolbox_dsl.yml +++ b/.github/workflows/test_toolbox_dsl.yml @@ -20,7 +20,9 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: From d21146f2789350e2e557b1a44b48cd65d4df8241 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:57:37 +0200 Subject: [PATCH 010/344] .github: workflows: check_consistency: enforce EOL at EOF --- .github/workflows/check_consistency.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index 0fd25b2a5..e447d86e9 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -27,3 +27,22 @@ jobs: - name: Check that no symlink is broken run: bin/run_ansible_toolbox.py repo validate_no_broken_link + + - name: Check that all source files end with EOL + run: | + # Find all source files and check they end with newline + files_without_eol=() + + while IFS= read -r -d '' file; do + if [[ -s "$file" ]] && [[ -n "$(tail -c1 "$file")" ]]; then + files_without_eol+=("$file") + fi + done < <(find . -path './.git' -prune -o \( -name "*.py" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" -o -name "*.md" \) -type f -print0) + + if [[ ${#files_without_eol[@]} -gt 0 ]]; then + echo "❌ Files missing final newline:" + printf '%s\n' "${files_without_eol[@]}" + echo "Fix with: echo >> filename" + exit 1 + fi + echo "✅ All source files end with newline" From 312652972e12ec2272cdab0713759dcad3c39ecd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 14:02:01 +0200 Subject: [PATCH 011/344] [projects] Add EOL at EOF --- projects/cluster/toolbox/rebuild_image/README.md | 2 +- projects/core/tests/README.md | 2 +- .../llm_d/orchestration/manifests/leaderworkersetoperator.yaml | 2 +- projects/matrix_benchmarking/subproject/PLUGINS.md | 2 +- projects/mcp_gateway/orchestration/presets.d/presets.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/cluster/toolbox/rebuild_image/README.md b/projects/cluster/toolbox/rebuild_image/README.md index b418ba72a..324f2ea26 100644 --- a/projects/cluster/toolbox/rebuild_image/README.md +++ b/projects/cluster/toolbox/rebuild_image/README.md @@ -55,4 +55,4 @@ result = rebuild_image( - **FORGE image rebuilds**: Rebuild FORGE container after code changes - **Dependency updates**: Trigger rebuilds when base images change - **Testing**: Validate Build configurations work correctly -- **CI/CD integration**: Automated rebuilds in pipelines \ No newline at end of file +- **CI/CD integration**: Automated rebuilds in pipelines diff --git a/projects/core/tests/README.md b/projects/core/tests/README.md index 44bbe0541..3f7edcc46 100644 --- a/projects/core/tests/README.md +++ b/projects/core/tests/README.md @@ -62,4 +62,4 @@ If tests fail, check: 1. That FORGE_HOME is properly set or discoverable 2. That expected projects exist with toolbox directories 3. That toolbox commands have `main.py` files with `run` functions -4. That project `__init__.py` files have docstrings \ No newline at end of file +4. That project `__init__.py` files have docstrings diff --git a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml index 788170f0d..ba27d5d27 100644 --- a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml +++ b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml @@ -5,4 +5,4 @@ metadata: spec: logLevel: Normal managementState: Managed - operatorLogLevel: Normal \ No newline at end of file + operatorLogLevel: Normal diff --git a/projects/matrix_benchmarking/subproject/PLUGINS.md b/projects/matrix_benchmarking/subproject/PLUGINS.md index 5499f7985..e98a1dc68 100644 --- a/projects/matrix_benchmarking/subproject/PLUGINS.md +++ b/projects/matrix_benchmarking/subproject/PLUGINS.md @@ -155,4 +155,4 @@ When creating a new visualization plugin: 5. Create plotting and analysis components 6. Test with sample data -For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. \ No newline at end of file +For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. diff --git a/projects/mcp_gateway/orchestration/presets.d/presets.yaml b/projects/mcp_gateway/orchestration/presets.d/presets.yaml index 55fe683ba..19c1267c3 100644 --- a/projects/mcp_gateway/orchestration/presets.d/presets.yaml +++ b/projects/mcp_gateway/orchestration/presets.d/presets.yaml @@ -61,4 +61,4 @@ large-scale: experiment.target: [gateway] experiment.duration_seconds: 240 experiment.warmup_seconds: 60 - experiment.tools_per_server: 10 \ No newline at end of file + experiment.tools_per_server: 10 From 2de1912da6fce645a8ec2b7b37666d1b48903009 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 14:26:16 +0200 Subject: [PATCH 012/344] [caliper] Factorize the S3 prefix code generation --- .../caliper/cli/orchestration_entrypoints.py | 12 ++--- projects/caliper/cli/s3_export.py | 54 +++++++++++++------ projects/caliper/cli/s3_import.py | 2 +- projects/caliper/orchestration/postprocess.py | 3 -- .../caliper/orchestration/step_logging.py | 9 ++-- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 5e8fba7d4..4e273e4b4 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -76,13 +76,13 @@ def s3_import_entrypoint( s3_config = postprocess_config.s3.import_ # Build import prefix using instance + directory - import_prefix_parts = [] - if s3_parent_config.instance: - import_prefix_parts.append(s3_parent_config.instance) - if s3_parent_config.directory: - import_prefix_parts.append(s3_parent_config.directory) + from projects.caliper.cli.s3_export import build_s3_prefix - import_prefix = "/".join(import_prefix_parts) if import_prefix_parts else "" + import_prefix = build_s3_prefix( + instance=s3_parent_config.instance, + directory=s3_parent_config.directory, + trailing_slash=False, + ) import_dir = output_dir / s3_config.output_dir log_s3_import_command( diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index 299305f17..5c500ea55 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -28,6 +28,38 @@ logger = logging.getLogger(__name__) +def build_s3_prefix( + instance: str | None = None, + directory: str | None = None, + upload_id: str | None = None, + trailing_slash: bool = True, +) -> str: + """Build S3 prefix from components, skipping empty parts. + + Args: + instance: Instance identifier (optional) + directory: Directory identifier (optional) + upload_id: Upload identifier (optional) + trailing_slash: Whether to add trailing slash (default: True) + + Returns: + Formatted S3 prefix path + """ + components = [] + if instance: + components.append(instance) + if directory: + components.append(directory) + if upload_id: + components.append(upload_id) + + if not components: + return "" + + prefix = "/".join(components) + return prefix + "/" if trailing_slash else prefix + + def list_ai_data_files(ai_data_dir: Path) -> list[Path]: """List all files in the AI evaluation export directory. @@ -287,14 +319,9 @@ def run_s3_export_with_explicit_paths( logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") # Construct the full S3 path: {instance}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") @@ -596,14 +623,9 @@ def run_s3_export( logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") # Construct the full S3 path: {instance}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index 5f2d8080a..e591ec4d6 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -221,7 +221,7 @@ def run_s3_import_with_explicit_params( Args: bucket: S3 bucket name - prefix: S3 prefix path (full path including s3.prefix/instance/directory) + prefix: S3 prefix path (full path including instance/directory) output_dir: Path to output directory for downloaded files vault: Vault name containing AWS credentials aws_credentials_file: AWS credentials file within vault diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ee285878e..e8dbe4b3f 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1234,10 +1234,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: # Log the CLI command to reproduce this step s3_parent_config = self.config.s3 - export_config = s3_parent_config.export export_path = f"{s3_parent_config.instance}/{s3_parent_config.directory}" - if export_config.prefix: - export_path += f"/{export_config.prefix}" log_s3_export_command( bucket=s3_parent_config.bucket, diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 474ed72c4..0ef4dc1ba 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -400,12 +400,9 @@ def log_s3_import_command( ) -> None: """Log the CLI command to reproduce the S3 import step.""" # Build prefix from instance and directory - prefix_parts = [] - if instance: - prefix_parts.append(instance) - if directory: - prefix_parts.append(directory) - prefix = "/".join(prefix_parts) if prefix_parts else "" + from projects.caliper.cli.s3_export import build_s3_prefix + + prefix = build_s3_prefix(instance=instance, directory=directory, trailing_slash=False) command_parts = [ f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' From 0fb91e5dc89fa5b31d009b8be173a60d70771190 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:02:11 +0200 Subject: [PATCH 013/344] [caliper] orchestration: postprocess: include the log file in the step status --- projects/caliper/orchestration/postprocess.py | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e8dbe4b3f..d840419a7 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -677,8 +677,20 @@ def _setup_paths(self) -> None: self.step_logs_dir = Path(env.ARTIFACT_DIR) self.step_logs_dir.mkdir(parents=True, exist_ok=True) - def _add_step(self, step_name: str, step_data: dict[str, Any]) -> None: + def _add_step( + self, step_name: str, step_data: dict[str, Any], log_file: Path | None = None + ) -> None: """Add a step result to the steps list.""" + if log_file: + # Make log file path relative to artifact root where logs are actually stored + try: + artifact_root = Path(env.ARTIFACT_DIR) + relative_log_path = log_file.relative_to(artifact_root) + step_data["log_file"] = str(relative_log_path) + except ValueError: + # If log file is not under artifact root, use absolute path + step_data["log_file"] = str(log_file) + self.steps.append({step_name: step_data}) def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, Any]) -> bool: @@ -760,7 +772,7 @@ def _build_result(self, final_status: str, test_block: dict[str, Any]) -> dict[s return { "final_status": final_status, "success": success_value, - "base_directory": str(self.visualize_output_dir) if self.visualize_output_dir else None, + "base_directory": str(Path(env.ARTIFACT_DIR)), "test_phase": test_block, "steps": self.steps, } @@ -770,7 +782,7 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - with step_logging("caliper_parse", self.step_logs_dir): + with step_logging("caliper_parse", self.step_logs_dir) as log_file: try: model, mod_str = parse_entrypoint( self.config, @@ -796,6 +808,7 @@ def _run_parse_step(self) -> None: "parse_cache_ref": model.parse_cache_ref, "completed_at": time.time(), }, + log_file, ) except Exception as e: # noqa: BLE001 self.parse_failed = True @@ -808,6 +821,7 @@ def _run_parse_step(self) -> None: "traceback": traceback.format_exc(), "completed_at": time.time(), }, + log_file, ) def _run_visualize_step(self) -> None: @@ -815,7 +829,7 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - with step_logging("caliper_visualize", self.step_logs_dir): + with step_logging("caliper_visualize", self.step_logs_dir) as log_file: try: mod_str, plugin = load_plugin_entrypoint( self.config, tree_root=self.tree_root, manifest_path=self.manifest_path @@ -879,6 +893,7 @@ def _run_visualize_step(self) -> None: "output_files": relative_paths, "completed_at": time.time(), }, + log_file, ) except Exception as e: # noqa: BLE001 @@ -892,6 +907,7 @@ def _run_visualize_step(self) -> None: "traceback": traceback.format_exc(), "completed_at": time.time(), }, + log_file, ) def _run_kpi_and_ai_data_steps(self) -> None: @@ -998,11 +1014,11 @@ def _run_artifacts_to_kpis_step( ) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: - with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir): + with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: result = _run_artifacts_to_kpis( self.config, plugin, model, output_dir, mod_str, self.tree_root ) - self._add_step("artifacts_to_kpis", result) + self._add_step("artifacts_to_kpis", result, log_file) self._check_step_result_and_set_failure("artifacts_to_kpis", result) else: self._add_step( @@ -1027,11 +1043,11 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - with step_logging("caliper_kpis_to_csv", self.step_logs_dir): + with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: # Path to the JSON file for reference in command logging kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv(self.config, plugin, model, output_dir, kpi_json_path) - self._add_step("kpis_to_csv", result) + self._add_step("kpis_to_csv", result, log_file) if result.get("status") == "failed": # CSV export failure doesn't affect overall status - it's supplementary logger.warning("KPI CSV export failed but continuing execution") @@ -1051,12 +1067,12 @@ def _run_artifacts_to_ai_data_step( ) return - with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir): + with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: try: result = _run_artifacts_to_ai_data( self.config, plugin, model, output_dir, mod_str, self.tree_root ) - self._add_step("artifacts_to_ai_data", result) + self._add_step("artifacts_to_ai_data", result, log_file) logger.info(f"AI eval export result: {result}") # Check if the result indicates failure or warning @@ -1065,7 +1081,7 @@ def _run_artifacts_to_ai_data_step( except Exception as e: logger.exception("AI eval export failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("artifacts_to_ai_data", step_result) + self._add_step("artifacts_to_ai_data", step_result, log_file) self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) def _run_s3_import_step(self, output_dir: Path) -> None: @@ -1081,7 +1097,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_import", self.step_logs_dir): + with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: try: result = s3_import_entrypoint(self.config, output_dir) @@ -1092,7 +1108,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: result.pop("import_dir"), output_dir ) - self._add_step("s3_import", result) + self._add_step("s3_import", result, log_file) logger.info(f"S3 import result: {result}") self._check_step_result_and_set_failure("s3_import", result) @@ -1100,7 +1116,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: except Exception as e: logger.exception("S3 import failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("s3_import", step_result) + self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: @@ -1143,7 +1159,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - with step_logging("caliper_analyse_kpis", self.step_logs_dir): + with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: try: # Convert relative path to absolute for analyze function current_kpis_path = output_dir / current_kpis_file @@ -1158,7 +1174,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: result["output_file"], output_dir ) - self._add_step("analyse_kpis", result) + self._add_step("analyse_kpis", result, log_file) logger.info(f"KPI analysis result: {result}") self._check_step_result_and_set_failure("analyse_kpis", result) @@ -1166,7 +1182,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: except Exception as e: logger.exception("KPI analysis failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("analyse_kpis", step_result) + self._add_step("analyse_kpis", step_result, log_file) self._check_step_result_and_set_failure("analyse_kpis", step_result) def _run_s3_export_step(self, output_dir: Path) -> None: @@ -1182,7 +1198,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_export", self.step_logs_dir): + with step_logging("caliper_s3_export", self.step_logs_dir) as log_file: try: # Collect file paths from previous steps kpis_file = None @@ -1253,7 +1269,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ai_data_dir=ai_data_dir, analysis_file=analysis_file, ) - self._add_step("s3_export", result) + self._add_step("s3_export", result, log_file) if result.get("status") == "success": # Format status for better readability @@ -1276,7 +1292,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" # Log programming errors more prominently - if isinstance(e, (AttributeError, NameError, TypeError)): + if isinstance(e, AttributeError | NameError | TypeError): logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") logger.exception("Full traceback for programming error:") else: @@ -1289,7 +1305,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: "exception_type": type(e).__name__, "completed_at": time.time(), } - self._add_step("s3_export", step_result) + self._add_step("s3_export", step_result, log_file) self._check_step_result_and_set_failure("s3_export", step_result) def _compute_final_status(self) -> str: From bd91cac75b61cd8bd1000b7e53255c689d641334 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:02:33 +0200 Subject: [PATCH 014/344] [caliper] orchestration: notification: add a link to the step log file --- .../caliper/orchestration/notification.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index c1e8fe46f..8901afdf6 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -27,6 +27,7 @@ class PostprocessStepResult: uploaded_files: int | None = None downloaded_files: int | None = None failed_files: int | None = None + log_file: str | None = None # relative path to step log file @dataclass @@ -76,7 +77,19 @@ def format_postprocess_status_notification( for step_name, step_result in sorted_steps: step_emoji = _get_step_emoji(step_result.status) - lines.append(f"- {step_emoji} **{step_name}**: `{step_result.status}`") + + # Create step name as link to log file if available + if step_result.log_file and get_file_link: + try: + log_url = get_file_link(step_result.log_file) + step_name_display = f"[**{step_name}**]({log_url})" + except Exception: + # Fallback to plain text if link generation fails + step_name_display = f"**{step_name}**" + else: + step_name_display = f"**{step_name}**" + + lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") # Add step message if available (but not for warning/failed steps to avoid duplication) if step_result.message and step_result.status not in ("warning", "failed", "failure"): @@ -208,7 +221,9 @@ def format_postprocess_status_notification( step_result, "paths", None ) if file_paths and get_file_link: - lines.extend(_format_step_file_links(step_name, file_paths, get_file_link)) + lines.extend( + _format_step_file_links(step_name, file_paths, get_file_link, step_result) + ) return "\n".join(lines) if lines else "" @@ -253,7 +268,7 @@ def _get_step_emoji(status: str) -> str: def _format_step_file_links( - step_name: str, file_paths: list[str], get_file_link: callable + step_name: str, file_paths: list[str], get_file_link: callable, step_result=None ) -> list[str]: """Format file paths as clickable links using the provided callback. @@ -261,6 +276,7 @@ def _format_step_file_links( step_name: Name of the step file_paths: List of relative file paths get_file_link: Callback function to generate URLs from file paths + step_result: Step result object containing additional metadata (optional) Returns: List of formatted link strings @@ -270,6 +286,9 @@ def _format_step_file_links( lines = [] + # Check if we need to combine output_dir with file paths (e.g., for visualize step) + output_dir = getattr(step_result, "output_dir", None) if step_result else None + # Group files by type for better organization file_groups = _group_files_by_type(file_paths) @@ -277,7 +296,13 @@ def _format_step_file_links( for file_type, files in file_groups.items(): for file_path in files: try: - file_url = get_file_link(file_path) + # Combine output_dir with file_path if available + if output_dir: + full_path = f"{output_dir}/{file_path}" + else: + full_path = file_path + + file_url = get_file_link(full_path) file_name = _get_display_name(file_path) emoji = "📊" if file_type == "visualization" else "📄" lines.append(f" - {emoji} [{file_name}]({file_url})") @@ -377,6 +402,7 @@ def parse_postprocess_result(status_data: dict) -> PostprocessResult | None: uploaded_files=step_data.get("uploaded_files"), downloaded_files=step_data.get("downloaded_files"), failed_files=step_data.get("failed_files"), + log_file=step_data.get("log_file"), ) return PostprocessResult( From f1ecbed3bde5d16e97cc79cd0159e5e9d8ac508f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:05:16 +0200 Subject: [PATCH 015/344] [core] ci_entrypoint: fournos_resolve: s/print/logger.info/ --- projects/core/ci_entrypoint/fournos_resolve.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/projects/core/ci_entrypoint/fournos_resolve.py b/projects/core/ci_entrypoint/fournos_resolve.py index 0d5a39fa1..f864243d8 100644 --- a/projects/core/ci_entrypoint/fournos_resolve.py +++ b/projects/core/ci_entrypoint/fournos_resolve.py @@ -183,11 +183,9 @@ def resolve_fournos_config( # Show the updated spec logger.info("Updated FournosJob spec:") - print("=" * 60) - print("Updated FournosJob spec:") - print("=" * 60) - print(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) - print("=" * 60) + logger.info("=" * 60) + logger.info(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) + logger.info("=" * 60) if dry_run: logger.info("DRY RUN: Not applying changes to cluster") From feea37de957f48501f7cb6ffa0b884de191e2cbc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:31:55 +0200 Subject: [PATCH 016/344] [skeleton] orchestration: disable the agent by default --- projects/skeleton/orchestration/config.yaml | 2 +- projects/skeleton/orchestration/presets.d/presets.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 91f2a4761..eb9bf99b2 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -105,7 +105,7 @@ caliper: include_ai_data: true agentic: - enabled: true + enabled: false model_key: qwen-3-6-35b on_failure: enabled: true diff --git a/projects/skeleton/orchestration/presets.d/presets.yaml b/projects/skeleton/orchestration/presets.d/presets.yaml index e49eca21f..211f236a8 100644 --- a/projects/skeleton/orchestration/presets.d/presets.yaml +++ b/projects/skeleton/orchestration/presets.d/presets.yaml @@ -16,3 +16,6 @@ medium_test: long_test: skeleton.test.duration_seconds: 300 # Long 5 minute test + +agent: + agentic.enabled: true From 4b2e3633100c037adf9a89bee42fd288b08be1ba Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:10 +0200 Subject: [PATCH 017/344] [caliper] cli: s3_import: don't show sample object keys --- projects/caliper/cli/s3_import.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index e591ec4d6..db6349fd8 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -276,12 +276,6 @@ def run_s3_import_with_explicit_params( objects = list_s3_objects(s3_client, bucket, import_s3_prefix, max_downloads) logger.info(f"Found {len(objects)} total objects in s3://{bucket}/{import_s3_prefix}") - if objects: - logger.info("Sample object keys:") - for i, obj in enumerate(objects[:5]): # Show first 5 objects - logger.info(f" {i + 1}: {obj['key']}") - if len(objects) > 5: - logger.info(f" ... and {len(objects) - 5} more objects") # Filter objects based on parameters logger.info( From e0f5b02b4f7f33bb404810ea7e6cb97e54817887 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:40 +0200 Subject: [PATCH 018/344] [caliper] orchestration: postprocess: pretty print the results --- projects/caliper/orchestration/postprocess.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index d840419a7..2bfd336b4 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import logging import time import traceback @@ -291,8 +292,6 @@ def _run_artifacts_to_kpis( kpis = plugin.compute_kpis(model) output_file.parent.mkdir(parents=True, exist_ok=True) - import json - # Transform flat KPI list into hierarchical structure hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) @@ -357,8 +356,6 @@ def _run_artifacts_to_ai_data( # Write main AI eval payload output_file.parent.mkdir(parents=True, exist_ok=True) - import json - with open(output_file, "w") as f: json.dump(payload, f, indent=2) @@ -478,7 +475,6 @@ def _export_test_entries_with_artifacts( # Write entry metadata entry_metadata_file = test_entry_dir / "entry_metadata.json" - import json with open(entry_metadata_file, "w") as f: json.dump(entry_info, f, indent=2) @@ -1073,7 +1069,9 @@ def _run_artifacts_to_ai_data_step( self.config, plugin, model, output_dir, mod_str, self.tree_root ) self._add_step("artifacts_to_ai_data", result, log_file) - logger.info(f"AI eval export result: {result}") + + logger.info("AI eval export result:") + logger.info(json.dumps(result, indent=2, default=str)) # Check if the result indicates failure or warning self._check_step_result_and_set_failure("artifacts_to_ai_data", result) @@ -1109,7 +1107,9 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) self._add_step("s3_import", result, log_file) - logger.info(f"S3 import result: {result}") + + logger.info("S3 import result:") + logger.info(json.dumps(result, indent=2, default=str)) self._check_step_result_and_set_failure("s3_import", result) @@ -1175,7 +1175,9 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: ) self._add_step("analyse_kpis", result, log_file) - logger.info(f"KPI analysis result: {result}") + + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) self._check_step_result_and_set_failure("analyse_kpis", result) From 823c81174cb50bb4aa7cbe3318326e3d9541e1c9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:53 +0200 Subject: [PATCH 019/344] [caliper] orchestration: step_logging: remove the log duplication --- projects/caliper/orchestration/step_logging.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 0ef4dc1ba..85cbb8f36 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -513,21 +513,6 @@ def log_analyze_command( ) -> None: """Log the CLI command to reproduce the analyze step.""" - command = f'caliper kpi analyze --current "{current_kpis_path}" --baseline-dir "{historical_kpis_dir}" --output "{output_path}" --plugin "{plugin_module}"' - - step_args = { - "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, - "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, - "output_path": str(output_path) if output_path else None, - "plugin_module": plugin_module, - } - - _log_command_banner( - "caliper_analyze_kpis", - command, - "Analyze current KPIs against historical data for regression detection", - step_args, - ) # Use the correct command structure: caliper kpi analyze command = "caliper kpi analyze" From 157c76d1a6a7c688a9069577e8080bef309d91ee Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:17:03 +0200 Subject: [PATCH 020/344] [llm_d] orchestration: config: save the reports in a dedicated directory --- projects/llm_d/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 36f12e78f..de1e08125 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -48,7 +48,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [] report_group: all visualize_config: projects/guidellm/postprocess/guidellm/visualize-groups.yaml From 1a2330f82bc21b9afe734a05209977cfd77a3abc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:17:16 +0200 Subject: [PATCH 021/344] [skeleton] orchestration: config: save the reports in a dedicated directory --- projects/skeleton/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index eb9bf99b2..9ee43c738 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -60,7 +60,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [summary_table, throughput_chart] report_group: null From eccf8448cdc6f909fda8f72fdcaadc6b51e6958d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:18:13 +0200 Subject: [PATCH 022/344] [caliper] orchestration: store all the step logs in the log file --- .../caliper/orchestration/step_logging.py | 62 +++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 85cbb8f36..7ef0f6d17 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -105,9 +105,12 @@ def step_logging_indexed( """ Context manager for step-specific logging with explicit step index. - Captures all logging output during the context and writes it to a dedicated + Captures ALL logging output during the context and writes it to a dedicated file named with the pattern: {step_index:03d}__{step_name}.log + This context manager temporarily configures the entire logging system to + ensure that ALL log messages from any logger get captured to the step log file. + Args: step_name: Name of the step (e.g., "caliper_parse", "caliper_visualize") step_index: Zero-based index of the step for ordering (e.g., 0, 1, 2...) @@ -121,9 +124,6 @@ def step_logging_indexed( logger.info("This will go to 000__caliper_parse.log") run_parse_operations() """ - # Ensure the global step handler is attached - _ensure_step_handler() - # Create the log file path log_filename = f"{step_index:03d}__{step_name}.log" log_file = output_dir / log_filename @@ -138,16 +138,55 @@ def step_logging_indexed( # Use same format as console output (no timestamp prefix, just the message) file_handler.setFormatter(logging.Formatter("%(message)s")) - # Store the file handler in thread-local storage - _step_local_handlers.file_handler = file_handler + # Get the root logger to capture everything + root_logger = logging.getLogger() + + # Save original state to restore later + original_handlers = root_logger.handlers.copy() + + # Note: We now use direct root logger capture instead of the StepLocalHandler mechanism + # This provides more reliable and comprehensive log capture try: + # Add our file handler directly to the root logger + root_logger.addHandler(file_handler) + + # Save propagation states and ensure loggers propagate to root + saved_logger_states = {} + + # Get all existing loggers from the logger registry + for logger_name in logging.getLogger().manager.loggerDict: + existing_logger = logging.getLogger(logger_name) + # Save original state (only propagate, don't touch levels) + saved_logger_states[logger_name] = { + "propagate": existing_logger.propagate, + } + # Only ensure propagation, don't change levels + existing_logger.propagate = True + yield log_file + finally: - # Clean up: close and remove the thread-local handler - if hasattr(_step_local_handlers, "file_handler"): - _step_local_handlers.file_handler.close() - del _step_local_handlers.file_handler + # Remove our file handler from root logger + if file_handler in root_logger.handlers: + root_logger.removeHandler(file_handler) + + # Restore original handlers completely + # (Remove any handlers that were added, restore original list) + root_logger.handlers.clear() + root_logger.handlers.extend(original_handlers) + + # Restore original logger propagation states + for logger_name, state in saved_logger_states.items(): + try: + existing_logger = logging.getLogger(logger_name) + existing_logger.propagate = state["propagate"] + except Exception: + # Ignore errors when restoring logger states + pass + + # Close the file handler + file_handler.close() @contextmanager @@ -213,7 +252,8 @@ def _log_command_banner( description: Brief description of what this step does step_args: Dictionary of step arguments for reference """ - logger = logging.getLogger(__name__) + # Use root logger to ensure step handler captures the banner + logger = logging.getLogger() LINE_WIDTH = 80 From c7d8c206923e49638f75961e713c7e75c4ff175b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:43:05 +0200 Subject: [PATCH 023/344] [caliper] orchestration: postprocess: visualize: show the output_dir --- projects/caliper/orchestration/postprocess.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2bfd336b4..6a7d5a8f5 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -92,6 +92,7 @@ def _resolve_paths( def _resolve_visualize_output_dir( raw: str | None, + base_dir: Path | None = None, ) -> Path: if raw is None or not str(raw).strip(): raise ValueError( @@ -100,7 +101,18 @@ def _resolve_visualize_output_dir( p = Path(raw).expanduser() if p.is_absolute(): return p.resolve() - raise ValueError("caliper.postprocess.visualize.output_dir must be an absolute path") + + # Handle relative paths by making them relative to base_dir (defaults to env.ARTIFACT_DIR) + if base_dir is None: + base_dir = env.ARTIFACT_DIR + + if base_dir is None: + raise ValueError( + "caliper.postprocess.visualize.output_dir must be an absolute path when no base directory is available" + ) + + resolved_path = (base_dir / p).resolve() + return resolved_path def _resolve_visualize_config_path( @@ -838,9 +850,14 @@ def _run_visualize_step(self) -> None: if self.visualize_output_dir is not None: output_dir = self.visualize_output_dir.expanduser().resolve() + logger.info(f"Using explicit visualize output directory: {output_dir}") else: output_dir = _resolve_visualize_output_dir( self.config.visualize.output_dir, + base_dir=env.ARTIFACT_DIR, + ) + logger.info( + f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" ) # Log command to reproduce this step @@ -881,12 +898,20 @@ def _run_visualize_step(self) -> None: # If path is not under output_dir, keep as-is relative_paths.append(str(path)) + # Calculate relative output_dir path from base_directory + try: + relative_output_dir = str(output_dir.relative_to(env.ARTIFACT_DIR)) + except ValueError: + # If output_dir is not under ARTIFACT_DIR, use absolute path + relative_output_dir = str(output_dir) + self._add_step( "visualize", { "status": "success", "plugin_module": mod_str, "output_files": relative_paths, + "output_dir": relative_output_dir, "completed_at": time.time(), }, log_file, From 0e4a726dfd7fa3af2a5e66c1656e002144ada375 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:46:20 +0200 Subject: [PATCH 024/344] [skeleton] postprocess: default: make more verbose --- .../default/plotting/summary_table.py | 18 +++++++++++- .../default/plotting/throughput_chart.py | 29 +++++++++++++++++-- .../skeleton/postprocess/default/plugin.py | 19 ++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/projects/skeleton/postprocess/default/plotting/summary_table.py b/projects/skeleton/postprocess/default/plotting/summary_table.py index 8e19ef2a7..e758b5e03 100644 --- a/projects/skeleton/postprocess/default/plotting/summary_table.py +++ b/projects/skeleton/postprocess/default/plotting/summary_table.py @@ -3,10 +3,13 @@ from __future__ import annotations import html as html_lib +import logging from pathlib import Path from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class SummaryTablePlot: """Generates an HTML summary table of test results.""" @@ -23,11 +26,19 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating summary table HTML report") + logger.info(f"Processing {len(model.unified_result_records)} result records") + rows = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): scenario = r.distinguishing_labels.get("scenario", "") tp = r.metrics.get("throughput", "") lat = r.metrics.get("latency_ms", "") + + logger.debug( + f"Row {i}: path={r.test_base_path}, scenario={scenario}, throughput={tp}, latency={lat}" + ) + rows.append( "" f"{html_lib.escape(str(r.test_base_path))}" @@ -37,6 +48,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: "" ) + logger.info(f"Generated {len(rows)} table rows") + table_html = ( "" "Skeleton sample — summary" @@ -50,5 +63,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: ) output_file = output_dir / "summary_table.html" + logger.info(f"Writing summary table to: {output_file}") output_file.write_text(table_html, encoding="utf-8") + file_size = output_file.stat().st_size + logger.info(f"Summary table HTML written successfully ({file_size} bytes)") return str(output_file) diff --git a/projects/skeleton/postprocess/default/plotting/throughput_chart.py b/projects/skeleton/postprocess/default/plotting/throughput_chart.py index 075ccd610..774d92f84 100644 --- a/projects/skeleton/postprocess/default/plotting/throughput_chart.py +++ b/projects/skeleton/postprocess/default/plotting/throughput_chart.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging from pathlib import Path import plotly.graph_objects as go from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class ThroughputChartPlot: """Generates a Plotly bar chart of throughput by scenario.""" @@ -24,21 +27,34 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating throughput chart HTML report") + logger.info( + f"Processing {len(model.unified_result_records)} result records for throughput data" + ) + xs: list[str] = [] ys: list[float] = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): label = str(r.distinguishing_labels.get("scenario") or r.test_base_path) raw = r.metrics.get("throughput", 0) try: y = float(raw) except (TypeError, ValueError): + logger.debug( + f"Record {i}: Could not convert throughput '{raw}' to float, using 0.0" + ) y = 0.0 + logger.debug(f"Record {i}: label='{label}', throughput={y}") xs.append(label) ys.append(y) + logger.info(f"Collected throughput data for {len(xs)} scenarios") + logger.info(f"Throughput range: {min(ys) if ys else 0} - {max(ys) if ys else 0}") + + logger.info("Creating Plotly bar chart") fig = go.Figure(data=[go.Bar(x=xs, y=ys)]) fig.update_layout( title="Throughput by scenario", @@ -49,5 +65,12 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: from projects.caliper.postprocess.helpers.visualization_utils import write_full_page_html output_file = output_dir / "throughput_chart.html" - write_full_page_html(fig, str(output_file), "Throughput Chart") - return str(output_file) + logger.info(f"Writing throughput chart to: {output_file}") + success = write_full_page_html(fig, str(output_file), "Throughput Chart") + if success: + file_size = output_file.stat().st_size + logger.info(f"Throughput chart HTML written successfully ({file_size} bytes)") + return str(output_file) + else: + logger.error(f"Failed to write throughput chart to {output_file}") + return None diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 14fe301b2..04a10eb16 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -15,6 +16,8 @@ from .parsing import SkeletonKpiHandler, SkeletonParser from .plotting import SummaryTablePlot, ThroughputChartPlot +logger = logging.getLogger(__name__) + class SkeletonDefaultPlugin(PostProcessingPlugin): """ @@ -47,16 +50,32 @@ def visualize( visualize_config: dict[str, Any] | None, ) -> list[str]: """Generate visualization reports.""" + logger.info("Starting skeleton plugin visualization") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Requested report IDs: {report_ids}") + logger.info(f"Group ID: {group_id}") + logger.info(f"Available plot types: {list(self.plots.keys())}") + logger.info(f"Model contains {len(model.unified_result_records)} result records") + output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] wanted = frozenset(report_ids or ()) + logger.info(f"Will generate plots for: {sorted(wanted)}") + for report_id in wanted: if report_id in self.plots: + logger.info(f"Generating plot: {report_id}") plot_class = self.plots[report_id] path = plot_class.generate(model, output_dir) + logger.info(f"Generated plot file: {path}") paths.append(path) + else: + logger.warning(f"Requested plot type '{report_id}' not available in this plugin") + logger.info( + f"Skeleton plugin visualization completed. Generated {len(paths)} files: {paths}" + ) return paths def kpi_catalog(self) -> list[dict[str, Any]]: From 530a52c00f39758a3c301b67e9df7a8b5f80671c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:03:48 +0200 Subject: [PATCH 025/344] [guidellm] postprocess: guidellm: use the logger --- .../guidellm/csv_export/kpi_csv_exporter.py | 11 +- .../guidellm/plotting/kpi_report.py | 7 +- .../guidellm/plotting/performance_analysis.py | 125 +++++++++--------- 3 files changed, 77 insertions(+), 66 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py index 4ba749ae8..e4e3420af 100644 --- a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py +++ b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +import logging from pathlib import Path from typing import Any @@ -13,6 +14,8 @@ create_csv_rows_from_kpi_record, ) +logger = logging.getLogger(__name__) + class KPICsvExporter: """Export KPI data to CSV format.""" @@ -62,7 +65,7 @@ def export_kpis_to_csv( rows = create_csv_rows_from_kpi_record(record) csv_rows.extend(rows) except Exception as e: - print(f"Warning: Failed to convert KPI record to CSV row: {e}") + logger.info(f"Warning: Failed to convert KPI record to CSV row: {e}") continue # Write CSV file @@ -90,14 +93,14 @@ def export_kpis_to_csv( # Summary info total_written = len(csv_rows) - print(f"Exported {total_written} KPI records to {output_path}") + logger.info(f"Exported {total_written} KPI records to {output_path}") if skipped_2d_count > 0: - print(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") + logger.info(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") # Count 2D KPI expansion total_2d_rows = sum(1 for row in csv_rows if getattr(row, "is_2d", False)) if total_2d_rows > 0: - print(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") + logger.info(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") return str(output_path) diff --git a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py index 98534c9f3..8eaf0007e 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py +++ b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from html import escape from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ create_report_filename, ) +logger = logging.getLogger(__name__) + def generate_kpi_report( records: list[Any], @@ -35,7 +38,7 @@ def generate_kpi_report( # Handle multiple records by creating sections for each test if len(records) > 1: - print(f"📊 Generating multi-test KPI report for {len(records)} test records") + logger.info(f"📊 Generating multi-test KPI report for {len(records)} test records") return _generate_multi_test_kpi_report(records, output_dir, report_title, report_number) # Use the single record to extract test conditions and KPIs @@ -460,7 +463,7 @@ def _generate_multi_test_kpi_report( test_sections = [] for i, record in enumerate(records, 1): - print(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") + logger.info(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") # Extract test condition labels test_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(record) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 549c0783c..d1794e2ef 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -18,6 +19,8 @@ save_figure, ) +logger = logging.getLogger(__name__) + def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: Any = 0.0) -> Any: """ @@ -51,18 +54,18 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data Returns: DataFrame with all benchmark metrics and distinguishing labels """ - print(f"📊 Converting {len(records)} records to analysis dataframe...") + logger.info(f"📊 Converting {len(records)} records to analysis dataframe...") data = [] # Get parameters that vary across all records for legend names - print("🔍 Analyzing parameter variations for meaningful legend names...") + logger.info("🔍 Analyzing parameter variations for meaningful legend names...") varying_params = get_varying_parameters(records) if varying_params: - print(f" Found varying parameters: {', '.join(sorted(varying_params))}") + logger.info(f" Found varying parameters: {', '.join(sorted(varying_params))}") else: - print(" No varying parameters found - using default naming") + logger.info(" No varying parameters found - using default naming") - print("📝 Processing records and extracting metrics...") + logger.info("📝 Processing records and extracting metrics...") for record in records: # Skip records without GuideLLM data or missing benchmarks if not record.run_identity.get("guidellm") or record.metrics.get("no_benchmarks_found"): @@ -77,7 +80,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if has_curves and isinstance(request_rates, list) and len(request_rates) > 0: # New format: expand performance curves into multiple data points - print( + logger.info( f" 🔄 Expanding performance curves for {legend_name} ({len(request_rates)} points)" ) curves = record.metrics.get("performance_curves", {}) @@ -157,7 +160,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) else: # Old format: single data point per record (backward compatibility) - print(f" 📊 Using scalar metrics for {legend_name}") + logger.info(f" 📊 Using scalar metrics for {legend_name}") # Extract scalar request_rate if it's a single value request_rate = ( request_rates[0] @@ -226,14 +229,14 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) if not data: - print("⚠️ No GuideLLM data found in records") + logger.info("⚠️ No GuideLLM data found in records") return pd.DataFrame() - print(f"✅ Successfully processed {len(data)} data points from {len(records)} records") + logger.info(f"✅ Successfully processed {len(data)} data points from {len(records)} records") df = pd.DataFrame(data) # Sort for consistent ordering - print("📋 Organizing data by configuration, concurrency, and request rate...") + logger.info("📋 Organizing data by configuration, concurrency, and request rate...") # Sort and fill any NaN values in numeric columns with 0 for consistent plotting df = df.sort_values( ["test_configuration", "request_concurrency", "request_rate", "rate_point_index"] @@ -259,7 +262,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data ] ) - print( + logger.info( f"🎯 Expanded {total_records} benchmark records into {len(df)} data points across {len(configs)} configurations:" ) for config in configs: @@ -270,7 +273,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if rate_points > 1 else f"{config_data['request_rate'].iloc[0]:.1f} req/s" ) - print(f" • {config}: {rate_points} rate points ({rate_range})") + logger.info(f" • {config}: {rate_points} rate points ({rate_range})") return df @@ -279,11 +282,11 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): """Create throughput scaling scatter plot.""" try: - print("📈 Creating throughput scaling scatter plot...") + logger.info("📈 Creating throughput scaling scatter plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for throughput scaling plot") + logger.info("⚠️ No data available for throughput scaling plot") return None title = f"Request Throughput vs Concurrency by Configuration{title_context}" @@ -314,22 +317,22 @@ def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ Throughput scaling plot created successfully") + logger.info("✅ Throughput scaling plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create throughput scaling plot: {e}") + logger.info(f"❌ Failed to create throughput scaling plot: {e}") return None def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = ""): """Create latency vs throughput scatter plot.""" try: - print("📈 Creating latency vs throughput trade-off plot...") + logger.info("📈 Creating latency vs throughput trade-off plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for latency vs throughput plot") + logger.info("⚠️ No data available for latency vs throughput plot") return None title = f"Latency vs Throughput Trade-off{title_context}" @@ -353,22 +356,22 @@ def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = "") showlegend=True, width=800, height=500, font={"size": 12}, title_font_size=16 ) - print("✅ Latency vs throughput plot created successfully") + logger.info("✅ Latency vs throughput plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create latency vs throughput plot: {e}") + logger.info(f"❌ Failed to create latency vs throughput plot: {e}") return None def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput vs concurrency line plot.""" try: - print("📈 Creating token throughput vs concurrency scaling plot...") + logger.info("📈 Creating token throughput vs concurrency scaling plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for token throughput vs concurrency plot") + logger.info("⚠️ No data available for token throughput vs concurrency plot") return None title = f"Token Throughput vs Concurrency{title_context}
Higher is better" @@ -399,22 +402,22 @@ def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput vs concurrency plot created successfully") + logger.info("✅ Token throughput vs concurrency plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput vs concurrency plot: {e}") + logger.info(f"❌ Failed to create token throughput vs concurrency plot: {e}") return None def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): """Create TTFT analysis line plot.""" try: - print("📈 Creating TTFT (Time to First Token) analysis plot...") + logger.info("📈 Creating TTFT (Time to First Token) analysis plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for TTFT analysis plot") + logger.info("⚠️ No data available for TTFT analysis plot") return None title = f"TTFT vs Concurrency{title_context}
Lower is better" @@ -440,23 +443,23 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ TTFT analysis plot created successfully") + logger.info("✅ TTFT analysis plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create TTFT analysis plot: {e}") + logger.info(f"❌ Failed to create TTFT analysis plot: {e}") return None def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput percentiles plot.""" try: - print("📈 Creating token throughput percentiles distribution plot...") + logger.info("📈 Creating token throughput percentiles distribution plot...") import plotly.express as px import plotly.graph_objects as go if df.empty: - print("⚠️ No data available for token throughput percentiles plot") + logger.info("⚠️ No data available for token throughput percentiles plot") return None title = f"Output Token Throughput Percentiles{title_context}
Higher is better" @@ -465,7 +468,9 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st # Get unique configurations and colors configurations = sorted(df["test_configuration"].unique()) - print(f" Plotting {len(configurations)} configurations with percentile distributions...") + logger.info( + f" Plotting {len(configurations)} configurations with percentile distributions..." + ) available_colors = px.colors.qualitative.Set1 color_map = { config: available_colors[i % len(available_colors)] @@ -480,7 +485,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ("P75", "output_tokens_per_second_p75", {"width": 3, "dash": "dash"}, 0.9), ("P90", "output_tokens_per_second_p90", {"width": 2, "dash": "dashdot"}, 0.8), ] - print(f" Adding {len(percentiles)} percentile lines per configuration...") + logger.info(f" Adding {len(percentiles)} percentile lines per configuration...") for config in configurations: config_df = df[df["test_configuration"] == config].sort_values("request_concurrency") @@ -510,11 +515,11 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput percentiles plot created successfully") + logger.info("✅ Token throughput percentiles plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput percentiles plot: {e}") + logger.info(f"❌ Failed to create token throughput percentiles plot: {e}") return None @@ -565,7 +570,7 @@ def generate_token_throughput_vs_concurrency( report_number: int | None = None, ) -> str | None: """Generate token throughput vs concurrency analysis and save to file.""" - print("\n🚀 Generating token throughput vs concurrency analysis...") + logger.info("\n🚀 Generating token throughput vs concurrency analysis...") df = create_dataframe_from_records(records) if df.empty: return None @@ -645,16 +650,16 @@ def generate_comprehensive_performance_report( # Create report identifier using core utilities display_title = create_report_title_display(report_title, report_number) - print(f"\n🚀 Starting {display_title} generation...") - print("=" * 70) + logger.info(f"\n🚀 Starting {display_title} generation...") + logger.info("=" * 70) df = create_dataframe_from_records(records) if df.empty: - print("❌ No data available for analysis") + logger.info("❌ No data available for analysis") return None - print("\n📊 Generating performance analysis plots...") - print(" This may take a moment as we create high-quality visualizations...") + logger.info("\n📊 Generating performance analysis plots...") + logger.info(" This may take a moment as we create high-quality visualizations...") # Generate all the plots as figures plot_functions = [ @@ -673,17 +678,17 @@ def generate_comprehensive_performance_report( report_dir = output_dir / report_dir_name report_dir.mkdir(exist_ok=True) - print(f"\n📁 Created report directory: {report_dir_name}") + logger.info(f"\n📁 Created report directory: {report_dir_name}") plots_data = [] for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - print(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") + logger.info(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") try: fig = plot_func(df, title_context) if fig: # Save as both PNG and HTML in the report directory filename = plot_name.lower().replace(" ", "_") - print(f"💾 Saving {plot_name} as PNG and HTML...") + logger.info(f"💾 Saving {plot_name} as PNG and HTML...") # Save PNG image width = 900 if "Percentiles" in plot_name else 800 @@ -704,26 +709,26 @@ def generate_comprehensive_performance_report( f"{report_dir_name}/{Path(html_path).name}", # HTML path ) ) - print(f"✅ {plot_name} saved as PNG and HTML") + logger.info(f"✅ {plot_name} saved as PNG and HTML") else: - print(f"⚠️ {plot_name} could not be created (no figure returned)") + logger.info(f"⚠️ {plot_name} could not be created (no figure returned)") except Exception as e: - print(f"❌ Failed to generate {plot_name}: {e}") + logger.info(f"❌ Failed to generate {plot_name}: {e}") - print(f"\n✅ Successfully generated {len(plots_data)} visualizations!") + logger.info(f"\n✅ Successfully generated {len(plots_data)} visualizations!") # Generate summary statistics - print("\n📊 Computing performance statistics and insights...") + logger.info("\n📊 Computing performance statistics and insights...") summary_stats = _generate_performance_summary(df) if summary_stats.get("best_tokens"): best = summary_stats["best_tokens"] - print(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") + logger.info(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") # Create comprehensive HTML report - print("\n📝 Assembling comprehensive HTML report...") - print(" 🔗 Creating report with images linking to interactive HTML versions...") + logger.info("\n📝 Assembling comprehensive HTML report...") + logger.info(" 🔗 Creating report with images linking to interactive HTML versions...") html_content = _create_comprehensive_html_report_with_images( plots_data, summary_stats, title_context, display_title @@ -734,28 +739,28 @@ def generate_comprehensive_performance_report( "performance_analysis", report_number, report_title, "html" ) report_file = output_dir / filename - print(f"\n💾 Writing {display_title} to: {report_file.name}") + logger.info(f"\n💾 Writing {display_title} to: {report_file.name}") report_file.write_text(html_content, encoding="utf-8") file_size_mb = report_file.stat().st_size / (1024 * 1024) - print(f"📄 Report complete! File size: {file_size_mb:.1f}MB") - print("=" * 70) - print(f"🎉 {display_title} ready: {report_file.name}") + logger.info(f"📄 Report complete! File size: {file_size_mb:.1f}MB") + logger.info("=" * 70) + logger.info(f"🎉 {display_title} ready: {report_file.name}") return str(report_file) except Exception as e: - print(f"❌ Failed to generate comprehensive performance report: {e}") + logger.info(f"❌ Failed to generate comprehensive performance report: {e}") return None def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: """Generate performance summary statistics from the dataframe.""" if df.empty: - print("⚠️ No data available for performance summary") + logger.info("⚠️ No data available for performance summary") return {} - print(" 🔍 Analyzing best performers across all metrics...") + logger.info(" 🔍 Analyzing best performers across all metrics...") # Find best performers best_tokens_idx = df["tokens_per_second"].idxmax() @@ -765,7 +770,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: best_ttft_idx = df["ttft_median_ms"].idxmin() # Configuration analysis - print(" 📊 Computing configuration performance rankings...") + logger.info(" 📊 Computing configuration performance rankings...") config_performance = ( df.groupby("test_configuration") .agg( @@ -779,7 +784,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: .sort_values("tokens_per_second", ascending=False) ) - print(f" ✅ Performance analysis complete for {len(config_performance)} configurations") + logger.info(f" ✅ Performance analysis complete for {len(config_performance)} configurations") return { "total_configurations": len(df["test_configuration"].unique()), From 80ccb5e47467f9a568799a6036bce33b9d1733df Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:04:10 +0200 Subject: [PATCH 026/344] [caliper] engine: parse: use the logger --- projects/caliper/engine/parse.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index dc0e6cce8..132e4428e 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from projects.caliper.engine.cache import ( @@ -18,6 +19,8 @@ ) from projects.caliper.engine.traverse import discover_test_bases +logger = logging.getLogger(__name__) + # Validation functions no longer needed - using per-test-base caching @@ -98,14 +101,14 @@ def run_parse( if all_warnings and force_report_partial: for w in all_warnings: - print(f"[parse warning] {w}") # noqa: T201 — CLI feedback + logger.warning(f"[parse warning] {w}") # Show parameter matrix if requested if show_parameter_matrix: - print() # noqa: T201 — CLI feedback + logger.info("") matrix_analysis = analyze_parameter_matrix(all_records) matrix_summary = format_parameter_matrix_summary(matrix_analysis) - print(matrix_summary) # noqa: T201 — CLI feedback - print() # noqa: T201 — CLI feedback + logger.info(matrix_summary) + logger.info("") return model From 9d4c7a7b2b4b78eae0be589c04c412dfb73652f5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:18:29 +0200 Subject: [PATCH 027/344] [caliper] relocate kpi_to_csv to the right location --- .../caliper/cli/orchestration_entrypoints.py | 17 +++++++ projects/caliper/engine/kpi/csv_export.py | 43 +++++++++++++++++ projects/caliper/engine/model.py | 46 +++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 10 ++-- 4 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 projects/caliper/engine/kpi/csv_export.py diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 4e273e4b4..9d2481688 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -175,3 +175,20 @@ def analyze_kpis_entrypoint( output_dir=output_dir, current_kpis_file=current_kpis_file, ) + + +def export_kpis_to_csv_entrypoint( + plugin, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, +) -> str: + """Entrypoint for orchestration to export KPIs to CSV.""" + from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv + + return export_kpis_to_csv( + plugin=plugin, + kpi_records=kpi_records, + output_path=output_path, + include_header_comments=include_header_comments, + ) diff --git a/projects/caliper/engine/kpi/csv_export.py b/projects/caliper/engine/kpi/csv_export.py new file mode 100644 index 000000000..baf9ef032 --- /dev/null +++ b/projects/caliper/engine/kpi/csv_export.py @@ -0,0 +1,43 @@ +"""Export KPIs to CSV format using plugin interface.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +def export_kpis_to_csv( + *, + plugin: object, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, +) -> str: + """ + Export KPI records to CSV format using the plugin's CSV export method. + + Args: + plugin: PostProcessingPlugin instance with export_kpis_to_csv method + kpi_records: List of KPI record dictionaries + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file as string + + Raises: + AttributeError: If plugin doesn't have export_kpis_to_csv method + """ + if not hasattr(plugin, "export_kpis_to_csv"): + raise AttributeError( + f"Plugin {plugin.__class__.__name__} does not implement export_kpis_to_csv method" + ) + + # Delegate to plugin-specific CSV export implementation + result_path = plugin.export_kpis_to_csv( + kpi_records=kpi_records, + output_path=output_path, + include_header_comments=include_header_comments, + ) + + return result_path diff --git a/projects/caliper/engine/model.py b/projects/caliper/engine/model.py index 3a786f69c..10fc42b04 100644 --- a/projects/caliper/engine/model.py +++ b/projects/caliper/engine/model.py @@ -105,6 +105,52 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Return canonical-shaped KPI dicts (pre-validated).""" return [] + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + # Default implementation: basic CSV export + import csv + + # Ensure parent directory exists before any file operations + output_path.parent.mkdir(parents=True, exist_ok=True) + + if not kpi_records: + output_path.touch() # Create empty file + return str(output_path) + + with open(output_path, "w", newline="", encoding="utf-8") as csvfile: + if include_header_comments: + csvfile.write("# KPI Export\n") + csvfile.write("# Generated by Caliper\n") + + if kpi_records: + # Get all possible fieldnames from all records + fieldnames = set() + for record in kpi_records: + fieldnames.update(record.keys()) + fieldnames = sorted(fieldnames) + + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for record in kpi_records: + writer.writerow(record) + + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Structured JSON for AI agent evaluation.""" return {"schema_version": "1", "run_id": "", "metrics": {}} diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 6a7d5a8f5..b721c045a 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -20,6 +20,7 @@ from projects.caliper.cli.orchestration_entrypoints import ( analyze_kpis_entrypoint, + export_kpis_to_csv_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, @@ -528,11 +529,10 @@ def _run_kpis_to_csv( output_path=output_file, ) - # Import and use the CSV exporter - from projects.guidellm.postprocess.guidellm.csv_export import quick_export_kpis_to_csv - - result_path = quick_export_kpis_to_csv( - records=kpi_records, + # Use Caliper entrypoint for CSV export + result_path = export_kpis_to_csv_entrypoint( + plugin=plugin, + kpi_records=kpi_records, output_path=output_file, include_header_comments=postprocess_config.kpi.kpis_to_csv.include_header_comments, ) From 7867d9dc07f35232fe468cf8c65376f6b2e4c99b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:18:58 +0200 Subject: [PATCH 028/344] [guidellm] postprocess: guidellm: plugin: expose the export_kpis_to_csv method --- .../guidellm/postprocess/guidellm/plugin.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index dfe8ff3ed..3e4c775c1 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -175,6 +175,27 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Compute KPI values from the unified model.""" return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format using GuideLLM's CSV exporter. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + from .csv_export import KPICsvExporter + + exporter = KPICsvExporter() + return exporter.export_kpis_to_csv(kpi_records, output_path, include_header_comments) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Build AI evaluation payload from the unified model.""" return self.ai_evaluator.build_payload(model, self) From 7c7137535f4b0277f68c49a7be70b508d7a07b45 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 22:22:42 +0200 Subject: [PATCH 029/344] [core] library: replot: show the correct artifacts dir --- projects/caliper/orchestration/postprocess.py | 34 ++++++------------- projects/caliper/orchestration/replot.py | 18 +++++----- projects/core/library/replot.py | 9 +++-- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index b721c045a..2c795f2fc 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -93,27 +93,18 @@ def _resolve_paths( def _resolve_visualize_output_dir( raw: str | None, - base_dir: Path | None = None, ) -> Path: if raw is None or not str(raw).strip(): - raise ValueError( - "caliper.postprocess.visualize.output_dir is required when no explicit visualize_output_directory is provided" - ) + # If empty, use env.ARTIFACT_DIR + return env.ARTIFACT_DIR + p = Path(raw).expanduser() if p.is_absolute(): + # If absolute, don't touch return p.resolve() - - # Handle relative paths by making them relative to base_dir (defaults to env.ARTIFACT_DIR) - if base_dir is None: - base_dir = env.ARTIFACT_DIR - - if base_dir is None: - raise ValueError( - "caliper.postprocess.visualize.output_dir must be an absolute path when no base directory is available" - ) - - resolved_path = (base_dir / p).resolve() - return resolved_path + else: + # If relative, use env.ARTIFACT_DIR / output_dir + return (env.ARTIFACT_DIR / p).resolve() def _resolve_visualize_config_path( @@ -854,7 +845,6 @@ def _run_visualize_step(self) -> None: else: output_dir = _resolve_visualize_output_dir( self.config.visualize.output_dir, - base_dir=env.ARTIFACT_DIR, ) logger.info( f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" @@ -937,12 +927,10 @@ def _run_kpi_and_ai_data_steps(self) -> None: return try: - # Determine output directory - if self.config.visualize.enabled and self.visualize_output_dir: - output_dir = Path(self.visualize_output_dir) - else: - output_dir = Path(self.artifacts_dir) / "postprocess_output" - output_dir.mkdir(parents=True, exist_ok=True) + # Determine output directory for KPI/AI data steps - use base artifact directory + output_dir = env.ARTIFACT_DIR + logger.info(f"KPI steps using base artifact directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) # Load plugin and model model, mod_str = parse_entrypoint( diff --git a/projects/caliper/orchestration/replot.py b/projects/caliper/orchestration/replot.py index 8ff5f6f69..86b6382a4 100644 --- a/projects/caliper/orchestration/replot.py +++ b/projects/caliper/orchestration/replot.py @@ -13,6 +13,7 @@ from projects.caliper.orchestration.postprocess import run_postprocess_from_orchestration_config from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome +from projects.core.library import env from projects.core.library import vault as vault_lib logger = logging.getLogger(__name__) @@ -304,15 +305,14 @@ def run_replot_from_orchestration_config( if artifact_directory.exists(): logger.info("Running post-processing...") - visualize_output_dir = artifact_directory / "postprocess_output" - visualize_output_dir.mkdir(parents=True, exist_ok=True) - - postprocess_result = run_postprocess_from_orchestration_config( - postprocess_config_raw=postprocess_config or {}, - artifacts_dir=artifact_directory, - visualize_output_dir=visualize_output_dir, - test_outcome=TestPhaseOutcome("SUCCESS"), - ) + with env.TempArtifactDir(artifact_directory / "postprocess_output"): + # Let Caliper handle output directory resolution from config + postprocess_result = run_postprocess_from_orchestration_config( + postprocess_config_raw=postprocess_config or {}, + artifacts_dir=artifact_directory, + visualize_output_dir=None, # Let Caliper resolve from config + test_outcome=TestPhaseOutcome("SUCCESS"), + ) else: logger.info("Artifacts directory not found") postprocess_result = { diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index b2e3cd7fb..81696785b 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -167,9 +167,12 @@ def caliper_replot_entrypoint( logger.info("Replot status:") logger.info(yaml.dump(status, indent=2)) - # Show where artifacts have been saved - final_artifact_dir = artifact_directory or env.ARTIFACT_DIR - logger.info(f"📁 Artifacts saved to: {final_artifact_dir}") + try: + logger.info( + f"📁 Artifacts saved to: {status['replot']['postprocess_result']['base_directory']}" + ) + except Exception: + pass # Check if replot was successful replot_status = status.get("replot", {}).get("status", "unknown") From 229cbfd1715b87d8996fdd94cec198a28f9b4343 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 22:57:46 +0200 Subject: [PATCH 030/344] [core] library: postprocess: use the right directories --- projects/core/library/postprocess.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index c6ab3186f..893c9e7f9 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -278,16 +278,13 @@ def run_postprocess_after_test( artifact_root_path = Path(artifact_root).resolve() if artifact_root is not None else None with env.NextArtifactDir("postprocessing"): - workspace = Path(env.ARTIFACT_DIR).resolve() logger.info( - "Running Caliper postprocess (artifacts=%s, workspace=%s, test_phase=%s)", + "Running Caliper postprocess (artifacts=%s, test_phase=%s)", artifact_root_path, - workspace, test_outcome.phase if test_outcome else "SUCCESS", ) status = run_orchestration_postprocess( artifact_dir=artifact_root_path, - visualize_output_dir=workspace, test_outcome=test_outcome, ) logger.info( @@ -415,11 +412,13 @@ def run_orchestration_postprocess( def postprocess_command(_ctx, artifact_dir: Path, output_dir: Path): """Run the post-processing pipeline.""" - status = run_orchestration_postprocess( - artifact_dir=artifact_dir, - test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), - visualize_output_dir=output_dir, - ) + # Set ARTIFACT_DIR temporarily so config resolution works with user's output directory + with env.TempArtifactDir(output_dir): + status = run_orchestration_postprocess( + artifact_dir=artifact_dir, + test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), + visualize_output_dir=None, # Let config resolution handle it + ) logger.info("Caliper postprocess status:\n" + yaml.dump(status, indent=2)) # Check success flag and return appropriate exit code From 6e5a351878961c2a4e5f776020d3d06503410608 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 07:28:25 +0200 Subject: [PATCH 031/344] [caliper] cli: commands: fix the s3-import command --- projects/caliper/cli/commands.py | 36 +++++++------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index f06881acf..5803dae0a 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -7,7 +7,7 @@ import click -from projects.caliper.cli.s3_import import run_s3_import +from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import @@ -446,11 +446,6 @@ def kpi_s3_import( verbose: bool, ) -> None: """Download historical KPI and analysis data from S3.""" - from projects.caliper.orchestration.postprocess_config import ( - CaliperOrchestrationPostprocessConfig, - CaliperOrchestrationS3ImportSection, - CaliperOrchestrationS3Section, - ) from projects.core.library import vault as vault_lib try: @@ -466,34 +461,19 @@ def kpi_s3_import( # Initialize vault system vault_lib.init(vaults=[vault] if vault else []) - # Parse bucket/prefix to extract instance and directory - prefix_parts = prefix.strip("/").split("/") - instance = prefix_parts[0] if len(prefix_parts) > 0 and prefix_parts[0] else None - directory = prefix_parts[1] if len(prefix_parts) > 1 else None - - s3_import_config = CaliperOrchestrationS3ImportSection( - enabled=True, - output_dir=output_dir.name, + # Run S3 import + result = run_s3_import_with_explicit_params( + bucket=bucket, + prefix=prefix, + output_dir=output_dir, + vault=vault, + aws_credentials_file=aws_credentials_file, include_kpis_json=include_kpis_json, include_kpis_csv=include_kpis_csv, include_ai_data=include_ai_data, max_downloads=max_downloads, ) - s3_config = CaliperOrchestrationS3Section( - bucket=bucket, - instance=instance, - directory=directory, - vault=vault, - aws_credentials_file=aws_credentials_file, - **{"import": s3_import_config}, - ) - - config = CaliperOrchestrationPostprocessConfig(s3=s3_config) - - # Run S3 import - result = run_s3_import(config, output_dir.parent) - if result["status"] == "success": click.echo("✅ S3 import completed successfully") downloaded_files = result.get("files", []) From 8aec9f50b26b36938e4e04d3c4ac3b69b72c34f1 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Fri, 10 Jul 2026 08:24:10 +0100 Subject: [PATCH 032/344] chore: Run linter Signed-off-by: Alberto Perdomo --- projects/caliper/orchestration/postprocess.py | 2 +- projects/llm_d/orchestration/ci.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ee285878e..c3ba2b234 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1279,7 +1279,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" # Log programming errors more prominently - if isinstance(e, (AttributeError, NameError, TypeError)): + if isinstance(e, AttributeError | NameError | TypeError): logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") logger.exception("Full traceback for programming error:") else: diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index d604f8b02..f8611771c 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -36,6 +36,7 @@ def init(): run.init() config.init(Path(__file__).parent) + def list_vaults() -> list[str]: """List all vaults (includes both mandatory and optional).""" all_vaults = vault.phase_vault_list_all() From 25bf9a330788b507f356c19bcd39fe5ccbfbbfc0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:01:06 +0200 Subject: [PATCH 033/344] [caliper] orchestration: postprocess: abort the post-process on parse failure --- projects/caliper/orchestration/postprocess.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2c795f2fc..2ea5152b1 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -646,13 +646,19 @@ def _execute_orchestration(self) -> dict[str, Any]: logger.info("Starting postprocessing steps") self._run_parse_step() logger.info(f"After parse step: parse_failed={self.parse_failed}") - self._run_visualize_step() - logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") - self._run_kpi_and_ai_data_steps() - logger.info( - f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" - ) - logger.info("All postprocessing steps completed") + + # Abort pipeline if parse failed - no point in running other steps without data + if self.parse_failed: + logger.error("Parse step failed - aborting remaining postprocessing steps") + logger.info("Pipeline aborted due to parse failure") + else: + self._run_visualize_step() + logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") + self._run_kpi_and_ai_data_steps() + logger.info( + f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" + ) + logger.info("All postprocessing steps completed") # Compute final status and build result final_status = self._compute_final_status() From 49d4cf255d51505c5313b6e47761e2afb806aee4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:07:35 +0200 Subject: [PATCH 034/344] [caliper] cli: commands: include the status file for the parse command --- projects/caliper/cli/commands.py | 47 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 5803dae0a..05e9a72e1 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -107,12 +107,19 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def parse_cmd( ctx: click.Context, no_cache: bool, cache_dir: Path | None, show_matrix: bool, + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -126,6 +133,9 @@ def parse_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status = {"success": False} + try: model = run_parse( base_dir=artifact_root, @@ -134,12 +144,43 @@ def parse_cmd( use_cache=not no_cache, show_parameter_matrix=show_matrix, ) + + status.update( + { + "success": True, + "plugin_module": mod, + "parsed_records": len(model.unified_result_records), + "cache_ref": str(model.parse_cache_ref) if model.parse_cache_ref else None, + } + ) + + click.echo( + f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" + ) + except Exception as e: # noqa: BLE001 + status.update( + { + "success": False, + "error": str(e), + } + ) click.echo(f"parse failed: {e}", err=True) + + # Write status file if requested + if status_file: + import yaml + + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) + + # Exit with non-zero code on failure + if not status["success"]: sys.exit(2) - click.echo( - f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" - ) @click.command("visualize") From 176cb9af96d3d10d47c325fcf0f8711ac5ef47fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:11:19 +0200 Subject: [PATCH 035/344] [caliper] orchestration: launch caliper as an external process --- projects/caliper/cli/commands.py | 8 +- projects/caliper/orchestration/cli_builder.py | 334 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 95 +++-- 3 files changed, 411 insertions(+), 26 deletions(-) create mode 100644 projects/caliper/orchestration/cli_builder.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 05e9a72e1..b54eb72e7 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -77,10 +77,10 @@ def _plugin_tuple(ctx: click.Context): root = _root_obj(ctx) try: - mod = resolve_plugin_module_string( - root.get("plugin_cli"), - root.get("postprocess_config"), - root.get("base_dir"), + mod, _manifest_path = resolve_plugin_module_string( + base_dir=root.get("base_dir"), + postprocess_config=root.get("postprocess_config"), + cli_plugin=root.get("plugin_cli"), ) if not mod: raise RuntimeError("Plugin module not found") diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py new file mode 100644 index 000000000..1563631de --- /dev/null +++ b/projects/caliper/orchestration/cli_builder.py @@ -0,0 +1,334 @@ +"""CLI command builder for Caliper orchestration fork/exec. + +Converts orchestration config objects to CLI argument lists for subprocess execution. +""" + +from __future__ import annotations + +import logging +import subprocess +import sys +from pathlib import Path + +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) + +logger = logging.getLogger(__name__) + + +def build_parse_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper parse. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "parse", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Parse-specific options + if not use_cache: + cmd.append("--no-cache") + + # Enable detailed parameter matrix output + cmd.append("--show-matrix") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_visualize_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_dir: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper visualize. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_dir: Output directory for visualization files + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "visualize", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Visualize-specific options + cmd.extend(["--output-dir", str(output_dir)]) + + if config.visualize.reports: + cmd.extend(["--reports", ",".join(config.visualize.reports)]) + + if config.visualize.report_group: + cmd.extend(["--report-group", config.visualize.report_group]) + + if config.visualize.visualize_config: + cmd.extend(["--visualize-config", str(config.visualize.visualize_config)]) + + # Include/exclude labels + for label in config.visualize.include_labels: + cmd.extend(["--include-label", label]) + + for label in config.visualize.exclude_labels: + cmd.extend(["--exclude-label", label]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def save_command_script(command: list[str], script_path: Path) -> None: + """Save executed command to a shell script for debugging. + + Args: + command: Command arguments list + script_path: Where to save the script + """ + script_path.parent.mkdir(parents=True, exist_ok=True) + + # Convert command to a readable shell script + script_content = "#!/bin/bash\n" + script_content += "# Generated command for debugging\n" + script_content += "# Run this command to reproduce the step manually\n\n" + + # Format command with argument pairs on separate lines + formatted_lines = _format_command_for_script(command) + script_content += formatted_lines + "\n" + + with open(script_path, "w", encoding="utf-8") as f: + f.write(script_content) + + # Make executable + script_path.chmod(0o755) + + +def _format_command_for_script(command: list[str]) -> str: + """Format command for script file with argument pairs on separate lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for shell script + """ + if not command: + return "" + + def escape_arg(arg: str) -> str: + """Escape shell arguments that need quoting.""" + if " " in arg or any(c in arg for c in ["$", "`", '"', "'"]): + return f'"{arg}"' + return arg + + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(escape_arg(command[i])) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) with line continuations + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f" {escape_arg(command[i])} {escape_arg(command[i + 1])}") + i += 2 + else: + # Boolean flag + lines.append(f" {escape_arg(command[i])}") + i += 1 + else: + # Standalone argument + lines.append(f" {escape_arg(command[i])}") + i += 1 + + # Join with line continuations + if len(lines) == 1: + return lines[0] + else: + return lines[0] + " \\\n" + " \\\n".join(lines[1:]) + + +def log_caliper_start_banner(command: list[str], script_path: Path, step_name: str) -> None: + """Log start banner and save command script for Caliper execution. + + Args: + command: Command arguments list + script_path: Where to save the executable script + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + """ + # Save command script for debugging + save_command_script(command, script_path) + + # Format command with argument pairs on new lines + formatted_command = _format_command_for_display(command) + + # Log banner indicating Caliper is starting + logger.info("=" * 60) + logger.info("🚀 STARTING CALIPER %s (fork/exec)", step_name.upper()) + logger.info("📄 Command:") + for line in formatted_command.splitlines(): + logger.info(" %s", line) + logger.info("=" * 60) + + +def _format_command_for_display(command: list[str]) -> str: + """Format command for readable display with argument pairs on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string + """ + if not command: + return "" + + # Start with base command (python -m module subcommand) + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return "\n".join(lines) + + +def handle_caliper_output_and_completion( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Write output to log file + with open(log_file, "w", encoding="utf-8") as log_f: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + status_yaml = yaml.dump(status_data, default_flow_style=False, sort_keys=False) + for line in status_yaml.strip().splitlines(): + logger.info(" %s", line) + + logger.info("=" * 60) + + return status_data diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2ea5152b1..36b64b230 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -28,6 +28,11 @@ s3_export_entrypoint, s3_import_entrypoint, ) +from projects.caliper.orchestration.cli_builder import ( + build_parse_command, + handle_caliper_output_and_completion, + log_caliper_start_banner, +) from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) @@ -787,48 +792,94 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - with step_logging("caliper_parse", self.step_logs_dir) as log_file: - try: - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, - ) + import subprocess + import tempfile - # Log command to reproduce this step - log_parse_command( - base_dir=self.tree_root, - plugin_module=mod_str, - use_cache=not self.config.parse.no_cache, - manifest_path=self.manifest_path, - ) + import yaml + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + use_cache=not self.config.parse.no_cache, + ) + + # Log start banner and save command script + script_path = env.ARTIFACT_DIR / "parse.sh" + log_caliper_start_banner(command, script_path, "parse") + + # Create log file path for stdout/stderr capture with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + step_index = _get_next_step_index(self.step_logs_dir) + log_file = self.step_logs_dir / f"{step_index:03d}__caliper_parse.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Execute CLI command with output capture + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion(result, log_file, status_file, "parse") + + if result.returncode == 0 and status_data and status_data.get("success", False): self._add_step( "parse", { "status": "success", - "plugin_module": mod_str, - "record_count": len(model.unified_result_records), - "parse_cache_ref": model.parse_cache_ref, + "plugin_module": status_data.get("plugin_module", "unknown"), + "record_count": status_data.get("parsed_records", 0), + "parse_cache_ref": status_data.get("cache_ref"), "completed_at": time.time(), }, log_file, ) - except Exception as e: # noqa: BLE001 + else: self.parse_failed = True - logger.exception("Caliper parse failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper parse failed: %s", error_msg) self._add_step( "parse", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, log_file, ) + except Exception as e: # noqa: BLE001 + self.parse_failed = True + logger.exception("Parse step execution failed") + self._add_step( + "parse", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_visualize_step(self) -> None: """Execute the visualize step if enabled.""" if not self.config.visualize.enabled: From 67a7b84528d239b455d935fc5639294edf807380 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:34:52 +0200 Subject: [PATCH 036/344] [caliper] cli: main: enable the logging on CLI command start --- projects/caliper/cli/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 167b32dc3..1bcb875d6 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import sys from pathlib import Path from urllib.parse import parse_qs, urlparse @@ -159,6 +160,13 @@ def artifacts_group(ctx: click.Context) -> None: def run_cli() -> None: """Invoke CLI; on missing required options, print subcommand help.""" + # Configure logging to capture INFO level messages for detailed output + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + force=True, # Override any existing configuration + ) + try: # standalone_mode=False returns exit codes instead of calling sys.exit; # propagate them so failures are non-zero (e.g. ctx.exit(1) from _exit_with_help). From 2684f205aab78d8fb73ae3d3b90bf6c21252821b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:50:20 +0200 Subject: [PATCH 037/344] [caliper] cli: commands: include the status file for the visualize command --- projects/caliper/cli/commands.py | 59 ++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index b54eb72e7..504427799 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -6,6 +6,7 @@ from pathlib import Path import click +import yaml from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export @@ -169,8 +170,6 @@ def parse_cmd( # Write status file if requested if status_file: - import yaml - try: status_file.parent.mkdir(parents=True, exist_ok=True) with open(status_file, "w", encoding="utf-8") as f: @@ -195,6 +194,12 @@ def parse_cmd( type=click.Path(path_type=Path), required=True, ) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def visualize_cmd( ctx: click.Context, @@ -204,6 +209,7 @@ def visualize_cmd( include_label: tuple[str, ...], exclude_label: tuple[str, ...], output_dir: Path, + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -217,6 +223,9 @@ def visualize_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status = {"success": False} + try: paths = run_visualize( base_dir=artifact_root, @@ -231,11 +240,51 @@ def visualize_cmd( use_cache=True, cache_path=None, ) + + status.update( + { + "success": True, + "plugin_module": mod, + "output_files": paths, + "output_dir": str(output_dir), + "generated_files": len(paths), + } + ) + + click.echo("Wrote: " + ", ".join(paths)) + except Exception as e: # noqa: BLE001 + import traceback + + error_details = { + "error": str(e), + "traceback": traceback.format_exc(), + } + + status.update( + { + "success": False, + **error_details, + } + ) + + # Print full traceback to stderr for visibility click.echo(f"visualize failed: {e}", err=True) - sys.exit(2) + click.echo("Full traceback:", err=True) + click.echo(traceback.format_exc(), err=True) - click.echo("Wrote: " + ", ".join(paths)) + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) + + # Exit with non-zero code on failure + if not status["success"]: + sys.exit(2) @click.command("list-reports") @@ -439,8 +488,6 @@ def kpi_analyze( result_data = json.load(f) # Convert to YAML for better readability - import yaml - yaml_output = yaml.dump(result_data, default_flow_style=False, indent=2) click.echo("\n📊 Analysis Results:") click.echo("=" * 50) From 9645f715bbfb0d861dfbc421b219236119f4b555 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:53:05 +0200 Subject: [PATCH 038/344] [caliper] orchestration: launch caliper visualize as an external process --- projects/caliper/orchestration/cli_builder.py | 11 +- projects/caliper/orchestration/postprocess.py | 149 +++++++++++------- 2 files changed, 100 insertions(+), 60 deletions(-) diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 1563631de..0e479c86a 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -112,7 +112,16 @@ def build_visualize_command( cmd.extend(["--report-group", config.visualize.report_group]) if config.visualize.visualize_config: - cmd.extend(["--visualize-config", str(config.visualize.visualize_config)]) + # Resolve the visualize config path properly (same logic as _resolve_visualize_config_path) + viz_config_path = config.visualize.visualize_config + if viz_config_path: + from projects.core.library import env + + viz_path = Path(viz_config_path).expanduser() + if not viz_path.is_absolute(): + # If relative, resolve relative to FORGE_HOME like the original does + viz_path = env.FORGE_HOME / viz_path + cmd.extend(["--visualize-config", str(viz_path.resolve())]) # Include/exclude labels for label in config.visualize.include_labels: diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 36b64b230..aeffc74eb 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -24,12 +24,12 @@ get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, - run_visualize_entrypoint, s3_export_entrypoint, s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( build_parse_command, + build_visualize_command, handle_caliper_output_and_completion, log_caliper_start_banner, ) @@ -47,9 +47,7 @@ log_ai_data_command, log_artifacts_to_kpis_command, log_kpis_to_csv_command, - log_parse_command, log_s3_export_command, - log_visualize_command, step_logging, ) from projects.core.library import env @@ -795,8 +793,6 @@ def _run_parse_step(self) -> None: import subprocess import tempfile - import yaml - # Create status file for CLI output with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: status_file = Path(status_f.name) @@ -831,7 +827,9 @@ def _run_parse_step(self) -> None: ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, "parse") + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, "parse" + ) if result.returncode == 0 and status_data and status_data.get("success", False): self._add_step( @@ -885,62 +883,74 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - with step_logging("caliper_visualize", self.step_logs_dir) as log_file: - try: - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path - ) + import subprocess + import tempfile - viz_cfg_path = _resolve_visualize_config_path( - self.config.visualize.visualize_config, - artifact_tree=self.tree_root, + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Resolve visualize output directory + if self.visualize_output_dir is not None: + output_dir = self.visualize_output_dir.expanduser().resolve() + logger.info(f"Using explicit visualize output directory: {output_dir}") + else: + output_dir = _resolve_visualize_output_dir( + self.config.visualize.output_dir, + ) + logger.info( + f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" ) - if self.visualize_output_dir is not None: - output_dir = self.visualize_output_dir.expanduser().resolve() - logger.info(f"Using explicit visualize output directory: {output_dir}") - else: - output_dir = _resolve_visualize_output_dir( - self.config.visualize.output_dir, - ) - logger.info( - f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" - ) + # Build CLI command + command = build_visualize_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not self.config.parse.no_cache, + ) - # Log command to reproduce this step - log_visualize_command( - base_dir=self.tree_root, - plugin_module=mod_str, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, - ) + # Log start banner and save command script + script_path = env.ARTIFACT_DIR / "visualize.sh" + log_caliper_start_banner(command, script_path, "visualize") - paths = run_visualize_entrypoint( - base_dir=self.tree_root, - plugin_module=mod_str, - plugin=plugin, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, - cache_path=self.cache_path, - ) + # Create log file path for stdout/stderr capture with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + + step_index = _get_next_step_index(self.step_logs_dir) + log_file = self.step_logs_dir / f"{step_index:03d}__caliper_visualize.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Execute CLI command with output capture + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, "visualize" + ) - # Convert paths to relative paths from output_dir + if result.returncode == 0 and status_data and status_data.get("success", False): + # Get output files and paths from status + output_files = status_data.get("output_files", []) + + # Convert absolute paths to relative paths from output_dir if needed relative_paths = [] - for path in paths: + for path in output_files: try: path_obj = Path(path) - relative_path = path_obj.relative_to(output_dir) - relative_paths.append(str(relative_path)) + if path_obj.is_absolute(): + relative_path = path_obj.relative_to(output_dir) + relative_paths.append(str(relative_path)) + else: + relative_paths.append(str(path)) except ValueError: # If path is not under output_dir, keep as-is relative_paths.append(str(path)) @@ -956,28 +966,49 @@ def _run_visualize_step(self) -> None: "visualize", { "status": "success", - "plugin_module": mod_str, + "plugin_module": status_data.get("plugin_module", "unknown"), "output_files": relative_paths, "output_dir": relative_output_dir, + "generated_files": status_data.get("generated_files", len(relative_paths)), "completed_at": time.time(), }, log_file, ) - - except Exception as e: # noqa: BLE001 + else: self.visualize_failed = True - logger.exception("Caliper visualize failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper visualize failed: %s", error_msg) self._add_step( "visualize", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, log_file, ) + except Exception as e: # noqa: BLE001 + self.visualize_failed = True + logger.exception("Visualize step execution failed") + self._add_step( + "visualize", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_kpi_and_ai_data_steps(self) -> None: """Execute KPI generation, CSV export, KPI export, and AI evaluation steps.""" if not self.config.kpi.enabled: From 811e50f97fc2c91a45559a376ee94746fac37c04 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:11:56 +0200 Subject: [PATCH 039/344] [caliper] cli: commands: include the status file for the kpi_generate_entrypoint command --- projects/caliper/cli/commands.py | 37 ++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 504427799..a20358eff 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -373,6 +373,9 @@ def ai_eval_export( @click.command("generate") @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_generate( ctx: click.Context, @@ -380,6 +383,7 @@ def kpi_generate( artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -389,14 +393,43 @@ def kpi_generate( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: run_kpi_generate( - base_dir=artifact_root, plugin_module=mod, plugin=plugin, output_path=output + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, ) + status_data = {"success": True, "output_file": str(output)} + click.echo(f"Generated {output}") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Generated {output}") @click.command("import") From 676e28bd80d3d403089470193e93c134c1dbbd93 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:13:33 +0200 Subject: [PATCH 040/344] [caliper] orchestration: launch caliper artifacts_to_kpis as an external process --- .../caliper/cli/orchestration_entrypoints.py | 19 +++ projects/caliper/orchestration/cli_builder.py | 45 ++++++++ projects/caliper/orchestration/postprocess.py | 108 ++++++++++++++---- 3 files changed, 150 insertions(+), 22 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 9d2481688..cde8b2e86 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -177,6 +177,25 @@ def analyze_kpis_entrypoint( ) +def kpi_generate_entrypoint( + base_dir: Path, + plugin_module: str, + plugin: Any, + output: Path, +) -> None: + """Entrypoint for orchestration to generate KPIs.""" + from projects.caliper.engine.kpi.generate import run_kpi_generate + + return run_kpi_generate( + base_dir=base_dir, + plugin_module=plugin_module, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, + ) + + def export_kpis_to_csv_entrypoint( plugin, kpi_records: list[dict[str, Any]], diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 0e479c86a..fd5bcdea6 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -65,6 +65,51 @@ def build_parse_command( return cmd +def build_kpi_generate_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi generate. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output file for KPI data + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "generate", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Generate-specific options + cmd.extend(["--output", str(output_file)]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def build_visualize_command( config: CaliperOrchestrationPostprocessConfig, tree_root: Path, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index aeffc74eb..e6d9d8088 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -11,6 +11,7 @@ import json import logging +import subprocess import time import traceback from pathlib import Path @@ -28,6 +29,7 @@ s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( + build_kpi_generate_command, build_parse_command, build_visualize_command, handle_caliper_output_and_completion, @@ -45,7 +47,6 @@ from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, log_ai_data_command, - log_artifacts_to_kpis_command, log_kpis_to_csv_command, log_s3_export_command, step_logging, @@ -55,6 +56,42 @@ logger = logging.getLogger(__name__) +def _execute_caliper_command( + command: list[str], + step_name: str, + script_file: Path, + log_file: Path, + status_file: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any]]: + """Execute a Caliper CLI command and return result and status data. + + Args: + command: CLI command arguments list + step_name: Name for logging banners (e.g., "PARSE", "VISUALIZE") + script_file: Path to save debug script + log_file: Path to save command output + status_file: Path to read status YAML from + + Returns: + Tuple of (subprocess result, parsed status data) + """ + # Log start banner and save script + log_caliper_start_banner(command, script_file, step_name) + + # Execute command with combined stdout/stderr + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name) + + return result, status_data + + def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. @@ -273,8 +310,9 @@ def _run_artifacts_to_kpis( output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, ) -> dict[str, Any]: - """Generate KPI JSON using the plugin's compute_kpis method.""" + """Generate KPI JSON using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} @@ -286,36 +324,62 @@ def _run_artifacts_to_kpis( } try: - # Write KPI JSON + # Prepare paths output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_artifacts_to_kpis_command( - base_dir=base_dir, - plugin_module=plugin_module, + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Create log file + log_file = output_dir / "000__caliper_kpi_generate.log" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, ) - kpis = plugin.compute_kpis(model) - output_file.parent.mkdir(parents=True, exist_ok=True) + # Execute command using generic function + script_file = env.ARTIFACT_DIR / "kpi_generate.sh" + result, status_data = _execute_caliper_command( + command=command, + step_name="KPI GENERATE", + script_file=script_file, + log_file=log_file, + status_file=status_file, + ) - # Transform flat KPI list into hierarchical structure - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - with open(output_file, "w") as f: - json.dump(hierarchical_data, f, indent=2) + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, output_dir), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } - logger.info( - f"Generated hierarchical KPI structure with {len(kpis)} total records in {output_file}" - ) - return { - "status": "success", - "kpi_count": len(kpis), - "output_file": _make_path_relative_to_base(output_file, output_dir), - "completed_at": time.time(), - } except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} From 9f1857eb6693401bb964d0fd5c5f65ea5bf9fc9a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:14:39 +0200 Subject: [PATCH 041/344] [caliper] orchestration: postprocess: factorize code --- projects/caliper/orchestration/postprocess.py | 109 +++++++----------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e6d9d8088..647ffeaba 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -59,24 +59,33 @@ def _execute_caliper_command( command: list[str], step_name: str, - script_file: Path, - log_file: Path, status_file: Path, -) -> tuple[subprocess.CompletedProcess, dict[str, Any]]: + step_logs_dir: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: """Execute a Caliper CLI command and return result and status data. Args: command: CLI command arguments list - step_name: Name for logging banners (e.g., "PARSE", "VISUALIZE") - script_file: Path to save debug script - log_file: Path to save command output + step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") status_file: Path to read status YAML from + step_logs_dir: Directory to create log file in Returns: - Tuple of (subprocess result, parsed status data) + Tuple of (subprocess result, parsed status data, log file path) """ + # Create log file path with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + + step_index = _get_next_step_index(step_logs_dir) + step_name_safe = step_name.replace(" ", "_") + log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Create script file for debugging + script_file = env.ARTIFACT_DIR / f"{step_name_safe}.sh" + # Log start banner and save script - log_caliper_start_banner(command, script_file, step_name) + log_caliper_start_banner(command, script_file, step_name.upper()) # Execute command with combined stdout/stderr result = subprocess.run( @@ -87,9 +96,9 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name) + status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name.upper()) - return result, status_data + return result, status_data, log_file def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: @@ -311,6 +320,7 @@ def _run_artifacts_to_kpis( plugin_module: str, base_dir: Path, manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: """Generate KPI JSON using fork/exec subprocess execution.""" @@ -331,9 +341,6 @@ def _run_artifacts_to_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "kpi_generate_status.yaml" - # Create log file - log_file = output_dir / "000__caliper_kpi_generate.log" - # Build CLI command command = build_kpi_generate_command( config=postprocess_config, @@ -344,13 +351,11 @@ def _run_artifacts_to_kpis( ) # Execute command using generic function - script_file = env.ARTIFACT_DIR / "kpi_generate.sh" - result, status_data = _execute_caliper_command( + result, status_data, log_file = _execute_caliper_command( command=command, - step_name="KPI GENERATE", - script_file=script_file, - log_file=log_file, + step_name="caliper kpi generate", status_file=status_file, + step_logs_dir=step_logs_dir, ) # Clean up temporary status file @@ -721,6 +726,7 @@ def _execute_orchestration(self) -> dict[str, Any]: else: self._run_visualize_step() logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") + self._run_kpi_and_ai_data_steps() logger.info( f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" @@ -854,7 +860,6 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - import subprocess import tempfile # Create status file for CLI output @@ -871,28 +876,12 @@ def _run_parse_step(self) -> None: use_cache=not self.config.parse.no_cache, ) - # Log start banner and save command script - script_path = env.ARTIFACT_DIR / "parse.sh" - log_caliper_start_banner(command, script_path, "parse") - - # Create log file path for stdout/stderr capture with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(self.step_logs_dir) - log_file = self.step_logs_dir / f"{step_index:03d}__caliper_parse.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Execute CLI command with output capture - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( - result, log_file, status_file, "parse" + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) if result.returncode == 0 and status_data and status_data.get("success", False): @@ -947,7 +936,6 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - import subprocess import tempfile # Create status file for CLI output @@ -977,28 +965,12 @@ def _run_visualize_step(self) -> None: use_cache=not self.config.parse.no_cache, ) - # Log start banner and save command script - script_path = env.ARTIFACT_DIR / "visualize.sh" - log_caliper_start_banner(command, script_path, "visualize") - - # Create log file path for stdout/stderr capture with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(self.step_logs_dir) - log_file = self.step_logs_dir / f"{step_index:03d}__caliper_visualize.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Execute CLI command with output capture - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( - result, log_file, status_file, "visualize" + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) if result.returncode == 0 and status_data and status_data.get("success", False): @@ -1177,7 +1149,14 @@ def _run_artifacts_to_kpis_step( if self.config.kpi.artifacts_to_kpis.enabled: with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: result = _run_artifacts_to_kpis( - self.config, plugin, model, output_dir, mod_str, self.tree_root + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, ) self._add_step("artifacts_to_kpis", result, log_file) self._check_step_result_and_set_failure("artifacts_to_kpis", result) From a514ea7db87054184b06376b4ea6018b5dd0c4a7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:42:03 +0200 Subject: [PATCH 042/344] [caliper] orchestration: launch caliper csv and ai-data as an external processes --- projects/caliper/cli/commands.py | 181 +++++++++- projects/caliper/cli/main.py | 2 + projects/caliper/cli/s3_export_cli.py | 52 ++- projects/caliper/engine/ai_eval.py | 12 +- projects/caliper/orchestration/cli_builder.py | 232 +++++++++++++ projects/caliper/orchestration/postprocess.py | 310 ++++++++++++------ 6 files changed, 671 insertions(+), 118 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index a20358eff..19dbabdd4 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -8,6 +8,7 @@ import click import yaml +from projects.caliper.cli.orchestration_entrypoints import export_kpis_to_csv_entrypoint from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export @@ -345,6 +346,10 @@ def list_reports_cmd( @click.command("ai-eval-export", hidden=True) @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option("--no-cache", is_flag=True, help="Disable parse cache") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def ai_eval_export( ctx: click.Context, @@ -352,6 +357,8 @@ def ai_eval_export( artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + no_cache: bool, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -361,12 +368,42 @@ def ai_eval_export( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: - run_ai_eval_export(base_dir=artifact_root, plugin_module=mod, plugin=plugin, output=output) + run_ai_eval_export( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=not no_cache, + ) + status_data = {"success": True, "output_file": str(output)} + click.echo(f"Exported {output}") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"ai-eval-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Exported {output}") # KPI commands @@ -432,6 +469,94 @@ def kpi_generate( sys.exit(3) +@click.command("csv-export") +@_workspace_cli_options +@click.option( + "--input", + "input_file", + type=click.Path(path_type=Path), + required=True, + help="Input KPI JSON file", +) +@click.option("--output", type=click.Path(path_type=Path), required=True, help="Output CSV file") +@click.option( + "--include-header-comments", is_flag=True, default=True, help="Include header comments in CSV" +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +@click.pass_context +def kpi_csv_export( + ctx: click.Context, + input_file: Path, + output: Path, + artifacts_dir: Path | None, + postprocess_config: Path | None, + plugin_module_override: str | None, + include_header_comments: bool, + status_file: Path | None, +) -> None: + _apply_workspace_cli_overrides( + ctx, + artifacts_dir=artifacts_dir, + postprocess_config=postprocess_config, + plugin_module_override=plugin_module_override, + ) + mod, plugin = _plugin_tuple(ctx) + + status_data = {"success": False} + + try: + # Read KPI JSON Lines file (one JSON object per line) + import json + + kpi_records = [] + with open(input_file, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: # Skip empty lines + kpi_record = json.loads(line) + kpi_records.append(kpi_record) + + # Export to CSV using the entrypoint + result_path = export_kpis_to_csv_entrypoint( + plugin=plugin, + kpi_records=kpi_records, + output_path=output, + include_header_comments=include_header_comments, + ) + + status_data = { + "success": True, + "output_file": str(result_path), + "kpi_count": len(kpi_records), + } + click.echo(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"kpi csv-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) + + @click.command("import") @click.option("--snapshot", type=click.Path(path_type=Path), required=True) @click.pass_context @@ -552,6 +677,9 @@ def kpi_analyze( "--aws-credentials-file", default="aws.credentials", help="Credentials file name within vault" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_s3_import( ctx: click.Context, @@ -565,10 +693,13 @@ def kpi_s3_import( vault: str, aws_credentials_file: str, verbose: bool, + status_file: Path | None, ) -> None: """Download historical KPI and analysis data from S3.""" from projects.core.library import vault as vault_lib + status_data = {"success": False} + try: # Ensure output directory exists output_dir.mkdir(parents=True, exist_ok=True) @@ -596,18 +727,54 @@ def kpi_s3_import( ) if result["status"] == "success": + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": result.get("files", []), + "file_count": len(result.get("files", [])), + } click.echo("✅ S3 import completed successfully") downloaded_files = result.get("files", []) click.echo(f"📄 Downloaded {len(downloaded_files)} files to {output_dir}") elif result["status"] == "empty": + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": [], + "file_count": 0, + "warning": "No objects found matching filters", + } click.echo(f"⚠️ No objects found matching filters in s3://{bucket}/{prefix}") else: - click.echo(f"❌ S3 import failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg} + click.echo(f"❌ S3 import failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) - except Exception as e: - click.echo(f"❌ S3 import failed: {e}", err=True) - sys.exit(2) + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"S3 import failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) # Artifacts commands diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 1bcb875d6..d67bc14e8 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -14,6 +14,7 @@ artifacts_export, artifacts_import, kpi_analyze, + kpi_csv_export, kpi_generate, kpi_import, kpi_s3_import, @@ -207,6 +208,7 @@ def run_cli() -> None: # Register KPI commands kpi_group.add_command(kpi_generate) +kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) kpi_group.add_command(kpi_analyze) kpi_group.add_command(kpi_s3_import) diff --git a/projects/caliper/cli/s3_export_cli.py b/projects/caliper/cli/s3_export_cli.py index 14853019f..82ca3643b 100644 --- a/projects/caliper/cli/s3_export_cli.py +++ b/projects/caliper/cli/s3_export_cli.py @@ -44,6 +44,9 @@ "--dry-run", is_flag=True, help="Show what would be uploaded without actually uploading" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def s3_export_cmd( ctx: click.Context, @@ -60,16 +63,24 @@ def s3_export_cmd( aws_credentials_file: str, dry_run: bool, verbose: bool, + status_file: Path | None, ) -> None: """Upload postprocess artifacts to S3.""" + import yaml + + status_data = {"success": False} + try: # Validate that at least one file/directory is provided if not any([kpis_file, csv_file, ai_data_dir, analysis_file]): + error_msg = "At least one file or directory must be specified" + status_data = {"success": False, "error": error_msg} click.echo("❌ Error: At least one file or directory must be specified", err=True) click.echo( " Use --kpis-file, --csv-file, --ai-data-dir, or --analysis-file", err=True ) - sys.exit(1) + if not status_file: + sys.exit(1) # Import S3 functions from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths @@ -116,7 +127,6 @@ def s3_export_cmd( ai_data_dir=ai_data_dir, analysis_file=analysis_file, bucket=bucket, - prefix=prefix, instance=instance, directory=directory, upload_id=upload_id, @@ -126,10 +136,18 @@ def s3_export_cmd( ) if result["status"] == "success": + status_data = { + "success": True, + "exported_path": result.get("exported_path", ""), + "uploaded_files": result.get("uploaded_files", 0), + "total_files": result.get("total_files", 0), + "dry_run": dry_run, + } if dry_run: click.echo("✅ Dry run completed - see upload plan") if "dry_run_file" in result: click.echo(f"📋 Upload plan saved to: {result['dry_run_file']}") + status_data["dry_run_file"] = result["dry_run_file"] else: click.echo("✅ S3 export completed successfully") if "uploaded_files" in result: @@ -138,9 +156,31 @@ def s3_export_cmd( if verbose and "exported_path" in result: click.echo(f"🌍 S3 location: {result['exported_path']}") else: - click.echo(f"❌ S3 export failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg} + click.echo(f"❌ S3 export failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) + + except Exception as e: # noqa: BLE001 + import traceback - except Exception as e: + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"❌ S3 export failed: {e}", err=True) - sys.exit(2) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + if not status_file: + sys.exit(2) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index 5a9d6953a..bafbf7447 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -27,6 +27,12 @@ def run_ai_eval_export( ) # Build base payload from plugin + if not hasattr(plugin, "build_ai_data_payload"): + raise AttributeError( + f"Plugin {plugin_module} does not support AI evaluation export. " + "Missing build_ai_data_payload method." + ) + build = plugin.build_ai_data_payload payload = build(model) @@ -75,7 +81,11 @@ def _export_test_entries_with_artifacts_engine( exported_entries = [] # Get the specific files we want to copy from the plugin - target_files = plugin.get_ai_eval_artifact_files(model) + if hasattr(plugin, "get_ai_eval_artifact_files"): + target_files = plugin.get_ai_eval_artifact_files(model) + else: + # Fallback: copy common test artifacts + target_files = ["*.log", "*.json", "*.yaml", "*.yml"] for idx, record in enumerate(model.unified_result_records): # Create directory for this test entry diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index fd5bcdea6..bfb78c716 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -184,6 +184,238 @@ def build_visualize_command( return cmd +def build_kpi_csv_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + input_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi csv-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + input_file: Input KPI JSON file + output_file: Output CSV file + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "csv-export", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # CSV export specific options + cmd.extend(["--input", str(input_file)]) + cmd.extend(["--output", str(output_file)]) + + if config.kpi.kpis_to_csv.include_header_comments: + cmd.append("--include-header-comments") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_ai_eval_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper ai-eval-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output directory for AI eval data + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "ai-eval-export", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # AI eval export specific options + cmd.extend(["--output", str(output_file)]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_import_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + output_dir: Path, +) -> list[str]: + """Build CLI command for caliper kpi s3-import. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + output_dir: Local output directory for downloads + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "s3-import", + ] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory + from projects.caliper.cli.s3_export import build_s3_prefix + + import_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if import_prefix: + cmd.extend(["--prefix", import_prefix]) + + # Create subdirectory for historical data (matches original behavior) + import_dir = output_dir / s3_config.import_.output_dir + cmd.extend(["--output-dir", str(import_dir)]) + + # Import options + if s3_config.import_.include_kpis_json: + cmd.append("--include-kpis-json") + if s3_config.import_.include_kpis_csv: + cmd.append("--include-kpis-csv") + if s3_config.import_.include_ai_data: + cmd.append("--include-ai-data") + + if s3_config.import_.max_downloads: + cmd.extend(["--max-downloads", str(s3_config.import_.max_downloads)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_export_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> list[str]: + """Build CLI command for caliper kpi s3-export. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + kpis_file: Optional KPIs JSON file + csv_file: Optional CSV file + ai_data_dir: Optional AI data directory + analysis_file: Optional analysis file + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "s3-export", + ] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory if available + from projects.caliper.cli.s3_export import build_s3_prefix + + export_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if export_prefix: + cmd.extend(["--prefix", export_prefix]) + + if s3_config.instance: + cmd.extend(["--instance", s3_config.instance]) + + if s3_config.directory: + cmd.extend(["--directory", s3_config.directory]) + + # File options + if kpis_file: + cmd.extend(["--kpis-file", str(kpis_file)]) + if csv_file: + cmd.extend(["--csv-file", str(csv_file)]) + if ai_data_dir: + cmd.extend(["--ai-data-dir", str(ai_data_dir)]) + if analysis_file: + cmd.extend(["--analysis-file", str(analysis_file)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def save_command_script(command: list[str], script_path: Path) -> None: """Save executed command to a shell script for debugging. diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 647ffeaba..ad608de18 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -21,16 +21,17 @@ from projects.caliper.cli.orchestration_entrypoints import ( analyze_kpis_entrypoint, - export_kpis_to_csv_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, - s3_export_entrypoint, - s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, + build_s3_export_command, + build_s3_import_command, build_visualize_command, handle_caliper_output_and_completion, log_caliper_start_banner, @@ -46,9 +47,6 @@ ) from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, - log_ai_data_command, - log_kpis_to_csv_command, - log_s3_export_command, step_logging, ) from projects.core.library import env @@ -96,7 +94,9 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name.upper()) + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, step_name.upper() + ) return result, status_data, log_file @@ -368,7 +368,7 @@ def _run_artifacts_to_kpis( if status_data.get("success"): return { "status": "success", - "output_file": _make_path_relative_to_base(output_file, output_dir), + "output_file": str(output_file), "completed_at": time.time(), } else: @@ -388,7 +388,7 @@ def _run_artifacts_to_kpis( return {"status": "failed", "error": str(e), "completed_at": time.time()} -# _run_s3_import function removed - now using s3_import_entrypoint directly +# _run_s3_import function removed - now using fork/exec subprocess execution directly def _run_artifacts_to_ai_data( @@ -398,55 +398,65 @@ def _run_artifacts_to_ai_data( output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export AI evaluation payload with structured directories and copied artifacts.""" + """Export AI evaluation payload using fork/exec subprocess execution.""" try: - # Always log the step header, even for skipped operations + # Create AI data directory and output file path ai_data_dir = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + ai_data_dir.mkdir(parents=True, exist_ok=True) output_file = ai_data_dir / "ai_data_payload.json" - log_ai_data_command( - base_dir=base_dir, - plugin_module=plugin_module, + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, + use_cache=True, # Default to using cache in orchestration ) - if not hasattr(plugin, "build_ai_data_payload"): + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): return { - "status": "skipped", - "reason": "plugin does not support AI evaluation", + "status": "success", + "output_file": status_data.get("output_file", str(ai_data_dir)), + "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), } - # Create AI evaluation directory structure using configured output directory - ai_data_dir.mkdir(parents=True, exist_ok=True) - - # Build payload from plugin - payload = plugin.build_ai_data_payload(model) - - # Export structured test entries with artifact copying - exported_entries = _export_test_entries_with_artifacts(model, ai_data_dir, base_dir, plugin) - - # Add exported entries info to payload - payload["exported_test_entries"] = exported_entries - - # Write main AI eval payload - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, "w") as f: - json.dump(payload, f, indent=2) - - logger.info(f"Generated AI evaluation payload in {output_file}") - logger.info(f"Exported {len(exported_entries)} test entries with artifacts") - - return { - "status": "success", - "output_file": _make_path_relative_to_base(output_file, output_dir), - "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), - "exported_entries": len(exported_entries), - "completed_at": time.time(), - } except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} @@ -566,8 +576,11 @@ def _run_kpis_to_csv( model, output_dir: Path, kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export KPI data to CSV format using the plugin's compute_kpis method.""" + """Export KPI data to CSV format using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} @@ -579,36 +592,59 @@ def _run_kpis_to_csv( } try: - # Compute KPIs from the model - kpi_records = plugin.compute_kpis(model) - # Create output file path output_file = output_dir / postprocess_config.kpi.kpis_to_csv.output output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_kpis_to_csv_command( - input_path=kpi_json_path, - output_path=output_file, + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" + + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + input_file=kpi_json_path, + output_file=output_file, ) - # Use Caliper entrypoint for CSV export - result_path = export_kpis_to_csv_entrypoint( - plugin=plugin, - kpi_records=kpi_records, - output_path=output_file, - include_header_comments=postprocess_config.kpi.kpis_to_csv.include_header_comments, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, ) - logger.info(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") - return { - "status": "success", - "kpi_count": len(kpi_records), - "output_file": _make_path_relative_to_base(result_path, output_dir), - "completed_at": time.time(), - } + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "kpi_count": status_data.get("kpi_count", 0), + "output_file": str(output_file), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} @@ -1186,7 +1222,16 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: # Path to the JSON file for reference in command logging kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - result = _run_kpis_to_csv(self.config, plugin, model, output_dir, kpi_json_path) + result = _run_kpis_to_csv( + self.config, + plugin, + model, + output_dir, + kpi_json_path, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) self._add_step("kpis_to_csv", result, log_file) if result.get("status") == "failed": # CSV export failure doesn't affect overall status - it's supplementary @@ -1210,7 +1255,14 @@ def _run_artifacts_to_ai_data_step( with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: try: result = _run_artifacts_to_ai_data( - self.config, plugin, model, output_dir, mod_str, self.tree_root + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, ) self._add_step("artifacts_to_ai_data", result, log_file) @@ -1241,25 +1293,59 @@ def _run_s3_import_step(self, output_dir: Path) -> None: with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: try: - result = s3_import_entrypoint(self.config, output_dir) + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_import_status.yaml" + + # Build CLI command + command = build_s3_import_command( + config=self.config, + status_file=status_file, + output_dir=output_dir, + ) - # Standardize field names and convert paths to relative - if "import_dir" in result: - # Rename import_dir to output_dir and make relative - result["output_dir"] = _make_path_relative_to_base( - result.pop("import_dir"), output_dir - ) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 import", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("s3_import", result, log_file) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + # Get the actual import directory (where files were downloaded) + import_dir = output_dir / self.config.s3.import_.output_dir + step_result = { + "status": "success", + "output_dir": str(import_dir), + "file_count": status_data.get("file_count", 0), + "completed_at": time.time(), + } + if status_data.get("warning"): + step_result["warning"] = status_data["warning"] + else: + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + + self._add_step("s3_import", step_result, log_file) logger.info("S3 import result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info(json.dumps(step_result, indent=2, default=str)) - self._check_step_result_and_set_failure("s3_import", result) + self._check_step_result_and_set_failure("s3_import", step_result) except Exception as e: logger.exception("S3 import failed") - step_result = {"status": "failed", "error": str(e)} + step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) @@ -1394,45 +1480,61 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ): analysis_file = output_dir / analyze_step["output_file"] - # Log the CLI command to reproduce this step - s3_parent_config = self.config.s3 - export_path = f"{s3_parent_config.instance}/{s3_parent_config.directory}" + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_export_status.yaml" - log_s3_export_command( - bucket=s3_parent_config.bucket, - export_path=export_path, + # Build CLI command + command = build_s3_export_command( + config=self.config, + status_file=status_file, kpis_file=kpis_file, csv_file=csv_file, ai_data_dir=ai_data_dir, analysis_file=analysis_file, ) - result = s3_export_entrypoint( - self.config, - output_dir, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 export", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) - self._add_step("s3_export", result, log_file) - if result.get("status") == "success": + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + step_result = { + "status": "success", + "exported_path": status_data.get("exported_path", ""), + "uploaded_files": status_data.get("uploaded_files", 0), + "total_files": status_data.get("total_files", 0), + "completed_at": time.time(), + } + # Format status for better readability - duration = result.get("duration", 0) - uploaded_files = result.get("uploaded_files", 0) - failed_files = result.get("failed_files", 0) - total_files = result.get("total_files", 0) - s3_path = result.get("exported_path", "") + uploaded_files = step_result.get("uploaded_files", 0) + total_files = step_result.get("total_files", 0) + s3_path = step_result.get("exported_path", "") logger.info( - f"S3 export completed successfully in {duration}s: " - f"{uploaded_files}/{total_files} files uploaded" - f"{f' ({failed_files} failed)' if failed_files > 0 else ''} " - f"to {s3_path}" + f"S3 export completed successfully: " + f"{uploaded_files}/{total_files} files uploaded to {s3_path}" ) else: - self._check_step_result_and_set_failure("s3_export", result) + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + self._check_step_result_and_set_failure("s3_export", step_result) + + self._add_step("s3_export", step_result, log_file) except Exception as e: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" From 3a5c54fa797c1d8a23f12fd3c53acbc7fca92fc6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 13:45:55 +0200 Subject: [PATCH 043/344] [caliper] orchestration: launch caliper analyse_kpis as an external process --- projects/caliper/cli/commands.py | 108 ++++++++++++++++++ projects/caliper/cli/main.py | 2 + projects/caliper/orchestration/cli_builder.py | 48 ++++++++ projects/caliper/orchestration/postprocess.py | 104 +++++++++++++---- 4 files changed, 239 insertions(+), 23 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 19dbabdd4..9cfd9ffaf 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -660,6 +660,114 @@ def kpi_analyze( sys.exit(3) +@click.command("analyse-kpis") +@_workspace_cli_options +@click.option( + "--output", + type=click.Path(path_type=Path), + required=True, + help="Output file for analysis results", +) +@click.option( + "--current-kpis-file", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Path to current KPIs JSON file", +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +@click.pass_context +def analyse_kpis_cmd( + ctx: click.Context, + output: Path, + current_kpis_file: Path, + artifacts_dir: Path | None, + postprocess_config: Path | None, + plugin_module_override: str | None, + status_file: Path | None, +) -> None: + """Analyze KPIs for orchestration (fork/exec).""" + import yaml + + _apply_workspace_cli_overrides( + ctx, + artifacts_dir=artifacts_dir, + postprocess_config=postprocess_config, + plugin_module_override=plugin_module_override, + ) + mod, plugin = _plugin_tuple(ctx) + base_dir: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + + try: + # Load manifest to get postprocess configuration + from projects.caliper.engine.plugin_config import resolve_manifest_path + from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, + ) + + manifest_path, manifest_data = resolve_manifest_path( + base_dir=base_dir, postprocess_config=_root_obj(ctx).get("postprocess_config") + ) + + # Create postprocess config object from manifest data + postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( + manifest_data.get("postprocess", {}) if manifest_data else {} + ) + + output_dir = output.parent + + # Run analysis using the entrypoint + from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint + + result = analyze_kpis_entrypoint( + postprocess_config=postprocess_config_obj, + plugin_module=mod, + base_dir=base_dir, + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + + status_data = { + "success": result.get("status") == "success", + "output_file": result.get("output_file"), + } + + if result.get("status") == "success": + click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") + else: + error_msg = result.get("error", "Unknown error") + status_data["error"] = error_msg + click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) + + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"❌ analyse-kpis failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + if not status_file: + sys.exit(2) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) + + @click.command("s3-import") @click.option("--bucket", required=True, help="S3 bucket name") @click.option("--prefix", default="", help="S3 object prefix/path") diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index d67bc14e8..0cccaeca4 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -11,6 +11,7 @@ from projects.caliper.cli.commands import ( ai_eval_export, + analyse_kpis_cmd, artifacts_export, artifacts_import, kpi_analyze, @@ -211,6 +212,7 @@ def run_cli() -> None: kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) kpi_group.add_command(kpi_analyze) +kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) # Register s3-export command under kpi group diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index bfb78c716..4018e6f95 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -346,6 +346,54 @@ def build_s3_import_command( return cmd +def build_analyse_kpis_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + current_kpis_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi analyse-kpis. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output file for analysis results + current_kpis_file: Current KPIs JSON file + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "analyse-kpis", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Analyse-kpis specific options + cmd.extend(["--output", str(output_file)]) + cmd.extend(["--current-kpis-file", str(current_kpis_file)]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def build_s3_export_command( config: CaliperOrchestrationPostprocessConfig, status_file: Path, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ad608de18..57339dab4 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -20,13 +20,13 @@ from pydantic import ValidationError from projects.caliper.cli.orchestration_entrypoints import ( - analyze_kpis_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, + build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -1349,6 +1349,71 @@ def _run_s3_import_step(self, output_dir: Path) -> None: self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) + +def _run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return {"status": "disabled", "reason": "analyze disabled", "completed_at": time.time()} + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": str(output_file), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + + except Exception as e: + logger.exception("KPI analysis failed in _run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time()} + def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: """Execute the KPI analysis step.""" if not self.config.analyze.enabled: @@ -1390,32 +1455,25 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: return with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: - try: - # Convert relative path to absolute for analyze function - current_kpis_path = output_dir / current_kpis_file - - result = analyze_kpis_entrypoint( - self.config, plugin_module, self.tree_root, output_dir, current_kpis_path - ) - - # Make output_file path relative - if "output_file" in result: - result["output_file"] = _make_path_relative_to_base( - result["output_file"], output_dir - ) + # current_kpis_file is now an absolute path (updated from relative path fix) + current_kpis_path = Path(current_kpis_file) - self._add_step("analyse_kpis", result, log_file) + result = _run_analyse_kpis( + postprocess_config=self.config, + output_dir=output_dir, + plugin_module=plugin_module, + base_dir=self.tree_root, + manifest_path=self.manifest_path, + current_kpis_file=current_kpis_path, + step_logs_dir=self.step_logs_dir, + ) - logger.info("KPI analysis result:") - logger.info(json.dumps(result, indent=2, default=str)) + self._add_step("analyse_kpis", result, log_file) - self._check_step_result_and_set_failure("analyse_kpis", result) + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) - except Exception as e: - logger.exception("KPI analysis failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("analyse_kpis", step_result, log_file) - self._check_step_result_and_set_failure("analyse_kpis", step_result) + self._check_step_result_and_set_failure("analyse_kpis", result) def _run_s3_export_step(self, output_dir: Path) -> None: """Execute the S3 export step.""" From e7e59188678cf2dc00455bb651daf4f7bcdf22e5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:06:11 +0200 Subject: [PATCH 044/344] [caliper] Improve --- projects/caliper/cli/commands.py | 153 +++-- .../caliper/cli/orchestration_entrypoints.py | 43 +- projects/caliper/cli/s3_import.py | 18 +- projects/caliper/engine/ai_eval.py | 81 ++- projects/caliper/engine/kpi/analyze.py | 25 +- projects/caliper/orchestration/cli_builder.py | 89 +-- projects/caliper/orchestration/postprocess.py | 634 +++++++++--------- .../caliper/orchestration/step_logging.py | 371 +--------- 8 files changed, 559 insertions(+), 855 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 9cfd9ffaf..583f20b29 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -147,19 +147,32 @@ def parse_cmd( show_parameter_matrix=show_matrix, ) + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + status.update( { "success": True, "plugin_module": mod, "parsed_records": len(model.unified_result_records), + "test_directories": test_directories, + "test_directories_count": len(test_directories), "cache_ref": str(model.parse_cache_ref) if model.parse_cache_ref else None, } ) click.echo( - f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" + f"Parsed {len(model.unified_result_records)} record(s) from {len(test_directories)} test directories; cache={model.parse_cache_ref}" ) + # Show test directories for user reference + if test_directories: + click.echo("📁 Test directories found:") + for test_dir in test_directories: + click.echo(f" • {test_dir}") + else: + click.echo("⚠️ No test directories found") + except Exception as e: # noqa: BLE001 status.update( { @@ -168,15 +181,15 @@ def parse_cmd( } ) click.echo(f"parse failed: {e}", err=True) - - # Write status file if requested - if status_file: - try: - status_file.parent.mkdir(parents=True, exist_ok=True) - with open(status_file, "w", encoding="utf-8") as f: - yaml.dump(status, f, default_flow_style=False, sort_keys=False) - except Exception as e: - click.echo(f"Failed to write status file {status_file}: {e}", err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) # Exit with non-zero code on failure if not status["success"]: @@ -273,15 +286,15 @@ def visualize_cmd( click.echo(f"visualize failed: {e}", err=True) click.echo("Full traceback:", err=True) click.echo(traceback.format_exc(), err=True) - - # Write status file if requested - if status_file: - try: - status_file.parent.mkdir(parents=True, exist_ok=True) - with open(status_file, "w", encoding="utf-8") as f: - yaml.dump(status, f, default_flow_style=False, sort_keys=False) - except Exception as e: - click.echo(f"Failed to write status file {status_file}: {e}", err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) # Exit with non-zero code on failure if not status["success"]: @@ -661,7 +674,6 @@ def kpi_analyze( @click.command("analyse-kpis") -@_workspace_cli_options @click.option( "--output", type=click.Path(path_type=Path), @@ -674,62 +686,70 @@ def kpi_analyze( required=True, help="Path to current KPIs JSON file", ) +@click.option( + "--historical-kpis-dir", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Directory containing historical KPI files", +) +@click.option( + "--plugin", + "plugin_module", + metavar="MODULE", + required=True, + help="Plugin module for KPI definitions and analysis rules", +) @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) -@click.pass_context def analyse_kpis_cmd( - ctx: click.Context, output: Path, current_kpis_file: Path, - artifacts_dir: Path | None, - postprocess_config: Path | None, - plugin_module_override: str | None, + historical_kpis_dir: Path, + plugin_module: str, status_file: Path | None, ) -> None: """Analyze KPIs for orchestration (fork/exec).""" import yaml - _apply_workspace_cli_overrides( - ctx, - artifacts_dir=artifacts_dir, - postprocess_config=postprocess_config, - plugin_module_override=plugin_module_override, - ) - mod, plugin = _plugin_tuple(ctx) - base_dir: Path = _root_obj(ctx)["base_dir"] - status_data = {"success": False} try: - # Load manifest to get postprocess configuration - from projects.caliper.engine.plugin_config import resolve_manifest_path + # Create a minimal postprocess config for analysis from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) - manifest_path, manifest_data = resolve_manifest_path( - base_dir=base_dir, postprocess_config=_root_obj(ctx).get("postprocess_config") - ) + # Create config with analyze enabled and using provided directories + postprocess_config_dict = { + "analyze": { + "enabled": True, + "output": output.name, # Use the provided output filename + "historical_kpis": str(historical_kpis_dir), # Use provided historical dir + } + } - # Create postprocess config object from manifest data postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( - manifest_data.get("postprocess", {}) if manifest_data else {} + postprocess_config_dict ) output_dir = output.parent - # Run analysis using the entrypoint + # Run analysis using the entrypoint (without base_dir since we have direct paths) from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint result = analyze_kpis_entrypoint( postprocess_config=postprocess_config_obj, - plugin_module=mod, - base_dir=base_dir, + plugin_module=plugin_module, + base_dir=output_dir, # Use output dir as base since we have absolute paths output_dir=output_dir, current_kpis_file=current_kpis_file, ) + # Check if the analysis function returned a proper result + if not isinstance(result, dict) or "status" not in result: + raise ValueError(f"Invalid result from analyze_kpis_entrypoint: {result}") + status_data = { "success": result.get("status") == "success", "output_file": result.get("output_file"), @@ -738,8 +758,14 @@ def analyse_kpis_cmd( if result.get("status") == "success": click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") else: - error_msg = result.get("error", "Unknown error") + # Show the full result for debugging if no clear error message + if "error" in result: + error_msg = result["error"] + else: + error_msg = f"Analysis failed with result: {result}" + status_data["error"] = error_msg + status_data.update(result) # Include all result data for debugging click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) if not status_file: sys.exit(1) @@ -834,28 +860,35 @@ def kpi_s3_import( max_downloads=max_downloads, ) + # Construct S3 path for status + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" + if result["status"] == "success": + downloaded_count = result.get("downloaded_files", 0) status_data = { "success": True, "output_dir": str(output_dir), - "downloaded_files": result.get("files", []), - "file_count": len(result.get("files", [])), + "downloaded_files": downloaded_count, + "file_count": downloaded_count, + "imported_path": s3_path, } click.echo("✅ S3 import completed successfully") - downloaded_files = result.get("files", []) - click.echo(f"📄 Downloaded {len(downloaded_files)} files to {output_dir}") - elif result["status"] == "empty": + click.echo(f"📄 Downloaded {downloaded_count} files to {output_dir}") + elif result["status"] == "warning": status_data = { "success": True, "output_dir": str(output_dir), - "downloaded_files": [], + "downloaded_files": 0, "file_count": 0, - "warning": "No objects found matching filters", + "warning": result.get("message", "No objects found matching filters"), + "imported_path": s3_path, } - click.echo(f"⚠️ No objects found matching filters in s3://{bucket}/{prefix}") + click.echo(f"⚠️ {result.get('message', 'No objects found matching filters')}") else: error_msg = result.get("error", "unknown error") - status_data = {"success": False, "error": error_msg} + status_data = {"success": False, "error": error_msg, "imported_path": s3_path} click.echo(f"❌ S3 import failed: {error_msg}", err=True) if not status_file: sys.exit(1) @@ -863,8 +896,18 @@ def kpi_s3_import( except Exception as e: # noqa: BLE001 import traceback + # Construct S3 path for error cases too + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" + full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} + status_data = { + "success": False, + "error": str(e), + "traceback": full_traceback, + "imported_path": s3_path, + } click.echo(f"S3 import failed: {e}", err=True) click.echo(f"Full traceback:\n{full_traceback}", err=True) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index cde8b2e86..5e319fd22 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -69,15 +69,13 @@ def s3_import_entrypoint( try: from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params - from projects.caliper.orchestration.step_logging import log_s3_import_command - - # Log command to reproduce this step - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.import_ # Build import prefix using instance + directory from projects.caliper.cli.s3_export import build_s3_prefix + s3_parent_config = postprocess_config.s3 + s3_config = postprocess_config.s3.import_ + import_prefix = build_s3_prefix( instance=s3_parent_config.instance, directory=s3_parent_config.directory, @@ -85,17 +83,6 @@ def s3_import_entrypoint( ) import_dir = output_dir / s3_config.output_dir - log_s3_import_command( - bucket=s3_parent_config.bucket, - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - output_dir=import_dir, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - # Use explicit parameters instead of config dict result = run_s3_import_with_explicit_params( bucket=s3_parent_config.bucket, @@ -168,13 +155,23 @@ def analyze_kpis_entrypoint( """Entrypoint for orchestration to call KPI analysis.""" from projects.caliper.engine.kpi.analyze import analyze_kpis - return analyze_kpis( - postprocess_config=postprocess_config, - plugin_module=plugin_module, - base_dir=base_dir, - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) + try: + result = analyze_kpis( + postprocess_config=postprocess_config, + plugin_module=plugin_module, + base_dir=base_dir, + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + return result + except Exception as e: + # Don't suppress the exception - let it bubble up with full traceback + import sys + import traceback + + print(f"❌ Exception in analyze_kpis_entrypoint: {e}", file=sys.stderr) + print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) + raise def kpi_generate_entrypoint( diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index db6349fd8..e5d3bc9c9 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -25,14 +25,13 @@ logger = logging.getLogger(__name__) -def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000) -> list[dict]: +def list_s3_objects(s3_client, bucket: str, prefix: str) -> list[dict]: """List objects in S3 bucket with given prefix. Args: s3_client: Configured boto3 S3 client bucket: S3 bucket name prefix: S3 key prefix to filter objects - max_objects: Maximum number of objects to return Returns: List of S3 object metadata dictionaries @@ -41,9 +40,8 @@ def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000 objects = [] paginator = s3_client.get_paginator("list_objects_v2") - for page in paginator.paginate( - Bucket=bucket, Prefix=prefix, PaginationConfig={"MaxItems": max_objects} - ): + # List ALL objects first - don't limit listing, limit downloads after filtering + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): if "Contents" in page: for obj in page["Contents"]: objects.append( @@ -273,7 +271,7 @@ def run_s3_import_with_explicit_params( } # List S3 objects (even in dry run mode to show what would be downloaded) - objects = list_s3_objects(s3_client, bucket, import_s3_prefix, max_downloads) + objects = list_s3_objects(s3_client, bucket, import_s3_prefix) logger.info(f"Found {len(objects)} total objects in s3://{bucket}/{import_s3_prefix}") @@ -293,6 +291,14 @@ def __init__(self): download_objects = filter_objects_for_download(objects, FilterConfig()) logger.info(f"After filtering: {len(download_objects)} objects match the import filters") + # Sort by last_modified timestamp in descending order (most recent first) + download_objects.sort(key=lambda obj: obj["last_modified"], reverse=True) + + # Apply max_downloads limit after filtering to get the most relevant files + if len(download_objects) > max_downloads: + logger.info(f"Limiting downloads to first {max_downloads} matching files") + download_objects = download_objects[:max_downloads] + if not download_objects: logger.warning("No objects match the import filters") return { diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index bafbf7447..f22cea9ad 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -103,29 +103,68 @@ def _export_test_entries_with_artifacts_engine( # Copy target files if they exist for target_file in target_files: - source_file = base_dir / target_file - if source_file.exists(): - # Create target directory structure in test entry dir - target_path = test_entry_dir / Path(target_file).name - target_path.parent.mkdir(parents=True, exist_ok=True) - - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "size_bytes": source_file.stat().st_size, - } + # Handle plugin-provided paths directly, expand fallback patterns + if hasattr(plugin, "get_ai_eval_artifact_files"): + # Plugin provided specific files - treat as direct paths + source_file = base_dir / target_file + if source_file.exists(): + # Create target directory structure in test entry dir + target_path = test_entry_dir / Path(target_file).name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(source_file, target_path) + entry_info["copied_files"].append( + { + "source": str(source_file), + "target": str(target_path), + "size_bytes": source_file.stat().st_size, + } + ) + logger.debug(f"Copied {source_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {source_file}: {e}") + entry_info["missing_files"].append( + {"file": str(source_file), "error": str(e)} + ) + else: + entry_info["missing_files"].append( + {"file": str(source_file), "error": "File does not exist"} ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) + # Fallback mode - expand glob patterns + import glob + + pattern_path = base_dir / target_file + matched_files = glob.glob(str(pattern_path)) + + if matched_files: + # Copy all matched files + for matched_file_str in matched_files: + matched_file = Path(matched_file_str) + target_path = test_entry_dir / matched_file.name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(matched_file, target_path) + entry_info["copied_files"].append( + { + "source": str(matched_file), + "target": str(target_path), + "size_bytes": matched_file.stat().st_size, + } + ) + logger.debug(f"Copied {matched_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {matched_file}: {e}") + entry_info["missing_files"].append( + {"file": str(matched_file), "error": str(e)} + ) + else: + # Pattern matched no files + entry_info["missing_files"].append( + {"file": str(pattern_path), "error": "Pattern matched no files"} + ) # Write entry metadata entry_metadata_file = test_entry_dir / "entry_metadata.json" diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 8b96ab459..40ae18680 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -156,16 +156,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Log command to reproduce this step - from projects.caliper.orchestration.step_logging import log_analyze_command - - log_analyze_command( - base_dir=base_dir, - plugin_module=plugin_module, - current_kpis_path=current_kpis_file, - historical_kpis_dir=historical_kpis_dir, - output_path=output_path, - ) # Call the core analysis function exit_code = run_kpi_analysis( @@ -237,12 +227,15 @@ def analyze_kpis( except Exception as e: error_msg = f"{type(e).__name__}: {str(e)}" logger.exception(f"Analysis step failed with {error_msg}") - return { - "status": "failed", - "error": error_msg, - "error_type": type(e).__name__, - "completed_at": time.time(), - } + + # Don't suppress the exception - provide detailed error info and re-raise + import traceback + + full_traceback = traceback.format_exc() + logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") + + # Re-raise the original exception so the caller can see the full details + raise RuntimeError(f"KPI analysis failed: {error_msg}") from e def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 4018e6f95..7e12fd4f4 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -7,7 +7,6 @@ import logging import subprocess -import sys from pathlib import Path from projects.caliper.orchestration.postprocess_config import ( @@ -16,6 +15,9 @@ logger = logging.getLogger(__name__) +# Base command for all Caliper CLI invocations +_CALIPER_BASE_CMD = ["caliper"] + def build_parse_command( config: CaliperOrchestrationPostprocessConfig, @@ -36,12 +38,7 @@ def build_parse_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "parse", - ] + cmd = _CALIPER_BASE_CMD + ["parse"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -84,13 +81,7 @@ def build_kpi_generate_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "generate", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "generate"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -131,12 +122,7 @@ def build_visualize_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "visualize", - ] + cmd = _CALIPER_BASE_CMD + ["visualize"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -151,7 +137,7 @@ def build_visualize_command( cmd.extend(["--output-dir", str(output_dir)]) if config.visualize.reports: - cmd.extend(["--reports", ",".join(config.visualize.reports)]) + cmd.extend(["--reports", config.visualize.reports]) if config.visualize.report_group: cmd.extend(["--report-group", config.visualize.report_group]) @@ -205,13 +191,7 @@ def build_kpi_csv_export_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "csv-export", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "csv-export"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -255,12 +235,7 @@ def build_ai_eval_export_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "ai-eval-export", - ] + cmd = _CALIPER_BASE_CMD + ["ai-eval-export"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -300,13 +275,7 @@ def build_s3_import_command( """ s3_config = config.s3 - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "s3-import", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-import"] # S3 configuration cmd.extend(["--bucket", s3_config.bucket]) @@ -353,40 +322,34 @@ def build_analyse_kpis_command( status_file: Path, output_file: Path, current_kpis_file: Path, + historical_kpis_dir: Path, ) -> list[str]: """Build CLI command for caliper kpi analyse-kpis. Args: config: Orchestration configuration - tree_root: Base directory for artifacts - manifest_path: Optional manifest file path + tree_root: Base directory for artifacts (unused, kept for compatibility) + manifest_path: Optional manifest file path (unused, kept for compatibility) status_file: Where to write status YAML output_file: Output file for analysis results current_kpis_file: Current KPIs JSON file + historical_kpis_dir: Directory containing historical KPI files Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "analyse-kpis", - ] - - # Workspace options - cmd.extend(["--artifacts-dir", str(tree_root)]) - - if manifest_path: - cmd.extend(["--postprocess-config", str(manifest_path)]) - - if config.plugin_module: - cmd.extend(["--plugin", config.plugin_module]) + cmd = _CALIPER_BASE_CMD + ["kpi", "analyse-kpis"] # Analyse-kpis specific options cmd.extend(["--output", str(output_file)]) cmd.extend(["--current-kpis-file", str(current_kpis_file)]) + cmd.extend(["--historical-kpis-dir", str(historical_kpis_dir)]) + + # Plugin module is required + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + else: + raise ValueError("Plugin module is required for KPI analysis") # Status file for orchestration cmd.extend(["--status-file", str(status_file)]) @@ -417,13 +380,7 @@ def build_s3_export_command( """ s3_config = config.s3 - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "s3-export", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-export"] # S3 configuration cmd.extend(["--bucket", s3_config.bucket]) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 57339dab4..024fb5e89 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -47,7 +47,6 @@ ) from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, - step_logging, ) from projects.core.library import env @@ -325,12 +324,18 @@ def _run_artifacts_to_kpis( """Generate KPI JSON using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } if not postprocess_config.kpi.artifacts_to_kpis.enabled: return { "status": "disabled", "reason": "kpi.artifacts_to_kpis disabled", "completed_at": time.time(), + "log_file": None, } try: @@ -370,12 +375,14 @@ def _run_artifacts_to_kpis( "status": "success", "output_file": str(output_file), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -385,7 +392,7 @@ def _run_artifacts_to_kpis( full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} # _run_s3_import function removed - now using fork/exec subprocess execution directly @@ -442,12 +449,14 @@ def _run_artifacts_to_ai_data( "output_file": status_data.get("output_file", str(ai_data_dir)), "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -457,7 +466,7 @@ def _run_artifacts_to_ai_data( full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} def _load_test_labels(test_dir: Path) -> dict[str, Any]: @@ -583,12 +592,18 @@ def _run_kpis_to_csv( """Export KPI data to CSV format using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } if not postprocess_config.kpi.kpis_to_csv.enabled: return { "status": "disabled", "reason": "kpi.kpis_to_csv disabled", "completed_at": time.time(), + "log_file": None, } try: @@ -630,12 +645,14 @@ def _run_kpis_to_csv( "kpi_count": status_data.get("kpi_count", 0), "output_file": str(output_file), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -645,7 +662,85 @@ def _run_kpis_to_csv( full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def _run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return { + "status": "disabled", + "reason": "analyze disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": str(output_file), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("KPI analysis failed in _run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} class CaliperPostprocessOrchestrator: @@ -927,6 +1022,8 @@ def _run_parse_step(self) -> None: "status": "success", "plugin_module": status_data.get("plugin_module", "unknown"), "record_count": status_data.get("parsed_records", 0), + "test_directories": status_data.get("test_directories", []), + "test_directories_count": status_data.get("test_directories_count", 0), "parse_cache_ref": status_data.get("cache_ref"), "completed_at": time.time(), }, @@ -1183,19 +1280,19 @@ def _run_artifacts_to_kpis_step( ) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: - with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: - result = _run_artifacts_to_kpis( - self.config, - plugin, - model, - output_dir, - mod_str, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("artifacts_to_kpis", result, log_file) - self._check_step_result_and_set_failure("artifacts_to_kpis", result) + result = _run_artifacts_to_kpis( + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_kpis", result, log_file) + self._check_step_result_and_set_failure("artifacts_to_kpis", result) else: self._add_step( "artifacts_to_kpis", @@ -1219,23 +1316,23 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: - # Path to the JSON file for reference in command logging - kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - result = _run_kpis_to_csv( - self.config, - plugin, - model, - output_dir, - kpi_json_path, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("kpis_to_csv", result, log_file) - if result.get("status") == "failed": - # CSV export failure doesn't affect overall status - it's supplementary - logger.warning("KPI CSV export failed but continuing execution") + # Path to the JSON file for reference in command logging + kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + result = _run_kpis_to_csv( + self.config, + plugin, + model, + output_dir, + kpi_json_path, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("kpis_to_csv", result, log_file) + if result.get("status") == "failed": + # CSV export failure doesn't affect overall status - it's supplementary + logger.warning("KPI CSV export failed but continuing execution") def _run_artifacts_to_ai_data_step( self, plugin: Any, model: Any, output_dir: Path, mod_str: str @@ -1252,31 +1349,31 @@ def _run_artifacts_to_ai_data_step( ) return - with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: - try: - result = _run_artifacts_to_ai_data( - self.config, - plugin, - model, - output_dir, - mod_str, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("artifacts_to_ai_data", result, log_file) + try: + result = _run_artifacts_to_ai_data( + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_ai_data", result, log_file) - logger.info("AI eval export result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info("AI eval export result:") + logger.info(json.dumps(result, indent=2, default=str)) - # Check if the result indicates failure or warning - self._check_step_result_and_set_failure("artifacts_to_ai_data", result) + # Check if the result indicates failure or warning + self._check_step_result_and_set_failure("artifacts_to_ai_data", result) - except Exception as e: - logger.exception("AI eval export failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("artifacts_to_ai_data", step_result, log_file) - self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) + except Exception as e: + logger.exception("AI eval export failed") + step_result = {"status": "failed", "error": str(e)} + self._add_step("artifacts_to_ai_data", step_result, None) + self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) def _run_s3_import_step(self, output_dir: Path) -> None: """Execute the S3 import step.""" @@ -1291,128 +1388,66 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: - try: - # Create temporary status file for subprocess communication - status_file = output_dir / "s3_import_status.yaml" - - # Build CLI command - command = build_s3_import_command( - config=self.config, - status_file=status_file, - output_dir=output_dir, - ) - - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper s3 import", - status_file=status_file, - step_logs_dir=self.step_logs_dir, - ) - - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass - - # Convert to expected format - if status_data.get("success"): - # Get the actual import directory (where files were downloaded) - import_dir = output_dir / self.config.s3.import_.output_dir - step_result = { - "status": "success", - "output_dir": str(import_dir), - "file_count": status_data.get("file_count", 0), - "completed_at": time.time(), - } - if status_data.get("warning"): - step_result["warning"] = status_data["warning"] - else: - step_result = { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } - - self._add_step("s3_import", step_result, log_file) - - logger.info("S3 import result:") - logger.info(json.dumps(step_result, indent=2, default=str)) - - self._check_step_result_and_set_failure("s3_import", step_result) - - except Exception as e: - logger.exception("S3 import failed") - step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} - self._add_step("s3_import", step_result, log_file) - self._check_step_result_and_set_failure("s3_import", step_result) - - -def _run_analyse_kpis( - postprocess_config: CaliperOrchestrationPostprocessConfig, - output_dir: Path, - plugin_module: str, - base_dir: Path, - manifest_path: Path | None, - current_kpis_file: Path, - step_logs_dir: Path, -) -> dict[str, Any]: - """Analyze KPIs using fork/exec subprocess execution.""" + try: + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_import_status.yaml" - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled", "completed_at": time.time()} + # Build CLI command + command = build_s3_import_command( + config=self.config, + status_file=status_file, + output_dir=output_dir, + ) - try: - # Prepare paths - output_file = output_dir / postprocess_config.analyze.output - output_file.parent.mkdir(parents=True, exist_ok=True) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 import", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - # Create temporary status file for subprocess communication - status_file = output_dir / "analyse_kpis_status.yaml" + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - # Build CLI command - command = build_analyse_kpis_command( - config=postprocess_config, - tree_root=base_dir, - manifest_path=manifest_path, - status_file=status_file, - output_file=output_file, - current_kpis_file=current_kpis_file, - ) + # Convert to expected format + if status_data.get("success"): + # Get the actual import directory (where files were downloaded) + import_dir = output_dir / self.config.s3.import_.output_dir + step_result = { + "status": "success", + "output_dir": str(import_dir), + "file_count": status_data.get("file_count", 0), + "completed_at": time.time(), + } + if status_data.get("warning"): + step_result["warning"] = status_data["warning"] + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] + else: + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper kpi analyse-kpis", - status_file=status_file, - step_logs_dir=step_logs_dir, - ) + self._add_step("s3_import", step_result, log_file) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass + logger.info("S3 import result:") + logger.info(json.dumps(step_result, indent=2, default=str)) - # Convert to expected format - if status_data.get("success"): - return { - "status": "success", - "output_file": str(output_file), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } + self._check_step_result_and_set_failure("s3_import", step_result) - except Exception as e: - logger.exception("KPI analysis failed in _run_analyse_kpis") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + except Exception as e: + logger.exception("S3 import failed") + step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} + self._add_step("s3_import", step_result, None) + self._check_step_result_and_set_failure("s3_import", step_result) def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: """Execute the KPI analysis step.""" @@ -1454,26 +1489,26 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: - # current_kpis_file is now an absolute path (updated from relative path fix) - current_kpis_path = Path(current_kpis_file) - - result = _run_analyse_kpis( - postprocess_config=self.config, - output_dir=output_dir, - plugin_module=plugin_module, - base_dir=self.tree_root, - manifest_path=self.manifest_path, - current_kpis_file=current_kpis_path, - step_logs_dir=self.step_logs_dir, - ) + # current_kpis_file is now an absolute path (updated from relative path fix) + current_kpis_path = Path(current_kpis_file) + + result = _run_analyse_kpis( + postprocess_config=self.config, + output_dir=output_dir, + plugin_module=plugin_module, + base_dir=self.tree_root, + manifest_path=self.manifest_path, + current_kpis_file=current_kpis_path, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("analyse_kpis", result, log_file) + log_file = result.pop("log_file", None) + self._add_step("analyse_kpis", result, log_file) - logger.info("KPI analysis result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) - self._check_step_result_and_set_failure("analyse_kpis", result) + self._check_step_result_and_set_failure("analyse_kpis", result) def _run_s3_export_step(self, output_dir: Path) -> None: """Execute the S3 export step.""" @@ -1488,132 +1523,131 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_export", self.step_logs_dir) as log_file: - try: - # Collect file paths from previous steps - kpis_file = None - csv_file = None - ai_data_dir = None - analysis_file = None - - # Get KPI JSON file from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if ( - artifacts_to_kpis_step - and artifacts_to_kpis_step.get("status") == "success" - and artifacts_to_kpis_step.get("output_file") - ): - kpis_file = output_dir / artifacts_to_kpis_step["output_file"] - - # Get CSV file from kpis_to_csv step - kpis_to_csv_step = self._get_step("kpis_to_csv") - if ( - kpis_to_csv_step - and kpis_to_csv_step.get("status") == "success" - and kpis_to_csv_step.get("output_file") - ): - csv_file = output_dir / kpis_to_csv_step["output_file"] - - # Get AI data directory from artifacts_to_ai_data step - ai_data_step = self._get_step("artifacts_to_ai_data") - logger.info(f"AI data step found: {ai_data_step}") - if ( - ai_data_step - and ai_data_step.get("status") == "success" - and ai_data_step.get("ai_data_dir") - ): - ai_data_dir = output_dir / ai_data_step["ai_data_dir"] - logger.info(f"AI data directory set to: {ai_data_dir}") - else: - logger.warning( - f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" - ) - - # Get analysis file from analyse_kpis step - analyze_step = self._get_step("analyse_kpis") - if ( - analyze_step - and analyze_step.get("status") in ("success", "warning") - and analyze_step.get("output_file") - ): - analysis_file = output_dir / analyze_step["output_file"] - - # Create temporary status file for subprocess communication - status_file = output_dir / "s3_export_status.yaml" - - # Build CLI command - command = build_s3_export_command( - config=self.config, - status_file=status_file, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, - ) - - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper s3 export", - status_file=status_file, - step_logs_dir=self.step_logs_dir, + try: + # Collect file paths from previous steps + kpis_file = None + csv_file = None + ai_data_dir = None + analysis_file = None + + # Get KPI JSON file from artifacts_to_kpis step + artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") + if ( + artifacts_to_kpis_step + and artifacts_to_kpis_step.get("status") == "success" + and artifacts_to_kpis_step.get("output_file") + ): + kpis_file = output_dir / artifacts_to_kpis_step["output_file"] + + # Get CSV file from kpis_to_csv step + kpis_to_csv_step = self._get_step("kpis_to_csv") + if ( + kpis_to_csv_step + and kpis_to_csv_step.get("status") == "success" + and kpis_to_csv_step.get("output_file") + ): + csv_file = output_dir / kpis_to_csv_step["output_file"] + + # Get AI data directory from artifacts_to_ai_data step + ai_data_step = self._get_step("artifacts_to_ai_data") + logger.info(f"AI data step found: {ai_data_step}") + if ( + ai_data_step + and ai_data_step.get("status") == "success" + and ai_data_step.get("ai_data_dir") + ): + ai_data_dir = output_dir / ai_data_step["ai_data_dir"] + logger.info(f"AI data directory set to: {ai_data_dir}") + else: + logger.warning( + f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass + # Get analysis file from analyse_kpis step + analyze_step = self._get_step("analyse_kpis") + if ( + analyze_step + and analyze_step.get("status") in ("success", "warning") + and analyze_step.get("output_file") + ): + analysis_file = output_dir / analyze_step["output_file"] - # Convert to expected format - if status_data.get("success"): - step_result = { - "status": "success", - "exported_path": status_data.get("exported_path", ""), - "uploaded_files": status_data.get("uploaded_files", 0), - "total_files": status_data.get("total_files", 0), - "completed_at": time.time(), - } + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_export_status.yaml" - # Format status for better readability - uploaded_files = step_result.get("uploaded_files", 0) - total_files = step_result.get("total_files", 0) - s3_path = step_result.get("exported_path", "") + # Build CLI command + command = build_s3_export_command( + config=self.config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) - logger.info( - f"S3 export completed successfully: " - f"{uploaded_files}/{total_files} files uploaded to {s3_path}" - ) - else: - step_result = { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } - self._check_step_result_and_set_failure("s3_export", step_result) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 export", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("s3_export", step_result, log_file) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - except Exception as e: - error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" + # Convert to expected format + if status_data.get("success"): + step_result = { + "status": "success", + "exported_path": status_data.get("exported_path", ""), + "uploaded_files": status_data.get("uploaded_files", 0), + "total_files": status_data.get("total_files", 0), + "completed_at": time.time(), + } - # Log programming errors more prominently - if isinstance(e, AttributeError | NameError | TypeError): - logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") - logger.exception("Full traceback for programming error:") - else: - logger.error(error_msg) - logger.exception("Full traceback:") + # Format status for better readability + uploaded_files = step_result.get("uploaded_files", 0) + total_files = step_result.get("total_files", 0) + s3_path = step_result.get("exported_path", "") + logger.info( + f"S3 export completed successfully: " + f"{uploaded_files}/{total_files} files uploaded to {s3_path}" + ) + else: step_result = { "status": "failed", - "error": error_msg, - "exception_type": type(e).__name__, + "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), } - self._add_step("s3_export", step_result, log_file) self._check_step_result_and_set_failure("s3_export", step_result) + self._add_step("s3_export", step_result, log_file) + + except Exception as e: + error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" + + # Log programming errors more prominently + if isinstance(e, AttributeError | NameError | TypeError): + logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") + logger.exception("Full traceback for programming error:") + else: + logger.error(error_msg) + logger.exception("Full traceback:") + + step_result = { + "status": "failed", + "error": error_msg, + "exception_type": type(e).__name__, + "completed_at": time.time(), + } + self._add_step("s3_export", step_result, None) + self._check_step_result_and_set_failure("s3_export", step_result) + def _compute_final_status(self) -> str: """Compute the final postprocessing status.""" # Debug logging to identify what's causing failures diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 7ef0f6d17..5354c48e0 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -237,372 +237,7 @@ def cleanup_step_logging(): del _step_local_handlers.file_handler -def _log_command_banner( - step_name: str, - command_line: str, - description: str = "", - step_args: dict[str, Any] | None = None, -) -> None: - """ - Log a command reproduction banner at the start of a step. - Args: - step_name: Name of the step (e.g., "caliper_parse") - command_line: The CLI command to reproduce this step - description: Brief description of what this step does - step_args: Dictionary of step arguments for reference - """ - # Use root logger to ensure step handler captures the banner - logger = logging.getLogger() - - LINE_WIDTH = 80 - - logger.info("") - logger.info("=" * LINE_WIDTH) - logger.info(f"CALIPER STEP: {step_name.upper()}") - if description: - logger.info(f"DESCRIPTION: {description}") - logger.info("=" * LINE_WIDTH) - logger.info(f"COMMAND TO REPRODUCE:\n{command_line}") - - if step_args: - logger.info("") - logger.info("STEP PARAMETERS:") - for key, value in step_args.items(): - if value is not None: - # Truncate very long values - value_str = str(value) - if len(value_str) > 100: - value_str = value_str[:97] + "..." - logger.info(f" {key}: {value_str}") - - logger.info("=" * LINE_WIDTH) - logger.info("") - - -def log_parse_command( - base_dir: Path, - plugin_module: str, - use_cache: bool, - manifest_path: Path | None = None, -) -> None: - """Log the CLI command to reproduce the parse step.""" - cache_flag = "" if use_cache else " --no-cache" - command = f'caliper parse --base-dir "{base_dir}" --plugin {plugin_module}{cache_flag}' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "use_cache": use_cache, - } - - # Only include manifest_path if it's provided and safe to log - if manifest_path: - step_args["manifest_path"] = str(manifest_path) - - _log_command_banner( - "caliper_parse", - command, - "Parse test results from artifact tree into unified model", - step_args, - ) - - -def log_visualize_command( - base_dir: Path, - plugin_module: str, - output_dir: Path, - reports_csv: str | None = None, - report_group: str | None = None, - visualize_config_path: Path | None = None, - include_pairs: tuple[str, ...] = (), - exclude_pairs: tuple[str, ...] = (), - use_cache: bool = True, -) -> None: - """Log the CLI command to reproduce the visualize step.""" - command_parts = [ - "caliper visualize", - f'--base-dir "{base_dir}"', - f'--output-dir "{output_dir}"', - f"--plugin {plugin_module}", - ] - - if not use_cache: - command_parts.append("--no-cache") - if reports_csv: - command_parts.append(f'--reports "{reports_csv}"') - if report_group: - command_parts.append(f'--report-group "{report_group}"') - if visualize_config_path: - command_parts.append(f'--visualize-config "{visualize_config_path}"') - - for include_pair in include_pairs: - command_parts.append(f'--include "{include_pair}"') - for exclude_pair in exclude_pairs: - command_parts.append(f'--exclude "{exclude_pair}"') - - command = " \\\n ".join(command_parts) - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_dir": str(output_dir), - "use_cache": use_cache, - } - - # Only include optional parameters if they're provided - if reports_csv: - step_args["reports_csv"] = reports_csv - if report_group: - step_args["report_group"] = report_group - if visualize_config_path: - step_args["visualize_config_path"] = str(visualize_config_path) - if include_pairs: - step_args["include_pairs"] = list(include_pairs) - if exclude_pairs: - step_args["exclude_pairs"] = list(exclude_pairs) - - _log_command_banner( - "caliper_visualize", - command, - "Generate visualization reports and plots from parsed data", - step_args, - ) - - -def log_artifacts_to_kpis_command( - base_dir: Path, - plugin_module: str, - output_file: Path, -) -> None: - """Log the CLI command to reproduce the KPI generation step.""" - command = f'caliper kpi generate --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_file": str(output_file), - } - - _log_command_banner( - "caliper_kpi_generate", command, "Generate KPI metrics from parsed test results", step_args - ) - - -def log_kpi_export_command( - input_path: Path, - target_system: str = "opensearch", -) -> None: - """Log the CLI command to reproduce the KPI export step.""" - command = f'caliper kpi export --input "{input_path}" --target {target_system}' - - step_args = { - "input_path": str(input_path), - "target_system": target_system, - } - - _log_command_banner( - "caliper_kpi_export", command, "Export KPI metrics to external monitoring system", step_args - ) - - -def log_ai_data_command( - base_dir: Path, - plugin_module: str, - output_file: Path, -) -> None: - """Log the CLI command to reproduce the AI evaluation export step.""" - command = f'caliper ai-eval export --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_file": str(output_file), - } - - _log_command_banner( - "caliper_ai_data_export", - command, - "Export AI evaluation payload with structured test directories and artifacts", - step_args, - ) - - -def log_s3_import_command( - bucket: str, - instance: str | None, - directory: str | None, - output_dir: Path, - include_kpis_json: bool = False, - include_kpis_csv: bool = False, - include_ai_data: bool = False, - max_downloads: int = 50, -) -> None: - """Log the CLI command to reproduce the S3 import step.""" - # Build prefix from instance and directory - from projects.caliper.cli.s3_export import build_s3_prefix - - prefix = build_s3_prefix(instance=instance, directory=directory, trailing_slash=False) - - command_parts = [ - f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' - ] - - # Add include flags if they are True (since default is False) - if include_kpis_json: - command_parts.append("--include-kpis-json") - if include_kpis_csv: - command_parts.append("--include-kpis-csv") - if include_ai_data: - command_parts.append("--include-ai-data") - if max_downloads != 50: - command_parts.append(f"--max-downloads {max_downloads}") - - command = " ".join(command_parts) - - step_args = { - "bucket": bucket, - "instance": instance, - "directory": directory, - "prefix": prefix, - "output_dir": str(output_dir), - "include_kpis_json": include_kpis_json, - "include_kpis_csv": include_kpis_csv, - "include_ai_data": include_ai_data, - "max_downloads": max_downloads, - } - - _log_command_banner( - "caliper_s3_import", - command, - "Import historical data from S3 for analysis", - step_args, - ) - - -def log_s3_export_command( - bucket: str, - export_path: str, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, -) -> None: - """Log the CLI command to reproduce the S3 export step.""" - command_parts = [f'caliper s3-export --bucket "{bucket}"'] - - # Add file paths - if kpis_file: - command_parts.append(f'--kpis-file "{kpis_file}"') - if csv_file: - command_parts.append(f'--csv-file "{csv_file}"') - if ai_data_dir: - command_parts.append(f'--ai-data-dir "{ai_data_dir}"') - if analysis_file: - command_parts.append(f'--analysis-file "{analysis_file}"') - - # Parse export_path once and use for both command and step_args - instance = None - directory = None - prefix = None - - if export_path: - # Parse the export path to get prefix, instance, directory - parts = export_path.split("/") - if len(parts) >= 3: - instance = parts[0] - directory = parts[1] - prefix = "/".join(parts[2:]) if len(parts) > 2 else "" - command_parts.append(f'--instance "{instance}"') - command_parts.append(f'--directory "{directory}"') - if prefix: - command_parts.append(f'--prefix "{prefix}"') - - command = " ".join(command_parts) - - step_args = { - "bucket": bucket, - "kpis_file": str(kpis_file) if kpis_file else None, - "csv_file": str(csv_file) if csv_file else None, - "ai_data_dir": str(ai_data_dir) if ai_data_dir else None, - "analysis_file": str(analysis_file) if analysis_file else None, - } - - # Add command line parameters to step_args (same values used in command) - if instance: - step_args["instance"] = instance - if directory: - step_args["directory"] = directory - if prefix: - step_args["prefix"] = prefix - - _log_command_banner( - "caliper_s3_export", - command, - "Export postprocess artifacts to S3", - step_args, - ) - - -def log_analyze_command( - base_dir: Path, - plugin_module: str, - current_kpis_path: Path | None = None, - historical_kpis_dir: Path | None = None, - output_path: Path | None = None, -) -> None: - """Log the CLI command to reproduce the analyze step.""" - - # Use the correct command structure: caliper kpi analyze - command = "caliper kpi analyze" - - # Add current KPI file path if provided - if current_kpis_path: - command += f' --current "{current_kpis_path}"' - - # Add the historical directory, not the specific file - if historical_kpis_dir: - command += f' --baseline-dir "{historical_kpis_dir}"' - - # Add output path if provided - if output_path: - command += f' --output "{output_path}"' - - # Add plugin module - command += f" --plugin {plugin_module}" - - # Add helpful context - command += "\n# Analyzes KPI for regressions" - command += "\n# The tool will automatically find the most recent kpis.json file in the baseline directory" - command += "\n# To re-run this analysis step independently, use the command above" - command += f"\n# Plugin: {plugin_module}" - command += f"\n# Base dir: {base_dir}" - - step_args = { - "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, - "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, - "output_path": str(output_path) if output_path else None, - "plugin_module": plugin_module, - "base_dir": str(base_dir), - } - - _log_command_banner( - "caliper_analyze", command, "Run regression analysis on test results", step_args - ) - - -def log_kpis_to_csv_command( - input_path: Path, - output_path: Path, -) -> None: - """Log the CLI command to reproduce the KPI CSV export step.""" - command = f'caliper kpi csv-export --input "{input_path}" --output "{output_path}"' - - step_args = { - "input_path": str(input_path), - "output_path": str(output_path), - } - - _log_command_banner( - "caliper_kpi_csv_export", command, "Export KPI metrics to CSV format", step_args - ) + + + From a55a40a7b261e053018bacdf910dfc9d57ab7c60 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:43:06 +0200 Subject: [PATCH 045/344] fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs --- .../base/workflows/task-forge-step.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index dcb29b30c..72e58e9c0 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,6 +13,8 @@ spec: type: string - name: kubeconfig-secret type: string + default: "fournos-clusterless-placeholder" + description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -31,6 +33,8 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig + - name: KUBECONFIG_SECRET_PARAM + value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -73,7 +77,24 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - export KUBECONFIG=$TARGET_KUBECONFIG + # Check if we have a kubeconfig for target cluster + if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then + echo "Setting up target cluster kubeconfig..." + export KUBECONFIG=$TARGET_KUBECONFIG + export CLUSTERLESS_MODE="false" + echo "Target cluster info:" + kubectl cluster-info + else + if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then + echo "Running in clusterless mode - no target cluster kubeconfig" + else + echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" + echo "Running in clusterless mode as fallback" + fi + export CLUSTERLESS_MODE="true" + echo "Hub cluster info:" + kubectl cluster-info + fi # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -126,3 +147,4 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) + optional: true From 9402f47336e9b5a0a9cd3e879249f285bfa6b7ad Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:49:17 +0200 Subject: [PATCH 046/344] [caliper] orchestration: postprocess: save the step scripts in a dedicated directory --- projects/caliper/orchestration/postprocess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 024fb5e89..4f0645274 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -78,8 +78,10 @@ def _execute_caliper_command( log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" log_file.parent.mkdir(parents=True, exist_ok=True) - # Create script file for debugging - script_file = env.ARTIFACT_DIR / f"{step_name_safe}.sh" + # Create script file for debugging in step_scripts directory with index prefix + step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" + step_scripts_dir.mkdir(parents=True, exist_ok=True) + script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" # Log start banner and save script log_caliper_start_banner(command, script_file, step_name.upper()) From 4dd1ee925e6a8ed7cbd5a64d83129b13de3ef643 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 15:02:19 +0200 Subject: [PATCH 047/344] [caliper] orchestration: postprocess: make the file paths relative --- projects/caliper/orchestration/postprocess.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 4f0645274..755f7888e 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -373,9 +373,11 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info(f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}") return { "status": "success", - "output_file": str(output_file), + "output_file": relative_path, "completed_at": time.time(), "log_file": log_file, } @@ -449,7 +451,7 @@ def _run_artifacts_to_ai_data( return { "status": "success", "output_file": status_data.get("output_file", str(ai_data_dir)), - "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), + "ai_data_dir": _make_path_relative_to_base(ai_data_dir, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -645,7 +647,7 @@ def _run_kpis_to_csv( return { "status": "success", "kpi_count": status_data.get("kpi_count", 0), - "output_file": str(output_file), + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -728,7 +730,7 @@ def _run_analyse_kpis( if status_data.get("success"): return { "status": "success", - "output_file": str(output_file), + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -1421,7 +1423,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: import_dir = output_dir / self.config.s3.import_.output_dir step_result = { "status": "success", - "output_dir": str(import_dir), + "output_dir": _make_path_relative_to_base(import_dir, env.ARTIFACT_DIR), "file_count": status_data.get("file_count", 0), "completed_at": time.time(), } @@ -1491,8 +1493,8 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - # current_kpis_file is now an absolute path (updated from relative path fix) - current_kpis_path = Path(current_kpis_file) + # Convert relative path back to absolute path + current_kpis_path = env.ARTIFACT_DIR / current_kpis_file result = _run_analyse_kpis( postprocess_config=self.config, @@ -1539,7 +1541,9 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and artifacts_to_kpis_step.get("status") == "success" and artifacts_to_kpis_step.get("output_file") ): - kpis_file = output_dir / artifacts_to_kpis_step["output_file"] + stored_output_file = artifacts_to_kpis_step["output_file"] + kpis_file = env.ARTIFACT_DIR / stored_output_file + logger.info(f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}") # Get CSV file from kpis_to_csv step kpis_to_csv_step = self._get_step("kpis_to_csv") @@ -1548,7 +1552,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and kpis_to_csv_step.get("status") == "success" and kpis_to_csv_step.get("output_file") ): - csv_file = output_dir / kpis_to_csv_step["output_file"] + csv_file = env.ARTIFACT_DIR / kpis_to_csv_step["output_file"] # Get AI data directory from artifacts_to_ai_data step ai_data_step = self._get_step("artifacts_to_ai_data") @@ -1558,7 +1562,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and ai_data_step.get("status") == "success" and ai_data_step.get("ai_data_dir") ): - ai_data_dir = output_dir / ai_data_step["ai_data_dir"] + ai_data_dir = env.ARTIFACT_DIR / ai_data_step["ai_data_dir"] logger.info(f"AI data directory set to: {ai_data_dir}") else: logger.warning( @@ -1572,7 +1576,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and analyze_step.get("status") in ("success", "warning") and analyze_step.get("output_file") ): - analysis_file = output_dir / analyze_step["output_file"] + analysis_file = env.ARTIFACT_DIR / analyze_step["output_file"] # Create temporary status file for subprocess communication status_file = output_dir / "s3_export_status.yaml" From 99bc553aefa9c7f76243969cf40bc15d07c6e877 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 15:04:00 +0200 Subject: [PATCH 048/344] [caliper] orchestration: postprocess: fix the header/footer banner in the logs --- projects/caliper/orchestration/postprocess.py | 160 +++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 755f7888e..6fb243660 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -33,7 +33,6 @@ build_s3_export_command, build_s3_import_command, build_visualize_command, - handle_caliper_output_and_completion, log_caliper_start_banner, ) from projects.caliper.orchestration.postprocess_config import ( @@ -53,6 +52,157 @@ logger = logging.getLogger(__name__) +def _format_command_for_log(command: list[str]) -> str: + """Format command for log file display with only --arguments on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for log file + """ + if not command: + return "" + + # Start with base command parts until we hit options + lines = [] + base_parts = [] + i = 0 + + # Collect all non-flag arguments into base_parts + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle --flag arguments on separate lines + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return " \\\n ".join(lines) + + +def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: + """Write step header to log file. + + Args: + log_file: Path to log file + command: Command that will be executed + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + formatted_command = _format_command_for_log(command) + header = f"""{"=" * 60} +🚀 STARTING {step_name} (fork/exec) +📄 Command: + {formatted_command} +{"=" * 60} + +""" + with open(log_file, "w", encoding="utf-8") as f: + f.write(header) + + +def _write_step_footer_to_log_file( + log_file: Path, result: subprocess.CompletedProcess, step_name: str +) -> None: + """Write step footer to log file. + + Args: + log_file: Path to log file + result: Subprocess result + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + footer = f""" + +{"=" * 60} +🏁 {step_name} COMPLETED +📊 Exit code: {result.returncode} +📄 Log file: {log_file} +{"=" * 60} +""" + with open(log_file, "a", encoding="utf-8") as f: + f.write(footer) + + +def _handle_caliper_output_and_completion_with_header( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + This version appends to the log file instead of overwriting it, since we've + already written a header. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Append output to log file (don't overwrite since we've written header) + with open(log_file, "a", encoding="utf-8") as log_f: + if result.stdout: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + if isinstance(status_data, dict): + for key, value in status_data.items(): + logger.info(" %s: %s", key, value) + else: + logger.info(" %s", status_data) + + logger.info("=" * 60) + + return status_data + + def _execute_caliper_command( command: list[str], step_name: str, @@ -86,6 +236,9 @@ def _execute_caliper_command( # Log start banner and save script log_caliper_start_banner(command, script_file, step_name.upper()) + # Write header to log file + _write_step_header_to_log_file(log_file, command, step_name.upper()) + # Execute command with combined stdout/stderr result = subprocess.run( command, @@ -95,10 +248,13 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( + status_data = _handle_caliper_output_and_completion_with_header( result, log_file, status_file, step_name.upper() ) + # Write footer to log file + _write_step_footer_to_log_file(log_file, result, step_name.upper()) + return result, status_data, log_file From fee94ecd2a9c606a29a4a25687cd80498521de1b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:25:35 +0200 Subject: [PATCH 049/344] [caliper] engine: visualize: better detect errors --- projects/caliper/engine/visualize.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index f85f6083c..e2947c151 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -89,12 +89,31 @@ def run_visualize( report_group=report_group, config=cfg, ) + + # Check if any report IDs were provided + if not ids: + raise ValueError( + "No report IDs provided. Use --reports with comma-separated report IDs " + "or --report-group with a valid visualize config file." + ) + output_dir.mkdir(parents=True, exist_ok=True) viz = plugin.visualize - return viz( + paths = viz( model, output_dir, ids, report_group, cfg, ) + + # Check for error conditions + if ids and not paths: + # Reports were requested but no files were generated + # This could be due to invalid plot types or other issues + raise ValueError( + f"No visualization files were generated despite requesting report IDs: {ids}. " + f"This may indicate unsupported plot types or other visualization errors." + ) + + return paths From 76e5862ada5b05846e69dbb1859f4ea47d2a0995 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:25:58 +0200 Subject: [PATCH 050/344] [guidellm] postprocess: guidellm: plugin: better detect errors --- projects/guidellm/postprocess/guidellm/plugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index 3e4c775c1..f570050df 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -145,9 +145,12 @@ def visualize( return paths # Generate reports using the registry + invalid_reports: list[str] = [] + for report_name in wanted: if report_name not in PLOT_REGISTRY: logger.warning("Unknown report '%s' requested", report_name) + invalid_reports.append(report_name) continue plot_config = PLOT_REGISTRY[report_name] @@ -165,6 +168,14 @@ def visualize( except Exception as e: logger.error("Error generating %s: %s", report_name, e) + # Check for invalid report names and raise error if found + if invalid_reports: + available_reports = list(PLOT_REGISTRY.keys()) + raise ValueError( + f"Invalid report types requested: {sorted(invalid_reports)}. " + f"Available types in this plugin: {sorted(available_reports)}" + ) + return paths def kpi_catalog(self) -> list[dict[str, Any]]: From d194b135a593c80d7452b73d13aeeb4557c942ce Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:26:09 +0200 Subject: [PATCH 051/344] [skeleton] postprocess: default: plugin: better detect errors --- projects/skeleton/postprocess/default/plugin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 04a10eb16..18b00075f 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -60,6 +60,7 @@ def visualize( output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] wanted = frozenset(report_ids or ()) + invalid_plots: list[str] = [] logger.info(f"Will generate plots for: {sorted(wanted)}") @@ -72,6 +73,15 @@ def visualize( paths.append(path) else: logger.warning(f"Requested plot type '{report_id}' not available in this plugin") + invalid_plots.append(report_id) + + # Check for invalid plot types and raise error if found + if invalid_plots: + available_types = list(self.plots.keys()) + raise ValueError( + f"Invalid plot types requested: {sorted(invalid_plots)}. " + f"Available types in this plugin: {sorted(available_types)}" + ) logger.info( f"Skeleton plugin visualization completed. Generated {len(paths)} files: {paths}" From 801a2f8d9a6a3b35324ca345bcf4af746146f929 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 20:22:36 +0200 Subject: [PATCH 052/344] [caliper] orchestration: postprocess: pretty print the ai-eval status --- projects/caliper/orchestration/postprocess.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 6fb243660..b364083f8 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -530,7 +530,9 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) - logger.info(f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}") + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) return { "status": "success", "output_file": relative_path, @@ -1699,7 +1701,9 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ): stored_output_file = artifacts_to_kpis_step["output_file"] kpis_file = env.ARTIFACT_DIR / stored_output_file - logger.info(f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}") + logger.info( + f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}" + ) # Get CSV file from kpis_to_csv step kpis_to_csv_step = self._get_step("kpis_to_csv") @@ -1712,7 +1716,8 @@ def _run_s3_export_step(self, output_dir: Path) -> None: # Get AI data directory from artifacts_to_ai_data step ai_data_step = self._get_step("artifacts_to_ai_data") - logger.info(f"AI data step found: {ai_data_step}") + + logger.info(f"AI data step found:\n{json.dumps(ai_data_step, indent=2)}") if ( ai_data_step and ai_data_step.get("status") == "success" From a2532c75eb540be1fcbd928b9e1409117ebddf5b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 21:30:16 +0200 Subject: [PATCH 053/344] [caliper] Final cleanup --- projects/caliper/cli/commands.py | 36 +-- .../caliper/cli/orchestration_entrypoints.py | 210 ------------------ projects/caliper/engine/kpi/analyze.py | 1 - projects/caliper/orchestration/postprocess.py | 40 ++-- .../caliper/orchestration/step_logging.py | 7 - 5 files changed, 47 insertions(+), 247 deletions(-) delete mode 100644 projects/caliper/cli/orchestration_entrypoints.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 583f20b29..3bcafe341 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -8,7 +8,6 @@ import click import yaml -from projects.caliper.cli.orchestration_entrypoints import export_kpis_to_csv_entrypoint from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export @@ -531,8 +530,10 @@ def kpi_csv_export( kpi_record = json.loads(line) kpi_records.append(kpi_record) - # Export to CSV using the entrypoint - result_path = export_kpis_to_csv_entrypoint( + # Export to CSV + from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv + + result_path = export_kpis_to_csv( plugin=plugin, kpi_records=kpi_records, output_path=output, @@ -735,20 +736,29 @@ def analyse_kpis_cmd( output_dir = output.parent - # Run analysis using the entrypoint (without base_dir since we have direct paths) - from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint + # Run analysis + from projects.caliper.engine.kpi.analyze import analyze_kpis - result = analyze_kpis_entrypoint( - postprocess_config=postprocess_config_obj, - plugin_module=plugin_module, - base_dir=output_dir, # Use output dir as base since we have absolute paths - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) + try: + result = analyze_kpis( + postprocess_config=postprocess_config_obj, + plugin_module=plugin_module, + base_dir=output_dir, # Use output dir as base since we have absolute paths + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + except Exception as e: + # Don't suppress the exception - let it bubble up with full traceback + import sys + import traceback + + print(f"❌ Exception in analyze_kpis: {e}", file=sys.stderr) + print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) + raise # Check if the analysis function returned a proper result if not isinstance(result, dict) or "status" not in result: - raise ValueError(f"Invalid result from analyze_kpis_entrypoint: {result}") + raise ValueError(f"Invalid result from analyze_kpis: {result}") status_data = { "success": result.get("status") == "success", diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py deleted file mode 100644 index 5e319fd22..000000000 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -Entrypoint functions for orchestration to call Caliper core functionality. - -This module provides well-defined interfaces between the orchestration layer -and Caliper's engine/CLI components. -""" - -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any - - -def parse_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, - use_cache: bool = True, -) -> tuple[Any, str]: - """Entrypoint for orchestration to call parse functionality.""" - from projects.caliper.engine.parse import run_parse - - mod_str, plugin = load_plugin_entrypoint(postprocess_config, tree_root, manifest_path) - - model = run_parse( - base_dir=tree_root, - plugin_module=mod_str, - plugin=plugin, - use_cache=use_cache, - ) - - return model, mod_str - - -def run_visualize_entrypoint(**kwargs) -> list[str]: - """Entrypoint for orchestration to call visualize functionality.""" - from projects.caliper.engine.visualize import run_visualize - - return run_visualize(**kwargs) - - -def load_plugin_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, -) -> tuple[str, Any]: - """Entrypoint for orchestration to load plugins.""" - from projects.caliper.engine.load_plugin import load_plugin - from projects.caliper.engine.plugin_config import resolve_plugin_module_string - - mod_str, _manifest = resolve_plugin_module_string( - base_dir=tree_root, - postprocess_config=manifest_path, - cli_plugin=postprocess_config.plugin_module, - ) - - return mod_str, load_plugin(mod_str) - - -def s3_import_entrypoint( - postprocess_config, - output_dir: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 import functionality.""" - - if not postprocess_config.s3.import_.enabled: - return {"status": "disabled", "reason": "s3_import disabled", "completed_at": time.time()} - - try: - from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params - - # Build import prefix using instance + directory - from projects.caliper.cli.s3_export import build_s3_prefix - - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.import_ - - import_prefix = build_s3_prefix( - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - trailing_slash=False, - ) - import_dir = output_dir / s3_config.output_dir - - # Use explicit parameters instead of config dict - result = run_s3_import_with_explicit_params( - bucket=s3_parent_config.bucket, - prefix=import_prefix, - output_dir=import_dir, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - return result - - except Exception as e: - import logging - - logger = logging.getLogger(__name__) - logger.exception("S3 import failed") - return {"status": "failed", "error": str(e), "completed_at": time.time()} - - -def s3_export_entrypoint( - postprocess_config, - output_dir: Path, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 export functionality with explicit paths.""" - from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths - - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.export - - return run_s3_export_with_explicit_paths( - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, - bucket=s3_parent_config.bucket, - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - upload_id=s3_config.upload_id, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - dry_run=s3_config.dry_run, - ) - - -def get_kpi_functions_entrypoint(plugin_module) -> dict: - """Entrypoint for orchestration to get KPI function metadata.""" - from projects.caliper.engine.kpi.decorators import get_kpi_functions - - try: - plugin_module_obj = __import__(plugin_module, fromlist=[""]) - return get_kpi_functions(plugin_module_obj) - except (ImportError, AttributeError): - return {} - - -def analyze_kpis_entrypoint( - postprocess_config, - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call KPI analysis.""" - from projects.caliper.engine.kpi.analyze import analyze_kpis - - try: - result = analyze_kpis( - postprocess_config=postprocess_config, - plugin_module=plugin_module, - base_dir=base_dir, - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) - return result - except Exception as e: - # Don't suppress the exception - let it bubble up with full traceback - import sys - import traceback - - print(f"❌ Exception in analyze_kpis_entrypoint: {e}", file=sys.stderr) - print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) - raise - - -def kpi_generate_entrypoint( - base_dir: Path, - plugin_module: str, - plugin: Any, - output: Path, -) -> None: - """Entrypoint for orchestration to generate KPIs.""" - from projects.caliper.engine.kpi.generate import run_kpi_generate - - return run_kpi_generate( - base_dir=base_dir, - plugin_module=plugin_module, - plugin=plugin, - output=output, - use_cache=True, - cache_path=None, - ) - - -def export_kpis_to_csv_entrypoint( - plugin, - kpi_records: list[dict[str, Any]], - output_path: Path, - include_header_comments: bool = True, -) -> str: - """Entrypoint for orchestration to export KPIs to CSV.""" - from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv - - return export_kpis_to_csv( - plugin=plugin, - kpi_records=kpi_records, - output_path=output_path, - include_header_comments=include_header_comments, - ) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 40ae18680..825762b62 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -156,7 +156,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Call the core analysis function exit_code = run_kpi_analysis( current_kpi_file=current_kpis_file, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index b364083f8..c0a594075 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -19,11 +19,6 @@ from pydantic import ValidationError -from projects.caliper.cli.orchestration_entrypoints import ( - get_kpi_functions_entrypoint, - load_plugin_entrypoint, - parse_entrypoint, -) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, build_analyse_kpis_command, @@ -329,9 +324,6 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() -# _load_plugin function removed - now using load_plugin_entrypoint - - def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: """ Transform flat KPI list into hierarchical JSON structure. @@ -355,7 +347,13 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) # Get KPI function metadata from the plugin module - kpi_functions = get_kpi_functions_entrypoint(model.plugin_module) + from projects.caliper.engine.kpi.decorators import get_kpi_functions + + try: + plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) + kpi_functions = get_kpi_functions(plugin_module_obj) + except (ImportError, AttributeError): + kpi_functions = {} for kpi in kpis: run_id = kpi.get("run_id", "unknown") @@ -1352,14 +1350,24 @@ def _run_kpi_and_ai_data_steps(self) -> None: output_dir.mkdir(parents=True, exist_ok=True) # Load plugin and model - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, + from projects.caliper.engine.load_plugin import load_plugin + from projects.caliper.engine.parse import run_parse + from projects.caliper.engine.plugin_config import resolve_plugin_module_string + + # Load plugin + mod_str, _manifest = resolve_plugin_module_string( + base_dir=self.tree_root, + postprocess_config=self.manifest_path, + cli_plugin=self.config.plugin_module, ) - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path + plugin = load_plugin(mod_str) + + # Parse model + model = run_parse( + base_dir=self.tree_root, + plugin_module=mod_str, + plugin=plugin, + use_cache=not self.config.parse.no_cache, ) # KPI JSON generation diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 5354c48e0..66cd83137 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -11,7 +11,6 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from typing import Any # Thread-local storage for step-specific handlers _step_local_handlers = threading.local() @@ -235,9 +234,3 @@ def cleanup_step_logging(): if hasattr(_step_local_handlers, "file_handler"): _step_local_handlers.file_handler.close() del _step_local_handlers.file_handler - - - - - - From 2ac95f6a9517812528042445560675a9af62f307 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 22:30:56 +0200 Subject: [PATCH 054/344] [rhoai] toolbox: wait_datasciencecluster_ready: about if the DSC isn't found --- .../rhoai/toolbox/wait_datasciencecluster_ready/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py index 27a1d1631..cd1a7f79f 100644 --- a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py +++ b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py @@ -73,6 +73,14 @@ def wait_for_datasciencecluster_ready(args, ctx): log_stdout=True, # Show the output check=False, ) + + # Abort if the DataScienceCluster is not found + if result.returncode != 0: + error_msg = f"DataScienceCluster {args.datasciencecluster_name} not found in namespace {args.namespace}" + if result.stderr: + error_msg += f": {result.stderr.strip()}" + raise RuntimeError(error_msg) + phase = result.stdout.strip() if result.stdout else None if phase == "Ready": From fedcd318f0c1f17a5a10147303400cde8fd1c4e3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 11 Jul 2026 08:30:49 +0200 Subject: [PATCH 055/344] [core] ci_entrypoint: prepare_ci: improve the status message --- projects/core/ci_entrypoint/prepare_ci.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 076d3b01e..11dad7411 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -783,12 +783,17 @@ def postchecks( """ artifact_dir = os.environ.get("ARTIFACT_DIR") + # Build command string for display + command_str = f"{project} {operation}" + if args: + command_str += f" {' '.join(args)}" + if not artifact_dir: # No artifact dir, just return simple status return ( - f"✅ {project} {operation} completed successfully" + f"✅ {command_str} completed successfully" if finish_reason == FinishReason.SUCCESS - else f"❌ {project} {operation} failed" + else f"❌ {command_str} failed" ) artifact_path = Path(artifact_dir) @@ -821,8 +826,8 @@ def postchecks( if finish_reason != FinishReason.SUCCESS or ( failures_file.exists() and failures_file.stat().st_size > 0 ): - status = f"❌ Execution of '{project} {operation}' failed after{duration_str}." + status = f"❌ Execution of '{command_str}' failed after{duration_str}." else: - status = f"✅ Execution of '{project} {operation}' succeeded after{duration_str}." + status = f"✅ Execution of '{command_str}' succeeded after{duration_str}." return status From 405ba90c557dd7f8072b8e3790ee0b1ef4ec34b9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 08:51:32 +0200 Subject: [PATCH 056/344] Revert "fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs" This reverts commit a55a40a7b261e053018bacdf910dfc9d57ab7c60. --- .../base/workflows/task-forge-step.yaml | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index 72e58e9c0..dcb29b30c 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,8 +13,6 @@ spec: type: string - name: kubeconfig-secret type: string - default: "fournos-clusterless-placeholder" - description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -33,8 +31,6 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig - - name: KUBECONFIG_SECRET_PARAM - value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -77,24 +73,7 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - # Check if we have a kubeconfig for target cluster - if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then - echo "Setting up target cluster kubeconfig..." - export KUBECONFIG=$TARGET_KUBECONFIG - export CLUSTERLESS_MODE="false" - echo "Target cluster info:" - kubectl cluster-info - else - if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then - echo "Running in clusterless mode - no target cluster kubeconfig" - else - echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" - echo "Running in clusterless mode as fallback" - fi - export CLUSTERLESS_MODE="true" - echo "Hub cluster info:" - kubectl cluster-info - fi + export KUBECONFIG=$TARGET_KUBECONFIG # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -147,4 +126,3 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) - optional: true From d6bbedabceda702adff37cbe84adab682d0b6f4a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 09:20:19 +0200 Subject: [PATCH 057/344] [foreign_testing] orchestration: foreign_testing: fix --- projects/foreign_testing/orchestration/foreign_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/foreign_testing/orchestration/foreign_testing.py b/projects/foreign_testing/orchestration/foreign_testing.py index e4451c05d..fd92e891c 100644 --- a/projects/foreign_testing/orchestration/foreign_testing.py +++ b/projects/foreign_testing/orchestration/foreign_testing.py @@ -3,7 +3,7 @@ import pathlib import shutil -from projects.core.ci_entrypoint import run_ci +from projects.core.ci_entrypoint import run_common from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def submit(project_path=None): launch_args = [*["--project-source", str(project_path)], *launch_args] logger.info(f"Submitting deployment with args: {launch_args}") - run_ci.execute_project_operation( + run_common.execute_project_operation( project, operation, launch_args, From 2006a4f2b4f9939126af7c5e01a8bdbf34b476fa Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 10:23:56 +0100 Subject: [PATCH 058/344] feat: Add ICSP for staging images Signed-off-by: Alberto Perdomo --- .../orchestration/config.d/platform.yaml | 4 + .../manifests/quay-registry-icsp.yaml | 9 -- .../manifests/rhoai-registry-icsp.yaml | 15 ++ projects/llm_d/orchestration/prepare_phase.py | 2 +- projects/llm_d/tests/test_profiles.py | 140 ++++++++++++++++- projects/rhoai/library/deploy.py | 146 +++++++++++++++--- vaults/psap-forge-staging-image-pull.yaml | 10 ++ 7 files changed, 287 insertions(+), 39 deletions(-) delete mode 100644 projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml create mode 100644 projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml create mode 100644 vaults/psap-forge-staging-image-pull.yaml diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index f9b2975e2..bf8ec298b 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -59,6 +59,10 @@ rhoai: vault: name: psap-rhoai-rc content: rhoai_rc.secret + staging_pull_secret: + vault: + name: psap-forge-staging-image-pull + content: .dockerconfigjson namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml deleted file mode 100644 index d18702599..000000000 --- a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy -metadata: - name: quay-registry -spec: - repositoryDigestMirrors: - - source: registry.redhat.io/rhoai - mirrors: - - quay.io/rhoai diff --git a/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml new file mode 100644 index 000000000..591e1e378 --- /dev/null +++ b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml @@ -0,0 +1,15 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: rhoai-registry +spec: + repositoryDigestMirrors: + - source: registry.redhat.io/rhoai + mirrors: + - quay.io/rhoai + - source: registry.redhat.io/rhaii + mirrors: + - registry.stage.redhat.io/rhaii + - source: registry.redhat.io/rhaii-early-access + mirrors: + - registry.stage.redhat.io/rhaii-early-access diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 3e3b63047..930f35e6a 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -40,7 +40,7 @@ ) logger = logging.getLogger(__name__) -RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" +RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "rhoai-registry-icsp.yaml" def verify_oc_access() -> None: diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 97a03cdfb..1d5f197c1 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import json from pathlib import Path import pytest @@ -259,7 +261,18 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", "display_name": "Red Hat OpenShift AI", "publisher": "RHOAI Development Catalog", - "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, }, "namespace": "redhat-ods-applications", "datasciencecluster_name": "default-dsc", @@ -325,12 +338,20 @@ def _fake_get_vault_content_path(vault_name: str, content_name: str): path = rhoai_deploy.custom_catalog_pull_secret_path( { - "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, } ) assert path == Path("/tmp/rhoai_rc.secret") - assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} + assert captured == { + "vault_name": "psap-rhoai-rc", + "content_name": "rhoai_rc.secret", + } def test_wait_for_rhoai_pull_secret_ready_treats_empty_mcp_status_as_not_ready( @@ -357,13 +378,124 @@ def _fake_oc(*args, **kwargs): monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) - monkeypatch.setattr(rhoai_deploy, "_decode_pull_secret", lambda _secret: "quay.io/rhoai") + monkeypatch.setattr( + rhoai_deploy, + "_decode_pull_secret", + lambda _secret: ( + "quay.io/rhoai registry.stage.redhat.io/rhaii " + "registry.stage.redhat.io/rhaii-early-access" + ), + ) monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) +def test_prepare_rhoai_pull_secret_merges_dockerconfigjson_registry_auths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _init_project_config() + + rc_pull_secret_path = tmp_path / "rhoai_rc.secret" + rc_pull_secret_path.write_text( + "rhoai+rhoai_external_readonly_bot:token-c", + encoding="utf-8", + ) + + staging_pull_secret_path = tmp_path / ".dockerconfigjson" + staging_pull_secret_path.write_text( + json.dumps( + { + "auths": { + "registry.stage.redhat.io/rhaii": {"auth": "token-a"}, + "registry.stage.redhat.io/rhaii-early-access": {"auth": "token-b"}, + } + } + ), + encoding="utf-8", + ) + + current_secret = { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps({"auths": {}}).encode("utf-8")).decode( + "utf-8" + ) + } + } + merged_payload: dict[str, object] = {} + + monkeypatch.setattr( + rhoai_deploy.vault, + "get_vault_content_path", + lambda vault_name, content_name: ( + rc_pull_secret_path + if (vault_name, content_name) == ("psap-rhoai-rc", "rhoai_rc.secret") + else staging_pull_secret_path + ), + ) + monkeypatch.setattr(rhoai_deploy, "oc_get_json", lambda *args, **kwargs: current_secret) + + def _fake_oc(*args, **kwargs): + if args[:2] == ("set", "data"): + from_file = next((arg for arg in args if str(arg).startswith("--from-file=")), "") + payload_path = Path(from_file.rsplit("=", 1)[1]) + merged_payload.update(json.loads(payload_path.read_text(encoding="utf-8"))) + return type("Result", (), {"stdout": "", "returncode": 0})() + if args[:2] == ("registry", "login"): + registry = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--registry=") + ) + auth_basic = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--auth-basic=") + ) + temp_path = Path( + next( + arg + for arg in args + if not str(arg).startswith("--") and arg != "registry" and arg != "login" + ) + ) + current = json.loads(temp_path.read_text(encoding="utf-8")) + current.setdefault("auths", {})[registry] = {"auth": auth_basic} + temp_path.write_text(json.dumps(current, indent=2, sort_keys=True), encoding="utf-8") + return type("Result", (), {"stdout": "", "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy, "wait_for_rhoai_pull_secret_ready", lambda **kwargs: None) + + rhoai_deploy.prepare_rhoai_pull_secret( + rhoai_deploy.RhoaiCustomCatalogConfig.model_validate( + { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, + } + ) + ) + + assert ( + merged_payload["auths"]["quay.io/rhoai"]["auth"] + == "rhoai+rhoai_external_readonly_bot:token-c" + ) + assert merged_payload["auths"]["registry.stage.redhat.io/rhaii"]["auth"] == "token-a" + assert ( + merged_payload["auths"]["registry.stage.redhat.io/rhaii-early-access"]["auth"] == "token-b" + ) + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index fa4c6203c..233479d87 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import json import logging import tempfile import time @@ -20,7 +21,11 @@ RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" RHOAI_PULL_SECRET_NAME = "pull-secret" -RHOAI_REGISTRY = "quay.io/rhoai" +RHOAI_CATALOG_REGISTRIES = ("quay.io/rhoai",) +RHOAI_REGISTRIES = ( + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", +) class _RhoaiCustomCatalogPullSecretVaultConfig(BaseModel): @@ -52,6 +57,7 @@ class RhoaiCustomCatalogConfig(BaseModel): display_name: str | None = None publisher: str | None = None pull_secret: _RhoaiCustomCatalogPullSecretConfig + staging_pull_secret: _RhoaiCustomCatalogPullSecretConfig | None = None class RhoaiOperatorConfig(BaseModel): @@ -149,6 +155,77 @@ def _decode_pull_secret(secret_data: dict[str, Any]) -> str: return base64.b64decode(encoded).decode("utf-8") +def _read_vault_pull_secret(pull_secret_path: Path) -> str | dict[str, Any]: + raw_content = pull_secret_path.read_text(encoding="utf-8").strip() + if not raw_content: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + try: + parsed = json.loads(raw_content) + except json.JSONDecodeError: + return raw_content + + if isinstance(parsed, dict) and "auths" in parsed: + return parsed + + return raw_content + + +def _apply_vault_pull_secret( + *, + current_secret: dict[str, Any], + pull_secret_path: Path, + registries: tuple[str, ...], + temp_path: Path, +) -> dict[str, Any]: + pull_secret_payload = _read_vault_pull_secret(pull_secret_path) + + if isinstance(pull_secret_payload, dict): + merged_secret = _merge_pull_secret_auths(current_secret, pull_secret_payload) + temp_path.write_text(json.dumps(merged_secret, indent=2, sort_keys=True), encoding="utf-8") + return merged_secret + + for registry in registries: + oc( + "registry", + "login", + f"--registry={registry}", + f"--auth-basic={pull_secret_payload}", + "--to", + str(temp_path), + ) + return json.loads(temp_path.read_text(encoding="utf-8")) + + +def _current_pull_secret_json() -> dict[str, Any]: + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + try: + return json.loads(decoded_secret) + except json.JSONDecodeError as exc: + raise RuntimeError("openshift-config/pull-secret is not valid dockerconfigjson") from exc + + +def _registries_present( + decoded_secret: str, registries: tuple[str, ...] = RHOAI_REGISTRIES +) -> bool: + return all(registry in decoded_secret for registry in registries) + + +def _merge_pull_secret_auths( + current_secret: dict[str, Any], pull_secret_payload: dict[str, Any] +) -> dict[str, Any]: + merged_secret = dict(current_secret) + merged_auths = dict(current_secret.get("auths", {})) + merged_auths.update(pull_secret_payload.get("auths", {})) + merged_secret["auths"] = merged_auths + return merged_secret + + def wait_for_rhoai_pull_secret_ready( *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 ) -> None: @@ -163,8 +240,8 @@ def wait_for_rhoai_pull_secret_ready( namespace=RHOAI_PULL_SECRET_NAMESPACE, ) decoded = _decode_pull_secret(secret) - if RHOAI_REGISTRY not in decoded: - raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + if not _registries_present(decoded, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): + raise RuntimeError("required RHOAI registries are not yet present in pull secret") mcp_status = oc( "get", @@ -180,7 +257,7 @@ def wait_for_rhoai_pull_secret_ready( logger.info( "RHOAI pull secret now contains %s and machine config pools are updated", - RHOAI_REGISTRY, + ", ".join(RHOAI_REGISTRIES), ) return except RuntimeError as exc: @@ -189,7 +266,7 @@ def wait_for_rhoai_pull_secret_ready( time.sleep(poll_interval_seconds) raise RuntimeError( - f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + f"Timed out waiting for RHOAI registries pull secret propagation: {last_error}" ) @@ -204,35 +281,54 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " f"'{custom_catalog.pull_secret.vault.name}'" ) - auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() - if not auth_basic: - raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") - current_secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded_secret = _decode_pull_secret(current_secret) + current_secret = _current_pull_secret_json() + current_secret_text = json.dumps(current_secret, sort_keys=True) - if RHOAI_REGISTRY in decoded_secret: - logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + required_registries = (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES) + + if _registries_present(current_secret_text, required_registries): + logger.info( + "RHOAI registries already present in cluster pull secret: %s", + ", ".join(required_registries), + ) return - logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + logger.info( + "Adding %s to cluster pull secret from %s", + ", ".join(required_registries), + pull_secret_path, + ) with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: - temp_file.write(decoded_secret) + temp_file.write(json.dumps(current_secret, indent=2, sort_keys=True)) temp_path = Path(temp_file.name) try: - oc( - "registry", - "login", - f"--registry={RHOAI_REGISTRY}", - f"--auth-basic={auth_basic}", - "--to", - str(temp_path), + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=pull_secret_path, + registries=RHOAI_CATALOG_REGISTRIES, + temp_path=temp_path, ) + + if custom_catalog.staging_pull_secret is not None: + staging_pull_secret_path = vault.get_vault_content_path( + custom_catalog.staging_pull_secret.vault.name, + custom_catalog.staging_pull_secret.vault.content, + ) + if staging_pull_secret_path is None: + raise RuntimeError( + "RHOAI staging pull secret content " + f"'{custom_catalog.staging_pull_secret.vault.content}' was not found in vault " + f"'{custom_catalog.staging_pull_secret.vault.name}'" + ) + + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=staging_pull_secret_path, + registries=RHOAI_REGISTRIES, + temp_path=temp_path, + ) oc( "set", "data", diff --git a/vaults/psap-forge-staging-image-pull.yaml b/vaults/psap-forge-staging-image-pull.yaml new file mode 100644 index 000000000..5979f3b4d --- /dev/null +++ b/vaults/psap-forge-staging-image-pull.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_FORGE_STAGING_IMAGE_PULL_SECRET_PATH +description: | + Vault containing registry credentials for staging image pulling + +content: + .dockerconfigjson: + description: | + Docker registry auth config used to extend the cluster pull secret + sample: | + {"auths":{"registry.stage.redhat.io":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii-early-access":{"auth":"base64-encoded-credentials"}}} From dfe4f004dc8f878cb1c35103d6441285f0ba75b1 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 11:57:07 +0100 Subject: [PATCH 059/344] fix: Mount staging vault secret Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/ci.py | 8 ++++++-- projects/llm_d/tests/test_profiles.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index f8611771c..b3748f679 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -28,6 +28,10 @@ from projects.llm_d.orchestration.test_phase import run as test_toolbox_run logger = logging.getLogger(__name__) +RHOAI_CUSTOM_CATALOG_VAULTS = [ + "psap-rhoai-rc", + "psap-forge-staging-image-pull", +] def init(): @@ -41,7 +45,7 @@ def list_vaults() -> list[str]: """List all vaults (includes both mandatory and optional).""" all_vaults = vault.phase_vault_list_all() if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): - return [*all_vaults, "psap-rhoai-rc"] + return [*all_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] return all_vaults @@ -59,7 +63,7 @@ def init_vaults_for_phase(phase: str) -> None: if phase == "prepare" and config.project.get_config( "platform.rhoai.custom_catalog.enabled", False ): - optional_vaults = [*optional_vaults, "psap-rhoai-rc"] + optional_vaults = [*optional_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 1d5f197c1..20b565954 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -201,17 +201,19 @@ def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided assert runtime_config.get_benchmark_keys() == ["short"] -def test_list_vaults_only_includes_rhoai_rc_vault_for_custom_catalog_runs() -> None: +def test_list_vaults_only_includes_rhoai_custom_catalog_vaults_for_custom_catalog_runs() -> None: _init_project_config() core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) assert "psap-rhoai-rc" not in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" not in llmd_ci.list_vaults() core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) assert "psap-rhoai-rc" in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" in llmd_ci.list_vaults() -def test_prepare_phase_adds_rhoai_rc_vault_only_for_custom_catalog_runs( +def test_prepare_phase_adds_rhoai_custom_catalog_vaults_only_for_custom_catalog_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: _init_project_config() @@ -230,7 +232,9 @@ def _fake_init(**kwargs): llmd_ci.init_vaults_for_phase("prepare") assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] + assert "psap-forge-staging-image-pull" not in calls[0]["optional_vaults"] assert "psap-rhoai-rc" in calls[1]["optional_vaults"] + assert "psap-forge-staging-image-pull" in calls[1]["optional_vaults"] def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( From 1444aac498555f8ac0183f4b08e9a87c50101c4f Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 12:30:56 +0100 Subject: [PATCH 060/344] fix: Loose substring check Signed-off-by: Alberto Perdomo --- projects/llm_d/tests/test_profiles.py | 40 +++++++++++++++++++++------ projects/rhoai/library/deploy.py | 32 ++++++++++++--------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 20b565954..6b9c948d4 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -373,7 +373,19 @@ def _fake_sleep(_seconds: float) -> None: clock["value"] += 0.1 def _fake_oc_get_json(*args, **kwargs): - return {"data": {".dockerconfigjson": "unused"}} + payload = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + return { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps(payload).encode("utf-8")).decode( + "utf-8" + ) + } + } def _fake_oc(*args, **kwargs): if args[:2] == ("get", "mcp"): @@ -382,20 +394,30 @@ def _fake_oc(*args, **kwargs): monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) - monkeypatch.setattr( - rhoai_deploy, - "_decode_pull_secret", - lambda _secret: ( - "quay.io/rhoai registry.stage.redhat.io/rhaii " - "registry.stage.redhat.io/rhaii-early-access" - ), - ) monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) +def test_registries_present_accepts_parent_registry_auth_entries() -> None: + pull_secret = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + + assert rhoai_deploy._registries_present( + pull_secret, + ( + "quay.io/rhoai", + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", + ), + ) + + def test_prepare_rhoai_pull_secret_merges_dockerconfigjson_registry_auths( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 233479d87..91c147547 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -210,10 +210,24 @@ def _current_pull_secret_json() -> dict[str, Any]: raise RuntimeError("openshift-config/pull-secret is not valid dockerconfigjson") from exc +def _normalize_registry_key(registry: str) -> str: + return registry.removeprefix("https://").removeprefix("http://").rstrip("/") + + +def _auth_entry_covers_registry(auth_key: str, required_registry: str) -> bool: + auth_key = _normalize_registry_key(auth_key) + required_registry = _normalize_registry_key(required_registry) + return auth_key == required_registry or required_registry.startswith(f"{auth_key}/") + + def _registries_present( - decoded_secret: str, registries: tuple[str, ...] = RHOAI_REGISTRIES + pull_secret: dict[str, Any], registries: tuple[str, ...] = RHOAI_REGISTRIES ) -> bool: - return all(registry in decoded_secret for registry in registries) + auth_keys = tuple(pull_secret.get("auths", {}).keys()) + return all( + any(_auth_entry_covers_registry(auth_key, registry) for auth_key in auth_keys) + for registry in registries + ) def _merge_pull_secret_auths( @@ -234,15 +248,9 @@ def wait_for_rhoai_pull_secret_ready( while time.monotonic() < deadline: try: - secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded = _decode_pull_secret(secret) - if not _registries_present(decoded, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): + pull_secret = _current_pull_secret_json() + if not _registries_present(pull_secret, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): raise RuntimeError("required RHOAI registries are not yet present in pull secret") - mcp_status = oc( "get", "mcp", @@ -283,11 +291,9 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) current_secret = _current_pull_secret_json() - current_secret_text = json.dumps(current_secret, sort_keys=True) - required_registries = (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES) - if _registries_present(current_secret_text, required_registries): + if _registries_present(current_secret, required_registries): logger.info( "RHOAI registries already present in cluster pull secret: %s", ", ".join(required_registries), From 4433216b9e185b27926d0624212b3164bfc8da27 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 15:02:53 +0100 Subject: [PATCH 061/344] fix: Wait for serving control plane before deploying llmisvc Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/test_phase.py | 58 +++++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f16034aad..f3716dd3e 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -2,11 +2,13 @@ import logging import sys +import time from pathlib import Path from typing import Any from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier +from projects.core.dsl.utils.k8s import oc from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -24,6 +26,13 @@ from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) +SERVING_CONTROL_PLANE_DEPLOYMENTS = ( + "kserve-controller-manager", + "llmisvc-controller-manager", + "model-serving-api", + "odh-model-controller", +) +SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 def create_test_labels() -> None: @@ -177,10 +186,13 @@ def deploy_inference_service() -> str: # Step 1: Ensure model cache is ready _prepare_model_cache() - # Step 2: Build and write inference service manifest + # Step 2: Wait for the serving control plane to settle before creating the service. + _wait_for_serving_control_plane_ready() + + # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() - # Step 3: Deploy the service and wait for endpoint + # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) endpoint_url = deploy_llmisvc.run( namespace=namespace, @@ -204,6 +216,48 @@ def _prepare_model_cache() -> None: prepare_model_cache() +def _wait_for_serving_control_plane_ready() -> None: + from projects.llm_d.orchestration import runtime_config + + rhoai_namespace = runtime_config.get_platform_config()["rhoai"]["namespace"] + + for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: + result = oc( + "wait", + "--for=condition=Available", + "--timeout=300s", + f"deployment/{deployment_name}", + "-n", + rhoai_namespace, + check=False, + ) + if result.returncode != 0: + status_result = oc( + "get", + "deployments", + "-n", + rhoai_namespace, + "-o", + "wide", + check=False, + ) + error_msg = ( + f"Timeout waiting for serving control plane deployment " + f"{deployment_name} in {rhoai_namespace}" + ) + if status_result.returncode == 0: + error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" + raise RuntimeError(error_msg) + + # The webhook deployments can report Available before leader election and informer + # startup finish, which makes the first LLMInferenceService create race the webhook. + logger.info( + "Waiting %ss for serving control plane stabilization", + SERVING_CONTROL_PLANE_STABILIZATION_SECONDS, + ) + time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) + + def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" from projects.llm_d.orchestration import runtime_config From d3ff07e70099d9fc6ff6cf37519f5d284f7e8b72 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Tue, 14 Jul 2026 16:36:02 +0100 Subject: [PATCH 062/344] chore: Fix review comments Signed-off-by: Alberto Perdomo --- .../toolbox/deploy_custom_catalog/main.py | 1 - .../kserve/toolbox/wait_kserve_ready/main.py | 81 +++++++++++++++++++ projects/llm_d/orchestration/test_phase.py | 55 +------------ 3 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 projects/kserve/toolbox/wait_kserve_ready/main.py diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 641184cb8..2b70724f0 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -41,7 +41,6 @@ def setup_directories(args, ctx): shell.mkdir("src") shell.mkdir("artifacts") ctx.display_name = args.display_name or args.catalog_source_name - ctx.publisher = args.publisher or "" return f"Prepared directories for {ctx.display_name}" diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py new file mode 100644 index 000000000..11e5ee1ef --- /dev/null +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import time + +from projects.core.dsl import entrypoint, execute_tasks, task +from projects.core.dsl.utils.k8s import oc + +SERVING_CONTROL_PLANE_DEPLOYMENTS = ( + "kserve-controller-manager", + "llmisvc-controller-manager", + "model-serving-api", + "odh-model-controller", +) +SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 + + +@entrypoint +def run(*, namespace: str) -> str: + """ + Wait for the KServe serving control plane deployments to be ready. + + Args: + namespace: Namespace where the serving control plane is deployed + """ + + task_args = {"namespace": namespace} + execute_tasks(task_args) + + return f"Serving control plane is ready in {namespace}" + + +@task +def wait_for_deployments(args, ctx): + """Wait for all serving control plane deployments to be available""" + + for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: + result = oc( + "wait", + "--for=condition=Available", + "--timeout=300s", + f"deployment/{deployment_name}", + "-n", + args.namespace, + check=False, + ) + if result.returncode != 0: + status_result = oc( + "get", + "deployments", + "-n", + args.namespace, + "-o", + "wide", + check=False, + ) + error_msg = ( + f"Timeout waiting for serving control plane deployment " + f"{deployment_name} in {args.namespace}" + ) + if status_result.returncode == 0: + error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" + raise RuntimeError(error_msg) + + return "All serving control plane deployments are available" + + +@task +def wait_for_stabilization(args, ctx): + """Wait for serving control plane to stabilize after becoming available""" + + # The webhook deployments can report Available before leader election and informer + # startup finish, which makes the first LLMInferenceService create race the webhook. + time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) + + return f"Waited {SERVING_CONTROL_PLANE_STABILIZATION_SECONDS}s for stabilization" + + +if __name__ == "__main__": + run.main() diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f3716dd3e..2ef047ef2 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -2,13 +2,11 @@ import logging import sys -import time from pathlib import Path from typing import Any from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier -from projects.core.dsl.utils.k8s import oc from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -18,6 +16,7 @@ from projects.guidellm.toolbox.run_smoke_request import main as run_smoke_request_command from projects.kserve.toolbox.capture_llmisvc_state import main as capture_llmisvc_state from projects.kserve.toolbox.deploy_llmisvc import main as deploy_llmisvc +from projects.kserve.toolbox.wait_kserve_ready import main as wait_kserve_ready from projects.llm_d.orchestration.prepare_phase import prepare_model_cache from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, @@ -26,13 +25,6 @@ from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) -SERVING_CONTROL_PLANE_DEPLOYMENTS = ( - "kserve-controller-manager", - "llmisvc-controller-manager", - "model-serving-api", - "odh-model-controller", -) -SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 def create_test_labels() -> None: @@ -187,7 +179,8 @@ def deploy_inference_service() -> str: _prepare_model_cache() # Step 2: Wait for the serving control plane to settle before creating the service. - _wait_for_serving_control_plane_ready() + rhoai_namespace = platform["rhoai"]["namespace"] + wait_kserve_ready.run(namespace=rhoai_namespace) # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() @@ -216,48 +209,6 @@ def _prepare_model_cache() -> None: prepare_model_cache() -def _wait_for_serving_control_plane_ready() -> None: - from projects.llm_d.orchestration import runtime_config - - rhoai_namespace = runtime_config.get_platform_config()["rhoai"]["namespace"] - - for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: - result = oc( - "wait", - "--for=condition=Available", - "--timeout=300s", - f"deployment/{deployment_name}", - "-n", - rhoai_namespace, - check=False, - ) - if result.returncode != 0: - status_result = oc( - "get", - "deployments", - "-n", - rhoai_namespace, - "-o", - "wide", - check=False, - ) - error_msg = ( - f"Timeout waiting for serving control plane deployment " - f"{deployment_name} in {rhoai_namespace}" - ) - if status_result.returncode == 0: - error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" - raise RuntimeError(error_msg) - - # The webhook deployments can report Available before leader election and informer - # startup finish, which makes the first LLMInferenceService create race the webhook. - logger.info( - "Waiting %ss for serving control plane stabilization", - SERVING_CONTROL_PLANE_STABILIZATION_SECONDS, - ) - time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) - - def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" from projects.llm_d.orchestration import runtime_config From a06b214ce5349206cb5d7de4925d2bce5a7eaaac Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 09:15:16 +0100 Subject: [PATCH 063/344] fix: Dry-run instead of time.sleep() Signed-off-by: Alberto Perdomo --- .../kserve/toolbox/wait_kserve_ready/main.py | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py index 11e5ee1ef..426cf2355 100644 --- a/projects/kserve/toolbox/wait_kserve_ready/main.py +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -2,9 +2,9 @@ from __future__ import annotations -import time +import yaml -from projects.core.dsl import entrypoint, execute_tasks, task +from projects.core.dsl import entrypoint, execute_tasks, retry, task from projects.core.dsl.utils.k8s import oc SERVING_CONTROL_PLANE_DEPLOYMENTS = ( @@ -13,7 +13,6 @@ "model-serving-api", "odh-model-controller", ) -SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 @entrypoint @@ -66,15 +65,46 @@ def wait_for_deployments(args, ctx): return "All serving control plane deployments are available" +@retry(attempts=40, delay=5, backoff=1.0) @task -def wait_for_stabilization(args, ctx): - """Wait for serving control plane to stabilize after becoming available""" - - # The webhook deployments can report Available before leader election and informer - # startup finish, which makes the first LLMInferenceService create race the webhook. - time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) - - return f"Waited {SERVING_CONTROL_PLANE_STABILIZATION_SECONDS}s for stabilization" +def probe_webhook_ready(args, ctx): + """Probe the admission webhook with a dry-run apply to confirm it is serving""" + + probe_manifest = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "LLMInferenceService", + "metadata": { + "name": "forge-webhook-probe", + "namespace": args.namespace, + }, + "spec": { + "model": { + "uri": "hf://probe/model", + "name": "probe", + }, + }, + } + + probe_dir = args.artifact_dir / "src" + probe_dir.mkdir(parents=True, exist_ok=True) + probe_path = probe_dir / "webhook-probe.yaml" + with open(probe_path, "w", encoding="utf-8") as f: + yaml.safe_dump(probe_manifest, f, sort_keys=False) + + result = oc( + "apply", + "--dry-run=server", + "-f", + str(probe_path), + "-n", + args.namespace, + check=False, + ) + + if result.returncode == 0: + return "Webhook is ready (dry-run accepted)" + + return False if __name__ == "__main__": From 9392eea2f72eb6d64a11f55f267b96849f13e990 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 11:00:17 +0100 Subject: [PATCH 064/344] feat: Improve GH comment support Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/pr_args.py | 32 +++++++++-- projects/llm_d/tests/test_pr_args.py | 34 +++++++++++ projects/rhoai/library/pr_args.py | 75 +++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 projects/rhoai/library/pr_args.py diff --git a/projects/llm_d/orchestration/pr_args.py b/projects/llm_d/orchestration/pr_args.py index 6811a4726..7c11fd048 100644 --- a/projects/llm_d/orchestration/pr_args.py +++ b/projects/llm_d/orchestration/pr_args.py @@ -2,14 +2,22 @@ """ llm_d Project PR Arguments Parser -llm_d relies on the framework preset mechanism from `/test fournos llm_d PRESET`. -No llm_d-specific PR directives are currently required. +Parses llm_d-specific directives from PR trigger comments. + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 """ from __future__ import annotations +import logging from typing import Any +from projects.rhoai.library.pr_args import get_rhoai_directive_handlers + +logger = logging.getLogger(__name__) + def get_supported_llm_d_directives() -> dict[str, str]: """ @@ -31,5 +39,21 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st Returns: Tuple of (configuration overrides dict, list of parsed directive lines) """ - _ = comment_text - return {}, [] + directive_handlers = get_rhoai_directive_handlers() + config_overrides: dict[str, Any] = {} + parsed_directives: list[str] = [] + + for raw_line in comment_text.splitlines(): + line = raw_line.strip() + if not line: + continue + + for prefix, handler in directive_handlers.items(): + if line.startswith(prefix): + result = handler(line) + config_overrides.update(result) + parsed_directives.append(line) + logger.info("Parsed llm_d directive: %s", line) + break + + return config_overrides, parsed_directives diff --git a/projects/llm_d/tests/test_pr_args.py b/projects/llm_d/tests/test_pr_args.py index 11ae6a13d..9545a9e30 100644 --- a/projects/llm_d/tests/test_pr_args.py +++ b/projects/llm_d/tests/test_pr_args.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from projects.llm_d.orchestration import pr_args @@ -22,3 +24,35 @@ def test_parse_project_directives_ignores_other_projects() -> None: assert overrides == {} assert directives == [] + + +def test_rhoai_rc_image_sets_custom_catalog_config() -> None: + image = "quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310" + overrides, directives = pr_args.parse_project_directives(f"/rhoai.rc-image {image}") + + assert overrides == { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + assert len(directives) == 1 + + +def test_rhoai_rc_image_without_image_raises() -> None: + with pytest.raises(ValueError, match="image cannot be empty"): + pr_args.parse_project_directives("/rhoai.rc-image") + + +def test_rhoai_rc_image_with_other_directives() -> None: + comment = """/test fournos llm_d smoke +/rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123""" + + overrides, directives = pr_args.parse_project_directives(comment) + + assert overrides["platform.rhoai.custom_catalog.enabled"] is True + assert ( + overrides["platform.rhoai.custom_catalog.image"] + == "quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + assert overrides["platform.operators.rhods-operator.channel"] == "beta" + assert len(directives) == 1 diff --git a/projects/rhoai/library/pr_args.py b/projects/rhoai/library/pr_args.py new file mode 100644 index 000000000..6d10dccd6 --- /dev/null +++ b/projects/rhoai/library/pr_args.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +RHOAI PR directive handlers. + +Provides directive handlers for RHOAI-specific PR trigger directives. +These handlers are consumed by project-level pr_args modules (e.g., llm_d). + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 +""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def handle_rhoai_rc_image_directive(line: str) -> dict[str, Any]: + """ + Handle /rhoai.rc-image directive for setting RHOAI RC catalog image. + + Format: /rhoai.rc-image IMAGE + + Args: + line: The directive line + + Returns: + Dictionary with custom catalog and operator channel configuration + + Raises: + ValueError: If image is empty + """ + image = line.removeprefix("/rhoai.rc-image").strip() + + if not image: + raise ValueError( + "Invalid /rhoai.rc-image directive: image cannot be empty. " + "Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + + return { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + + +def get_rhoai_directive_handlers() -> dict[str, callable]: + """ + Get a mapping of RHOAI directive prefixes to their handler functions. + + Returns: + Dictionary mapping directive prefixes to handler functions + """ + return { + "/rhoai.rc-image": handle_rhoai_rc_image_directive, + } + + +def get_supported_rhoai_directives() -> dict[str, str]: + """ + Get a dictionary of supported RHOAI directives and their descriptions. + + Returns: + Dictionary mapping directive names to detailed descriptions + """ + return { + "/rhoai.rc-image": """Set RHOAI RC catalog image for testing release candidates. + Format: /rhoai.rc-image IMAGE + Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 + Effect: Enables custom catalog with the given image and sets operator channel to beta.""", + } From 1d335e895a9eae8e5552f2c6945bc428ef70d2bc Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 11:13:07 +0100 Subject: [PATCH 065/344] fix: Cleanup ztunnels CRD Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/cleanup_phase.py | 28 ++++++++++++++++++- .../orchestration/config.d/platform.yaml | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index 0bf1e0e86..a6cfafd62 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -5,7 +5,7 @@ from projects.cluster.toolbox.cleanup_operators import ( main as cleanup_operators_command, ) -from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_resource_exists from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) @@ -73,6 +73,8 @@ def cleanup_operators(*, dry_run: bool = False) -> None: dry_run=dry_run, ) + _cleanup_stale_crds(platform, dry_run=dry_run) + def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[dict]: """Identify which operators should be cleaned up based on configuration.""" @@ -112,3 +114,27 @@ def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[ logger.info(f"Identified {len(operators_to_delete)} operators for deletion") return operators_to_delete + + +def _cleanup_stale_crds(platform: dict, *, dry_run: bool = False) -> None: + """Delete CRDs listed under cleanup_crds in operator specs. + + Some operators (e.g. rhcl-operator) leave CRDs with stored API versions + that block reinstallation on the next run. + """ + operators = platform.get("operators", {}) + if not isinstance(operators, dict): + return + + for package_name, operator_spec in operators.items(): + if not isinstance(operator_spec, dict): + continue + for crd in operator_spec.get("cleanup_crds", []): + if not oc_resource_exists("crd", crd): + logger.info("CRD %s not present, skipping", crd) + continue + if dry_run: + logger.info("[DRY RUN] Would delete CRD %s (from %s)", crd, package_name) + continue + logger.info("Deleting stale CRD %s (from %s)", crd, package_name) + oc("delete", "crd", crd, "--ignore-not-found=true", "--timeout=60s", check=False) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index bf8ec298b..b9defea06 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -41,6 +41,8 @@ operators: namespace: openshift-operators channel: stable source: redhat-operators + cleanup_crds: + - ztunnels.sailoperator.io rhods-operator: display_name: Red Hat OpenShift AI namespace: redhat-ods-operator From d8564dba5e969cd06f20b0ad5e409518497f4fe8 Mon Sep 17 00:00:00 2001 From: Harshith u <44830597+Harshith-umesh@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:02:46 -0400 Subject: [PATCH 066/344] Add Harshith-umesh to OWNERS and reviewers list --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 3fc847a58..ca8ee10bb 100644 --- a/OWNERS +++ b/OWNERS @@ -6,6 +6,7 @@ approvers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh reviewers: - ashishkamra - albertoperdomo2 @@ -13,3 +14,4 @@ reviewers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh From ced07f86e2ff5cd59e7b60a9ea17c3dff5ee047c Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 09:40:38 +0100 Subject: [PATCH 067/344] feat: Isolate benchmark workloads with per-deployment lifecycle Signed-off-by: Alberto Perdomo --- projects/llm_d/docs/benchmark-isolation.md | 109 +++++++++++++ projects/llm_d/orchestration/ci.py | 9 +- projects/llm_d/orchestration/cleanup_phase.py | 6 +- .../llm_d/orchestration/runtime_config.py | 67 +++++--- projects/llm_d/orchestration/test_phase.py | 69 ++++---- projects/llm_d/tests/test_profiles.py | 152 +++++++++++++++++- 6 files changed, 337 insertions(+), 75 deletions(-) create mode 100644 projects/llm_d/docs/benchmark-isolation.md diff --git a/projects/llm_d/docs/benchmark-isolation.md b/projects/llm_d/docs/benchmark-isolation.md new file mode 100644 index 000000000..c6a357c71 --- /dev/null +++ b/projects/llm_d/docs/benchmark-isolation.md @@ -0,0 +1,109 @@ +# Benchmark Isolation: Per-Workload Deployment + +## Why each benchmark needs a fresh deployment + +The llm_d test harness deploys an LLMInferenceService (LLMISVC) for each +benchmark workload rather than sharing a single deployment across workloads. +Three factors drive this design: + +### 1. `--max-model-len` affects performance + +The vLLM serving engine's `--max-model-len` parameter controls the maximum +sequence length and directly impacts memory allocation, scheduling behavior, +and throughput. Different workloads have different token distributions: + +- `short` uses `prompt_tokens=256, output_tokens=128` +- `heavy-heterogeneous` uses prompts up to 30,000 tokens + +Running `heavy-heterogeneous` against a server configured with +`--max-model-len=4096` would truncate or reject requests, while running +`short` against `--max-model-len=32768` wastes GPU memory on unused KV-cache +blocks. Workloads can provide `deployment_overrides` to set the right +`--max-model-len` for their token distribution. + +### 2. KV-cache pollution across workloads + +When multiple benchmarks share a deployment, residual KV-cache entries from +the first workload can affect the performance measurements of subsequent +workloads. The current mitigation relies on guidellm's prefixed prompts to +generate unique cache keys per benchmark. This works for standard load +generation but is not a guarantee of isolation. + +### 3. Future trace-replay workloads + +Trace-replay workloads (replaying production request traces) will require +starting the vLLM server with `VLLM_DEV_MODE=1` so that the KV-cache can +be explicitly cleared between runs. This is a deployment-time environment +variable, not a runtime toggle, reinforcing the need for per-workload +deployment isolation. + +## Architecture: 3D RunSpec matrix + +The test matrix is a Cartesian product of three dimensions: + +``` +model_name x deployment_profile x benchmark_key +``` + +Each combination (a `RunSpec`) gets: +1. Its own Kubernetes namespace +2. A fresh LLMISVC deployment with the profile's vLLM args +3. A smoke test to verify the deployment is healthy +4. A single benchmark run +5. Full teardown (LLMISVC, benchmark job, PVC, namespace resources) + +### Example + +``` +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.deployment_profile: [approximate-prefix-cache, distributed-default] +/var runtime.benchmark_key: [short, multi-turn] +``` + +Produces 1 x 2 x 2 = 4 isolated test runs, each with its own namespace: + +| RunSpec | Model | Profile | Benchmark | +|---------|-------|---------|-----------| +| 1 | Qwen3-0.6B | approximate-prefix-cache | short | +| 2 | Qwen3-0.6B | approximate-prefix-cache | multi-turn | +| 3 | Qwen3-0.6B | distributed-default | short | +| 4 | Qwen3-0.6B | distributed-default | multi-turn | + +## Backward compatibility + +| Scenario | `benchmark_key` value | Matrix | Behavior | +|----------|-----------------------|--------|----------| +| Smoke-only | `null` | M x P | No benchmark, just smoke test | +| Single benchmark | `"short"` | M x P x 1 | One benchmark per deploy (same as before) | +| Multiple benchmarks | `["a", "b"]` | M x P x 2 | Each gets its own deployment | + +When `benchmark_key` is null or a single scalar, the matrix degrades to the +previous 2D behavior with no overhead. + +## Per-workload deployment overrides + +Workloads can override deployment profile settings via `deployment_overrides` +in `config.d/workloads.yaml`. This dict is deep-merged on top of the resolved +deployment profile when building the LLMISVC manifest. + +```yaml +benchmarks: + heavy-heterogeneous: + args: + backend_type: openai_http + rate_type: concurrent + rate: [300, 200, 100, 50, 1] + data: prompt_tokens=8000,... + deployment_overrides: + vllm_args: + - --max-model-len=32768 + + trace-replay: + args: + backend_type: openai_http + ... + deployment_overrides: + env: + - name: VLLM_DEV_MODE + value: "1" +``` diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index b3748f679..6dc049806 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -107,10 +107,17 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" + from projects.llm_d.orchestration import runtime_config + # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) - return test_toolbox_run() + exit_code = 0 + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + with env.NextArtifactDir(run_spec.artifact_dirname): + exit_code = max(exit_code, test_toolbox_run()) + return exit_code @main.command() diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index a6cfafd62..1fac3327c 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -30,11 +30,7 @@ def cleanup_namespace(*, namespace: str | None = None) -> None: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() inference_service_name = platform["inference_service"]["name"] - benchmark_job_names = runtime_config.get_benchmark_job_names() - # Multiple benchmarks may share a job_name (defaults to "guidellm-benchmark"); - # the broad labeled sweep below deletes every forge-labeled job/pod/pvc - # regardless, so a single representative name is enough for the named pass. - benchmark_name = benchmark_job_names[0] if benchmark_job_names else None + benchmark_name = runtime_config.get_benchmark_job_name() if not oc_resource_exists("namespace", namespace): return diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index bc1ce1adf..dabbe22af 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -28,6 +28,8 @@ class RunSpec: model_slug: str deployment_profile_name: str deployment_profile_slug: str + benchmark_key: str | None + benchmark_slug: str | None namespace: str artifact_dirname: str namespace_is_managed: bool @@ -99,14 +101,14 @@ def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: return [value] if value else [] -def _deep_merge(base: Any, override: Any) -> Any: +def deep_merge(base: Any, override: Any) -> Any: if not isinstance(base, dict) or not isinstance(override, dict): return copy.deepcopy(override) merged = copy.deepcopy(base) for key, value in override.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = _deep_merge(merged[key], value) + merged[key] = deep_merge(merged[key], value) else: merged[key] = copy.deepcopy(value) return merged @@ -235,7 +237,7 @@ def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: benchmark_args = benchmark.get("args", {}) workload_args = workload_defaults.get("args", {}) if workload_args: - benchmark["args"] = _deep_merge(workload_args, benchmark_args) + benchmark["args"] = deep_merge(workload_args, benchmark_args) return benchmark @@ -253,22 +255,20 @@ def get_benchmark_config() -> dict[str, Any] | None: return _resolve_benchmark_config(benchmark_keys[0]) -def get_benchmark_configs() -> list[tuple[str, dict[str, Any]]]: - return [ - (benchmark_key, _resolve_benchmark_config(benchmark_key)) - for benchmark_key in get_benchmark_keys() - ] +def get_benchmark_deployment_overrides() -> dict[str, Any]: + """Return deployment overrides from the active benchmark config, if any.""" + benchmark = get_benchmark_config() + if benchmark is None: + return {} + return benchmark.get("deployment_overrides", {}) -def get_benchmark_job_names() -> list[str]: - """Distinct k8s benchmark job names for the active run, in order. - - A single benchmark_key collapses to one name; multiple benchmarks that - share a default job_name dedupe to one entry. Empty when benchmarking - is disabled, so cleanup paths can treat it uniformly. - """ - all_names = [benchmark.get("job_name") for _, benchmark in get_benchmark_configs()] - return list(dict.fromkeys(name for name in all_names if name)) +def get_benchmark_job_name() -> str | None: + """The k8s benchmark job name for the active run, or None when benchmarking is disabled.""" + benchmark = get_benchmark_config() + if benchmark is None: + return None + return benchmark.get("job_name") def get_deployment_profile_name() -> str: @@ -291,10 +291,10 @@ def get_deployment_profile() -> dict[str, Any]: defaults = copy.deepcopy(deployment_config.get("defaults", {})) scheduler_manifest = profile.pop("scheduler_manifest", None) - resolved_profile = _deep_merge(defaults, profile) + resolved_profile = deep_merge(defaults, profile) if scheduler_manifest: scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - resolved_profile = _deep_merge(resolved_profile, scheduler_data) + resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile @@ -312,6 +312,10 @@ def get_run_specs() -> list[RunSpec]: _get_runtime_value("deployment_profile"), "runtime.deployment_profile", ) + benchmark_keys = _normalize_string_or_list( + _get_runtime_value("benchmark_key"), + "runtime.benchmark_key", + ) if not model_names: raise ValueError("runtime.model_name must be set to a model name or list of model names") @@ -320,25 +324,37 @@ def get_run_specs() -> list[RunSpec]: "runtime.deployment_profile must be set to a deployment profile or list of profiles" ) - combinations = list(product(model_names, profile_names)) + if benchmark_keys: + benchmark_entries = [ + (key, slugify_identifier(key, max_length=24)) for key in benchmark_keys + ] + else: + benchmark_entries = [(None, None)] + + combinations = list(product(model_names, profile_names, benchmark_entries)) base_namespace = _derive_base_namespace() namespace_max_length = get_platform_config()["cluster"]["namespace"]["max_length"] # Compute base managed state once (ignores per-spec overrides) namespace_is_managed = get_namespace_is_managed() run_specs: list[RunSpec] = [] - for model_name, profile_name in combinations: + for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) if len(combinations) == 1: namespace = base_namespace artifact_dirname = "llmd_run" else: + ns_parts = [base_namespace, model_slug, profile_slug] + dir_parts = ["llmd_run", profile_slug, model_slug] + if bench_slug is not None: + ns_parts.append(bench_slug) + dir_parts.append(bench_slug) namespace = truncate_k8s_name( - f"{base_namespace}-{model_slug}-{profile_slug}", + "-".join(ns_parts), max_length=namespace_max_length, ) - artifact_dirname = f"llmd_run_{profile_slug}_{model_slug}" + artifact_dirname = "_".join(dir_parts) run_specs.append( RunSpec( @@ -346,6 +362,8 @@ def get_run_specs() -> list[RunSpec]: model_slug=model_slug, deployment_profile_name=profile_name, deployment_profile_slug=profile_slug, + benchmark_key=bench_key, + benchmark_slug=bench_slug, namespace=namespace, artifact_dirname=artifact_dirname, namespace_is_managed=namespace_is_managed, @@ -364,12 +382,14 @@ def activate_run_spec(run_spec: RunSpec): "model_name": _get_runtime_value("model_name"), "deployment_profile": _get_runtime_value("deployment_profile"), "namespace_override": _get_runtime_value("namespace_override"), + "benchmark_key": _get_runtime_value("benchmark_key"), } prev_managed_override = _namespace_is_managed_override config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) config.project.set_config("runtime.namespace_override", run_spec.namespace) + config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) _namespace_is_managed_override = run_spec.namespace_is_managed try: yield @@ -377,6 +397,7 @@ def activate_run_spec(run_spec: RunSpec): config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) config.project.set_config("runtime.namespace_override", saved["namespace_override"]) + config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) _namespace_is_managed_override = prev_managed_override diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2ef047ef2..0af65e07e 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -40,11 +40,8 @@ def create_test_labels() -> None: "deployment_profile": deployment_profile, } - # Add guidellm configuration information if benchmark_keys: - labels["guidellm_loadshape"] = ( - benchmark_keys[0] if len(benchmark_keys) == 1 else benchmark_keys - ) + labels["guidellm_loadshape"] = benchmark_keys[0] write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) @@ -222,6 +219,10 @@ def _build_inference_service_manifest() -> Path: deployment_profile = runtime_config.get_deployment_profile() model_cache = runtime_config.get_model_cache_config() + benchmark_overrides = runtime_config.get_benchmark_deployment_overrides() + if benchmark_overrides: + deployment_profile = runtime_config.deep_merge(deployment_profile, benchmark_overrides) + # Build the InferenceService manifest manifest = render_inference_service_from_parts( config_dir=config_dir, @@ -267,30 +268,29 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: def run_guidellm_benchmark(*, endpoint_url: str) -> None: - # Load config where it's consumed from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() - benchmark_configs = runtime_config.get_benchmark_configs() - - if not benchmark_configs: - return # Skip if benchmark is disabled - - for benchmark_key, benchmark in benchmark_configs: - guidellm_args = build_guidellm_args(benchmark) - if not any(arg.startswith("--processor=") for arg in guidellm_args): - guidellm_args.append(f"--processor={runtime_config.get_model_name()}") - artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" - with env.NextArtifactDir(artifact_name): - run_guidellm_benchmark_command.run( - endpoint_url=endpoint_url, - name=benchmark.get("job_name"), - namespace=namespace, - image=benchmark.get("image"), - timeout=benchmark.get("timeout_seconds"), - pvc_size=benchmark.get("pvc_size"), - guidellm_args=guidellm_args, - ) + benchmark = runtime_config.get_benchmark_config() + + if benchmark is None: + return + + benchmark_key = runtime_config.get_benchmark_keys()[0] + guidellm_args = build_guidellm_args(benchmark) + if not any(arg.startswith("--processor=") for arg in guidellm_args): + guidellm_args.append(f"--processor={runtime_config.get_model_name()}") + artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" + with env.NextArtifactDir(artifact_name): + run_guidellm_benchmark_command.run( + endpoint_url=endpoint_url, + name=benchmark.get("job_name"), + namespace=namespace, + image=benchmark.get("image"), + timeout=benchmark.get("timeout_seconds"), + pvc_size=benchmark.get("pvc_size"), + guidellm_args=guidellm_args, + ) def capture_inference_service_state() -> None: @@ -318,7 +318,6 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" - # Load config where it's consumed from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() @@ -326,18 +325,12 @@ def cleanup_test_resources() -> None: inference_service = platform["inference_service"] smoke = platform["smoke"] - # Narrow cleanup (no broad sweep): cover every benchmark job name. Benchmarks - # typically share a single job_name, so this dedupes to one call; when - # benchmarking is disabled, run once with None so the IS/smoke cleanup still - # runs. oc delete is idempotent across iterations. - benchmark_job_names = runtime_config.get_benchmark_job_names() or [None] - for benchmark_job_name in benchmark_job_names: - cleanup_test_resources_command.run( - namespace=namespace, - inference_service_name=inference_service["name"], - smoke_pod_name=smoke["pod_name"], - benchmark_job_name=benchmark_job_name, - ) + cleanup_test_resources_command.run( + namespace=namespace, + inference_service_name=inference_service["name"], + smoke_pod_name=smoke["pod_name"], + benchmark_job_name=runtime_config.get_benchmark_job_name(), + ) def capture_namespace_events_after_test( diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 6b9c948d4..c4df70be7 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -165,10 +165,31 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non "multi-turn", ] - benchmark_configs = dict(runtime_config.get_benchmark_configs()) - assert benchmark_configs["concurrent-1k-1k"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["heavy-heterogeneous"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["multi-turn"]["args"]["request_type"] == "text_completions" + run_specs = runtime_config.get_run_specs() + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + benchmark = runtime_config.get_benchmark_config() + assert benchmark["args"]["request_type"] == "text_completions" + + +def test_release_preset_produces_3_run_specs() -> None: + _init_project_config() + + core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 3 + assert [spec.benchmark_key for spec in run_specs] == [ + "concurrent-1k-1k", + "heavy-heterogeneous", + "multi-turn", + ] + assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) + assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) + + namespaces = [spec.namespace for spec in run_specs] + assert len(namespaces) == len(set(namespaces)) def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: @@ -544,6 +565,116 @@ def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: ] assert run_specs[0].model_slug == "openai-gpt-oss-120b" assert run_specs[2].model_slug == "qwen-qwen3-0-6b" + # With a single benchmark_key (or null), benchmark fields reflect the scalar + assert all(spec.benchmark_key is not None or spec.benchmark_slug is None for spec in run_specs) + + +def test_3d_matrix_with_multiple_benchmark_keys() -> None: + _init_project_config() + + core_config.project.set_config( + "runtime.model_name", + "[openai/gpt-oss-120b, Qwen/Qwen3-0.6B]", + ) + core_config.project.set_config( + "runtime.deployment_profile", + "[distributed-default, precise-prefix-cache]", + ) + core_config.project.set_config( + "runtime.benchmark_key", + "[concurrent-1k-1k, multi-turn]", + ) + + run_specs = runtime_config.get_run_specs() + + # 2 models x 2 profiles x 2 benchmarks = 8 specs + assert len(run_specs) == 8 + assert all(spec.benchmark_key is not None for spec in run_specs) + assert all(spec.benchmark_slug is not None for spec in run_specs) + + assert run_specs[0].model_name == "openai/gpt-oss-120b" + assert run_specs[0].deployment_profile_name == "distributed-default" + assert run_specs[0].benchmark_key == "concurrent-1k-1k" + + # Verify all namespaces are unique + namespaces = [spec.namespace for spec in run_specs] + assert len(namespaces) == len(set(namespaces)) + + # Verify all artifact dirnames are unique + dirnames = [spec.artifact_dirname for spec in run_specs] + assert len(dirnames) == len(set(dirnames)) + + +def test_null_benchmark_key_produces_smoke_only_specs() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key is None + assert run_specs[0].benchmark_slug is None + + +def test_single_benchmark_key_backward_compatible() -> None: + _init_project_config() + + core_config.project.apply_preset("smoke") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key == "short" + assert run_specs[0].benchmark_slug == "short" + assert run_specs[0].artifact_dirname == "llmd_run" + + +def test_activate_run_spec_sets_benchmark_key() -> None: + _init_project_config() + + core_config.project.set_config( + "runtime.benchmark_key", + "[concurrent-1k-1k, multi-turn]", + ) + + run_specs = runtime_config.get_run_specs() + + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + keys = runtime_config.get_benchmark_keys() + assert keys == [run_spec.benchmark_key] + assert runtime_config.get_benchmark_config() is not None + + +def test_benchmark_deployment_overrides() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + core_config.project.config["workloads"]["benchmarks"]["concurrent-1k-1k"][ + "deployment_overrides" + ] = {"vllm_args": ["--max-model-len=8192"]} + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {"vllm_args": ["--max-model-len=8192"]} + + +def test_benchmark_deployment_overrides_empty_when_not_set() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} + + +def test_benchmark_deployment_overrides_empty_when_no_benchmark() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} def test_runtime_rejects_legacy_model_key_path() -> None: @@ -665,12 +796,14 @@ def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> N assert "scheduler" not in manifest["spec"]["router"] -def test_benchmark_job_names_collapse_shared_default() -> None: +def test_benchmark_job_name_from_activated_spec() -> None: _init_project_config() core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") - # Three benchmark_keys, all sharing the workload default job_name -> one entry. - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] + + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" def test_smoke_preset_benchmark_behavior() -> None: @@ -679,7 +812,10 @@ def test_smoke_preset_benchmark_behavior() -> None: core_config.project.apply_preset("smoke") # The smoke preset enables the short benchmark assert runtime_config.get_benchmark_keys() == ["short"] - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] + + run_spec = runtime_config.get_run_specs()[0] + with runtime_config.activate_run_spec(run_spec): + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" def test_run_spec_preserves_namespace_managed_flag() -> None: From b8b04e35c9f490444c28aa8715da3ca73fcb2350 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:43:06 +0200 Subject: [PATCH 068/344] fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs --- .../base/workflows/task-forge-step.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index dcb29b30c..90f746be6 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,6 +13,8 @@ spec: type: string - name: kubeconfig-secret type: string + default: "fournos-clusterless-placeholder" + description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -31,6 +33,8 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig + - name: KUBECONFIG_SECRET_PARAM + value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -73,7 +77,21 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - export KUBECONFIG=$TARGET_KUBECONFIG + # Check if we have a kubeconfig for target cluster + if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then + echo "Setting up target cluster kubeconfig..." + export KUBECONFIG=$TARGET_KUBECONFIG + export CLUSTERLESS_MODE="false" + + else + if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then + echo "Running in clusterless mode - no target cluster kubeconfig" + else + echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" + echo "Running in clusterless mode as fallback" + fi + export CLUSTERLESS_MODE="true" + fi # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -126,3 +144,4 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) + optional: true From b4b4437e7c5d05d383ff4e6e5d6d884ee09f88eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 11:45:01 +0200 Subject: [PATCH 069/344] [skeleton] orchestration: ci: add the launcher stub --- projects/skeleton/orchestration/ci.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/skeleton/orchestration/ci.py b/projects/skeleton/orchestration/ci.py index 6dfef28ac..d8efa8e5c 100755 --- a/projects/skeleton/orchestration/ci.py +++ b/projects/skeleton/orchestration/ci.py @@ -102,6 +102,17 @@ def preflight(ctx) -> int: return 0 +@main.command() +@click.pass_context +@ci_lib.safe_ci_entrypoint +def launcher(ctx) -> int: + """Launcher job - Deploys fournos jobs.""" + + logger.warning("Nothing so far for the launcher job") + + return 0 + + main.add_command(caliper_export_entrypoint) main.add_command(caliper_replot_entrypoint) From 889f94efea9208e163014d1cb776eb321187d87b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:33:43 +0200 Subject: [PATCH 070/344] fournos: gitops: base/workflows/pipeline-launcher: new pipeline to invoke a clusterless launcher --- .../base/workflows/pipeline-launcher.yaml | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 fournos/gitops/base/workflows/pipeline-launcher.yaml diff --git a/fournos/gitops/base/workflows/pipeline-launcher.yaml b/fournos/gitops/base/workflows/pipeline-launcher.yaml new file mode 100644 index 000000000..bca8f0fe8 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-launcher.yaml @@ -0,0 +1,77 @@ +# Single-step pipeline: run only (no prepare, no cleanup). +# Use on pre-configured clusters where setup/teardown is handled externally. +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: forge-launcher + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-preflight + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: preflight + - name: job-step-name + value: 00__preflight + + - name: 01-launcher + runAfter: [00-preflight] + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: launcher + - name: job-step-name + value: 01__launcher + + finally: + - name: 02-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 02__export-artifacts From 7bf1e0e39f3bfc116e493ae5159297a1e4b16a00 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:34:52 +0200 Subject: [PATCH 071/344] [fournos_launcher] orchestration: pr_args: add support for the /clusterless directive --- .../fournos_launcher/orchestration/pr_args.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index b0003acea..4d7a3300b 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -40,6 +40,11 @@ def get_supported_fournos_directives() -> dict[str, str]: Format: /exclusive true|false Example: /exclusive false Effect: Sets fournos.job.exclusive in configuration (default: true).""", + "/clusterless": """Enable clusterless mode for FOURNOS job execution. + Format: /clusterless + Example: /clusterless + Effect: Sets fournos.job.exclusive=false and fournos.job.clusterless=true. + Note: Clusterless and exclusive modes are mutually exclusive.""", "/pipeline": """Set FOURNOS pipeline name for job execution. Format: /pipeline pipeline_name Example: /pipeline llm-load-test @@ -115,6 +120,21 @@ def handle_exclusive_directive(line: str) -> dict[str, str]: return {"fournos.job.exclusive": exclusive_value == "true"} +def handle_clusterless_directive(line: str) -> dict[str, str]: + """ + Handle /clusterless directive for enabling clusterless mode. + + Format: /clusterless + + Args: + line: The directive line + + Returns: + Dictionary with clusterless configuration (sets exclusive=false, clusterless=true) + """ + return {"fournos.job.exclusive": False, "fournos.job.clusterless": True} + + def handle_pipeline_directive(line: str) -> dict[str, str]: """ Handle /pipeline directive for setting pipeline name. @@ -291,6 +311,7 @@ def get_fournos_directive_handlers() -> dict[str, callable]: return { "/cluster": handle_cluster_directive, "/exclusive": handle_exclusive_directive, + "/clusterless": handle_clusterless_directive, "/pipeline": handle_pipeline_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, @@ -324,6 +345,14 @@ def parse_fournos_directives( required_directives=None, # No required directives for FOURNOS ) + # Validate directive conflicts + if config_overrides.get("fournos.job.clusterless") and config_overrides.get( + "fournos.job.exclusive" + ): + raise ValueError( + "Conflicting directives: /clusterless and /exclusive true cannot both be used" + ) + # Log successful parses at info level for FOURNOS for directive in parsed_directives: logger.info(f"Parsed FOURNOS directive: {directive}") From d74d9f12f16327158a0bed48550f217052d00245 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:35:51 +0200 Subject: [PATCH 072/344] [fournos_launcher] orchestration: config: add clusterless field --- projects/fournos_launcher/orchestration/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 4ff135edc..5be5ca092 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -25,6 +25,7 @@ fournos: pipeline_name: forge-test-only ci_label: null # will be generated from the CI job UID exclusive: true + clusterless: null env: OPENSHIFT_CI_job: [JOB_TYPE, JOB_NAME, JOB_SPEC, OPENSHIFT_CI, JOB_NAME_SAFE, BUILD_ID] OPENSHIFT_CI_git_pr: [PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_REF, REPO_NAME, REPO_OWNER, PULL_BASE_SHA, JOB_NAME, PULL_TITLE, PULL_REFS, PULL_HEAD_REF] From f5cb9c5e5ae9790c004348b5261be935f2fe7b4f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:41:57 +0200 Subject: [PATCH 073/344] [fournos_launcher] orchestration: pr_args: add the /fournos directive --- .../fournos_launcher/orchestration/pr_args.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index 4d7a3300b..270d83dd8 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -45,6 +45,11 @@ def get_supported_fournos_directives() -> dict[str, str]: Example: /clusterless Effect: Sets fournos.job.exclusive=false and fournos.job.clusterless=true. Note: Clusterless and exclusive modes are mutually exclusive.""", + "/fournos": """Set FOURNOS namespace environment. + Format: /fournos environment + Example: /fournos wip + /fournos staging + Effect: Sets fournos.namespace to psap-automation-{environment}.""", "/pipeline": """Set FOURNOS pipeline name for job execution. Format: /pipeline pipeline_name Example: /pipeline llm-load-test @@ -135,6 +140,38 @@ def handle_clusterless_directive(line: str) -> dict[str, str]: return {"fournos.job.exclusive": False, "fournos.job.clusterless": True} +def handle_fournos_directive(line: str) -> dict[str, str]: + """ + Handle /fournos directive for setting FOURNOS namespace environment. + + Format: /fournos environment + + Args: + line: The directive line + + Returns: + Dictionary with namespace configuration + + Raises: + ValueError: If environment is not wip or staging + """ + FOURNOS_NAMESPACE_BASE = "psap-automation" + VALID_ENVIRONMENTS = {"wip", "staging"} + + environment = line.removeprefix("/fournos ").strip() + + if not environment: + raise ValueError(f"Invalid /fournos directive: environment cannot be empty in '{line}'") + + if environment not in VALID_ENVIRONMENTS: + raise ValueError( + f"Invalid /fournos directive: environment must be one of {VALID_ENVIRONMENTS}, got '{environment}' in '{line}'" + ) + + namespace = f"{FOURNOS_NAMESPACE_BASE}-{environment}" + return {"fournos.namespace": namespace} + + def handle_pipeline_directive(line: str) -> dict[str, str]: """ Handle /pipeline directive for setting pipeline name. @@ -312,6 +349,7 @@ def get_fournos_directive_handlers() -> dict[str, callable]: "/cluster": handle_cluster_directive, "/exclusive": handle_exclusive_directive, "/clusterless": handle_clusterless_directive, + "/fournos": handle_fournos_directive, "/pipeline": handle_pipeline_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, From f2662b1263d0c5c83d5f07301ee77a12ae4f772e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:51:38 +0200 Subject: [PATCH 074/344] [fournos_launcher] orchestration: submit: /cluster not mandatory if clusterless --- projects/fournos_launcher/orchestration/submit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index dbbcf5944..6c99a9b84 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -258,9 +258,11 @@ def submit_job(): # Validate required configuration before job submission cluster_name = config.project.get_config("cluster.name") - if not cluster_name: + clusterless_mode = config.project.get_config("fournos.job.clusterless") + + if not cluster_name and not clusterless_mode: raise ValueError( - "/cluster (cluster.name) must be set - cannot submit job without target cluster" + "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" ) # Get GPU hardware configuration From 32345a81ffaeed336804c9a4e61dcbdad2a720ed Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 13:53:57 +0200 Subject: [PATCH 075/344] [fournos_launcher] orchestration: more logs and checks for clusterless --- .../fournos_launcher/orchestration/cli.py | 22 ++++++++++++++++--- .../fournos_launcher/orchestration/submit.py | 8 +++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/projects/fournos_launcher/orchestration/cli.py b/projects/fournos_launcher/orchestration/cli.py index 4acd19eea..ff3eda17d 100755 --- a/projects/fournos_launcher/orchestration/cli.py +++ b/projects/fournos_launcher/orchestration/cli.py @@ -53,13 +53,29 @@ def submit(ctx, cluster, project, args, namespace, override, commit): logger.warning(f"Ignoring invalid override format: {override_str} (expected key=value)") # Override config values if provided + clusterless_mode = config.project.get_config("fournos.job.clusterless") + if cluster: config.project.set_config("cluster.name", cluster) else: cluster = config.project.get_config("cluster.name") - if not cluster: - raise ValueError("--cluster or cluster.name is mandatory") - logger.info(f"Using cluster {cluster}") + + if not cluster and not clusterless_mode: + raise ValueError( + "--cluster or cluster.name is mandatory (unless clusterless mode is enabled)" + ) + + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: + raise ValueError( + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" + ) + + if cluster: + logger.info(f"Using cluster {cluster}") + else: + logger.info("Using clusterless mode") if project: config.project.set_config("ci_job.project", project) diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index 6c99a9b84..fbe5ac3b8 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -265,6 +265,13 @@ def submit_job(): "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" ) + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: + raise ValueError( + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" + ) + # Get GPU hardware configuration gpu_count = config.project.get_config("fournos.job.hardware.gpu_count") gpu_type = config.project.get_config("fournos.job.hardware.gpu_type") @@ -296,6 +303,7 @@ def submit_job(): "env": env_dict, "ci_label": config.project.get_config("fournos.job.ci_label"), "exclusive": config.project.get_config("fournos.job.exclusive"), + "clusterless": config.project.get_config("fournos.job.clusterless"), "gpu_count": gpu_count, "gpu_type": gpu_type, } From c51e0c5eea701b08ee3b10f2bb1eead18847fa62 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 13:58:25 +0200 Subject: [PATCH 076/344] [fournos_launcher] toolbox: submit_and_wait: add support for spec.clusterless --- .../toolbox/submit_and_wait/main.py | 19 ++++++++++++++----- .../submit_and_wait/templates/job.yaml.j2 | 4 ++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 88f544a80..40e4917b6 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -47,9 +47,9 @@ def __init__( @entrypoint def run( - cluster_name: str, project: str, *, + cluster_name: str | None = None, args: list = None, variables_overrides: dict = None, job_name: str = "", @@ -63,13 +63,14 @@ def run( exclusive: bool = True, gpu_count: int = None, gpu_type: str = None, + clusterless: bool = False, ): """ Submit a FOURNOS job and wait for completion Args: - cluster_name: Name of the target cluster for the FOURNOS job project: The project name to execute (e.g., 'llm_d', 'skeleton') + cluster_name: Name of the target cluster for the FOURNOS job (None when clusterless=True) args: List of arguments to pass to the project (default: empty list) variables_overrides: Dictionary of config variables to override (default: empty dict) job_name: Custom name for the FOURNOS job (auto-generated if empty) @@ -83,12 +84,13 @@ def run( exclusive: Whether the job should run exclusively on its nodes (default: True) gpu_count: Number of GPUs required for the job (default: None) gpu_type: Type of GPU required for the job (default: None) + clusterless: Whether to enable clusterless mode (makes cluster_name optional, forces exclusive=False) (default: False) Examples: # Called by entrypoint with config values: run( + "llm_d", cluster_name="my-cluster", - project="llm_d", args=["test", "--verbose"], variables_overrides={"model": "mistral", "replicas": 2}, namespace="my-fournos-jobs", @@ -101,6 +103,10 @@ def run( gpu_type="nvidia-tesla-v100", ) """ + # Enforce clusterless=True always uses exclusive=False to prevent conflicts + if clusterless: + exclusive = False + # Set defaults if args is None: args = [] @@ -123,8 +129,11 @@ def run( def validate_inputs(args, ctx): """Validate input parameters""" - if not args.cluster_name: - raise ValueError("cluster_name is required") + if not args.cluster_name and not args.clusterless: + raise ValueError("cluster_name is required unless clusterless mode is enabled") + + if args.clusterless and args.exclusive: + raise ValueError("Clusterless mode and exclusive mode cannot both be enabled") if not args.project: raise ValueError("project is required") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 index 89ab7c5f8..07a84ea9b 100644 --- a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 +++ b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 @@ -10,7 +10,11 @@ metadata: spec: owner: {{ args.owner }} displayName: {{ args.display_name }} + {% if args.clusterless -%} + clusterless: {{ args.clusterless }} + {%- else -%} cluster: {{ args.cluster_name }} + {%- endif %} exclusive: {{ args.exclusive }} pipeline: {{ args.pipeline_name }} {% if args.gpu_count and args.gpu_type -%} From b0bb1d5f6d38fbe074bc50e93e669f65c88144de Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 14:06:33 +0200 Subject: [PATCH 077/344] [projects] Add Changelog files --- .../CHANGELOG-orchestration.md | 31 +++++++++++++++++++ .../fournos_launcher/CHANGELOG-toolbox.md | 19 ++++++++++++ 2 files changed, 50 insertions(+) diff --git a/projects/fournos_launcher/CHANGELOG-orchestration.md b/projects/fournos_launcher/CHANGELOG-orchestration.md index 1430e84d9..e83eb539e 100644 --- a/projects/fournos_launcher/CHANGELOG-orchestration.md +++ b/projects/fournos_launcher/CHANGELOG-orchestration.md @@ -1,5 +1,36 @@ # Fournos Launcher Orchestration Changelog +## 2026-07-13 - Clusterless Mode and New Directives + +### New PR Directives +- **Clusterless Directive**: Added `/clusterless` directive to enable clusterless mode execution + - **Effect**: Sets `fournos.job.exclusive=false` and `fournos.job.clusterless=true` + - **Validation**: Prevents conflicting usage with `/exclusive true` +- **Fournos Environment Directive**: Added `/fournos` directive for namespace targeting + - **Format**: `/fournos wip` or `/fournos staging` + - **Effect**: Sets `fournos.namespace` to `psap-automation-{environment}` + - **Validation**: Restricts to `wip` and `staging` environments only + +### Enhanced Validation +- **Mutual Exclusivity**: Added comprehensive validation to prevent `clusterless=true` and `exclusive=true` conflicts + - **PR Parser Level**: Catches directive conflicts during comment parsing + - **CLI Level**: Validates configuration before command execution + - **Submit Level**: Final validation before job submission +- **Optional Cluster**: Made `/cluster` directive optional when `/clusterless` is used + - **Smart Validation**: Requires cluster specification only when not in clusterless mode + - **Clear Error Messages**: Improved error messaging to explain clusterless option + +### Files Modified +- `pr_args.py` - Added `/clusterless` and `/fournos` directive handlers with validation +- `cli.py` - Enhanced cluster requirement logic and added conflict validation +- `submit.py` - Added clusterless/exclusive mutual exclusivity checks + +### Benefits +- **Flexible Execution**: Enables both cluster-targeted and clusterless job execution modes +- **Environment Targeting**: Simplified namespace selection for different deployment environments +- **Robust Validation**: Prevents configuration conflicts with clear, actionable error messages +- **Backward Compatibility**: Maintains existing directive functionality while adding new capabilities + ## 2026-06-30 - Notification System Integration ### Job Completion Notifications diff --git a/projects/fournos_launcher/CHANGELOG-toolbox.md b/projects/fournos_launcher/CHANGELOG-toolbox.md index e4b0ee2bb..f16dc0ea9 100644 --- a/projects/fournos_launcher/CHANGELOG-toolbox.md +++ b/projects/fournos_launcher/CHANGELOG-toolbox.md @@ -1,5 +1,24 @@ # Fournos Launcher Toolbox Changelog +## 2026-07-13 - Clusterless Mode Support + +### submit_and_wait + +#### Clusterless Execution Support +- **Clusterless Parameter**: Added `clusterless: bool = False` parameter to entrypoint function +- **Template Updates**: Enhanced job.yaml.j2 template to conditionally render `clusterless: true` or `cluster: ` fields +- **Validation Logic**: Added validation to ensure clusterless and exclusive modes are mutually exclusive +- **Optional Cluster**: Made `cluster_name` parameter optional when `clusterless=True` + +#### Files Modified +- `main.py` - Added clusterless parameter support and validation logic +- `templates/job.yaml.j2` - Updated FournosJob template with conditional cluster/clusterless field rendering + +#### Benefits +- Enables clusterless job execution for improved resource flexibility +- Maintains backward compatibility with existing cluster-based workflows +- Provides clear validation for conflicting configuration options + ## 2026-06-24 - Job Management Improvements ### submit_and_wait From fcddf87678fd456127f33f2ebdd34d42d3161ef5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 14 Jul 2026 09:05:47 +0200 Subject: [PATCH 078/344] fournos: gitops: base/workflows/pipeline-prepare-test: fix --- fournos/gitops/base/workflows/pipeline-prepare-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml index 0070f0b0b..40a5a0216 100644 --- a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml +++ b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml @@ -57,7 +57,7 @@ spec: value: 02__preflight - name: 03-test - runAfter: [01-preflight] + runAfter: [02-preflight] taskRef: name: forge-step kind: Task From db896543a065ae8f51e13135ed4162165f4b5b02 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 14 Jul 2026 09:07:58 +0200 Subject: [PATCH 079/344] fournos: gitops: base/workflows/kustomization: add missing pipelines and alpha order --- fournos/gitops/base/workflows/kustomization.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fournos/gitops/base/workflows/kustomization.yaml b/fournos/gitops/base/workflows/kustomization.yaml index d56669a87..2212ab4a0 100644 --- a/fournos/gitops/base/workflows/kustomization.yaml +++ b/fournos/gitops/base/workflows/kustomization.yaml @@ -5,9 +5,11 @@ resources: # - resolve-job.yaml - task-forge-step.yaml - pipeline-full.yaml - - pipeline-test-only.yaml + - pipeline-launcher.yaml + - pipeline-prepare-only.yaml + - pipeline-prepare-test.yaml - pipeline-replot.yaml - + - pipeline-test-only.yaml commonLabels: app.kubernetes.io/name: forge-on-fournos app.kubernetes.io/component: workflows From b8f75a9d0ac8c6308903b554319059c096dc2450 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 15:27:36 +0100 Subject: [PATCH 080/344] feat: Add timeout to best effort delete Signed-off-by: Alberto Perdomo --- projects/llm_d/toolbox/cleanup_test_resources/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 90e12d77a..c4f0a461f 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -220,7 +220,7 @@ def wait_for_workload_pods_deletion(args, ctx): def _best_effort_delete(description: str, *oc_args: str) -> None: """Best effort deletion of Kubernetes resources with timeout""" try: - oc(*oc_args, check=False) + oc(*oc_args, "--timeout=120s", check=False) except subprocess.TimeoutExpired: logger.warning("Timed out deleting %s: oc %s", description, " ".join(oc_args)) From 7bbcf33aebd2a07a7ac84473397a9c69bd40bc79 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 18:01:43 +0100 Subject: [PATCH 081/344] chore: Fix truncate_k8s_name Signed-off-by: Alberto Perdomo --- projects/core/dsl/utils/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/projects/core/dsl/utils/__init__.py b/projects/core/dsl/utils/__init__.py index 296b0cccd..80effeb96 100644 --- a/projects/core/dsl/utils/__init__.py +++ b/projects/core/dsl/utils/__init__.py @@ -39,5 +39,13 @@ def slugify_identifier(value: str, *, max_length: int = 63) -> str: def truncate_k8s_name(value: str, *, max_length: int = 63) -> str: - """Truncate a string to Kubernetes name length limits.""" - return value[:max_length].rstrip("-") + """Truncate a string to Kubernetes name length limits. + + When truncation occurs, a short hash suffix is appended to avoid + collisions between names that share a long common prefix. + """ + if len(value) <= max_length: + return value + hash_suffix = hashlib.sha256(value.encode()).hexdigest()[:6] + prefix = value[: max_length - len(hash_suffix) - 1].rstrip("-") + return f"{prefix}-{hash_suffix}" From 22cf89482fbe03c84e150427ff0e2ef367a5493e Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 18:02:21 +0100 Subject: [PATCH 082/344] feat: Enhance delete best effort Signed-off-by: Alberto Perdomo --- projects/llm_d/docs/benchmark-isolation.md | 4 +++- projects/llm_d/toolbox/cleanup_test_resources/main.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/docs/benchmark-isolation.md b/projects/llm_d/docs/benchmark-isolation.md index c6a357c71..3fae5d2b0 100644 --- a/projects/llm_d/docs/benchmark-isolation.md +++ b/projects/llm_d/docs/benchmark-isolation.md @@ -84,7 +84,9 @@ previous 2D behavior with no overhead. Workloads can override deployment profile settings via `deployment_overrides` in `config.d/workloads.yaml`. This dict is deep-merged on top of the resolved -deployment profile when building the LLMISVC manifest. +deployment profile when building the LLMISVC manifest. Lists (like `vllm_args`) +are **replaced entirely**, not concatenated — this avoids duplicate or +conflicting flags when the override supplies a different set of arguments. ```yaml benchmarks: diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index c4f0a461f..9afa1fb88 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -218,9 +218,15 @@ def wait_for_workload_pods_deletion(args, ctx): def _best_effort_delete(description: str, *oc_args: str) -> None: - """Best effort deletion of Kubernetes resources with timeout""" + """Best effort deletion of Kubernetes resources with timeout.""" try: - oc(*oc_args, "--timeout=120s", check=False) + result = oc(*oc_args, "--timeout=300s", check=False) + if result.returncode != 0: + logger.warning( + "Failed to delete %s (rc=%d), capturing describe", description, result.returncode + ) + describe_args = tuple(a for a in oc_args if a != "delete") + oc("describe", *describe_args, check=False) except subprocess.TimeoutExpired: logger.warning("Timed out deleting %s: oc %s", description, " ".join(oc_args)) From cbff983d574ff22a0028fae4e3b94833b7546c31 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 10 Jul 2026 11:49:08 -0400 Subject: [PATCH 083/344] Port model-furnace features to forge/rhaiis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CSV dashboard export (csv_export.py): generates consolidated_dashboard.csv from PSAP JSON files matching model-furnace's exact column schema - S3 sync (s3_dashboard.py): download/append/re-upload consolidated CSV and upload profiler traces, using new psap-forge-dashboard-s3 vault - Regression detection (analyze.py): geometric mean comparison with configurable thresholds per metric, ported from model-furnace's compare_results.py - Slack webhook notifications (send.py): regression/improvement alerts via Incoming Webhook (psap-forge-rhaiis-slack vault), with Bot Token fallback - Profiler toolbox tasks: enable/disable gate, copy traces from pod, and verify_profiler_prereqs (webhook pod, TLS, ConfigMap checks) - Test phase wiring: profiler prereq check → gate toggle → benchmark → trace copy → PSAP payload → CSV export → S3 sync → regression analysis → Slack → profiler trace upload → MLflow metadata - Config: new vault definitions, profiler section, csv_dashboard and analyze sections in rhaiis config Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 246 +++++++++++- projects/core/notifications/send.py | 151 ++++++++ .../rhaiis/orchestration/config.d/rhaiis.yaml | 7 + .../orchestration/config.d/workloads.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 17 + .../rhaiis/orchestration/runtime_config.py | 4 + projects/rhaiis/orchestration/test_phase.py | 355 ++++++++++++++++-- projects/rhaiis/postprocess/csv_export.py | 184 +++++++++ projects/rhaiis/postprocess/s3_dashboard.py | 175 +++++++++ .../toolbox/copy_profiler_traces/main.py | 67 ++++ .../toolbox/enable_profiler_gate/main.py | 50 +++ .../toolbox/verify_profiler_prereqs/main.py | 127 +++++++ vaults/psap-forge-dashboard-s3.yaml | 16 + vaults/psap-forge-rhaiis-slack.yaml | 10 + 14 files changed, 1389 insertions(+), 24 deletions(-) create mode 100644 projects/rhaiis/postprocess/csv_export.py create mode 100644 projects/rhaiis/postprocess/s3_dashboard.py create mode 100644 projects/rhaiis/toolbox/copy_profiler_traces/main.py create mode 100644 projects/rhaiis/toolbox/enable_profiler_gate/main.py create mode 100644 projects/rhaiis/toolbox/verify_profiler_prereqs/main.py create mode 100644 vaults/psap-forge-dashboard-s3.yaml create mode 100644 vaults/psap-forge-rhaiis-slack.yaml diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 825762b62..f2ef0b4bd 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,14 +2,258 @@ from __future__ import annotations +import csv import json import logging +import tempfile import time +from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Optional + +import numpy as np logger = logging.getLogger(__name__) +PROFILE_MAP = { + (1000, 1000): "profile1", + (512, 2048): "profile2", + (2048, 128): "profile3", + (8000, 1000): "profile4", +} + +METRICS = { + "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, + "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, + "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, + "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, + "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, +} + + +@dataclass +class RegressionResult: + profile: str + metric_column: str + metric_label: str + current_value: float + baseline_value: float + pct_diff: float + is_regression: bool + is_improvement: bool = False + + +def geometric_mean(values) -> Optional[float]: + positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + if len(positive) == 0: + return None + return float(np.exp(np.mean(np.log(positive)))) + + +def compare_runs( + current_csv_path: str, + consolidated_csv_path: str, + compare_version: str, + current_version: str, + restrict_profiles: Optional[list[str]] = None, +) -> tuple[list[RegressionResult], str]: + """Compare current run metrics against a baseline version. + + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. + """ + import pandas as pd + + current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") + baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") + + for col in current_df.columns: + if current_df[col].dtype == object: + current_df[col] = current_df[col].str.strip() + for col in baseline_df.columns: + if baseline_df[col].dtype == object: + baseline_df[col] = baseline_df[col].str.strip() + + if current_df.empty: + return [], "Current run produced no benchmark data" + + model = current_df["model"].iloc[0] + accelerator = current_df["accelerator"].iloc[0] + tp = current_df["TP"].iloc[0] + + logger.info("Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, compare_version, model, accelerator, tp) + + def _assign_profile(row): + try: + isl = int(float(row["prompt toks"])) + osl = int(float(row["output toks"])) + except (ValueError, TypeError): + return None + return PROFILE_MAP.get((isl, osl)) + + current_df["_profile"] = current_df.apply(_assign_profile, axis=1) + baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) + + baseline_filtered = baseline_df[ + (baseline_df["version"] == compare_version) + & (baseline_df["model"] == model) + & (baseline_df["accelerator"] == accelerator) + & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) + ] + + if baseline_filtered.empty: + return [], f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}" + + current_profiles = set(current_df["_profile"].dropna().unique()) + baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) + common_profiles = sorted(current_profiles & baseline_profiles) + + if restrict_profiles: + common_profiles = [p for p in common_profiles if p in restrict_profiles] + + if not common_profiles: + return [], "No common ISL/OSL profiles between current and baseline" + + results: list[RegressionResult] = [] + + for profile in common_profiles: + cur = current_df[current_df["_profile"] == profile].copy() + base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() + + cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") + base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") + + common_conc = set(cur["intended concurrency"].dropna().unique()) & set(base["intended concurrency"].dropna().unique()) + common_conc.discard(1) + + if not common_conc: + continue + + cur_common = cur[cur["intended concurrency"].isin(common_conc)] + base_common = base[base["intended concurrency"].isin(common_conc)] + + for col, meta in METRICS.items(): + if col not in cur_common.columns or col not in base_common.columns: + continue + + cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() + base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() + + cur_gm = geometric_mean(cur_vals) + base_gm = geometric_mean(base_vals) + + if cur_gm is None or base_gm is None: + continue + + pct_diff = ((cur_gm - base_gm) / base_gm) * 100 + threshold = meta["threshold"] + + if meta["higher_is_better"]: + is_regression = pct_diff < -threshold + is_improvement = pct_diff > threshold + else: + is_regression = pct_diff > threshold + is_improvement = pct_diff < -threshold + + results.append(RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + )) + + regressions = [r for r in results if r.is_regression] + logger.info("Comparison complete: %d metric comparisons, %d regressions", + len(results), len(regressions)) + return results, "" + + +def run_regression_analysis( + current_csv_path: Path, + consolidated_csv_path: Path, + compare_version: str, + current_version: str, + output_file: Path, +) -> dict[str, Any]: + """Run regression analysis comparing current run against baseline. + + Returns dict with status, regressions, improvements, and skip_reason. + """ + try: + results, skip_reason = compare_runs( + str(current_csv_path), + str(consolidated_csv_path), + compare_version, + current_version, + ) + + if skip_reason: + analysis = { + "status": "skipped", + "reason": skip_reason, + "current_version": current_version, + "compare_version": compare_version, + } + else: + regressions = [r for r in results if r.is_regression] + improvements = [r for r in results if r.is_improvement] + + analysis = { + "status": "completed", + "current_version": current_version, + "compare_version": compare_version, + "total_comparisons": len(results), + "regression_count": len(regressions), + "improvement_count": len(improvements), + "regressions": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in regressions + ], + "improvements": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in improvements + ], + "all_results": [ + { + "profile": r.profile, + "metric": r.metric_label, + "metric_column": r.metric_column, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + "is_regression": r.is_regression, + "is_improvement": r.is_improvement, + } + for r in results + ], + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis, f, indent=2) + + return analysis + + except Exception as e: + logger.exception("Regression analysis failed") + return {"status": "failed", "error": str(e)} + def run_kpi_analysis( current_kpi_file: Path, diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index ab6f64acc..10415559f 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -16,6 +16,7 @@ GITHUB_APP_PEM_FILE = "topsail-bot.2024-09-18.private-key.pem" GITHUB_APP_CLIENT_ID_FILE = "topsail-bot.clientid" SLACK_TOKEN_FILE = "topsail-bot.slack-token" +SLACK_WEBHOOK_URL_FILE = "slack-webhook-url" DEFAULT_REPO_OWNER = "openshift-psap" DEFAULT_REPO_NAME = "forge" @@ -714,6 +715,156 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok +def _send_via_webhook(webhook_url: str, blocks: list[dict]) -> bool: + """Post a Slack message via Incoming Webhook URL.""" + from urllib.error import URLError + from urllib.request import Request, urlopen + + body = json.dumps({"blocks": blocks}) + try: + req = Request( + webhook_url, + data=body.encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + urlopen(req, timeout=15) # noqa: S310 + return True + except (URLError, OSError) as e: + logger.warning("Slack webhook POST failed: %s", e) + return False + + +def _get_webhook_url(vault_name: str) -> str | None: + """Read the webhook URL from a vault file.""" + from projects.core.library import vault as vault_lib + + path = vault_lib.get_vault_content_path(vault_name, SLACK_WEBHOOK_URL_FILE) + if path is None or not path.exists(): + logger.warning("Webhook URL file not found in vault %s", vault_name) + return None + return path.read_text().strip() + + +def send_regression_notification( + analysis_result: dict, + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, +) -> bool: + """Send regression/improvement Slack notification from analysis results. + + Uses a webhook URL (from *webhook_vault*) when available, falling + back to the Bot-Token path (from *notification_vault*). + + Args: + analysis_result: Output from run_regression_analysis() + model: Model name for display + accelerator: Accelerator name for display + job_id: Job identifier + slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + + Returns: + True if notification sent successfully + """ + status = analysis_result.get("status") + if status == "skipped": + reason = analysis_result.get("reason", "unknown") + logger.info("Regression analysis skipped: %s", reason) + return True + + if status != "completed": + return True + + regressions = analysis_result.get("regressions", []) + improvements = analysis_result.get("improvements", []) + + if not regressions and not improvements: + logger.info("No regressions or improvements to report") + return True + + current_version = analysis_result.get("current_version", "") + compare_version = analysis_result.get("compare_version", "") + + if regressions and improvements: + icon = ":warning:" + headline = "Performance regressions and improvements detected" + elif regressions: + icon = ":warning:" + headline = "Performance regressions detected" + else: + icon = ":large_green_circle:" + headline = "Performance improvements detected" + + detail_lines = [] + all_results = analysis_result.get("all_results", []) + profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + + for profile in profiles: + profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + detail_lines.append(f"\n*{profile}:*") + for r in profile_results: + if r.get("is_regression"): + direction = "dropped" if r["pct_diff"] < 0 else "increased" + detail_lines.append( + f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + else: + detail_lines.append( + f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + + details = "\n".join(detail_lines) + + import re as _re + + if slack_user and _re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + message = ( + f"{icon} *{headline}*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"*Changes:*\n{details}" + ) + + if dry_run: + logger.info("DRY RUN regression notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) + + def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 4965747f1..c2d0676d8 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,3 +36,10 @@ accelerator_env_vars: nvidia: {} amd: VLLM_ROCM_USE_AITER: "1" + +profiler: + enabled: false + s3_bucket: psap-dashboard-data + s3_prefix: pytorch-profiles/rhaiis + vault: psap-forge-dashboard-s3 + labels: [] diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 95231ac5a..113e55d49 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,8 +5,8 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [1, 50, 100, 200, 300] - max_seconds: 450 + rates: [1] + max_seconds: 45 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 6c932e4bb..7323c7d0e 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -1,6 +1,9 @@ vaults: - psap-forge-notifications - psap-forge-mlflow-export +- psap-forge-aws-s3-export +- psap-forge-dashboard-s3 +- psap-forge-rhaiis-slack benchmarks: guidellm: @@ -19,6 +22,11 @@ tests: rhaiis: model_key: qwen3-0_6b workload_key: profile1 + run_benchmark: true + warmup: true + version: "" + compare_version: "" + slack_user: "" caliper: postprocess: @@ -34,6 +42,15 @@ caliper: artifacts_to_kpis: enabled: true output: kpis.json + csv_dashboard: + enabled: false + s3_bucket: psap-dashboard-data + s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv + vault: psap-forge-dashboard-s3 + analyze: + enabled: false + historical_kpis: historical_kpis + output: regression_analysis.json export: from: null diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index d17c4c7d0..44ff22f21 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -67,6 +67,10 @@ def get_test_workload_key() -> str: return config.project.get_config("tests.rhaiis.workload_key") +def get_profiler_config() -> dict: + return dict(config.project.get_config("rhaiis.profiler", {}) or {}) + + def merge_vllm_args( defaults: dict, model: dict, diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 762dbc2b1..b324f7f6f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -112,27 +112,60 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - logger.info("Running benchmark at rates=%s", rates) - - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) - - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}", - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - ) + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + + if profiler_enabled: + logger.info("Skipping warmup -- profiler step provides its own warmup") + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=workload_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + else: + logger.info("Running benchmark at rates=%s", rates) + + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) + + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-{deployment_name}", + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) finally: _capture_and_cleanup(deployment_name, namespace) @@ -141,6 +174,18 @@ def _run_test( except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) + try: + _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args) + except Exception: + logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + + profiler_cfg = runtime_config.get_profiler_config() + if profiler_cfg.get("enabled", False): + try: + _upload_profiler_traces(model_cfg, accelerator, vllm_args, profiler_cfg) + except Exception: + logger.warning("Profiler trace upload failed; continuing", exc_info=True) + try: _set_mlflow_metadata( model_key, @@ -254,6 +299,274 @@ def _generate_psap_payload( ) +def _generate_and_sync_dashboard_csv( + model_cfg: dict, + accelerator: str, + workload_key: str, + vllm_args: dict, +) -> None: + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv + + psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) + if not psap_files: + logger.warning("No PSAP JSON files found, skipping dashboard CSV") + return + + version = config.project.get_config("tests.rhaiis.version", "") + if not version: + logger.info("No version configured, skipping dashboard CSV generation") + return + + csv_path = None + for psap_file in psap_files: + output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" + csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path) + logger.info("Generated dashboard CSV: %s", csv_path) + + if not csv_path: + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): + logger.info("Dashboard CSV S3 sync not enabled") + return + + from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 + + sync_result = sync_csv_to_s3( + csv_path, + s3_bucket=csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data"), + s3_key=csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv"), + vault_name=csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Dashboard CSV sync result: %s", sync_result) + + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not compare_version: + return + + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator) + + +def _run_regression_check( + csv_path, + compare_version: str, + current_version: str, + model_cfg: dict, + accelerator: str, +) -> None: + import tempfile + from pathlib import Path + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.caliper.engine.kpi.analyze import run_regression_analysis + from projects.core.library import config + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") + s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if not credentials_path: + logger.warning("AWS credentials not available, skipping regression analysis") + return + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + except Exception as e: + logger.warning("Could not download consolidated CSV for regression check: %s", e) + return + + try: + output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" + analysis = run_regression_analysis( + current_csv_path=Path(str(csv_path)), + consolidated_csv_path=Path(consolidated_path), + compare_version=compare_version, + current_version=current_version, + output_file=output_file, + ) + + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + from projects.core.notifications.send import send_regression_notification + + send_regression_notification( + analysis, + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=env.ARTIFACT_DIR.rsplit("/", 1)[-1] if env.ARTIFACT_DIR else "", + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-rhaiis-slack", + notification_vault="psap-forge-notifications", + ) + except Exception: + logger.warning("Regression analysis failed; continuing", exc_info=True) + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def _run_warmup_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, + model_cfg: dict, + workload: dict, + benchmark_timeout: int, +) -> None: + """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + + warmup_rates = [200] + warmup_max_seconds = 60 + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=warmup_rates, + max_seconds=warmup_max_seconds, + ) + + logger.info("Running warmup (concurrency=200, duration=60s)") + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-warmup-{deployment_name}", + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + logger.info("Warmup completed") + except Exception: + logger.warning("Warmup failed; continuing with benchmark", exc_info=True) + + +def _run_profiler_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, + model_cfg: dict, + workload: dict, + workload_key: str, + benchmark_timeout: int, +) -> None: + """Run profiler-gated benchmarks: verify prereqs → enable gate → benchmark → disable gate → copy traces.""" + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + from projects.rhaiis.toolbox.copy_profiler_traces.main import run as copy_profiler_traces + from projects.rhaiis.toolbox.enable_profiler_gate.main import run as enable_profiler_gate + from projects.rhaiis.toolbox.verify_profiler_prereqs.main import run as verify_profiler_prereqs + + logger.info("Verifying profiler prerequisites") + verify_profiler_prereqs(namespace=namespace) + + profiler_cfg = runtime_config.get_profiler_config() + labels = profiler_cfg.get("labels", []) + if not labels: + logger.info("No profiler labels configured, skipping profiler step") + return + + profiler_max_seconds = profiler_cfg.get("max_seconds", 60) + + for label in labels: + logger.info("Profiling label=%s", label) + + gate_value = label if isinstance(label, str) else str(label) + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + gate_value=gate_value, + ) + + profiler_rates = profiler_cfg.get("rates", [1]) + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=profiler_rates, + max_seconds=profiler_max_seconds, + ) + + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-profiler-{deployment_name}-{gate_value}", + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + finally: + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + disable=True, + ) + + logger.info("Copying profiler traces from pod") + try: + copy_profiler_traces(name=deployment_name, namespace=namespace) + except Exception: + logger.warning("Failed to copy profiler traces", exc_info=True) + + +def _upload_profiler_traces( + model_cfg: dict, + accelerator: str, + vllm_args: dict, + profiler_cfg: dict, +) -> None: + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 + + traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces" + if not traces_dir.exists() or not any(traces_dir.iterdir()): + logger.info("No profiler traces to upload") + return + + version = config.project.get_config("tests.rhaiis.version", "") + if not version: + logger.info("No version configured, skipping profiler trace upload") + return + + result = upload_profiler_traces_to_s3( + traces_dir, + model_name=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + tp_size=int(vllm_args.get("tensor-parallel-size", 1)), + version=version, + profile_labels=profiler_cfg.get("labels", []), + s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), + s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), + vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Profiler trace upload result: %s", result) + + def _capture_and_cleanup(deployment_name: str, namespace: str) -> None: from projects.rhaiis.toolbox.capture_isvc_state.main import run as capture_isvc_state diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py new file mode 100644 index 000000000..42c56fb7a --- /dev/null +++ b/projects/rhaiis/postprocess/csv_export.py @@ -0,0 +1,184 @@ +"""Generate consolidated_dashboard.csv from PSAP JSON files. + +Matches the CSV schema from model-furnace's visualization/parse_furnace_results.py. +""" + +from __future__ import annotations + +import csv +import json +import logging +import re +from pathlib import Path + +logger = logging.getLogger(__name__) + +FIELDNAMES = [ + "run", "accelerator", "model", "version", "prompt toks", "output toks", "TP", + "measured concurrency", "intended concurrency", "measured rps", + "output_tok/sec", "total_tok/sec", "prompt_token_count_mean", + "prompt_token_count_p99", "output_token_count_mean", + "output_token_count_p99", "ttft_median", "ttft_p95", "ttft_p1", "ttft_p999", + "tpot_median", "tpot_p95", "tpot_p99", "tpot_p999", "tpot_p1", + "itl_median", "itl_p95", "itl_p999", "itl_p1", + "request_latency_median", "request_latency_min", "request_latency_max", + "successful_requests", "errored_requests", "uuid", + "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", + "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", +] + + +def find_psap_files(artifact_dir: Path) -> list[Path]: + """Glob for PSAP_perf*.json files in artifact_dir (recursive).""" + return sorted(artifact_dir.rglob("PSAP_perf*.json")) + + +def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path) -> Path: + """Read a PSAP JSON file and produce a dashboard CSV. + + Args: + psap_json_path: Path to the PSAP payload JSON file. + version: Version string for the CSV rows. + output_path: Where to write the CSV. + + Returns: + Path to the generated CSV file. + """ + with open(psap_json_path, encoding="utf-8") as f: + payload = json.load(f) + + model = payload.get("model", "") + accelerator = payload.get("accelerator_type", "") + tp = payload.get("accelerator_count", 1) + image_tag = payload.get("container_image_tag", "") + runtime_args = json.dumps(payload.get("inference_server_args", {})) + guidellm_start_ms = payload.get("guidellm_start_time_ms") + guidellm_end_ms = payload.get("guidellm_end_time_ms") + + report = payload.get("report", {}) + guidellm_version = report.get("metadata", {}).get("guidellm_version", "") + benchmarks = report.get("benchmarks", []) + + data_str = "" + args = report.get("args", {}) + if isinstance(args, dict): + data_list = args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + prompt_toks = tokens.get("prompt_tokens", "") + output_toks = tokens.get("output_tokens", "") + + rows = [] + for bench in benchmarks: + metrics = bench.get("metrics", {}) + row = _extract_row( + metrics=metrics, + model=model, + accelerator=accelerator, + version=version, + tp=tp, + prompt_toks=prompt_toks, + output_toks=output_toks, + image_tag=image_tag, + runtime_args=runtime_args, + guidellm_start_ms=guidellm_start_ms, + guidellm_end_ms=guidellm_end_ms, + guidellm_version=guidellm_version, + ) + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + logger.info("Generated dashboard CSV with %d rows: %s", len(rows), output_path) + return output_path + + +def _extract_row( + *, + metrics: dict, + model: str, + accelerator: str, + version: str, + tp: int, + prompt_toks: str, + output_toks: str, + image_tag: str, + runtime_args: str, + guidellm_start_ms, + guidellm_end_ms, + guidellm_version: str, +) -> dict: + def _pct(metric_name: str, pct: str) -> float: + return float( + metrics.get(metric_name, {}) + .get("successful", {}) + .get("percentiles", {}) + .get(pct, 0) + ) + + def _stat(metric_name: str, stat: str) -> float: + return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, 0)) + + request_totals = metrics.get("request_totals", {}) + successful = int(request_totals.get("successful", 0)) + errored = int(request_totals.get("errored", 0)) + + concurrency_mean = _stat("request_concurrency", "mean") + intended_concurrency = _stat("request_concurrency", "max") + + output_tok_sec = _stat("output_token_throughput", "mean") + total_tok_sec = _stat("total_token_throughput", "mean") + measured_rps = _stat("request_rate", "mean") + + return { + "run": "", + "accelerator": accelerator, + "model": model, + "version": version, + "prompt toks": prompt_toks, + "output toks": output_toks, + "TP": tp, + "measured concurrency": concurrency_mean, + "intended concurrency": intended_concurrency, + "measured rps": measured_rps, + "output_tok/sec": output_tok_sec, + "total_tok/sec": total_tok_sec, + "prompt_token_count_mean": _stat("prompt_token_count", "mean"), + "prompt_token_count_p99": _pct("prompt_token_count", "p99"), + "output_token_count_mean": _stat("output_token_count", "mean"), + "output_token_count_p99": _pct("output_token_count", "p99"), + "ttft_median": _pct("time_to_first_token_ms", "p50") / 1000.0, + "ttft_p95": _pct("time_to_first_token_ms", "p95") / 1000.0, + "ttft_p1": _pct("time_to_first_token_ms", "p1") / 1000.0, + "ttft_p999": _pct("time_to_first_token_ms", "p999") / 1000.0, + "tpot_median": _pct("time_per_output_token_ms", "p50") / 1000.0, + "tpot_p95": _pct("time_per_output_token_ms", "p95") / 1000.0, + "tpot_p99": _pct("time_per_output_token_ms", "p99") / 1000.0, + "tpot_p999": _pct("time_per_output_token_ms", "p999") / 1000.0, + "tpot_p1": _pct("time_per_output_token_ms", "p1") / 1000.0, + "itl_median": _pct("inter_token_latency_ms", "p50") / 1000.0, + "itl_p95": _pct("inter_token_latency_ms", "p95") / 1000.0, + "itl_p999": _pct("inter_token_latency_ms", "p999") / 1000.0, + "itl_p1": _pct("inter_token_latency_ms", "p1") / 1000.0, + "request_latency_median": _pct("request_latency_ms", "p50") / 1000.0, + "request_latency_min": _stat("request_latency_ms", "min") / 1000.0, + "request_latency_max": _stat("request_latency_ms", "max") / 1000.0, + "successful_requests": successful, + "errored_requests": errored, + "uuid": "", + "ttft_mean": _stat("time_to_first_token_ms", "mean") / 1000.0, + "ttft_p99": _pct("time_to_first_token_ms", "p99") / 1000.0, + "itl_mean": _stat("inter_token_latency_ms", "mean") / 1000.0, + "itl_p99": _pct("inter_token_latency_ms", "p99") / 1000.0, + "runtime_args": runtime_args, + "guidellm_start_time_ms": guidellm_start_ms or "", + "guidellm_end_time_ms": guidellm_end_ms or "", + "image_tag": image_tag, + "guidellm_version": guidellm_version, + } diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py new file mode 100644 index 000000000..efa2c5c3e --- /dev/null +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -0,0 +1,175 @@ +"""S3 dashboard CSV sync and profiler trace upload for rhaiis.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def sync_csv_to_s3( + csv_path: Path, + *, + s3_bucket: str = "psap-dashboard-data", + s3_key: str = "staging/rhaiis-dashboard/consolidated_dashboard.csv", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Download consolidated CSV from S3, append new rows, re-upload. + + Ported from model-furnace's ModelValidator._sync_csv_to_s3(). + """ + try: + import pandas as pd + except ImportError: + return {"status": "failed", "error": "pandas not available"} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + if dry_run: + logger.info("DRY RUN: Would sync %s to s3://%s/%s", csv_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + + try: + s3.download_file(s3_bucket, s3_key, consolidated_path) + logger.info("Downloaded s3://%s/%s", s3_bucket, s3_key) + except Exception as e: + if hasattr(e, "response") and e.response.get("Error", {}).get("Code") == "404": + logger.warning("Consolidated CSV not found on S3, uploading new CSV") + s3.upload_file(str(csv_path), s3_bucket, s3_key) + return {"status": "success", "action": "created", "rows_added": -1} + raise + + new_df = pd.read_csv(csv_path) + if new_df.empty: + logger.warning("No data rows in generated CSV %s", csv_path) + return {"status": "skipped", "reason": "empty CSV"} + + existing_df = pd.read_csv(consolidated_path, on_bad_lines="warn") + merged_df = pd.concat([existing_df, new_df], ignore_index=True) + merged_df.to_csv(consolidated_path, index=False) + rows_added = len(new_df) + logger.info("Appended %d rows (%d total)", rows_added, len(merged_df)) + + s3.upload_file(consolidated_path, s3_bucket, s3_key) + logger.info("Uploaded consolidated CSV to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "action": "appended", "rows_added": rows_added} + + except Exception as e: + logger.error("S3 CSV sync failed: %s", e) + return {"status": "failed", "error": str(e)} + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def upload_profiler_traces_to_s3( + traces_dir: Path, + *, + model_name: str, + accelerator: str, + tp_size: int, + version: str, + profile_labels: list[str], + s3_bucket: str = "psap-dashboard-data", + s3_prefix: str = "pytorch-profiles/rhaiis", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload profiler traces to S3. + + Ported from model-furnace's ModelValidator._upload_profiler_traces_to_s3(). + Path: s3://{bucket}/{prefix}/{accel}/{model}/tp{N}/{version}/{profile_label}/ + """ + if not traces_dir.exists(): + return {"status": "skipped", "reason": f"traces dir not found: {traces_dir}"} + + trace_files = [ + f for f in traces_dir.iterdir() + if f.name.startswith("trace") and "rank0" in f.name + and f.suffix in (".json", ".gz") + ] + if not trace_files: + return {"status": "skipped", "reason": "no trace files found"} + + if dry_run: + logger.info("DRY RUN: Would upload %d traces to S3", len(trace_files)) + return {"status": "success", "dry_run": True, "trace_count": len(trace_files)} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + accel_folder = _accelerator_s3_folder(accelerator) + model_encoded = model_name.replace("/", "--") + uploaded = 0 + + try: + s3 = create_s3_client(credentials_path) + + for trace_file in trace_files: + matched_label = None + for label in profile_labels: + if label in trace_file.name: + matched_label = label + break + if not matched_label: + matched_label = profile_labels[0] if len(profile_labels) == 1 else None + if not matched_label: + logger.warning("Could not match trace %s to a profile label", trace_file.name) + continue + + s3_key = ( + f"{s3_prefix}/{accel_folder}/{model_encoded}" + f"/tp{tp_size}/{version}/{matched_label}/{trace_file.name}" + ) + + try: + s3.head_object(Bucket=s3_bucket, Key=s3_key) + s3.copy_object( + Bucket=s3_bucket, + CopySource={"Bucket": s3_bucket, "Key": s3_key}, + Key=s3_key + ".bak", + ) + logger.info("Backed up existing trace at s3://%s/%s", s3_bucket, s3_key) + except Exception: + pass + + s3.upload_file(str(trace_file), s3_bucket, s3_key) + logger.info("Uploaded trace to s3://%s/%s", s3_bucket, s3_key) + uploaded += 1 + + return {"status": "success", "uploaded": uploaded} + except Exception as e: + logger.error("Trace upload failed: %s", e) + return {"status": "failed", "error": str(e), "uploaded": uploaded} + + +def _accelerator_s3_folder(accelerator: str) -> str: + upper = accelerator.upper() + if upper.startswith("H200"): + return "H200" + if upper.startswith("B200"): + return "B200" + if upper.startswith("MI300"): + return "MI300x" + return accelerator diff --git a/projects/rhaiis/toolbox/copy_profiler_traces/main.py b/projects/rhaiis/toolbox/copy_profiler_traces/main.py new file mode 100644 index 000000000..0d9331aad --- /dev/null +++ b/projects/rhaiis/toolbox/copy_profiler_traces/main.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str): + return execute_tasks(locals()) + + +@task +def setup_directories(args, context): + shell.mkdir("artifacts/traces") + return "Traces directory created" + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def list_trace_files(args, context): + result = shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'ls /tmp/trace_*.json* 2>/dev/null || echo NO_TRACES'", + check=False, + log_stdout=False, + ) + + if "NO_TRACES" in result.stdout or not result.stdout.strip(): + raise RuntimeError(f"No profiler traces found in pod {context.pod_name}") + + trace_list = result.stdout.strip() + context.trace_count = len(trace_list.splitlines()) + return f"Found {context.trace_count} trace files" + + +@task +def copy_traces(args, context): + traces_dir = args.artifact_dir / "artifacts/traces" + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'cd /tmp && tar cf - trace_*.json*' " + f"| tar --no-same-owner -xf - -C {traces_dir}", + ) + return f"Copied {context.trace_count} trace files to {traces_dir}" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py new file mode 100644 index 000000000..69468b405 --- /dev/null +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str, gate_value: str = "0", disable: bool = False): + return execute_tasks(locals()) + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def toggle_profiler_gate(args, context): + if args.disable: + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'rm -f /tmp/profiler_gate'", + check=False, + ) + return f"Profiler gate disabled on {context.pod_name}" + + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + f"-- sh -c 'echo {args.gate_value} > /tmp/profiler_gate'", + ) + return f"Profiler gate enabled on {context.pod_name} (value={args.gate_value})" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py new file mode 100644 index 000000000..dc2e95e50 --- /dev/null +++ b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Verify that the vLLM profiler webhook infrastructure is ready. + +Ported from model-furnace's oc_commands.verify_profiler_prerequisites(). +Checks webhook pod, MutatingWebhookConfiguration, TLS secret, and +ConfigMap before any profiler run is attempted. +""" + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + +MIN_CA_BUNDLE_LENGTH = 100 + + +@entrypoint +def run(*, namespace: str): + return execute_tasks(locals()) + + +@task +def check_webhook_pod(args, context): + result = shell.run( + "oc get pods -n vllm-profiler -l app=env-injector " + "-o jsonpath='{.items[0].status.phase}'", + check=False, + ) + phase = result.stdout.strip().strip("'") + if result.returncode != 0 or phase != "Running": + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook pod not running (status: {phase or 'not found'}). " + "Deploy it with: ./scripts/deploy.sh" + ) + return f"FAIL: webhook pod status={phase or 'not found'}" + return "OK: webhook pod is Running" + + +@task +def check_webhook_target_namespace(args, context): + result = shell.run( + "oc get deployment env-injector -n vllm-profiler " + "-o jsonpath='{.spec.template.spec.containers[0]" + '.env[?(@.name=="TARGET_NAMESPACE")].value}\'', + check=False, + ) + target_ns = result.stdout.strip().strip("'") + if result.returncode != 0 or not target_ns: + context.errors = getattr(context, "errors", []) + context.errors.append("Could not read webhook TARGET_NAMESPACE from deployment") + return "FAIL: could not read TARGET_NAMESPACE" + if target_ns != args.namespace: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook TARGET_NAMESPACE is '{target_ns}' but deployment namespace " + f"is '{args.namespace}'. Update manifests.yaml and run: " + "oc apply -f manifests.yaml" + ) + return f"FAIL: TARGET_NAMESPACE={target_ns} != {args.namespace}" + return f"OK: TARGET_NAMESPACE matches '{args.namespace}'" + + +@task +def check_mutating_webhook(args, context): + result = shell.run( + "oc get mutatingwebhookconfiguration env-injector-webhook " + "-o jsonpath='{.webhooks[0].clientConfig.caBundle}'", + check=False, + ) + ca_bundle = result.stdout.strip().strip("'") + if result.returncode != 0 or len(ca_bundle) < MIN_CA_BUNDLE_LENGTH: + context.errors = getattr(context, "errors", []) + context.errors.append( + "MutatingWebhookConfiguration missing or caBundle is empty. " + "Run: bash scripts/gen-certs.sh && bash scripts/patch-ca-bundle.sh" + ) + return "FAIL: MutatingWebhookConfiguration missing or bad caBundle" + return "OK: MutatingWebhookConfiguration has valid caBundle" + + +@task +def check_tls_secret(args, context): + result = shell.run( + "oc get secret env-injector-certs -n vllm-profiler -o name", + check=False, + ) + if result.returncode != 0: + context.errors = getattr(context, "errors", []) + context.errors.append( + "TLS secret 'env-injector-certs' not found in vllm-profiler namespace" + ) + return "FAIL: TLS secret missing" + return "OK: TLS secret exists" + + +@task +def check_configmap(args, context): + result = shell.run( + f"oc get configmap env-injector-files -n {args.namespace} " + "-o jsonpath='{.data}'", + check=False, + ) + if result.returncode != 0 or "sitecustomize.py" not in result.stdout: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"ConfigMap 'env-injector-files' not found in {args.namespace}. " + "Create it with: oc apply -k " + ) + return f"FAIL: ConfigMap missing in {args.namespace}" + return f"OK: ConfigMap exists in {args.namespace}" + + +@task +def raise_on_errors(args, context): + errors = getattr(context, "errors", []) + if errors: + raise ValueError( + "Profiler prerequisites not met:\n - " + "\n - ".join(errors) + ) + return "All profiler prerequisites verified" + + +if __name__ == "__main__": + run.main() diff --git a/vaults/psap-forge-dashboard-s3.yaml b/vaults/psap-forge-dashboard-s3.yaml new file mode 100644 index 000000000..145fc0e70 --- /dev/null +++ b/vaults/psap-forge-dashboard-s3.yaml @@ -0,0 +1,16 @@ +env_key: PSAP_FORGE_DASHBOARD_S3_SECRET_PATH +description: | + AWS credentials for rhaiis dashboard CSV sync (psap-dashboard-data) + and profiler trace uploads. Ported from model-furnace s3-uploader.conf. + +content: + aws.credentials: + description: | + AWS credentials file (INI format). The key must have + s3:GetObject, s3:PutObject, s3:CopyObject on: + - psap-dashboard-data (CSV dashboard, profiler traces) + - psap-model-furnace (vLLM logs) + sample: | + [default] + aws_access_key_id = AKIA... + aws_secret_access_key = ..... diff --git a/vaults/psap-forge-rhaiis-slack.yaml b/vaults/psap-forge-rhaiis-slack.yaml new file mode 100644 index 000000000..838d41964 --- /dev/null +++ b/vaults/psap-forge-rhaiis-slack.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_FORGE_RHAIIS_SLACK_SECRET_PATH +description: | + Slack Incoming Webhook for rhaiis regression/comparison alerts. + Ported from model-furnace's slack.conf. + +content: + slack-webhook-url: + description: | + Plain-text file containing a single Slack Incoming Webhook URL. + Example: https://hooks.slack.com/services/T.../B.../... From 579b8aa1a0f3fd7ff9828a2e900e7dcceb384ea8 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 19:44:26 -0400 Subject: [PATCH 084/344] Fix CSV schema alignment with model-furnace and add run UUID - Fix CSV export to match model-furnace field extraction exactly: accelerator parsing, runtime_args formatting, metric key paths, raw ms values (no /1000), median/p01 instead of p50/p1, and add missing columns (DP, dataset, spec_decoding, etc.) - Generate a single UUID4 per run and thread it through PSAP payload (experiment_id), dashboard CSV (uuid column), predictor log upload (logs/{uuid}.log on S3), and regression notifications (job_id) - Add predictor pod log upload to s3://psap-model-furnace/logs/ Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 50 +++++- projects/rhaiis/postprocess/csv_export.py | 180 +++++++++++++------- projects/rhaiis/postprocess/parser.py | 3 +- projects/rhaiis/postprocess/s3_dashboard.py | 35 ++++ 4 files changed, 199 insertions(+), 69 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b324f7f6f..52b65f02f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import uuid as _uuid_mod from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels @@ -48,6 +49,9 @@ def _run_test( namespace: str, deployment_name: str | None = None, ) -> int: + run_uuid = str(_uuid_mod.uuid4()) + logger.info("Run UUID: %s", run_uuid) + model_cfg = runtime_config.get_model(model_key) workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() @@ -170,15 +174,20 @@ def _run_test( _capture_and_cleanup(deployment_name, namespace) try: - _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key) + _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args) + _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + try: + _upload_predictor_log(run_uuid) + except Exception: + logger.warning("Predictor log upload failed; continuing", exc_info=True) + profiler_cfg = runtime_config.get_profiler_config() if profiler_cfg.get("enabled", False): try: @@ -266,6 +275,8 @@ def _generate_psap_payload( vllm_image: str, vllm_args: dict, workload_key: str, + *, + run_uuid: str = "", ) -> None: from pathlib import Path @@ -288,6 +299,7 @@ def _generate_psap_payload( vllm_args=vllm_args, accelerator=accelerator, workload_key=workload_key, + run_uuid=run_uuid, ) output_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "results" write_psap_payload( @@ -304,6 +316,8 @@ def _generate_and_sync_dashboard_csv( accelerator: str, workload_key: str, vllm_args: dict, + *, + run_uuid: str = "", ) -> None: from pathlib import Path @@ -323,7 +337,7 @@ def _generate_and_sync_dashboard_csv( csv_path = None for psap_file in psap_files: output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" - csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path) + csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path, run_uuid=run_uuid) logger.info("Generated dashboard CSV: %s", csv_path) if not csv_path: @@ -349,7 +363,7 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator) + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) def _run_regression_check( @@ -358,6 +372,8 @@ def _run_regression_check( current_version: str, model_cfg: dict, accelerator: str, + *, + run_uuid: str = "", ) -> None: import tempfile from pathlib import Path @@ -403,7 +419,7 @@ def _run_regression_check( analysis, model=model_cfg.get("hf_model_id", ""), accelerator=accelerator, - job_id=env.ARTIFACT_DIR.rsplit("/", 1)[-1] if env.ARTIFACT_DIR else "", + job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", @@ -416,6 +432,30 @@ def _run_regression_check( os.unlink(consolidated_path) +def _upload_predictor_log(run_uuid: str) -> None: + """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 + + log_path = Path(env.ARTIFACT_DIR) / "artifacts" / "inferenceservice.pods.logs" + if not log_path.exists(): + logger.info("No predictor pod log found at %s, skipping upload", log_path) + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + result = upload_predictor_log_to_s3( + log_path, + run_uuid=run_uuid, + vault_name=vault_name, + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Predictor log upload result: %s", result) + + def _run_warmup_step( *, deployment_name: str, diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py index 42c56fb7a..93270ac27 100644 --- a/projects/rhaiis/postprocess/csv_export.py +++ b/projects/rhaiis/postprocess/csv_export.py @@ -25,35 +25,65 @@ "successful_requests", "errored_requests", "uuid", "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", + "DP", "dataset", "spec_decoding", "prefix_caching", + "turns", "prefix_tokens", "prefix_count", "request_type", ] +def _parse_accelerator_key(acc_key: str) -> tuple[str, str | None]: + """Return (chip_family, cluster_tag) from an accelerator key. + + Convention: keys follow ``_`` (e.g. H200_HERA). + Keys without an underscore have no cluster tag. + """ + if not acc_key: + return "", None + if "_" in acc_key: + chip_family, cluster_tag = acc_key.split("_", 1) + return chip_family, cluster_tag.lower() + return acc_key, None + + +def _format_runtime_args(inference_server_args: dict, trtllm_config: dict | None = None) -> str: + """Format args into semicolon-separated ``key: value`` string matching model-furnace.""" + if not inference_server_args and not trtllm_config: + return "" + parts = [] + for key, value in (inference_server_args or {}).items(): + formatted_key = key.replace("_", "-") + formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + parts.append(f"{formatted_key}: {formatted_value}") + for key, value in (trtllm_config or {}).items(): + formatted_key = f"trtllm.{key.replace('_', '-')}" + formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + parts.append(f"{formatted_key}: {formatted_value}") + return "; ".join(parts) + + def find_psap_files(artifact_dir: Path) -> list[Path]: """Glob for PSAP_perf*.json files in artifact_dir (recursive).""" return sorted(artifact_dir.rglob("PSAP_perf*.json")) -def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path) -> Path: - """Read a PSAP JSON file and produce a dashboard CSV. - - Args: - psap_json_path: Path to the PSAP payload JSON file. - version: Version string for the CSV rows. - output_path: Where to write the CSV. - - Returns: - Path to the generated CSV file. - """ +def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "") -> Path: + """Read a PSAP JSON file and produce a dashboard CSV.""" with open(psap_json_path, encoding="utf-8") as f: payload = json.load(f) model = payload.get("model", "") - accelerator = payload.get("accelerator_type", "") - tp = payload.get("accelerator_count", 1) + acc_key = payload.get("accelerator_type", "") + acc, cluster_tag = _parse_accelerator_key(acc_key) + + inference_server_args = payload.get("inference_server_args", {}) + tp = ( + inference_server_args.get("tensor-parallel-size") + or inference_server_args.get("tp-size") + or inference_server_args.get("tp_size") + or payload.get("accelerator_count", 1) + ) + image_tag = payload.get("container_image_tag", "") - runtime_args = json.dumps(payload.get("inference_server_args", {})) - guidellm_start_ms = payload.get("guidellm_start_time_ms") - guidellm_end_ms = payload.get("guidellm_end_time_ms") + runtime_args = _format_runtime_args(inference_server_args, payload.get("trtllm_config")) report = payload.get("report", {}) guidellm_version = report.get("metadata", {}).get("guidellm_version", "") @@ -67,16 +97,32 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path data_str = data_list[0] if isinstance(data_list, list) else str(data_list) tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) - prompt_toks = tokens.get("prompt_tokens", "") - output_toks = tokens.get("output_tokens", "") + prompt_toks = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + output_toks = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + start_times = [] + end_times = [] + for bench in benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + guidellm_start_ms = int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") + guidellm_end_ms = int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") rows = [] for bench in benchmarks: metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + model_name = f"{acc}-{cluster_tag}-{model}-{tp}" if cluster_tag else f"{acc}-{model}-{tp}" + row = _extract_row( metrics=metrics, + strategy=strategy, + model_name=model_name, model=model, - accelerator=accelerator, + accelerator=acc, version=version, tp=tp, prompt_toks=prompt_toks, @@ -86,6 +132,7 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path guidellm_start_ms=guidellm_start_ms, guidellm_end_ms=guidellm_end_ms, guidellm_version=guidellm_version, + run_uuid=run_uuid, ) rows.append(row) @@ -102,83 +149,90 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path def _extract_row( *, metrics: dict, + strategy: dict, + model_name: str, model: str, accelerator: str, version: str, - tp: int, - prompt_toks: str, - output_toks: str, + tp, + prompt_toks, + output_toks, image_tag: str, runtime_args: str, guidellm_start_ms, guidellm_end_ms, guidellm_version: str, + run_uuid: str = "", ) -> dict: - def _pct(metric_name: str, pct: str) -> float: - return float( + def _pct(metric_name: str, pct: str): + return ( metrics.get(metric_name, {}) .get("successful", {}) .get("percentiles", {}) - .get(pct, 0) + .get(pct) ) - def _stat(metric_name: str, stat: str) -> float: - return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, 0)) + def _stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("successful", {}).get(stat) - request_totals = metrics.get("request_totals", {}) - successful = int(request_totals.get("successful", 0)) - errored = int(request_totals.get("errored", 0)) - - concurrency_mean = _stat("request_concurrency", "mean") - intended_concurrency = _stat("request_concurrency", "max") + def _total_stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("total", {}).get(stat) - output_tok_sec = _stat("output_token_throughput", "mean") - total_tok_sec = _stat("total_token_throughput", "mean") - measured_rps = _stat("request_rate", "mean") + request_totals = metrics.get("request_totals", {}) + successful = request_totals.get("successful", 0) + errored = request_totals.get("errored", 0) return { - "run": "", + "run": model_name, "accelerator": accelerator, "model": model, "version": version, "prompt toks": prompt_toks, "output toks": output_toks, "TP": tp, - "measured concurrency": concurrency_mean, - "intended concurrency": intended_concurrency, - "measured rps": measured_rps, - "output_tok/sec": output_tok_sec, - "total_tok/sec": total_tok_sec, + "measured concurrency": _stat("request_concurrency", "mean"), + "intended concurrency": strategy.get("streams"), + "measured rps": _stat("requests_per_second", "mean"), + "output_tok/sec": _total_stat("output_tokens_per_second", "mean"), + "total_tok/sec": _total_stat("tokens_per_second", "mean"), "prompt_token_count_mean": _stat("prompt_token_count", "mean"), "prompt_token_count_p99": _pct("prompt_token_count", "p99"), "output_token_count_mean": _stat("output_token_count", "mean"), "output_token_count_p99": _pct("output_token_count", "p99"), - "ttft_median": _pct("time_to_first_token_ms", "p50") / 1000.0, - "ttft_p95": _pct("time_to_first_token_ms", "p95") / 1000.0, - "ttft_p1": _pct("time_to_first_token_ms", "p1") / 1000.0, - "ttft_p999": _pct("time_to_first_token_ms", "p999") / 1000.0, - "tpot_median": _pct("time_per_output_token_ms", "p50") / 1000.0, - "tpot_p95": _pct("time_per_output_token_ms", "p95") / 1000.0, - "tpot_p99": _pct("time_per_output_token_ms", "p99") / 1000.0, - "tpot_p999": _pct("time_per_output_token_ms", "p999") / 1000.0, - "tpot_p1": _pct("time_per_output_token_ms", "p1") / 1000.0, - "itl_median": _pct("inter_token_latency_ms", "p50") / 1000.0, - "itl_p95": _pct("inter_token_latency_ms", "p95") / 1000.0, - "itl_p999": _pct("inter_token_latency_ms", "p999") / 1000.0, - "itl_p1": _pct("inter_token_latency_ms", "p1") / 1000.0, - "request_latency_median": _pct("request_latency_ms", "p50") / 1000.0, - "request_latency_min": _stat("request_latency_ms", "min") / 1000.0, - "request_latency_max": _stat("request_latency_ms", "max") / 1000.0, + "ttft_median": _stat("time_to_first_token_ms", "median"), + "ttft_p95": _pct("time_to_first_token_ms", "p95"), + "ttft_p1": _pct("time_to_first_token_ms", "p01"), + "ttft_p999": _pct("time_to_first_token_ms", "p999"), + "tpot_median": _stat("time_per_output_token_ms", "median"), + "tpot_p95": _pct("time_per_output_token_ms", "p95"), + "tpot_p99": _pct("time_per_output_token_ms", "p99"), + "tpot_p999": _pct("time_per_output_token_ms", "p999"), + "tpot_p1": _pct("time_per_output_token_ms", "p01"), + "itl_median": _stat("inter_token_latency_ms", "median"), + "itl_p95": _pct("inter_token_latency_ms", "p95"), + "itl_p999": _pct("inter_token_latency_ms", "p999"), + "itl_p1": _pct("inter_token_latency_ms", "p01"), + "request_latency_median": _stat("request_latency", "median"), + "request_latency_min": _stat("request_latency", "min"), + "request_latency_max": _stat("request_latency", "max"), "successful_requests": successful, "errored_requests": errored, - "uuid": "", - "ttft_mean": _stat("time_to_first_token_ms", "mean") / 1000.0, - "ttft_p99": _pct("time_to_first_token_ms", "p99") / 1000.0, - "itl_mean": _stat("inter_token_latency_ms", "mean") / 1000.0, - "itl_p99": _pct("inter_token_latency_ms", "p99") / 1000.0, + "uuid": run_uuid, + "ttft_mean": _stat("time_to_first_token_ms", "mean"), + "ttft_p99": _pct("time_to_first_token_ms", "p99"), + "itl_mean": _stat("inter_token_latency_ms", "mean"), + "itl_p99": _pct("inter_token_latency_ms", "p99"), "runtime_args": runtime_args, "guidellm_start_time_ms": guidellm_start_ms or "", "guidellm_end_time_ms": guidellm_end_ms or "", "image_tag": image_tag, "guidellm_version": guidellm_version, + "DP": "", + "dataset": "", + "spec_decoding": "", + "prefix_caching": "", + "turns": "", + "prefix_tokens": "", + "prefix_count": "", + "request_type": "", } diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 31059e2ea..c0151a298 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -117,6 +117,7 @@ def generate_psap_payload( vllm_args: dict[str, Any], accelerator: str, workload_key: str, + run_uuid: str = "", ) -> dict[str, Any]: report = json.loads(benchmarks_json_path.read_text(encoding="utf-8")) @@ -135,7 +136,7 @@ def generate_psap_payload( guidellm_end = int(max(ends) * 1000) return { - "experiment_id": str(uuid.uuid4()).upper(), + "experiment_id": run_uuid or str(uuid.uuid4()).upper(), "experiment_type": "perf", "model": model_id, "inference_server": "vllm", diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index efa2c5c3e..253c176b8 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -164,6 +164,41 @@ def upload_profiler_traces_to_s3( return {"status": "failed", "error": str(e), "uploaded": uploaded} +def upload_predictor_log_to_s3( + log_path: Path, + *, + run_uuid: str, + s3_bucket: str = "psap-model-furnace", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload the vLLM predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + if not log_path.exists(): + return {"status": "skipped", "reason": f"log not found: {log_path}"} + + s3_key = f"logs/{run_uuid}.log" + + if dry_run: + logger.info("DRY RUN: Would upload %s to s3://%s/%s", log_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + try: + s3 = create_s3_client(credentials_path) + s3.upload_file(str(log_path), s3_bucket, s3_key) + logger.info("Uploaded predictor log to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "s3_key": s3_key} + except Exception as e: + logger.error("Predictor log upload failed: %s", e) + return {"status": "failed", "error": str(e)} + + def _accelerator_s3_folder(accelerator: str) -> str: upper = accelerator.upper() if upper.startswith("H200"): From 373743da1ee0cefe9ad7700ce7d0125a2f1630bf Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 20:16:00 -0400 Subject: [PATCH 085/344] Fix: keep kubectl in forge-core container image The openshift-client tarball includes both oc and kubectl, but the Containerfile was deleting kubectl after extraction. Pipeline tasks that reference kubectl (e.g. 00-pre-cleanup) fail with exit code 127. Co-authored-by: Cursor --- projects/core/image/Containerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/core/image/Containerfile b/projects/core/image/Containerfile index 200c61e53..50935add0 100644 --- a/projects/core/image/Containerfile +++ b/projects/core/image/Containerfile @@ -37,9 +37,9 @@ RUN pip install --no-cache-dir uv # Install OpenShift CLI (oc) RUN curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz" && \ tar -xzf openshift-client-linux.tar.gz && \ - chmod +x oc && \ - mv oc /usr/local/bin/ && \ - rm -f openshift-client-linux.tar.gz kubectl + chmod +x oc kubectl && \ + mv oc kubectl /usr/local/bin/ && \ + rm -f openshift-client-linux.tar.gz # Optimize uv for container environments ENV UV_COMPILE_BYTECODE=1 From fd8edec3613d21a91b57c3c80acc455637587015 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 20:19:31 -0400 Subject: [PATCH 086/344] Revert "Fix: keep kubectl in forge-core container image" This reverts commit e17f1712338e68a29db7f0f13db20be0692efd8c. --- projects/core/image/Containerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/core/image/Containerfile b/projects/core/image/Containerfile index 50935add0..200c61e53 100644 --- a/projects/core/image/Containerfile +++ b/projects/core/image/Containerfile @@ -37,9 +37,9 @@ RUN pip install --no-cache-dir uv # Install OpenShift CLI (oc) RUN curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz" && \ tar -xzf openshift-client-linux.tar.gz && \ - chmod +x oc kubectl && \ - mv oc kubectl /usr/local/bin/ && \ - rm -f openshift-client-linux.tar.gz + chmod +x oc && \ + mv oc /usr/local/bin/ && \ + rm -f openshift-client-linux.tar.gz kubectl # Optimize uv for container environments ENV UV_COMPILE_BYTECODE=1 From 3d3de4e0ba5e3636f7ec42968634a01ed54b5b7a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:28:37 -0400 Subject: [PATCH 087/344] Fix accelerator label and exclude warmup from PSAP payload - Use gpu_type (e.g. H200) instead of platform name (nvidia) for PSAP payload, CSV, and S3 artifact paths - Exclude warmup benchmarks.json from PSAP payload generation to prevent warmup results contaminating the dashboard CSV Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 52b65f02f..da7c1688f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -55,6 +55,7 @@ def _run_test( model_cfg = runtime_config.get_model(model_key) workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator deploy_cfg = runtime_config.get_deploy_config() benchmark_cfg = runtime_config.get_benchmark_config() @@ -174,12 +175,12 @@ def _run_test( _capture_and_cleanup(deployment_name, namespace) try: - _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) + _generate_psap_payload(model_cfg, gpu_type, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args, run_uuid=run_uuid) + _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) @@ -191,7 +192,7 @@ def _run_test( profiler_cfg = runtime_config.get_profiler_config() if profiler_cfg.get("enabled", False): try: - _upload_profiler_traces(model_cfg, accelerator, vllm_args, profiler_cfg) + _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: logger.warning("Profiler trace upload failed; continuing", exc_info=True) @@ -282,15 +283,16 @@ def _generate_psap_payload( from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - matches = list( + all_matches = list( Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") ) + matches = [m for m in all_matches if "warmup" not in str(m)] if not matches: logger.warning( "benchmarks.json not found under %s, skipping PSAP payload", env.ARTIFACT_DIR ) return - benchmarks_json = matches[0] + benchmarks_json = matches[-1] payload = generate_psap_payload( benchmarks_json_path=benchmarks_json, From 718c8eadc9f34eeee8f599bb0c1d67de4d50f5d2 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:39:40 -0400 Subject: [PATCH 088/344] Only generate PSAP payload from main benchmark results Track the main benchmark artifact directory explicitly by index instead of globbing for all benchmarks.json files. Warmup and profiler benchmark results are no longer included in the PSAP payload or dashboard CSV. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 44 ++++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index da7c1688f..67f910c93 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -147,6 +147,7 @@ def _run_test( benchmark_timeout=benchmark_timeout, ) + main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") else: @@ -162,6 +163,7 @@ def _run_test( max_seconds=max_seconds, ) + pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", name=f"guidellm-{deployment_name}", @@ -171,18 +173,26 @@ def _run_test( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, ) + from pathlib import Path + candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) + if candidates: + main_benchmark_dir = candidates[0] finally: _capture_and_cleanup(deployment_name, namespace) - try: - _generate_psap_payload(model_cfg, gpu_type, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) - except Exception: - logger.warning("PSAP payload generation failed; continuing", exc_info=True) + if main_benchmark_dir: + try: + _generate_psap_payload( + model_cfg, gpu_type, vllm_image, vllm_args, workload_key, + run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, + ) + except Exception: + logger.warning("PSAP payload generation failed; continuing", exc_info=True) - try: - _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + try: + _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) try: _upload_predictor_log(run_uuid) @@ -278,21 +288,25 @@ def _generate_psap_payload( workload_key: str, *, run_uuid: str = "", + benchmark_dir: Path | None = None, ) -> None: from pathlib import Path from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - all_matches = list( - Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") - ) - matches = [m for m in all_matches if "warmup" not in str(m)] - if not matches: + if benchmark_dir: + benchmarks_json = benchmark_dir / "artifacts" / "results" / "benchmarks.json" + else: + matches = list( + Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") + ) + benchmarks_json = matches[-1] if matches else None + + if not benchmarks_json or not benchmarks_json.exists(): logger.warning( - "benchmarks.json not found under %s, skipping PSAP payload", env.ARTIFACT_DIR + "benchmarks.json not found, skipping PSAP payload" ) return - benchmarks_json = matches[-1] payload = generate_psap_payload( benchmarks_json_path=benchmarks_json, From f0add8b790858b38505cb7177009d618a5101bc5 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:53:20 -0400 Subject: [PATCH 089/344] Add cluster_tag config for run ID in dashboard CSV When rhaiis.cluster_tag is set (e.g. "hera2"), the PSAP payload accelerator_type becomes "H200_HERA2", which csv_export splits into accelerator=H200 and run=H200-hera2-model-TP. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index c2d0676d8..2116f1acf 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -1,5 +1,6 @@ namespace: forge-rhaiis accelerator: nvidia +cluster_tag: "" gpu_types: nvidia: h200 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 67f910c93..83efeacc4 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -56,6 +56,9 @@ def _run_test( workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + from projects.core.library import config as _cfg + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() deploy_cfg = runtime_config.get_deploy_config() benchmark_cfg = runtime_config.get_benchmark_config() @@ -183,14 +186,14 @@ def _run_test( if main_benchmark_dir: try: _generate_psap_payload( - model_cfg, gpu_type, vllm_image, vllm_args, workload_key, + model_cfg, accelerator_key, vllm_image, vllm_args, workload_key, run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, ) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) + _generate_and_sync_dashboard_csv(model_cfg, accelerator_key, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) From 01e1209de06ff04189f5359a11538d532962a57e Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 13:57:35 -0400 Subject: [PATCH 090/344] Fix vllm_args merge order, default configs, and log upload bucket - merge_vllm_args now applies rhaiis.vllm_args last so configOverrides like tensor-parallel-size take precedence over model defaults - Remove tensor-parallel-size from rhaiis.yaml defaults (models define it) - Move namespace and image_pull_secret into rhaiis.yaml defaults - Fix predictor log upload bucket to psap-dashboard-data Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 8 +++----- projects/rhaiis/orchestration/runtime_config.py | 6 +++--- projects/rhaiis/postprocess/s3_dashboard.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 2116f1acf..4a43129e7 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -1,4 +1,4 @@ -namespace: forge-rhaiis +namespace: kserve-e2e-perf accelerator: nvidia cluster_tag: "" @@ -16,8 +16,8 @@ deploy: cpu_request: "4" memory_request: "16Gi" storage_source: hf - storage_pvc: model-pvc-2 - image_pull_secret: "" + storage_pvc: model-pvc + image_pull_secret: npalaska-image-pull service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 @@ -28,8 +28,6 @@ vllm_args: no-enable-log-requests: true no-enable-prefix-caching: true uvicorn-log-level: debug - max-model-len: 8192 - tensor-parallel-size: 1 env_vars: {} diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 44ff22f21..1abbc2492 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -72,13 +72,13 @@ def get_profiler_config() -> dict: def merge_vllm_args( - defaults: dict, + overrides: dict, model: dict, workload: dict, ) -> dict: - merged = dict(defaults) - merged.update(model.get("vllm_args", {})) + merged = dict(model.get("vllm_args", {})) merged.update(workload.get("vllm_args", {})) + merged.update(overrides) return merged diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index 253c176b8..d8b3705a0 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -168,7 +168,7 @@ def upload_predictor_log_to_s3( log_path: Path, *, run_uuid: str, - s3_bucket: str = "psap-model-furnace", + s3_bucket: str = "psap-dashboard-data", vault_name: str = "psap-forge-dashboard-s3", credentials_file: str = "aws.credentials", dry_run: bool = False, From d3a33660761555faf460f5309324a0d9d9348f70 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 14:17:55 -0400 Subject: [PATCH 091/344] Fix predictor log path to match toolbox task artifact dir The capture_isvc_state toolbox task writes logs under its own sub-directory (NNN__capture_isvc_state/artifacts/), not the top-level ARTIFACT_DIR. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 83efeacc4..f638572ee 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -458,9 +458,10 @@ def _upload_predictor_log(run_uuid: str) -> None: from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 - log_path = Path(env.ARTIFACT_DIR) / "artifacts" / "inferenceservice.pods.logs" - if not log_path.exists(): - logger.info("No predictor pod log found at %s, skipping upload", log_path) + matches = sorted(Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs")) + log_path = matches[-1] if matches else None + if not log_path or not log_path.exists(): + logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) return csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) From 542d404a59f84fe321128f611ee7df93c80d6d76 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 15:27:25 -0400 Subject: [PATCH 092/344] Make warmup and profiler configs explicit and match model-furnace - Add rhaiis.warmup config section (rate: 200, max_seconds: 60) - Add profiler rates/max_seconds to config (200/200, matching model-furnace) - Read warmup params from config instead of hardcoding Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 6 ++++++ projects/rhaiis/orchestration/test_phase.py | 11 +++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 4a43129e7..96f1d7def 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,9 +36,15 @@ accelerator_env_vars: amd: VLLM_ROCM_USE_AITER: "1" +warmup: + rate: 200 + max_seconds: 60 + profiler: enabled: false s3_bucket: psap-dashboard-data s3_prefix: pytorch-profiles/rhaiis vault: psap-forge-dashboard-s3 labels: [] + rates: [200] + max_seconds: 200 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f638572ee..c641cb21b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -491,18 +491,21 @@ def _run_warmup_step( run as run_guidellm_benchmark, ) - warmup_rates = [200] - warmup_max_seconds = 60 + from projects.core.library import config + + warmup_cfg = config.project.get_config("rhaiis.warmup", {}) + warmup_rate = warmup_cfg.get("rate", 200) + warmup_max_seconds = warmup_cfg.get("max_seconds", 60) guidellm_args = runtime_config.build_guidellm_args( benchmark_cfg=benchmark_cfg, model_id=model_cfg["hf_model_id"], data=workload["data"], - rates=warmup_rates, + rates=[warmup_rate], max_seconds=warmup_max_seconds, ) - logger.info("Running warmup (concurrency=200, duration=60s)") + logger.info("Running warmup (concurrency=%d, duration=%ds)", warmup_rate, warmup_max_seconds) try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", From 6cf58cb3b0abe5a77ef5453381bc86127249df97 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 15:49:24 -0400 Subject: [PATCH 093/344] Auto-generate profiler labels from workload data Derives labels like isl1000_osl1000 from the workload's data field instead of requiring manual rhaiis.profiler.labels config, matching model-furnace's get_profile_label() behavior. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index c641cb21b..ef119d5d9 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -546,8 +546,8 @@ def _run_profiler_step( profiler_cfg = runtime_config.get_profiler_config() labels = profiler_cfg.get("labels", []) if not labels: - logger.info("No profiler labels configured, skipping profiler step") - return + labels = [_derive_profiler_label(workload)] + logger.info("Auto-generated profiler label from workload: %s", labels[0]) profiler_max_seconds = profiler_cfg.get("max_seconds", 60) @@ -594,6 +594,15 @@ def _run_profiler_step( logger.warning("Failed to copy profiler traces", exc_info=True) +def _derive_profiler_label(workload: dict) -> str: + """Auto-generate a profiler label like 'isl1000_osl1000' from the workload data string.""" + data = workload.get("data", "") + params = dict(item.split("=", 1) for item in data.split(",") if "=" in item) + isl = params.get("prompt_tokens", "0") + osl = params.get("output_tokens", "0") + return f"isl{isl}_osl{osl}" + + def _upload_profiler_traces( model_cfg: dict, accelerator: str, From 8cd83a769c67a7cf6fd7af489340567d616a0153 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:01:21 -0400 Subject: [PATCH 094/344] Add multi-workload iteration support Deploy model once, then iterate over workload_keys running warmup/profiler/benchmark/CSV for each. Backward compatible: empty workload_keys falls back to single workload_key. Co-authored-by: Cursor --- projects/rhaiis/orchestration/cli.py | 2 +- .../orchestration/config.d/workloads.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 188 +++++++++++------- projects/rhaiis/orchestration/test_rhaiis.py | 6 +- 5 files changed, 120 insertions(+), 81 deletions(-) diff --git a/projects/rhaiis/orchestration/cli.py b/projects/rhaiis/orchestration/cli.py index a6cc2e626..b65a07c54 100644 --- a/projects/rhaiis/orchestration/cli.py +++ b/projects/rhaiis/orchestration/cli.py @@ -113,7 +113,7 @@ def test( try: test_phase.run( model_key=model_key, - workload_key=workload_key, + workload_keys=[workload_key], namespace=namespace, deployment_name=deployment_name, ) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 113e55d49..0ef8e0079 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -10,8 +10,8 @@ profile1: profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" - rates: [1, 50, 100, 200, 300] - max_seconds: 450 + rates: [1] + max_seconds: 45 profile3: data: "prompt_tokens=2048,output_tokens=128" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 7323c7d0e..f5aa9224c 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -22,6 +22,7 @@ tests: rhaiis: model_key: qwen3-0_6b workload_key: profile1 + workload_keys: [] run_benchmark: true warmup: true version: "" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ef119d5d9..0fbd21e36 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -13,14 +13,14 @@ def run( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: return run_and_postprocess( do_test, model_key=model_key, - workload_key=workload_key, + workload_keys=workload_keys, namespace=namespace, deployment_name=deployment_name, ) @@ -29,14 +29,14 @@ def run( def do_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: with env.NextArtifactDir("testing"): return _run_test( model_key=model_key, - workload_key=workload_key, + workload_keys=workload_keys, namespace=namespace, deployment_name=deployment_name, ) @@ -45,15 +45,11 @@ def do_test( def _run_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: - run_uuid = str(_uuid_mod.uuid4()) - logger.info("Run UUID: %s", run_uuid) - model_cfg = runtime_config.get_model(model_key) - workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator from projects.core.library import config as _cfg @@ -67,21 +63,12 @@ def _run_test( vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload) + first_workload = runtime_config.get_workload(workload_keys[0]) + vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) - rates = workload.get("rates", [1]) - max_seconds = workload.get("max_seconds", 180) - - _create_test_labels(model_key, workload_key, accelerator, vllm_args) - - logger.info( - "Testing model=%s workload=%s accelerator=%s", model_cfg["name"], workload_key, accelerator - ) + _create_test_labels(model_key, workload_keys[0], accelerator, vllm_args) - from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( - run as run_guidellm_benchmark, - ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) @@ -120,68 +107,118 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - from projects.core.library import config - - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) - run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) - - if profiler_enabled: - logger.info("Skipping warmup -- profiler step provides its own warmup") - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, + logger.info( + "Running %d workload(s): %s", len(workload_keys), workload_keys, + ) + for wl_key in workload_keys: + _run_workload( + model_key=model_key, + workload_key=wl_key, model_cfg=model_cfg, - workload=workload, - workload_key=workload_key, - benchmark_timeout=benchmark_timeout, - ) - elif warmup_enabled: - _run_warmup_step( + accelerator=accelerator, + accelerator_key=accelerator_key, + gpu_type=gpu_type, + vllm_image=vllm_image, + vllm_args=vllm_args, + benchmark_cfg=benchmark_cfg, deployment_name=deployment_name, namespace=namespace, endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, benchmark_timeout=benchmark_timeout, ) + finally: + _capture_and_cleanup(deployment_name, namespace) - main_benchmark_dir = None - if not run_benchmark: - logger.info("run_benchmark=false, skipping main benchmark") - else: - logger.info("Running benchmark at rates=%s", rates) + return 0 - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) +def _run_workload( + *, + model_key: str, + workload_key: str, + model_cfg: dict, + accelerator: str, + accelerator_key: str, + gpu_type: str, + vllm_image: str, + vllm_args: dict, + benchmark_cfg: dict, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_timeout: int, +) -> None: + """Run a single workload: warmup/profiler, benchmark, PSAP/CSV, uploads.""" + run_uuid = str(_uuid_mod.uuid4()) + logger.info("=== Workload %s (UUID: %s) ===", workload_key, run_uuid) - pre_index = env.next_artifact_index() - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}", - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - ) - from pathlib import Path - candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) - if candidates: - main_benchmark_dir = candidates[0] - finally: - _capture_and_cleanup(deployment_name, namespace) + workload = runtime_config.get_workload(workload_key) + rates = workload.get("rates", [1]) + max_seconds = workload.get("max_seconds", 180) + + from projects.core.library import config + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + + if profiler_enabled: + logger.info("Skipping warmup -- profiler step provides its own warmup") + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=workload_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + main_benchmark_dir = None + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + else: + logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) + + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) + + pre_index = env.next_artifact_index() + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-{deployment_name}-{workload_key}", + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + from pathlib import Path + candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) + if candidates: + main_benchmark_dir = candidates[0] if main_benchmark_dir: try: @@ -202,8 +239,7 @@ def _run_test( except Exception: logger.warning("Predictor log upload failed; continuing", exc_info=True) - profiler_cfg = runtime_config.get_profiler_config() - if profiler_cfg.get("enabled", False): + if profiler_enabled: try: _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: @@ -226,8 +262,6 @@ def _run_test( except Exception: logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) - return 0 - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict diff --git a/projects/rhaiis/orchestration/test_rhaiis.py b/projects/rhaiis/orchestration/test_rhaiis.py index e2a125b89..69be8db73 100644 --- a/projects/rhaiis/orchestration/test_rhaiis.py +++ b/projects/rhaiis/orchestration/test_rhaiis.py @@ -14,8 +14,12 @@ namespace="rhaiis.namespace", ) def test(_cfg): + workload_keys = config.project.get_config("tests.rhaiis.workload_keys", []) + if not workload_keys: + workload_keys = [_cfg.workload_key] + test_phase.run( model_key=_cfg.model_key, - workload_key=_cfg.workload_key, + workload_keys=workload_keys, namespace=_cfg.namespace, ) From 8773870cfc758925773c9b60ebea134336710a19 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:11:40 -0400 Subject: [PATCH 095/344] Restructure multi-workload into two phases Phase 1: warmup/profiler for all workloads, then upload traces Phase 2: benchmark + PSAP/CSV/log for each workload MLflow metadata set once after all workloads complete. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 118 +++++++++++--------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0fbd21e36..31c746c96 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -107,11 +107,52 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + logger.info( "Running %d workload(s): %s", len(workload_keys), workload_keys, ) + + # Phase 1: warmup or profiler for ALL workloads first + for wl_key in workload_keys: + workload = runtime_config.get_workload(wl_key) + if profiler_enabled: + logger.info("Running profiler for workload=%s", wl_key) + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=wl_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + logger.info("Running warmup for workload=%s", wl_key) + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + if profiler_enabled: + try: + _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) + except Exception: + logger.warning("Profiler trace upload failed; continuing", exc_info=True) + + # Phase 2: benchmark + post-processing for ALL workloads for wl_key in workload_keys: - _run_workload( + _run_workload_benchmark( model_key=model_key, workload_key=wl_key, model_cfg=model_cfg, @@ -126,13 +167,33 @@ def _run_test( endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, ) + + try: + first_workload = runtime_config.get_workload(workload_keys[0]) + first_rates = first_workload.get("rates", [1]) + first_max_seconds = first_workload.get("max_seconds", 180) + _set_mlflow_metadata( + model_key, + ",".join(workload_keys), + model_cfg, + accelerator, + vllm_image, + vllm_args, + benchmark_cfg, + first_rates, + first_max_seconds, + namespace, + deployment_name, + ) + except Exception: + logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) finally: _capture_and_cleanup(deployment_name, namespace) return 0 -def _run_workload( +def _run_workload_benchmark( *, model_key: str, workload_key: str, @@ -148,9 +209,9 @@ def _run_workload( endpoint_url: str, benchmark_timeout: int, ) -> None: - """Run a single workload: warmup/profiler, benchmark, PSAP/CSV, uploads.""" + """Run benchmark and post-processing for a single workload.""" run_uuid = str(_uuid_mod.uuid4()) - logger.info("=== Workload %s (UUID: %s) ===", workload_key, run_uuid) + logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) rates = workload.get("rates", [1]) @@ -161,33 +222,7 @@ def _run_workload( run as run_guidellm_benchmark, ) - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) - - if profiler_enabled: - logger.info("Skipping warmup -- profiler step provides its own warmup") - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=workload_key, - benchmark_timeout=benchmark_timeout, - ) - elif warmup_enabled: - _run_warmup_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - benchmark_timeout=benchmark_timeout, - ) main_benchmark_dir = None if not run_benchmark: @@ -239,29 +274,6 @@ def _run_workload( except Exception: logger.warning("Predictor log upload failed; continuing", exc_info=True) - if profiler_enabled: - try: - _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) - except Exception: - logger.warning("Profiler trace upload failed; continuing", exc_info=True) - - try: - _set_mlflow_metadata( - model_key, - workload_key, - model_cfg, - accelerator, - vllm_image, - vllm_args, - benchmark_cfg, - rates, - max_seconds, - namespace, - deployment_name, - ) - except Exception: - logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict From 2d5256b88188416029ecebfa695afaefb726ee35 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:31:19 -0400 Subject: [PATCH 096/344] Include workload key in warmup and profiler pod names guidellm-warmup-{model}-{workload_key} guidellm-profiler-{model}-{workload_key}-{gate_value} Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 31c746c96..6a405205a 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -141,6 +141,7 @@ def _run_test( benchmark_cfg=benchmark_cfg, model_cfg=model_cfg, workload=workload, + workload_key=wl_key, benchmark_timeout=benchmark_timeout, ) @@ -530,6 +531,7 @@ def _run_warmup_step( benchmark_cfg: dict, model_cfg: dict, workload: dict, + workload_key: str, benchmark_timeout: int, ) -> None: """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" @@ -555,7 +557,7 @@ def _run_warmup_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-warmup-{deployment_name}", + name=f"guidellm-warmup-{deployment_name}-{workload_key}", namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, @@ -619,7 +621,7 @@ def _run_profiler_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-profiler-{deployment_name}-{gate_value}", + name=f"guidellm-profiler-{deployment_name}-{workload_key}-{gate_value}", namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, From 4ae0e91c57361c176b94e29899660266bcaef79f Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:39:38 -0400 Subject: [PATCH 097/344] Add HF_TOKEN to GuideLLM job template Pulls from storage-config secret (optional) to avoid HuggingFace rate limits when downloading tokenizers. Co-authored-by: Cursor --- .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8324631c4..7a74cdd08 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -26,6 +26,12 @@ spec: env: - name: USER value: guidellm + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: storage-config + key: HF_TOKEN + optional: true volumeMounts: - name: home mountPath: /home/guidellm From cbd0f6e2a0b1e32be6d9ca22acda28502d40b03c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:30:42 -0400 Subject: [PATCH 098/344] Use prefix-workload-model naming for GuideLLM jobs Format: guidellm-{warmup|bench|profiler}-{workload_key}-{model} Trims model name to fit K8s 63-char limit while keeping the workload key fully visible. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 6a405205a..804f61ba4 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -9,6 +9,16 @@ logger = logging.getLogger(__name__) +_K8S_NAME_MAX = 63 + + +def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: + """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" + base = f"{prefix}-{workload_key}-" + available = _K8S_NAME_MAX - len(base) + model = deployment_name[:available] if available > 0 else "" + return f"{base}{model}".rstrip("-") + def run( *, @@ -244,7 +254,7 @@ def _run_workload_benchmark( pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}-{workload_key}", + name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), namespace=namespace, image=benchmark_image, timeout=benchmark_timeout, @@ -557,7 +567,7 @@ def _run_warmup_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-warmup-{deployment_name}-{workload_key}", + name=_guidellm_job_name("guidellm-warmup", workload_key, deployment_name), namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, @@ -621,7 +631,7 @@ def _run_profiler_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-profiler-{deployment_name}-{workload_key}-{gate_value}", + name=_guidellm_job_name("guidellm-profiler", workload_key, deployment_name), namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, From ef3468841fec88b4cadf20f7f88d5f8e1dce32f6 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:49:26 -0400 Subject: [PATCH 099/344] Share a single UUID across all workloads in a multi-workload run Generate the UUID once in _run_test and pass it to each _run_workload_benchmark call so all workloads in the same job share the same identifier in CSV, PSAP payload, S3 logs, and regression notifications. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 804f61ba4..940371512 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -85,6 +85,9 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready + run_uuid = str(_uuid_mod.uuid4()) + logger.info("Run UUID for this job: %s", run_uuid) + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -177,6 +180,7 @@ def _run_test( namespace=namespace, endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, + run_uuid=run_uuid, ) try: @@ -219,9 +223,9 @@ def _run_workload_benchmark( namespace: str, endpoint_url: str, benchmark_timeout: int, + run_uuid: str, ) -> None: """Run benchmark and post-processing for a single workload.""" - run_uuid = str(_uuid_mod.uuid4()) logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) From bbc54d39dc85256e6cd36e5a17a1332eb27333e7 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:54:47 -0400 Subject: [PATCH 100/344] Fix predictor log upload ordering _upload_predictor_log was called inside _run_workload_benchmark, before _capture_and_cleanup had run, so the log file didn't exist yet. Move the upload to _run_test, after capture_isvc_state creates the log file. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 940371512..0e9e2f7de 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -205,6 +205,11 @@ def _run_test( finally: _capture_and_cleanup(deployment_name, namespace) + try: + _upload_predictor_log(run_uuid) + except Exception: + logger.warning("Predictor log upload failed; continuing", exc_info=True) + return 0 @@ -284,11 +289,6 @@ def _run_workload_benchmark( except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) - try: - _upload_predictor_log(run_uuid) - except Exception: - logger.warning("Predictor log upload failed; continuing", exc_info=True) - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict From 0dd147fb36c6df165a1141545ab8fb2385533a48 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 11:47:19 -0400 Subject: [PATCH 101/344] Add vllm-profiler/enabled label to InferenceService template The env-injector webhook requires this label on the pod to inject the profiler sitecustomize.py and PYTHONPATH. Without it, the profiler hook never loads and no traces are generated. Co-authored-by: Cursor --- .../deploy_kserve_isvc/templates/inferenceservice.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index baf29eadd..3d04aa276 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -10,6 +10,7 @@ metadata: prometheus.io/port: "8080" labels: opendatahub.io/dashboard: "true" + vllm-profiler/enabled: "true" name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: From 771212cb90a597d18517930b7e668f1f63d28de0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:13:11 -0400 Subject: [PATCH 102/344] Fix profiler trace upload path Traces are saved in per-task artifact subdirectories like *__copy_profiler_traces/artifacts/traces/, not under the parent ARTIFACT_DIR. Glob for trace files across all copy_profiler_traces subdirectories and consolidate if needed before uploading. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0e9e2f7de..f7bb2913b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -676,11 +676,20 @@ def _upload_profiler_traces( from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 - traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces" - if not traces_dir.exists() or not any(traces_dir.iterdir()): + trace_files = sorted(Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*")) + if not trace_files: logger.info("No profiler traces to upload") return + traces_dir = trace_files[0].parent + if len(set(f.parent for f in trace_files)) > 1: + traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces_combined" + traces_dir.mkdir(parents=True, exist_ok=True) + for f in trace_files: + import shutil + shutil.copy2(f, traces_dir / f.name) + logger.info("Found %d profiler trace files in %s", len(trace_files), traces_dir) + version = config.project.get_config("tests.rhaiis.version", "") if not version: logger.info("No version configured, skipping profiler trace upload") From 6d8a7c5f505f4a83c041b2a1652d99104b185bbd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:28:39 -0400 Subject: [PATCH 103/344] Allow passing run UUID via tests.rhaiis.run_uuid config override If provided, uses the given UUID instead of generating a new one. Enables correlating FournosJob names with their run UUID for traceability across CSV, S3 logs, and pipeline outputs. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f7bb2913b..a1658de6f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -85,7 +85,8 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - run_uuid = str(_uuid_mod.uuid4()) + from projects.core.library import config as _cfg + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) benchmark_timeout = benchmark_cfg.get("timeout", 14400) From bb3b6e7179e85078465861db2c744d0489e4f5bd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:33:16 -0400 Subject: [PATCH 104/344] Annotate FournosJob with run UUID for traceability After generating the UUID, annotate the FournosJob resource with rhaiis.run-uuid so the UUID is visible via oc get without needing to dig through pipeline logs. Fully automatic, no user action needed. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index a1658de6f..55880662b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -89,6 +89,20 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) + import os, subprocess + fjob_name = os.environ.get("FJOB_NAME", "") + fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") + if fjob_name: + try: + subprocess.run( + ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, + f"rhaiis.run-uuid={run_uuid}", "--overwrite"], + check=False, capture_output=True, timeout=10, + ) + logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) + except Exception: + logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) From b08e37888e63ff733f4d5f187fb1810c01894f3a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:45:32 -0400 Subject: [PATCH 105/344] Fix FournosJob UUID annotation to target management cluster The test pod's KUBECONFIG points to the target cluster (hera), but FournosJob resources live on the management cluster. Strip KUBECONFIG from the env so oc uses in-cluster service account credentials. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 55880662b..88132e33d 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -94,10 +94,11 @@ def _run_test( fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: try: + mgmt_env = {k: v for k, v in os.environ.items() if k != "KUBECONFIG"} subprocess.run( ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, f"rhaiis.run-uuid={run_uuid}", "--overwrite"], - check=False, capture_output=True, timeout=10, + check=False, capture_output=True, timeout=10, env=mgmt_env, ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: From f272a57f749ee492e0b425c3d46dea1aa81b2922 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 19:14:03 -0400 Subject: [PATCH 106/344] Fix profiler trace S3 upload when labels are auto-generated When rhaiis.profiler.labels is empty, labels are auto-generated from workload data at runtime but were not passed to the upload function. Infer them from trace filenames (_run{LABEL}_range pattern) so traces are correctly matched and uploaded to S3. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 88132e33d..0331ce572 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -681,6 +681,25 @@ def _derive_profiler_label(workload: dict) -> str: return f"isl{isl}_osl{osl}" +def _infer_profile_labels_from_traces(trace_files: list) -> list[str]: + """Extract unique profile labels from trace filenames. + + Filenames follow the pattern: trace_rank{R}_pid{P}_run{LABEL}_range{S}-{E}.json + The {LABEL} portion is the gate value used during profiling (e.g. 'isl1000_osl1000'). + """ + import re + + pattern = re.compile(r"_run(.+?)_range\d+-\d+") + labels: list[str] = [] + seen: set[str] = set() + for f in trace_files: + m = pattern.search(f.name) + if m and m.group(1) not in seen: + seen.add(m.group(1)) + labels.append(m.group(1)) + return labels + + def _upload_profiler_traces( model_cfg: dict, accelerator: str, @@ -711,13 +730,18 @@ def _upload_profiler_traces( logger.info("No version configured, skipping profiler trace upload") return + profile_labels = profiler_cfg.get("labels", []) + if not profile_labels: + profile_labels = _infer_profile_labels_from_traces(trace_files) + logger.info("Inferred profile labels from trace filenames: %s", profile_labels) + result = upload_profiler_traces_to_s3( traces_dir, model_name=model_cfg.get("hf_model_id", ""), accelerator=accelerator, tp_size=int(vllm_args.get("tensor-parallel-size", 1)), version=version, - profile_labels=profiler_cfg.get("labels", []), + profile_labels=profile_labels, s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), From 9ecf78fd945aba3e70babfe7036ae5968db69272 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 12:57:18 -0400 Subject: [PATCH 107/344] Add profile4 workload (ISL=8000, OSL=1000) Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 0ef8e0079..be6c74c0a 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -20,8 +20,7 @@ profile3: profile4: data: "prompt_tokens=8000,output_tokens=1000" - rates: [1, 25, 50, 75, 100] - max_seconds: 450 + rates: [1] + max_seconds: 45 samples: 50 - vllm_args: - max-model-len: 10000 + From 2c101a91db99980582fbfafac86966255fae7e57 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:14:36 -0400 Subject: [PATCH 108/344] Increase wait_copy_pod_ready timeout to 5 minutes IBM VPC block storage volume re-attachment can take 3+ minutes. The previous 2-minute window (24x5s) was insufficient. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 91cbcc445..5a8253abf 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -264,7 +264,7 @@ def create_copy_pod(args, ctx): return f"Created copy pod {ctx.benchmark_name}-copy" -@retry(attempts=24, delay=5, backoff=1.0) +@retry(attempts=60, delay=5, backoff=1.0) @task def wait_copy_pod_ready(args, ctx): """Wait for copy pod to be ready""" From 28193bf4d2a4c2afbc4f1c2c8b48fd5b50829eef Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:30:16 -0400 Subject: [PATCH 109/344] Set USER env var in vLLM container to fix getpwuid crash on OpenShift OpenShift assigns arbitrary UIDs without /etc/passwd entries, causing PyTorch's getpass.getuser() -> pwd.getpwuid() to raise KeyError. Setting USER bypasses the pwd lookup entirely. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index baabd0852..42d53cd73 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -43,6 +43,8 @@ spec: {%- endif %} {%- endfor %} env: + - name: USER + value: "vllm" {%- if args.storage_source == "hf" %} - name: HF_HUB_OFFLINE value: "0" From b6d8c06712777207193548c6f0798dde5b63add9 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:50:43 -0400 Subject: [PATCH 110/344] Add ISVC/SR cleanup to post_cleanup hook Ensures InferenceService and ServingRuntime are deleted during Tekton finally step, covering cases where the test pod is killed before its own try/finally cleanup runs. Co-authored-by: Cursor --- projects/rhaiis/orchestration/prepare_rhaiis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index aa8c3b06a..5f7421035 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -48,6 +48,8 @@ def cleanup(): ns = runtime_config.get_namespace() logger.info(f"Cleaning up rhaiis benchmark resources in {ns}") + oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) logger.info("Cleanup complete") From b9e23804fc510411111b0efb72e3b1fcc1918b78 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 16:39:55 -0400 Subject: [PATCH 111/344] Update profile1 workload: rate=50, max_seconds=150 Needed to trigger regression comparison (concurrency=1 is excluded). Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index be6c74c0a..c413b505e 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,8 +5,8 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [1] - max_seconds: 45 + rates: [50] + max_seconds: 150 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" From f53d950c66e00d0345ed902b36a614beff0ff80b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 20:52:55 -0400 Subject: [PATCH 112/344] Add AI agent analysis integration for regression reports Port Model Furnace agent analysis to Forge: request AI analysis when severe regressions are detected, generate HTML report, upload to S3, and include report link in Slack notifications. Configure agent gateway URL for in-cluster access from management cluster. Co-authored-by: Cursor --- projects/core/notifications/send.py | 5 + .../rhaiis/orchestration/config.d/rhaiis.yaml | 5 + projects/rhaiis/orchestration/test_phase.py | 107 ++++++ projects/rhaiis/postprocess/agent.py | 306 ++++++++++++++++++ 4 files changed, 423 insertions(+) create mode 100644 projects/rhaiis/postprocess/agent.py diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 10415559f..0e96228c5 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -755,6 +755,7 @@ def send_regression_notification( webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, + report_url: str = "", ) -> bool: """Send regression/improvement Slack notification from analysis results. @@ -770,6 +771,7 @@ def send_regression_notification( webhook_vault: Vault containing slack-webhook-url file notification_vault: Fallback vault for Bot-Token Slack credentials dry_run: Log only, don't send + report_url: Optional presigned URL to an agent analysis report Returns: True if notification sent successfully @@ -834,6 +836,8 @@ def send_regression_notification( else: user_line = "" + report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + message = ( f"{icon} *{headline}*\n" f"{user_line}" @@ -841,6 +845,7 @@ def send_regression_notification( f"*Model:* {model}\n" f"*Accelerator:* {accelerator}\n" f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"{report_line}" f"*Changes:*\n{details}" ) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 96f1d7def..545ad6076 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -48,3 +48,8 @@ profiler: labels: [] rates: [200] max_seconds: 200 + +agent_analysis: + enabled: false + severity_threshold: 10 + url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0331ce572..5f0894ccd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -509,6 +509,15 @@ def _run_regression_check( ) if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + report_url = "" + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if agent_cfg.get("enabled", False): + report_url = _run_agent_analysis( + analysis, model_cfg, accelerator, + current_version, compare_version, run_uuid, + severity_threshold=agent_cfg.get("severity_threshold", 10), + ) + from projects.core.notifications.send import send_regression_notification send_regression_notification( @@ -519,6 +528,7 @@ def _run_regression_check( slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", + report_url=report_url, ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) @@ -528,6 +538,103 @@ def _run_regression_check( os.unlink(consolidated_path) +def _run_agent_analysis( + analysis: dict, + model_cfg: dict, + accelerator: str, + current_version: str, + compare_version: str, + run_uuid: str, + *, + severity_threshold: int = 10, +) -> str: + """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.agent import ( + AGENT_SEVERITY_THRESHOLD, + build_pr_followup_prompt, + check_agent_connectivity, + markdown_to_html, + request_agent_analysis, + send_followup, + ) + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + agent_url = agent_cfg.get("url", "") + if not agent_url: + logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") + return "" + + threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD + severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] + if not severe: + logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) + return "" + + ok, detail = check_agent_connectivity(agent_url) + if not ok: + logger.warning("Agent not reachable, skipping analysis: %s", detail) + return "" + + tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + model = model_cfg.get("hf_model_id", "") + improvements = analysis.get("improvements", []) + + agent_response = request_agent_analysis( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe, + job_id=run_uuid, + improvements=improvements if improvements else None, + agent_url=agent_url, + ) + if not agent_response: + return "" + + pr_prompt = build_pr_followup_prompt(current_version, compare_version) + pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) + if pr_analysis: + agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" + + html_content = markdown_to_html( + agent_response, run_uuid, model, current_version, compare_version, + ) + html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" + html_path.write_text(html_content, encoding="utf-8") + logger.info("Agent analysis saved to %s", html_path) + + try: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if credentials_path: + s3 = create_s3_client(credentials_path) + s3_bucket = "psap-dashboard-data" + s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" + s3.upload_file( + str(html_path), s3_bucket, s3_key, + ExtraArgs={"ContentType": "text/html"}, + ) + report_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": s3_bucket, "Key": s3_key}, + ExpiresIn=2592000, + ) + logger.info("Agent analysis uploaded to S3, presigned URL generated") + return report_url + except Exception: + logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) + + return "" + + def _upload_predictor_log(run_uuid: str) -> None: """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" from pathlib import Path diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py new file mode 100644 index 000000000..22732edc7 --- /dev/null +++ b/projects/rhaiis/postprocess/agent.py @@ -0,0 +1,306 @@ +"""Client for the PSAP AI agent's /v1/stream endpoint. + +Sends regression context as a natural-language prompt and collects the +streamed AI response into a markdown string. Ported from model-furnace. +""" + +import json +import logging +import uuid +from typing import Optional +from urllib.error import URLError +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +DEFAULT_AGENT_URL = ( + "https://psap-agent-staging-psap-ai-agent.apps.ocp4.intlab.redhat.com/v1/stream" +) +AGENT_TIMEOUT_SECONDS = 600 +AGENT_HEALTH_TIMEOUT = 15 +AGENT_SEVERITY_THRESHOLD = 10 + + +def check_agent_connectivity(agent_url: str = "") -> tuple[bool, str]: + """Verify that the agent endpoint is reachable. + + Tries a lightweight GET to the agent health URL. Returns (ok, detail). + """ + url = agent_url or DEFAULT_AGENT_URL + base_url = url.rsplit("/", 2)[0] + health_url = f"{base_url}/health" + + try: + req = Request(health_url, method="GET") + resp = urlopen(req, timeout=AGENT_HEALTH_TIMEOUT) # noqa: S310 + status = resp.getcode() + if status and status < 400: + return True, f"Agent reachable at {health_url} (HTTP {status})" + return False, f"Agent returned HTTP {status} at {health_url}" + except (URLError, OSError) as e: + return False, f"Cannot reach agent at {health_url}: {e}" + except Exception as e: + return False, f"Unexpected error checking agent at {health_url}: {e}" + + +def _build_prompt( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + improvements: Optional[list] = None, +) -> str: + """Build a natural-language prompt for the agent from regression context.""" + lines = [ + f"Analyze the performance regression between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}.", + "", + "The following metrics regressed significantly (>10%):", + ] + for r in severe_regressions: + direction = "dropped" if r["pct_diff"] < 0 else "increased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + if improvements: + lines.append("") + lines.append("The following metrics improved significantly:") + for r in improvements: + direction = "increased" if r["pct_diff"] > 0 else "decreased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + lines.append("") + lines.append( + f"Check if the runtime args differ between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}. " + ) + lines.append("") + lines.append( + "Provide root cause analysis using pytorch profiler traces, vLLM logs, " + "and vLLM source code where available." + ) + return "\n".join(lines) + + +def build_pr_followup_prompt( + current_version: str, + compare_version: str, +) -> str: + """Build a followup prompt asking the agent to identify relevant PRs.""" + return ( + "Based on your previous analysis, use the compare_vllm_versions and " + "get_vllm_pull_request tools to identify which specific pull requests " + f"between vLLM {compare_version} and {current_version} most likely " + "explain the performance differences you found. For each PR you " + "identify, include its GitHub link and a brief explanation of why " + "it's relevant to the regressions or improvements you observed." + ) + + +def request_agent_analysis( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + job_id: str = "", + improvements: Optional[list] = None, + agent_url: str = "", +) -> Optional[str]: + """Send regression context to the PSAP agent and return its analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url or DEFAULT_AGENT_URL + + prompt = _build_prompt( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe_regressions, + improvements=improvements, + ) + + session_key = f"forge-rhaiis-{job_id or uuid.uuid4()}" + body = { + "message": prompt, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Requesting agent analysis for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent analysis received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent analysis for job %s: %s", job_id, e) + return None + + +def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[str]: + """Send a followup message to the agent on an existing session. + + Reuses the same thread_id/session_id as request_agent_analysis so the + agent retains full conversation context from the initial analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url or DEFAULT_AGENT_URL + + session_key = f"forge-rhaiis-{job_id}" + body = { + "message": message, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Sending agent followup for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent followup received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty followup response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent followup request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent followup for job %s: %s", job_id, e) + return None + + +def _collect_response(resp) -> Optional[str]: + """Read the NDJSON stream and extract the final AI message content.""" + collected_tokens = [] + final_message = None + + for raw_line in resp: + line = raw_line.decode("utf-8", errors="replace").strip() + if not line or line == "[DONE]": + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + event_type = event.get("type") + content = event.get("content") + + if event_type == "token" and isinstance(content, str): + collected_tokens.append(content) + elif ( + event_type == "message" + and isinstance(content, dict) + and content.get("type") == "ai" + and content.get("content") + ): + final_message = content["content"] + + if final_message: + return final_message + if collected_tokens: + return "".join(collected_tokens) + return None + + +def markdown_to_html( + md_text: str, + job_id: str, + model: str, + current_version: str, + compare_version: str, +) -> str: + """Convert markdown analysis to a self-contained HTML page.""" + try: + import markdown as md_lib + + body = md_lib.markdown(md_text, extensions=["tables", "fenced_code"]) + except ImportError: + import html as html_lib + + body = f"
{html_lib.escape(md_text)}
" + + from datetime import datetime, timezone + + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + return f""" + + + + +Analysis — {job_id} + + + +

Regression Analysis Report

+
+ Job: {job_id} + Model: {model} + Versions: {current_version} vs {compare_version} + Generated: {timestamp} +
+{body} + +""" From d91ae9158603be700c05bdefc3800fbe11d40898 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 21:02:40 -0400 Subject: [PATCH 113/344] Add standalone agent analysis mode (no benchmark required) When run_benchmark=false but agent_analysis.enabled=true, download existing data from the S3 consolidated CSV and run regression check + agent analysis against historical data. This allows testing the agent analysis pipeline without running a new benchmark. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 89 +++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 5f0894ccd..1de90d513 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -263,6 +263,10 @@ def _run_workload_benchmark( main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") + try: + _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) else: logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) @@ -462,6 +466,91 @@ def _generate_and_sync_dashboard_csv( _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) +def _run_standalone_analysis( + model_cfg: dict, + accelerator: str, + vllm_args: dict, + *, + run_uuid: str = "", +) -> None: + """Run regression check + agent analysis using existing S3 data (no new benchmark).""" + import tempfile + from pathlib import Path + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if not agent_cfg.get("enabled", False): + logger.info("Standalone analysis skipped: agent_analysis not enabled") + return + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not version or not compare_version: + logger.info("Standalone analysis skipped: version or compare_version not configured") + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") + s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if not credentials_path: + logger.warning("AWS credentials not available, skipping standalone analysis") + return + + consolidated_path = None + current_csv_path = None + try: + import pandas as pd + + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + + df = pd.read_csv(consolidated_path, on_bad_lines="warn") + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].str.strip() + + model_id = model_cfg.get("hf_model_id", "") + tp = str(vllm_args.get("tensor-parallel-size", 1)) + + current_rows = df[ + (df["version"] == version) + & (df["model"] == model_id) + & (df["accelerator"] == accelerator) + & (df["TP"].astype(str) == tp) + ] + + if current_rows.empty: + logger.warning( + "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", + version, model_id, accelerator, tp, + ) + return + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + current_csv_path = tmp.name + current_rows.to_csv(tmp, index=False) + + logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) + _run_regression_check( + current_csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + if current_csv_path and os.path.exists(current_csv_path): + os.unlink(current_csv_path) + + def _run_regression_check( csv_path, compare_version: str, From 9b7a673c6cc272185b514276f68c31975ae87d79 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:06:54 -0400 Subject: [PATCH 114/344] Skip warmup when run_benchmark and profiler are both disabled Warmup only makes sense as a pre-benchmark step. When both run_benchmark=false and profiler.enabled=false, skip warmup regardless of its own setting to avoid deploying a model needlessly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 1de90d513..4fc6211fd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -140,7 +140,12 @@ def _run_test( profiler_cfg = runtime_config.get_profiler_config() profiler_enabled = profiler_cfg.get("enabled", False) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = ( + config.project.get_config("tests.rhaiis.warmup", True) + and run_benchmark + and not profiler_enabled + ) logger.info( "Running %d workload(s): %s", len(workload_keys), workload_keys, From 3cabbb44276f3c984a382792c49c2dd55b7fe1bd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:24:38 -0400 Subject: [PATCH 115/344] Use chip family (H200) not accelerator_key (H200_HERA2) for CSV filtering The cluster_tag suffix is only used in the CSV run ID column. The accelerator column stores the chip family (e.g. H200). Strip the cluster_tag when filtering S3 data and when passing to regression check / notifications. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 4fc6211fd..697c317ef 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -418,7 +418,7 @@ def _generate_psap_payload( def _generate_and_sync_dashboard_csv( model_cfg: dict, - accelerator: str, + accelerator_key: str, workload_key: str, vllm_args: dict, *, @@ -429,6 +429,9 @@ def _generate_and_sync_dashboard_csv( from projects.core.library import config from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv + # Use chip family only (e.g. "H200") — cluster_tag is only for run ID in the CSV + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) if not psap_files: logger.warning("No PSAP JSON files found, skipping dashboard CSV") @@ -473,7 +476,7 @@ def _generate_and_sync_dashboard_csv( def _run_standalone_analysis( model_cfg: dict, - accelerator: str, + accelerator_key: str, vllm_args: dict, *, run_uuid: str = "", @@ -506,6 +509,9 @@ def _run_standalone_analysis( logger.warning("AWS credentials not available, skipping standalone analysis") return + # CSV stores just the chip family (e.g. "H200"), not the full accelerator_key with cluster tag + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + consolidated_path = None current_csv_path = None try: From 6827edf4c5037fe89ba22eaef6878d505699df18 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:50:30 -0400 Subject: [PATCH 116/344] Rename gpu_types key 'amd' to 'amd_mi300x' for clarity Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 545ad6076..66df54d5c 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -4,7 +4,7 @@ cluster_tag: "" gpu_types: nvidia: h200 - amd: mi300x + amd_mi300x: mi300x l40s: l40s images: From 813a0989b02578ca2da7c952d8ed054dd6a9ad5d Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:06:33 -0400 Subject: [PATCH 117/344] Fix TP column comparison: CSV stores floats (2.0), convert to int before string match Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 697c317ef..427241d60 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -534,7 +534,7 @@ def _run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].astype(str) == tp) + & (df["TP"].astype(float).astype(int).astype(str) == tp) ] if current_rows.empty: From 09e2bc9674d960b260d33d20dc9c8c7c74f395ab Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:24:38 -0400 Subject: [PATCH 118/344] Skip vLLM deployment when only standalone analysis is needed When both run_benchmark=false and profiler.enabled=false, the pipeline only needs S3 data for regression analysis. Skip the InferenceService deployment entirely and run standalone analysis directly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 427241d60..7cd267030 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -104,6 +104,22 @@ def _run_test( except Exception: logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + + # Standalone analysis only — no deployment needed + if not run_benchmark and not profiler_enabled: + logger.info("run_benchmark=false and profiler=false, running standalone analysis only") + for wl_key in workload_keys: + try: + _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) + return 0 + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -136,11 +152,6 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - from projects.core.library import config - - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) - run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) warmup_enabled = ( config.project.get_config("tests.rhaiis.warmup", True) and run_benchmark From cf3ba112c2ae4a489b743898ab2044118785343b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:30:50 -0400 Subject: [PATCH 119/344] Handle NaN values in TP column during CSV filtering The TP column can have NaN for rows with missing data. Fill with -1 before int conversion so those rows are excluded from the filter. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 7cd267030..356beba99 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -545,7 +545,7 @@ def _run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].astype(float).astype(int).astype(str) == tp) + & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) ] if current_rows.empty: From 2fd8142527189cb2e89d76d34973410817845876 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 13:20:47 -0400 Subject: [PATCH 120/344] Fix workload_keys filtering, add TP/dashboard to Slack notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire restrict_profiles through standalone analysis → regression check → compare_runs so workload_keys actually filters which profiles appear in Slack notifications (previously all S3 profiles were reported) - Remove redundant per-key loop in standalone analysis path - Add TP/DP parallelism info to Slack notification message - Add pre-filtered RHAIIS dashboard link to Slack notification - Rename "Model Furnace" references to "Fournos" in message text Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 2 + projects/core/notifications/send.py | 73 +++++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 24 +++++-- 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index f2ef0b4bd..78f33839b 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -178,6 +178,7 @@ def run_regression_analysis( compare_version: str, current_version: str, output_file: Path, + restrict_profiles: Optional[list[str]] = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. @@ -189,6 +190,7 @@ def run_regression_analysis( str(consolidated_csv_path), compare_version, current_version, + restrict_profiles=restrict_profiles, ) if skip_reason: diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 0e96228c5..babe3b77d 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -745,6 +745,54 @@ def _get_webhook_url(vault_name: str) -> str | None: return path.read_text().strip() +DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +PROFILE_DISPLAY_NAMES = { + "profile1": "Profile A: Balanced (1k/1k)", + "profile2": "Profile B: Variable Workload (512/2k)", + "profile3": "Profile C: Large Prompt (2k/128)", + "profile4": "Profile D: Prefill Heavy (8k/1k)", +} + + +def _build_dashboard_url( + *, + model: str = "", + accelerator: str = "", + current_version: str = "", + compare_version: str = "", + profiles: list[str] | None = None, + tp: str = "", +) -> str: + """Build a RHAIIS dashboard URL with filters pre-selected.""" + from urllib.parse import quote, urlencode + + params: dict[str, str] = {"view": "RHAIIS Dashboard"} + + if accelerator: + params["accelerators"] = accelerator + if model: + params["models"] = model + + versions = ",".join(v for v in [current_version, compare_version] if v) + if versions: + params["versions"] = versions + + if profiles: + display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] + params["profile"] = ",".join(display_names) + + if tp: + try: + params["tp_sizes"] = f"{float(tp):.1f}" + except (ValueError, TypeError): + params["tp_sizes"] = tp + + params["section"] = "performance_plots" + + return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" + + def send_regression_notification( analysis_result: dict, *, @@ -756,6 +804,8 @@ def send_regression_notification( notification_vault: str | None = None, dry_run: bool = False, report_url: str = "", + tp: str = "", + dp: str = "", ) -> bool: """Send regression/improvement Slack notification from analysis results. @@ -772,6 +822,8 @@ def send_regression_notification( notification_vault: Fallback vault for Bot-Token Slack credentials dry_run: Log only, don't send report_url: Optional presigned URL to an agent analysis report + tp: Tensor parallelism size for display + dp: Data parallelism size for display Returns: True if notification sent successfully @@ -838,14 +890,35 @@ def send_regression_notification( report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + parallelism_line = "" + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + if parallelism_parts: + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" + + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + profiles=profiles if profiles else None, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + message = ( f"{icon} *{headline}*\n" f"{user_line}" f"*Job:* `{job_id}`\n" f"*Model:* {model}\n" f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" f"*Versions:* {current_version} vs {compare_version} (baseline)\n" f"{report_line}" + f"{dashboard_line}" f"*Changes:*\n{details}" ) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 356beba99..ea9e63dc5 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -113,11 +113,13 @@ def _run_test( # Standalone analysis only — no deployment needed if not run_benchmark and not profiler_enabled: logger.info("run_benchmark=false and profiler=false, running standalone analysis only") - for wl_key in workload_keys: - try: - _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) + try: + _run_standalone_analysis( + model_cfg, accelerator_key, vllm_args, + run_uuid=run_uuid, restrict_profiles=workload_keys or None, + ) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) return 0 benchmark_timeout = benchmark_cfg.get("timeout", 14400) @@ -482,7 +484,7 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) def _run_standalone_analysis( @@ -491,6 +493,7 @@ def _run_standalone_analysis( vllm_args: dict, *, run_uuid: str = "", + restrict_profiles: list[str] | None = None, ) -> None: """Run regression check + agent analysis using existing S3 data (no new benchmark).""" import tempfile @@ -561,7 +564,8 @@ def _run_standalone_analysis( logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) _run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, + current_csv_path, compare_version, version, model_cfg, accelerator, + run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -581,6 +585,8 @@ def _run_regression_check( accelerator: str, *, run_uuid: str = "", + restrict_profiles: list[str] | None = None, + vllm_args: dict | None = None, ) -> None: import tempfile from pathlib import Path @@ -617,6 +623,7 @@ def _run_regression_check( compare_version=compare_version, current_version=current_version, output_file=output_file, + restrict_profiles=restrict_profiles, ) if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: @@ -631,6 +638,7 @@ def _run_regression_check( from projects.core.notifications.send import send_regression_notification + _vllm = vllm_args or {} send_regression_notification( analysis, model=model_cfg.get("hf_model_id", ""), @@ -640,6 +648,8 @@ def _run_regression_check( webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", report_url=report_url, + tp=str(_vllm.get("tensor-parallel-size", "")), + dp=str(_vllm.get("data-parallel-size", "")), ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) From b244b49436d0e46e84475295aed66822584111a0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 17 Jul 2026 00:13:23 -0400 Subject: [PATCH 121/344] Address PR review: move RHAIIS-specific code out of core/caliper - Move PROFILE_MAP, METRICS from caliper engine to rhaiis postprocess - Make compare_runs/run_regression_analysis accept profile_map and metrics as parameters instead of hardcoded constants - Move send_regression_notification, dashboard URL builder, and profile display names from core/notifications to rhaiis postprocess - Consolidate Slack webhook into psap-forge-notifications vault, remove separate psap-forge-rhaiis-slack vault - Remove hardcoded DEFAULT_AGENT_URL from agent.py, require caller to provide URL from config - Add docstring to enable_profiler_gate run function - Clean up vault descriptions to be project-generic Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 29 +-- projects/core/notifications/send.py | 198 ---------------- .../rhaiis/orchestration/config.d/rhaiis.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 1 - projects/rhaiis/orchestration/test_phase.py | 7 +- projects/rhaiis/postprocess/agent.py | 15 +- projects/rhaiis/postprocess/regression.py | 224 ++++++++++++++++++ .../toolbox/enable_profiler_gate/main.py | 11 + vaults/psap-forge-dashboard-s3.yaml | 7 +- vaults/psap-forge-notifications.yaml | 5 + vaults/psap-forge-rhaiis-slack.yaml | 10 - 11 files changed, 268 insertions(+), 243 deletions(-) create mode 100644 projects/rhaiis/postprocess/regression.py delete mode 100644 vaults/psap-forge-rhaiis-slack.yaml diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 78f33839b..847fbf159 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -15,21 +15,6 @@ logger = logging.getLogger(__name__) -PROFILE_MAP = { - (1000, 1000): "profile1", - (512, 2048): "profile2", - (2048, 128): "profile3", - (8000, 1000): "profile4", -} - -METRICS = { - "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, - "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, - "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, - "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, - "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, -} - @dataclass class RegressionResult: @@ -55,10 +40,16 @@ def compare_runs( consolidated_csv_path: str, compare_version: str, current_version: str, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], restrict_profiles: Optional[list[str]] = None, ) -> tuple[list[RegressionResult], str]: """Compare current run metrics against a baseline version. + Args: + profile_map: Mapping of (input_tokens, output_tokens) -> profile name. + metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. """ import pandas as pd @@ -89,7 +80,7 @@ def _assign_profile(row): osl = int(float(row["output toks"])) except (ValueError, TypeError): return None - return PROFILE_MAP.get((isl, osl)) + return profile_map.get((isl, osl)) current_df["_profile"] = current_df.apply(_assign_profile, axis=1) baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) @@ -132,7 +123,7 @@ def _assign_profile(row): cur_common = cur[cur["intended concurrency"].isin(common_conc)] base_common = base[base["intended concurrency"].isin(common_conc)] - for col, meta in METRICS.items(): + for col, meta in metrics.items(): if col not in cur_common.columns or col not in base_common.columns: continue @@ -178,6 +169,8 @@ def run_regression_analysis( compare_version: str, current_version: str, output_file: Path, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], restrict_profiles: Optional[list[str]] = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. @@ -190,6 +183,8 @@ def run_regression_analysis( str(consolidated_csv_path), compare_version, current_version, + profile_map=profile_map, + metrics=metrics, restrict_profiles=restrict_profiles, ) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index babe3b77d..dfc494765 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -745,204 +745,6 @@ def _get_webhook_url(vault_name: str) -> str | None: return path.read_text().strip() -DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" - -PROFILE_DISPLAY_NAMES = { - "profile1": "Profile A: Balanced (1k/1k)", - "profile2": "Profile B: Variable Workload (512/2k)", - "profile3": "Profile C: Large Prompt (2k/128)", - "profile4": "Profile D: Prefill Heavy (8k/1k)", -} - - -def _build_dashboard_url( - *, - model: str = "", - accelerator: str = "", - current_version: str = "", - compare_version: str = "", - profiles: list[str] | None = None, - tp: str = "", -) -> str: - """Build a RHAIIS dashboard URL with filters pre-selected.""" - from urllib.parse import quote, urlencode - - params: dict[str, str] = {"view": "RHAIIS Dashboard"} - - if accelerator: - params["accelerators"] = accelerator - if model: - params["models"] = model - - versions = ",".join(v for v in [current_version, compare_version] if v) - if versions: - params["versions"] = versions - - if profiles: - display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] - params["profile"] = ",".join(display_names) - - if tp: - try: - params["tp_sizes"] = f"{float(tp):.1f}" - except (ValueError, TypeError): - params["tp_sizes"] = tp - - params["section"] = "performance_plots" - - return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" - - -def send_regression_notification( - analysis_result: dict, - *, - model: str = "", - accelerator: str = "", - job_id: str = "", - slack_user: str = "", - webhook_vault: str | None = None, - notification_vault: str | None = None, - dry_run: bool = False, - report_url: str = "", - tp: str = "", - dp: str = "", -) -> bool: - """Send regression/improvement Slack notification from analysis results. - - Uses a webhook URL (from *webhook_vault*) when available, falling - back to the Bot-Token path (from *notification_vault*). - - Args: - analysis_result: Output from run_regression_analysis() - model: Model name for display - accelerator: Accelerator name for display - job_id: Job identifier - slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials - dry_run: Log only, don't send - report_url: Optional presigned URL to an agent analysis report - tp: Tensor parallelism size for display - dp: Data parallelism size for display - - Returns: - True if notification sent successfully - """ - status = analysis_result.get("status") - if status == "skipped": - reason = analysis_result.get("reason", "unknown") - logger.info("Regression analysis skipped: %s", reason) - return True - - if status != "completed": - return True - - regressions = analysis_result.get("regressions", []) - improvements = analysis_result.get("improvements", []) - - if not regressions and not improvements: - logger.info("No regressions or improvements to report") - return True - - current_version = analysis_result.get("current_version", "") - compare_version = analysis_result.get("compare_version", "") - - if regressions and improvements: - icon = ":warning:" - headline = "Performance regressions and improvements detected" - elif regressions: - icon = ":warning:" - headline = "Performance regressions detected" - else: - icon = ":large_green_circle:" - headline = "Performance improvements detected" - - detail_lines = [] - all_results = analysis_result.get("all_results", []) - profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) - - for profile in profiles: - profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] - detail_lines.append(f"\n*{profile}:*") - for r in profile_results: - if r.get("is_regression"): - direction = "dropped" if r["pct_diff"] < 0 else "increased" - detail_lines.append( - f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " - f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" - ) - else: - detail_lines.append( - f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " - f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" - ) - - details = "\n".join(detail_lines) - - import re as _re - - if slack_user and _re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" - - report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" - - parallelism_line = "" - parallelism_parts = [] - if tp: - parallelism_parts.append(f"TP={tp}") - if dp: - parallelism_parts.append(f"DP={dp}") - if parallelism_parts: - parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" - - dashboard_url = _build_dashboard_url( - model=model, - accelerator=accelerator, - current_version=current_version, - compare_version=compare_version, - profiles=profiles if profiles else None, - tp=tp, - ) - dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" - - message = ( - f"{icon} *{headline}*\n" - f"{user_line}" - f"*Job:* `{job_id}`\n" - f"*Model:* {model}\n" - f"*Accelerator:* {accelerator}\n" - f"{parallelism_line}" - f"*Versions:* {current_version} vs {compare_version} (baseline)\n" - f"{report_line}" - f"{dashboard_line}" - f"*Changes:*\n{details}" - ) - - if dry_run: - logger.info("DRY RUN regression notification:\n%s", message) - return True - - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) - - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 66df54d5c..12da18a24 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -4,7 +4,7 @@ cluster_tag: "" gpu_types: nvidia: h200 - amd_mi300x: mi300x + amd: mi300x l40s: l40s images: @@ -17,7 +17,7 @@ deploy: memory_request: "16Gi" storage_source: hf storage_pvc: model-pvc - image_pull_secret: npalaska-image-pull + image_pull_secret: "" service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index f5aa9224c..1442b8c3a 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -3,7 +3,6 @@ vaults: - psap-forge-mlflow-export - psap-forge-aws-s3-export - psap-forge-dashboard-s3 -- psap-forge-rhaiis-slack benchmarks: guidellm: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ea9e63dc5..20038e4bd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -594,6 +594,7 @@ def _run_regression_check( from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials from projects.caliper.engine.kpi.analyze import run_regression_analysis from projects.core.library import config + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") @@ -623,6 +624,8 @@ def _run_regression_check( compare_version=compare_version, current_version=current_version, output_file=output_file, + profile_map=PROFILE_MAP, + metrics=METRICS, restrict_profiles=restrict_profiles, ) @@ -636,7 +639,7 @@ def _run_regression_check( severity_threshold=agent_cfg.get("severity_threshold", 10), ) - from projects.core.notifications.send import send_regression_notification + from projects.rhaiis.postprocess.regression import send_regression_notification _vllm = vllm_args or {} send_regression_notification( @@ -645,7 +648,7 @@ def _run_regression_check( accelerator=accelerator, job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-rhaiis-slack", + webhook_vault="psap-forge-notifications", notification_vault="psap-forge-notifications", report_url=report_url, tp=str(_vllm.get("tensor-parallel-size", "")), diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py index 22732edc7..d08140310 100644 --- a/projects/rhaiis/postprocess/agent.py +++ b/projects/rhaiis/postprocess/agent.py @@ -13,20 +13,17 @@ logger = logging.getLogger(__name__) -DEFAULT_AGENT_URL = ( - "https://psap-agent-staging-psap-ai-agent.apps.ocp4.intlab.redhat.com/v1/stream" -) AGENT_TIMEOUT_SECONDS = 600 AGENT_HEALTH_TIMEOUT = 15 AGENT_SEVERITY_THRESHOLD = 10 -def check_agent_connectivity(agent_url: str = "") -> tuple[bool, str]: +def check_agent_connectivity(agent_url: str) -> tuple[bool, str]: """Verify that the agent endpoint is reachable. Tries a lightweight GET to the agent health URL. Returns (ok, detail). """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url base_url = url.rsplit("/", 2)[0] health_url = f"{base_url}/health" @@ -109,15 +106,15 @@ def request_agent_analysis( compare_version: str, tp: str, severe_regressions: list, + agent_url: str, job_id: str = "", improvements: Optional[list] = None, - agent_url: str = "", ) -> Optional[str]: """Send regression context to the PSAP agent and return its analysis. Returns the agent's response as a markdown string, or None on failure. """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url prompt = _build_prompt( model=model, @@ -168,7 +165,7 @@ def request_agent_analysis( return None -def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[str]: +def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: """Send a followup message to the agent on an existing session. Reuses the same thread_id/session_id as request_agent_analysis so the @@ -176,7 +173,7 @@ def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[st Returns the agent's response as a markdown string, or None on failure. """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url session_key = f"forge-rhaiis-{job_id}" body = { diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py new file mode 100644 index 000000000..d992e859d --- /dev/null +++ b/projects/rhaiis/postprocess/regression.py @@ -0,0 +1,224 @@ +"""RHAIIS-specific profile/metric definitions and Slack notification for regression analysis.""" + +from __future__ import annotations + +import logging +import re + +logger = logging.getLogger(__name__) + +PROFILE_MAP = { + (1000, 1000): "profile1", + (512, 2048): "profile2", + (2048, 128): "profile3", + (8000, 1000): "profile4", +} + +METRICS = { + "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, + "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, + "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, + "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, + "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, +} + +DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +PROFILE_DISPLAY_NAMES = { + "profile1": "Profile A: Balanced (1k/1k)", + "profile2": "Profile B: Variable Workload (512/2k)", + "profile3": "Profile C: Large Prompt (2k/128)", + "profile4": "Profile D: Prefill Heavy (8k/1k)", +} + + +def _build_dashboard_url( + *, + model: str = "", + accelerator: str = "", + current_version: str = "", + compare_version: str = "", + profiles: list[str] | None = None, + tp: str = "", +) -> str: + """Build a RHAIIS dashboard URL with filters pre-selected.""" + from urllib.parse import quote, urlencode + + params: dict[str, str] = {"view": "RHAIIS Dashboard"} + + if accelerator: + params["accelerators"] = accelerator + if model: + params["models"] = model + + versions = ",".join(v for v in [current_version, compare_version] if v) + if versions: + params["versions"] = versions + + if profiles: + display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] + params["profile"] = ",".join(display_names) + + if tp: + try: + params["tp_sizes"] = f"{float(tp):.1f}" + except (ValueError, TypeError): + params["tp_sizes"] = tp + + params["section"] = "performance_plots" + + return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" + + +def send_regression_notification( + analysis_result: dict, + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, + report_url: str = "", + tp: str = "", + dp: str = "", +) -> bool: + """Send regression/improvement Slack notification from analysis results. + + Uses a webhook URL (from *webhook_vault*) when available, falling + back to the Bot-Token path (from *notification_vault*). + + Args: + analysis_result: Output from run_regression_analysis() + model: Model name for display + accelerator: Accelerator name for display + job_id: Job identifier + slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + report_url: Optional presigned URL to an agent analysis report + tp: Tensor parallelism size for display + dp: Data parallelism size for display + + Returns: + True if notification sent successfully + """ + from projects.core.notifications.send import ( + _get_webhook_url, + _send_via_webhook, + send_notification, + ) + + status = analysis_result.get("status") + if status == "skipped": + reason = analysis_result.get("reason", "unknown") + logger.info("Regression analysis skipped: %s", reason) + return True + + if status != "completed": + return True + + regressions = analysis_result.get("regressions", []) + improvements = analysis_result.get("improvements", []) + + if not regressions and not improvements: + logger.info("No regressions or improvements to report") + return True + + current_version = analysis_result.get("current_version", "") + compare_version = analysis_result.get("compare_version", "") + + if regressions and improvements: + icon = ":warning:" + headline = "Performance regressions and improvements detected" + elif regressions: + icon = ":warning:" + headline = "Performance regressions detected" + else: + icon = ":large_green_circle:" + headline = "Performance improvements detected" + + detail_lines = [] + all_results = analysis_result.get("all_results", []) + profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + + for profile in profiles: + profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + detail_lines.append(f"\n*{profile}:*") + for r in profile_results: + if r.get("is_regression"): + direction = "dropped" if r["pct_diff"] < 0 else "increased" + detail_lines.append( + f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + else: + detail_lines.append( + f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + + details = "\n".join(detail_lines) + + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + + parallelism_line = "" + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + if parallelism_parts: + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" + + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + profiles=profiles if profiles else None, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + + message = ( + f"{icon} *{headline}*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"{report_line}" + f"{dashboard_line}" + f"*Changes:*\n{details}" + ) + + if dry_run: + logger.info("DRY RUN regression notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py index 69468b405..b1783b09c 100644 --- a/projects/rhaiis/toolbox/enable_profiler_gate/main.py +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -10,6 +10,17 @@ @entrypoint def run(*, name: str, namespace: str, gate_value: str = "0", disable: bool = False): + """Toggle the vLLM profiler gate file on a KServe predictor pod. + + Writes (or removes) /tmp/profiler_gate inside the predictor container + to enable or disable on-demand profiling for a running inference service. + + Args: + name: KServe InferenceService name used to locate the predictor pod. + namespace: Kubernetes namespace where the InferenceService is deployed. + gate_value: Value written to the gate file (default "0"). + disable: When True, removes the gate file instead of creating it. + """ return execute_tasks(locals()) diff --git a/vaults/psap-forge-dashboard-s3.yaml b/vaults/psap-forge-dashboard-s3.yaml index 145fc0e70..ada9243ed 100644 --- a/vaults/psap-forge-dashboard-s3.yaml +++ b/vaults/psap-forge-dashboard-s3.yaml @@ -1,15 +1,14 @@ env_key: PSAP_FORGE_DASHBOARD_S3_SECRET_PATH description: | - AWS credentials for rhaiis dashboard CSV sync (psap-dashboard-data) - and profiler trace uploads. Ported from model-furnace s3-uploader.conf. + AWS credentials for dashboard CSV sync and data uploads + to S3 buckets used by Forge projects. content: aws.credentials: description: | AWS credentials file (INI format). The key must have s3:GetObject, s3:PutObject, s3:CopyObject on: - - psap-dashboard-data (CSV dashboard, profiler traces) - - psap-model-furnace (vLLM logs) + - psap-dashboard-data (CSV dashboard data, profiler traces) sample: | [default] aws_access_key_id = AKIA... diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 12ae01cc8..5646df832 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -12,3 +12,8 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot + slack-webhook-url: + description: | + Plain-text file containing a Slack Incoming Webhook URL + for regression/comparison alerts. + Example: https://hooks.slack.com/services/T.../B.../... diff --git a/vaults/psap-forge-rhaiis-slack.yaml b/vaults/psap-forge-rhaiis-slack.yaml deleted file mode 100644 index 838d41964..000000000 --- a/vaults/psap-forge-rhaiis-slack.yaml +++ /dev/null @@ -1,10 +0,0 @@ -env_key: PSAP_FORGE_RHAIIS_SLACK_SECRET_PATH -description: | - Slack Incoming Webhook for rhaiis regression/comparison alerts. - Ported from model-furnace's slack.conf. - -content: - slack-webhook-url: - description: | - Plain-text file containing a single Slack Incoming Webhook URL. - Example: https://hooks.slack.com/services/T.../B.../... From a65612d3e4b1b9554844103e5506aadbc94c2bc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 14:09:49 +0200 Subject: [PATCH 122/344] [claude] Add hook for running ruff after file modifications --- .claude/settings.json | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..b9eecce75 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,36 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + }, + { + "matcher": "Edit", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + } + ] + } +} From d1752eb1db17ae1caa82564f0f840391fb513ed5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:10 +0200 Subject: [PATCH 123/344] [guidellm] toolbox: run_smoke_request: improve the troubleshooting when the Pod doesn't start --- .../toolbox/run_smoke_request/main.py | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index 703bef5fe..db3a7d742 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -5,7 +5,7 @@ import json import logging -from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, task from projects.core.dsl.utils import write_json, write_text from projects.core.dsl.utils.k8s import ( oc, @@ -129,7 +129,7 @@ def create_smoke_pod(args, ctx): return f"Created smoke pod {ctx.pod_name}" -@retry(attempts=60, delay=5, backoff=1.0) +@retry(attempts=12, delay=5, backoff=1.0) @task def wait_smoke_pod_running(args, ctx): """Wait for smoke pod to be running""" @@ -224,10 +224,61 @@ def execute_smoke_request(args, ctx): return "Smoke request completed successfully" +@always +@task +def capture_smoke_pod_debug_info(args, ctx): + """Capture pod YAML and description for debugging (always runs)""" + + if not hasattr(ctx, "pod_name"): + return "No pod name available for debug capture" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + try: + # Capture pod YAML + yaml_result = oc( + "get", + "pod", + ctx.pod_name, + "-n", + args.namespace, + "-o", + "yaml", + check=False, + ) + if yaml_result.returncode == 0 and yaml_result.stdout: + write_text(artifacts_dir / f"{ctx.pod_name}.yaml", yaml_result.stdout) + + # Capture pod description + desc_result = oc( + "describe", + "pod", + ctx.pod_name, + "-n", + args.namespace, + check=False, + ) + if desc_result.returncode == 0 and desc_result.stdout: + write_text(artifacts_dir / f"{ctx.pod_name}.description.txt", desc_result.stdout) + + return f"Captured debug info for pod {ctx.pod_name}" + + except Exception as e: + # Don't fail the task if debug capture fails + logger.warning(f"Failed to capture debug info for pod {ctx.pod_name}: {e}") + return f"Failed to capture debug info: {e}" + + +@always @task def cleanup_smoke_pod(args, ctx): """Delete the smoke pod after test completion""" + if not hasattr(ctx, "pod_name"): + return "No pod name available for cleanup" + oc( "delete", "pod", From 0117b588ffd991a1f0e80c1fa1c6ccab5a09a936 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:30 +0200 Subject: [PATCH 124/344] [guidellm] toolbox: run_smoke_request: use a FQDN quay image for curl --- projects/guidellm/toolbox/run_smoke_request/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index db3a7d742..fda906902 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -24,7 +24,7 @@ def run( namespace: str, endpoint_url: str, pod_name: str = "llm-d-smoke", - client_image: str = "curlimages/curl:8.11.1", + client_image: str = "quay.io/curl/curl:8.21.0", endpoint_path: str = "/v1/completions", request_timeout_seconds: int = 60, served_model_name: str, From c16641230063daf725084b63f0a549c8e4fcf888 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:19:29 +0200 Subject: [PATCH 125/344] [rhoai] library: deploy: add the _rhoai suffix when deploying the custom catalog --- projects/rhoai/library/deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 91c147547..f62bd84ba 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -125,6 +125,7 @@ def deploy_rhoai_custom_catalog(*, custom_catalog: RhoaiCustomCatalogConfig) -> catalog_image=custom_catalog.image, display_name=custom_catalog.display_name or custom_catalog.name, publisher=custom_catalog.publisher or "", + artifact_dirname_suffix="rhoai", ) From df4e2ab4b961e68260d84103789b820bc85c55b8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:53 +0200 Subject: [PATCH 126/344] [llm_d] orchestration: config: use a FQDN quay image for curl --- projects/llm_d/orchestration/config.d/platform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index b9defea06..32325769a 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -92,6 +92,6 @@ artifacts: smoke: pod_name: llm-d-smoke - client_image: curlimages/curl:8.11.1 + client_image: quay.io/curl/curl:8.21.0 endpoint_path: /v1/completions request_timeout_seconds: 60 From 3fde22169b74082f63a2374a7e59b04d6a8762e4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:38:36 +0200 Subject: [PATCH 127/344] [core] library: config: add a helper method to generate the variables_override file --- projects/core/library/config.py | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index d06fca5e2..d0d0437cb 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -519,3 +519,57 @@ def reload(orchestration_dir, *, apply_config_overrides=True): init(orchestration_dir, apply_config_overrides=apply_config_overrides) return project + + +def write_variables_override(presets=None, variables_dict=None): + """Write configuration overrides to the variables_override.yaml file in ARTIFACT_DIR. + + This is a top-level function that doesn't require project config to be initialized. + + Args: + presets: List of presets to set in project.args + variables_dict: Dictionary of additional configuration paths and values + """ + from pathlib import Path + + if not env.ARTIFACT_DIR: + raise ValueError( + "env.ARTIFACT_DIR is not initialized. " + "Call env.init() before using write_variables_override()" + ) + + artifact_dir = Path(env.ARTIFACT_DIR) + override_file = artifact_dir / VARIABLE_OVERRIDES_FILENAME + + # Ensure the directory exists + override_file.parent.mkdir(parents=True, exist_ok=True) + + # Load existing overrides or create empty structure + if override_file.exists(): + with open(override_file) as f: + loaded = yaml.safe_load(f) + if loaded is not None and not isinstance(loaded, dict): + raise ValueError( + f"Override file {override_file} must contain a YAML dictionary, got {type(loaded).__name__}" + ) + overrides = loaded or {} + else: + overrides = {} + + # Set presets in project.args + if presets: + overrides["project.args"] = presets + logger.info(f"write_variables_override: project.args --> {presets}") + + # Add additional variables to top level + if variables_dict: + for jsonpath, value in variables_dict.items(): + overrides[jsonpath] = value + logger.info(f"write_variables_override: {jsonpath} --> {value}") + + # Write back to file + with open(override_file, "w") as f: + yaml.dump(overrides, f, indent=4, default_flow_style=False, sort_keys=False) + + total_items = (1 if presets else 0) + (len(variables_dict) if variables_dict else 0) + logger.info(f"write_variables_override: wrote {total_items} override(s) to {override_file}") From 9142731938d5b5a46a8b9b156e13e82fa78c5c01 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:39:37 +0200 Subject: [PATCH 128/344] [llm_d] orchestration: ci: allow passing the presets to apply before launching a command --- projects/llm_d/CHANGELOG-orchestration.md | 16 ++++++++++++++++ projects/llm_d/orchestration/ci.py | 20 +++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/CHANGELOG-orchestration.md b/projects/llm_d/CHANGELOG-orchestration.md index 5ccfc73a7..46fdeddcd 100644 --- a/projects/llm_d/CHANGELOG-orchestration.md +++ b/projects/llm_d/CHANGELOG-orchestration.md @@ -1,5 +1,21 @@ # LLM-D Orchestration Changelog +## 2026-07-16 - Preset Configuration Support + +### Configuration Management +- **Preset CLI Option**: Added `--preset` flag to CI entrypoint for preset configuration before execution + - **Multiple Presets**: Supports multiple preset values via repeated `--preset` flags + - **Early Initialization**: Sets preset configuration during environment initialization, before config loading + - **Variables Override Integration**: Uses framework helper to generate proper override files + +### Files Modified +- `projects/llm_d/orchestration/ci.py` - Added `--presets` CLI option and initialization logic + +### Benefits +- **Flexible Testing**: Enables running same test suite with different preset configurations +- **Command-Line Control**: Provides runtime preset selection without modifying configuration files +- **Framework Integration**: Leverages core configuration override system for consistent behavior + ## 2026-06-26 - Orchestration & Cleanup Improvements ### New Features diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index b3748f679..25398162a 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -34,10 +34,15 @@ ] -def init(): +def init(presets=None): """Initialize LLM-D orchestration environment""" env.init() run.init() + + # Set presets configuration if provided + if presets: + config.write_variables_override(presets=presets) + config.init(Path(__file__).parent) @@ -69,12 +74,21 @@ def init_vaults_for_phase(phase: str) -> None: @click.group(cls=ci_lib.HelpfulGroup) +@click.option("--preset", multiple=True, help="Set preset configuration before starting") @click.pass_context @ci_lib.safe_ci_function -def main(ctx): +def main(ctx, preset): """LLM-D Project CI Operations for FORGE.""" ctx.ensure_object(types.SimpleNamespace) - init() + + presets_list = list(preset) + if presets_list and "," in presets_list[0]: + lst = presets_list.pop(0) + presets_list = lst.split(",") + presets_list + + logger.info(f"Setting preset configuration from CLI: {presets_list}") + + init(presets_list) if ctx.invoked_subcommand == "resolve-fournos-config": logger.info("No need to initialize the vaults for the resolve step") From e044d84dda329dd4affb0fb421277660404d0a80 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Fri, 17 Jul 2026 14:36:15 +0100 Subject: [PATCH 129/344] feat: Do not retry Signed-off-by: Alberto Perdomo --- .../toolbox/cleanup_test_resources/main.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 9afa1fb88..475bf49ec 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -6,7 +6,7 @@ import subprocess from projects.core.dsl import entrypoint, execute_tasks, retry, shell, task -from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_get_json logger = logging.getLogger("DSL") @@ -164,35 +164,38 @@ def delete_llm_d_labeled_resources(args, ctx): def delete_inference_service(args, ctx): """Delete the LLM inference service""" - _best_effort_delete( - "llminferenceservice", + result = oc( "delete", "llminferenceservice", args.inference_service_name, "-n", args.namespace, "--ignore-not-found=true", + "--timeout=30s", + check=False, ) + if result.returncode != 0: + logger.error( + "Failed to delete llminferenceservice/%s (rc=%d), capturing describe", + args.inference_service_name, + result.returncode, + ) + oc( + "describe", + "llminferenceservice", + args.inference_service_name, + "-n", + args.namespace, + check=False, + ) + raise RuntimeError( + f"Failed to delete llminferenceservice/{args.inference_service_name} " + f"in {args.namespace} (rc={result.returncode})" + ) return f"Deleted llminferenceservice {args.inference_service_name}" -@retry(attempts=90, delay=10, backoff=1.0) -@task -def wait_for_inference_service_deletion(args, ctx): - """Wait for the llminferenceservice to be deleted""" - - if not oc_resource_exists( - "llminferenceservice", args.inference_service_name, namespace=args.namespace - ): - return f"llminferenceservice/{args.inference_service_name} deleted from {args.namespace}" - - return ( - False, - f"Waiting for llminferenceservice/{args.inference_service_name} deletion in {args.namespace}", - ) - - @retry(attempts=90, delay=10, backoff=1.0) @task def wait_for_workload_pods_deletion(args, ctx): From cbc8a59db076dbd2ad32531755b18620805dee25 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 17 Jul 2026 13:13:08 -0400 Subject: [PATCH 130/344] Make HF_TOKEN secret name configurable in guidellm benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The secret name was hardcoded to "storage-config". Now it's driven by the benchmarks.guidellm.hf_token_secret config key and conditionally injected—if the key is empty or omitted, no HF_TOKEN env var is added to the K8s job manifest. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 3 +++ .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 4 +++- projects/guidellm/toolbox/run_guidellm_benchmark/utils.py | 3 +++ projects/rhaiis/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 3 +++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 5a8253abf..1d301988b 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -33,6 +33,7 @@ def run( timeout: int = 900, pvc_size: str = "1Gi", guidellm_args: list[str] | None = None, + hf_token_secret: str = "", ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -45,6 +46,7 @@ def run( timeout: Timeout in seconds to wait for job completion pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. """ execute_tasks(locals()) @@ -139,6 +141,7 @@ def create_guidellm_resources_task(args, ctx): image=ctx.image, endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, + hf_token_secret=args.hf_token_secret, ), ) return f"GuideLLM benchmark {ctx.benchmark_name} created" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 7a74cdd08..8f1ef15b6 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -26,12 +26,14 @@ spec: env: - name: USER value: guidellm +{% if hf_token_secret %} - name: HF_TOKEN valueFrom: secretKeyRef: - name: storage-config + name: {{ hf_token_secret }} key: HF_TOKEN optional: true +{% endif %} volumeMounts: - name: home mountPath: /home/guidellm diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 55e81d035..7cab6a2c2 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -175,6 +175,7 @@ def render_guidellm_job_from_parts( image: str, endpoint_url: str, guidellm_args: list[str], + hf_token_secret: str = "", ) -> dict[str, Any]: """Render a GuideLL-M job manifest from individual components. @@ -184,6 +185,7 @@ def render_guidellm_job_from_parts( image: Container image for GuideLLM endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. Returns: Job manifest as dict @@ -195,6 +197,7 @@ def render_guidellm_job_from_parts( "namespace": namespace, "name": name, "image": image, + "hf_token_secret": hf_token_secret, }, ) manifest = yaml.safe_load(rendered_yaml) diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 1442b8c3a..c0477ff2f 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -11,6 +11,7 @@ benchmarks: image: ghcr.io/vllm-project/guidellm:v0.6.0 timeout: 14400 pvc_size: 10Gi + hf_token_secret: "storage-config" args: backend_type: openai_http rate_type: concurrent diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 20038e4bd..180cd46e7 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -307,6 +307,7 @@ def _run_workload_benchmark( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) from pathlib import Path candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) @@ -824,6 +825,7 @@ def _run_warmup_step( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) logger.info("Warmup completed") except Exception: @@ -888,6 +890,7 @@ def _run_profiler_step( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) finally: enable_profiler_gate( From bdba8e98895583e50b822ab7f5ab6433139f9c80 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 14:17:21 +0200 Subject: [PATCH 131/344] [kserve] toolbox: deploy_llmisvc: accept a dry-run parameter to only build the llmisvc src --- .../kserve/toolbox/deploy_llmisvc/main.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 5b27231bf..a34c14412 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -6,7 +6,15 @@ import yaml -from projects.core.dsl import always, entrypoint, execute_tasks, on_failure, retry, task +from projects.core.dsl import ( + EarlyReturn, + always, + entrypoint, + execute_tasks, + on_failure, + retry, + task, +) from projects.core.dsl.utils import write_text from projects.core.dsl.utils.k8s import ( oc, @@ -28,6 +36,7 @@ def run( namespace: str, inference_service_manifest_path: str, gateway_status_address_name: str = "gateway-external", + dry_run: bool = False, ) -> str: """ Deploy an LLMInferenceService and wait for its endpoint. @@ -36,6 +45,7 @@ def run( namespace: Namespace used by llm_d inference_service_manifest_path: Path to the InferenceService YAML manifest file gateway_status_address_name: Gateway status address name for endpoint resolution + dry_run: If True, only prepare the manifest without deploying """ # Load manifest to extract the service name @@ -48,9 +58,13 @@ def run( "inference_service_manifest_path": inference_service_manifest_path, "inference_service_name": inference_service_name, "gateway_status_address_name": gateway_status_address_name, + "dry_run": dry_run, } context = execute_tasks(task_args) + if dry_run: + return "" + # Ensure endpoint_url is available endpoint_url = getattr(context, "endpoint_url", None) if not endpoint_url: @@ -86,6 +100,16 @@ def copy_manifest_to_src(args, ctx): return f"Copied manifest from {original_path} to {src_path}" +@task +def check_dry_run(args, ctx): + """Check if dry-run mode is enabled and return early if so""" + if args.dry_run: + return EarlyReturn( + f"Dry-run completed: Prepared manifest for {args.inference_service_name}" + ) + return "Proceeding with full deployment" + + @task def delete_existing_service(args, ctx): """Delete existing LLMInferenceService""" @@ -154,6 +178,9 @@ def wait_pods_appear(args, ctx): def capture_llmisv_description(args, ctx): """Capture LLMISV description with events and status for failure analysis""" + if args.dry_run: + return "Dry-run, nothing to do" + try: # Ensure artifacts directory exists artifacts_dir = args.artifact_dir / "artifacts" @@ -188,6 +215,9 @@ def capture_llmisv_description(args, ctx): def capture_replicaset_description(args, ctx): """Capture ReplicaSet description for pod creation failure analysis""" + if args.dry_run: + return "Dry-run, nothing to do" + try: # Ensure artifacts directory exists artifacts_dir = args.artifact_dir / "artifacts" From 8e7bfaa2253706913c5272e769b115d35c0d3db5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:11:00 +0200 Subject: [PATCH 132/344] [kserve] toolbox: deploy_llmisvc: update to avoid building a custom dict for the args --- .../kserve/toolbox/deploy_llmisvc/main.py | 124 +++++++----------- 1 file changed, 48 insertions(+), 76 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index a34c14412..cf064757c 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -48,44 +48,32 @@ def run( dry_run: If True, only prepare the manifest without deploying """ - # Load manifest to extract the service name - manifest = load_yaml(Path(inference_service_manifest_path)) - inference_service_name = manifest["metadata"]["name"] - - # Pass only the required arguments to avoid including manifest content in logs - task_args = { - "namespace": namespace, - "inference_service_manifest_path": inference_service_manifest_path, - "inference_service_name": inference_service_name, - "gateway_status_address_name": gateway_status_address_name, - "dry_run": dry_run, - } - context = execute_tasks(task_args) + ctx = execute_tasks(locals()) if dry_run: - return "" + return ctx.src_manifest_path # Ensure endpoint_url is available - endpoint_url = getattr(context, "endpoint_url", None) + endpoint_url = getattr(ctx, "endpoint_url", None) if not endpoint_url: raise RuntimeError("Failed to resolve gateway endpoint URL after deployment") - # Log service description for visibility - service_description = getattr(context, "service_description", None) - if service_description: - print(f"Deployed: {service_description}") - return endpoint_url @task def copy_manifest_to_src(args, ctx): - """Copy inference service manifest to src directory for inspection and use""" + """Copy inference service manifest to src directory and extract service name""" import shutil # Get the original manifest path original_path = Path(args.inference_service_manifest_path) + # Load manifest to extract the service name + manifest = load_yaml(original_path) + ctx.inference_service_name = manifest["metadata"]["name"] + ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}" + # Ensure the src directory exists src_dir = args.artifact_dir / "src" src_dir.mkdir(parents=True, exist_ok=True) @@ -97,16 +85,14 @@ def copy_manifest_to_src(args, ctx): # Store the src path in context for other tasks to use ctx.src_manifest_path = str(src_path) - return f"Copied manifest from {original_path} to {src_path}" + return f"Copied manifest from {original_path} to {src_path} (service: {ctx.inference_service_name})" @task def check_dry_run(args, ctx): """Check if dry-run mode is enabled and return early if so""" if args.dry_run: - return EarlyReturn( - f"Dry-run completed: Prepared manifest for {args.inference_service_name}" - ) + return EarlyReturn(f"Dry-run completed: Prepared manifest for {ctx.inference_service_name}") return "Proceeding with full deployment" @@ -114,7 +100,7 @@ def check_dry_run(args, ctx): def delete_existing_service(args, ctx): """Delete existing LLMInferenceService""" - name = args.inference_service_name + name = ctx.inference_service_name oc( "delete", "llminferenceservice", @@ -124,8 +110,7 @@ def delete_existing_service(args, ctx): "--ignore-not-found=true", check=False, ) - ctx.service_name = name - ctx.selector = f"app.kubernetes.io/name={name}" + return f"Deleted existing LLMInferenceService {name}" @@ -134,13 +119,11 @@ def delete_existing_service(args, ctx): def wait_old_pods_gone(args, ctx): """Wait for old llm-d pods to disappear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") - - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) + pods = oc_get_json( + "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + ) if not pods or not pods.get("items"): - return f"Old pods gone for {service_name}" + return f"Old pods gone for {ctx.inference_service_name}" return False # Retry @@ -154,7 +137,7 @@ def apply_inference_service(args, ctx): # Load and apply the manifest from src manifest = load_yaml(Path(src_manifest_path)) oc_apply(src_manifest_path, manifest) - return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.service_name}" + return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.inference_service_name}" @on_failure(on_wait_pods_appear_failure) @@ -163,13 +146,11 @@ def apply_inference_service(args, ctx): def wait_pods_appear(args, ctx): """Wait for llm-d pods to appear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") - - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) + pods = oc_get_json( + "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + ) if pods and pods.get("items"): - return f"Pods appeared for {service_name}" + return f"Pods appeared for {ctx.inference_service_name}" return False # Retry @@ -186,8 +167,10 @@ def capture_llmisv_description(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" # Capture LLMISV description result = oc( @@ -223,16 +206,17 @@ def capture_replicaset_description(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" # Get replicasets for the service rs_result = oc( "get", "replicaset", "-l", - selector, + ctx.selector, "-n", args.namespace, "-o", @@ -246,16 +230,18 @@ def capture_replicaset_description(args, ctx): if rs_result.stdout.strip(): # Describe each replicaset for rs_name in rs_result.stdout.strip().split("\n"): - if rs_name.strip(): - rs_desc_result = oc( - "describe", - rs_name.strip(), - "-n", - args.namespace, - log_stdout=False, - check=False, - ) - replicaset_descriptions.append(rs_desc_result.stdout) + if not rs_name.strip(): + continue + + rs_desc_result = oc( + "describe", + rs_name.strip(), + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + replicaset_descriptions.append(rs_desc_result.stdout) # Save all replicaset descriptions rs_desc_path = artifacts_dir / "replicaset_description.txt" @@ -275,7 +261,7 @@ def capture_replicaset_description(args, ctx): def query_service_status(args, ctx): """Query the status of the LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query only the Ready condition status result = oc( @@ -302,7 +288,7 @@ def query_service_status(args, ctx): def query_service_message(args, ctx): """Query detailed message from LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query the Ready condition details result = oc( @@ -339,8 +325,7 @@ def query_service_message(args, ctx): def wait_service_ready(args, ctx): """Wait for LLMInferenceService to be ready""" - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + service_name = ctx.inference_service_name # Query the current status and show diagnostic info result = oc( @@ -359,7 +344,7 @@ def wait_service_ready(args, ctx): "get", "pods", "-l", - selector, + ctx.selector, "-n", args.namespace, log_stdout=True, # Show pod status in logs @@ -395,7 +380,7 @@ def resolve_endpoint_task(args, ctx): endpoint_url = try_resolve_endpoint_url( namespace=args.namespace, - inference_service_name=args.inference_service_name, + inference_service_name=ctx.inference_service_name, gateway_status_address_name=args.gateway_status_address_name, ) if endpoint_url: @@ -405,19 +390,6 @@ def resolve_endpoint_task(args, ctx): return False # Retry -@task -def deploy_llmisvc_task(args, ctx): - """Deploy the llm_d inference service and resolve its endpoint""" - - # All work is done by the individual tasks - service_info = getattr( - ctx, "service_description", f"LLM Service '{args.inference_service_name}'" - ) - return ( - f"LLMInferenceService deployment completed: {service_info} - Endpoint: {ctx.endpoint_url}" - ) - - def try_resolve_endpoint_url( *, namespace: str, inference_service_name: str, gateway_status_address_name: str ) -> str | None: From 76de0e39d1015c065f1885ea903dd0f8bd4b7539 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 14:27:02 +0200 Subject: [PATCH 133/344] [llm_d] orchestration: add a kserve dry-run flag --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 80 ++++++++++++------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 6dd9496ca..95ac92ebb 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,3 +5,4 @@ smoke_request_key: default benchmark_key: null job_name: null namespace_override: null +kserve_dry_run: false diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2ef047ef2..451043985 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -7,7 +7,7 @@ from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier -from projects.core.library import env +from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt from projects.core.orchestration.utils.k8s import ensure_namespace @@ -17,6 +17,7 @@ from projects.kserve.toolbox.capture_llmisvc_state import main as capture_llmisvc_state from projects.kserve.toolbox.deploy_llmisvc import main as deploy_llmisvc from projects.kserve.toolbox.wait_kserve_ready import main as wait_kserve_ready +from projects.llm_d.orchestration import runtime_config from projects.llm_d.orchestration.prepare_phase import prepare_model_cache from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, @@ -29,7 +30,6 @@ def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() deployment_profile = runtime_config.get_deployment_profile_name() @@ -52,6 +52,13 @@ def create_test_labels() -> None: def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" + + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + if dry_run: + ret = do_test() + logger.info("Kserve dry-run mode enabled - Skipping caliper post-processing") + return ret + return run_and_postprocess(do_test) @@ -74,8 +81,6 @@ def _run_finalizer( logger.exception("Ignoring %s failure after primary test failure", description) return finalizer_exc - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() capture_namespace_events = platform["artifacts"]["capture_namespace_events"] @@ -107,18 +112,19 @@ def _run_finalizer( def do_test() -> int: # Load minimal config needed for orchestration flow - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() - - # Ensure namespace exists before starting any deployments - ensure_namespace( - namespace, - labels={ - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - }, - ) + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + + if not dry_run: + # Ensure namespace exists before starting any deployments + ensure_namespace( + namespace, + labels={ + "app.kubernetes.io/managed-by": "forge", + "forge.openshift.io/project": "llm_d", + }, + ) endpoint_url: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None @@ -131,6 +137,10 @@ def do_test() -> int: endpoint_url = deploy_inference_service() + if dry_run: + logging.warning("Running in dry-run mode, skipping the rest of the test steps") + return 0 + if not endpoint_url: raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") run_smoke_request(endpoint_url=endpoint_url) @@ -146,6 +156,9 @@ def do_test() -> int: logging.warning("Caught a SignalInterrupt, skipping the finalizers") do_finalizers = False + if dry_run: + do_finalizers = False + if do_finalizers: primary_exc, finalizer_exc = run_finalizers( endpoint_url, primary_exc, finalizer_exc @@ -169,37 +182,50 @@ def deploy_inference_service() -> str: logger.info("Starting LLMInferenceService deployment") # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # Step 1: Ensure model cache is ready - _prepare_model_cache() + # Check if dry-run mode is enabled early + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + + # Step 1: Ensure model cache is ready (skip in dry-run) + if not dry_run: + _prepare_model_cache() + else: + logger.info("Skipping model cache preparation - dry-run mode enabled") # Step 2: Wait for the serving control plane to settle before creating the service. - rhoai_namespace = platform["rhoai"]["namespace"] - wait_kserve_ready.run(namespace=rhoai_namespace) + if not dry_run: + rhoai_namespace = platform["rhoai"]["namespace"] + wait_kserve_ready.run(namespace=rhoai_namespace) # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) + endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), gateway_status_address_name=gateway["status_address_name"], + dry_run=dry_run, ) + if dry_run: + llmisvc_manifest_path = endpoint_url + logger.info("Dry-run completed: LLMInferenceService manifest prepared:") + logger.info(llmisvc_manifest_path) + return llmisvc_manifest_path + logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url def _prepare_model_cache() -> None: """Ensure model cache PVC is ready for deployment.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() logger.info("Preparing model cache for model: %s", model_name) @@ -211,7 +237,6 @@ def _prepare_model_cache() -> None: def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" - from projects.llm_d.orchestration import runtime_config config_dir = runtime_config.get_config_dir() namespace = runtime_config.get_namespace() @@ -245,7 +270,6 @@ def _build_inference_service_manifest() -> Path: def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() @@ -267,9 +291,6 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: def run_guidellm_benchmark(*, endpoint_url: str) -> None: - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() benchmark_configs = runtime_config.get_benchmark_configs() @@ -295,7 +316,6 @@ def run_guidellm_benchmark(*, endpoint_url: str) -> None: def capture_inference_service_state() -> None: # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() @@ -318,8 +338,12 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config + + # Skip cleanup when in dry-run mode + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + if dry_run: + logger.info("Skipping cleanup_test_resources - dry-run mode enabled") + return namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() From 9b6ab212990beefabf867803461b9385683c5392 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:17:26 +0200 Subject: [PATCH 134/344] [guidellm] toolbox: run_smoke_request: save to file directly --- projects/guidellm/toolbox/run_smoke_request/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index fda906902..5905beff2 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -238,7 +238,7 @@ def capture_smoke_pod_debug_info(args, ctx): try: # Capture pod YAML - yaml_result = oc( + oc( "get", "pod", ctx.pod_name, @@ -248,20 +248,17 @@ def capture_smoke_pod_debug_info(args, ctx): "yaml", check=False, ) - if yaml_result.returncode == 0 and yaml_result.stdout: - write_text(artifacts_dir / f"{ctx.pod_name}.yaml", yaml_result.stdout) # Capture pod description - desc_result = oc( + oc( "describe", "pod", ctx.pod_name, "-n", args.namespace, check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.description.txt", ) - if desc_result.returncode == 0 and desc_result.stdout: - write_text(artifacts_dir / f"{ctx.pod_name}.description.txt", desc_result.stdout) return f"Captured debug info for pod {ctx.pod_name}" From c02d8e34f56030b547b33e537b34a4aaa1745dfe Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 13:00:58 +0300 Subject: [PATCH 135/344] feat: add nightly pipeline to forge using clusterless pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a clusterless nightly pipeline that automatically detects new image versions and triggers test runs when a new version is found. Architecture: - Core nightly package (projects/core/nightly/) defines abstract interfaces (ImageReceiver, NightlyVerifier) with built-in retry logic. Projects implement these to define how versions are fetched and how run history is checked. - Phase handlers (receive_image.py, confirm.py) use polymorphism to call the project's implementations without knowing the concrete types. - Auto-discovery in the project's resolvers/ package — adding a new source is just dropping a new file with a NAME attribute. mcp_gateway implementation: - GHCRReceiver: finds the latest commit with a published ghcr.io image - MLflowVerifier: queries MLflow experiment runs to find the last tested version - CronJob triggers a clusterless FournosJob nightly Pipeline flow: 1. CronJob creates a clusterless FournosJob with pipeline: "nightly" 2. receive-image task: calls project's get_receiver().get_latest_version() 3. confirm-nightly task: calls project's get_verifier().get_last_tested_version(), compares versions, creates a full FournosJob if new version detected Also generalizes task-forge-step env var export to loop over all spec.env keys instead of hardcoding specific variable names. Co-authored-by: Cursor --- .../gitops/base/workflows/kustomization.yaml | 2 + .../base/workflows/pipeline-nightly.yaml | 63 +++++++++ .../base/workflows/task-forge-step.yaml | 8 +- projects/agentic_tools/ci_base.py | 17 ++- projects/core/nightly/__init__.py | 22 +++ projects/core/nightly/base_receiver.py | 46 ++++++ projects/core/nightly/base_verifier.py | 46 ++++++ projects/core/nightly/confirm.py | 131 ++++++++++++++++++ projects/core/nightly/receive_image.py | 42 ++++++ .../mcp_gateway/orchestration/config.yaml | 7 + projects/mcp_gateway/resolvers/__init__.py | 66 +++++++++ .../mcp_gateway/resolvers/cronjob-ghcr.yaml | 50 +++++++ .../mcp_gateway/resolvers/ghcr_receiver.py | 70 ++++++++++ .../mcp_gateway/resolvers/mlflow_verifier.py | 121 ++++++++++++++++ 14 files changed, 686 insertions(+), 5 deletions(-) create mode 100644 fournos/gitops/base/workflows/pipeline-nightly.yaml create mode 100644 projects/core/nightly/__init__.py create mode 100644 projects/core/nightly/base_receiver.py create mode 100644 projects/core/nightly/base_verifier.py create mode 100644 projects/core/nightly/confirm.py create mode 100644 projects/core/nightly/receive_image.py create mode 100644 projects/mcp_gateway/resolvers/__init__.py create mode 100644 projects/mcp_gateway/resolvers/cronjob-ghcr.yaml create mode 100644 projects/mcp_gateway/resolvers/ghcr_receiver.py create mode 100644 projects/mcp_gateway/resolvers/mlflow_verifier.py diff --git a/fournos/gitops/base/workflows/kustomization.yaml b/fournos/gitops/base/workflows/kustomization.yaml index 2212ab4a0..2bdff8912 100644 --- a/fournos/gitops/base/workflows/kustomization.yaml +++ b/fournos/gitops/base/workflows/kustomization.yaml @@ -6,10 +6,12 @@ resources: - task-forge-step.yaml - pipeline-full.yaml - pipeline-launcher.yaml + - pipeline-nightly.yaml - pipeline-prepare-only.yaml - pipeline-prepare-test.yaml - pipeline-replot.yaml - pipeline-test-only.yaml + commonLabels: app.kubernetes.io/name: forge-on-fournos app.kubernetes.io/component: workflows diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml new file mode 100644 index 000000000..18f729de1 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -0,0 +1,63 @@ +# Nightly pipeline: clusterless job that checks for new image versions +# and conditionally triggers a full test FournosJob. +# +# Flow: receive-image (resolve latest version) → confirm-nightly (compare with S3, create FournosJob if new) +# +# This pipeline runs as a clusterless FournosJob (no target cluster needed). +# Both tasks use the standard forge-step, dispatching to phases registered +# in the project's ci.py (backed by shared handlers in projects/core/nightly/). +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: nightly + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-receive-image + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: receive-image + - name: job-step-name + value: 00__receive-image + + - name: 01-confirm-nightly + runAfter: [00-receive-image] + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: confirm-nightly + - name: job-step-name + value: 01__confirm-nightly diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index 90f746be6..d8d9e0fc9 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -106,10 +106,10 @@ spec: fi } - # Export environment variables from FournosJob spec - export_env_var_from_fjob PULL_BASE_SHA - export_env_var_from_fjob PULL_PULL_SHA - export_env_var_from_fjob PULL_NUMBER + # Export all environment variables from FournosJob spec.env + for var_name in $(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r '.spec.env // {} | keys[]'); do + export_env_var_from_fjob "$var_name" + done if [[ "${PULL_PULL_SHA:-}" ]]; then echo diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index d868f6fc6..853848660 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -134,7 +134,7 @@ def list_vaults(self) -> list[str]: def should_init_vaults(self, phase: str) -> bool: """Return False to skip vault init for certain phases.""" - return phase != "resolve-fournos-config" + return phase not in ("resolve-fournos-config", "receive-image", "confirm-nightly") def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. @@ -176,6 +176,17 @@ class path like ``projects.foo.notifications.MyProvider``) and # Build # ------------------------------------------------------------------ + _NIGHTLY_PHASES: dict[str, PhaseSpec] = { + "receive-image": PhaseSpec( + func="projects.core.nightly.receive_image:run", + help="Nightly: resolve latest available image version.", + ), + "confirm-nightly": PhaseSpec( + func="projects.core.nightly.confirm:run", + help="Nightly: check S3 for last run, create FournosJob if new version.", + ), + } + def build(self) -> click.Group: """Assemble and return the fully-wired click.Group.""" app = self @@ -200,6 +211,10 @@ def main(ctx): for cmd_name, spec in self.phases.items(): _register_phase_command(main, cmd_name, spec) + for cmd_name, spec in self._NIGHTLY_PHASES.items(): + if cmd_name not in self.phases: + _register_phase_command(main, cmd_name, spec) + main.add_command( create_fournos_resolve_entrypoint( vault_list_func=self.list_vaults, diff --git a/projects/core/nightly/__init__.py b/projects/core/nightly/__init__.py new file mode 100644 index 000000000..183c607a2 --- /dev/null +++ b/projects/core/nightly/__init__.py @@ -0,0 +1,22 @@ +"""Shared nightly pipeline handlers. + +Architecture: + Core (this package) defines abstract interfaces and the orchestration logic. + Each project implements concrete receivers/verifiers in its resolvers/ package + and exposes them via get_receiver() and get_verifier() factory functions. + + The core handlers import the project's resolvers module at runtime and call + those factories — pure polymorphism, no knowledge of concrete implementations. + +Interfaces: + - ImageReceiver: fetches the latest version from a registry + - NightlyVerifier: checks if a version was already tested + +Handlers: + - receive_image.run(): calls project's get_receiver().get_latest_version() + - confirm.run(): calls project's get_verifier().get_last_tested_version() + +Projects register these as phases in ci_base: + "receive-image" -> projects.core.nightly.receive_image:run + "confirm-nightly" -> projects.core.nightly.confirm:run +""" diff --git a/projects/core/nightly/base_receiver.py b/projects/core/nightly/base_receiver.py new file mode 100644 index 000000000..ba93c82c0 --- /dev/null +++ b/projects/core/nightly/base_receiver.py @@ -0,0 +1,46 @@ +"""Abstract interface for image receivers. + +Projects implement this to define how the latest available version +is fetched from a container registry (GHCR, Quay, etc.). +""" + +from __future__ import annotations + +import logging +import time +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + + +class ImageReceiver(ABC): + """Fetches the latest available version from a container registry.""" + + @abstractmethod + def _fetch_version(self) -> str: + """Query the registry and return the latest version string. + + Raises on failure (network, auth, no tags found, etc.). + """ + ... + + def get_latest_version(self) -> str: + """Call _fetch_version with retry on transient failures.""" + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_version() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "Receiver attempt %d/%d failed: %s. Retrying in %ds...", + attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError( + f"Receiver failed after {MAX_ATTEMPTS} attempts" + ) from last_error diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py new file mode 100644 index 000000000..8d1212aa1 --- /dev/null +++ b/projects/core/nightly/base_verifier.py @@ -0,0 +1,46 @@ +"""Abstract interface for nightly verifiers. + +Projects implement this to define how run history is checked +(S3, MLflow, etc.) to determine if a version was already tested. +""" + +from __future__ import annotations + +import logging +import time +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + + +class NightlyVerifier(ABC): + """Checks historical run data to determine if a version was already tested.""" + + @abstractmethod + def _fetch_last_version(self) -> str: + """Return the version string of the last successful run. + + Returns empty string if no previous run exists. + """ + ... + + def get_last_tested_version(self) -> str: + """Call _fetch_last_version with retry on transient failures.""" + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_last_version() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "Verifier attempt %d/%d failed: %s. Retrying in %ds...", + attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError( + f"Verifier failed after {MAX_ATTEMPTS} attempts" + ) from last_error diff --git a/projects/core/nightly/confirm.py b/projects/core/nightly/confirm.py new file mode 100644 index 000000000..b677b0176 --- /dev/null +++ b/projects/core/nightly/confirm.py @@ -0,0 +1,131 @@ +"""Nightly confirm handler. + +Loads the project's resolvers module to get a NightlyVerifier instance. +Compares the resolved version (from receive-image step) against what +the verifier reports as the last tested version. Creates a FournosJob +if a new version is detected. + +The core never knows how or where history is stored — it just calls +get_last_tested_version() on whatever the project provides. +""" + +from __future__ import annotations + +import importlib +import json +import logging +import os +import subprocess +from pathlib import Path + +from projects.core.library import config + +logger = logging.getLogger(__name__) + + +def _read_resolved_version() -> str: + """Read the version written by the receive-image step.""" + artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) + version_file = artifact_base / "resolved_version.txt" + + if not version_file.exists(): + raise FileNotFoundError( + f"resolved_version.txt not found at {version_file}. " + "Did the receive-image step run successfully?" + ) + + version = version_file.read_text().strip() + if not version: + raise RuntimeError("resolved_version.txt is empty") + return version + + +def _create_fournos_job(project: str, version: str) -> None: + """Create a FournosJob for the full test run.""" + namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") + cluster = config.project.get_config("nightly.cluster") + version_key = config.project.get_config("nightly.version_key") + pipeline = config.project.get_config("nightly.pipeline") + preset = config.project.get_config("nightly.preset", "") + owner = config.project.get_config("nightly.owner", "nightly-pipeline") + + args_list = [preset] if preset else [] + + fjob_manifest = { + "apiVersion": "fournos.dev/v1", + "kind": "FournosJob", + "metadata": { + "generateName": f"nightly-{project.replace('_', '-')}-", + "namespace": namespace, + }, + "spec": { + "cluster": cluster, + "displayName": f"nightly {project} {version}", + "owner": owner, + "pipeline": pipeline, + "exclusive": True, + "executionEngine": { + "forge": { + "project": project, + "args": args_list, + "configOverrides": { + version_key: version, + }, + }, + }, + }, + } + + manifest_json = json.dumps(fjob_manifest) + logger.info( + "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", + project, version, cluster, pipeline, + ) + + result = subprocess.run( + ["oc", "apply", "-f", "-"], + input=manifest_json, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise RuntimeError(f"Failed to create FournosJob: {result.stderr}") + + logger.info("FournosJob created: %s", result.stdout.strip()) + + +def run(): + """Entrypoint for the confirm-nightly phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + image_version = _read_resolved_version() + logger.info("New version available: %s", image_version) + + module = importlib.import_module(f"projects.{project}.resolvers") + verifier = module.get_verifier() + + logger.info("Using verifier: %s", type(verifier).__name__) + last_version = verifier.get_last_tested_version() + logger.info("Last tested version: '%s'", last_version or "") + + if image_version == last_version: + logger.info("Version %s already tested. No new run needed.", image_version) + print("RESULT: NO-OP") + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "result.txt").write_text("NO-OP") + return + + logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") + + _create_fournos_job(project, image_version) + + result_msg = f"FournosJob created for {project} version {image_version}" + logger.info("RESULT: %s", result_msg) + print(f"RESULT: {result_msg}") + + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "result.txt").write_text(result_msg) diff --git a/projects/core/nightly/receive_image.py b/projects/core/nightly/receive_image.py new file mode 100644 index 000000000..0bc2d3979 --- /dev/null +++ b/projects/core/nightly/receive_image.py @@ -0,0 +1,42 @@ +"""Nightly receive-image handler. + +Loads the project's resolvers module and calls get_receiver() to obtain +an ImageReceiver instance. The core never knows what registry or strategy +the project uses — it just calls get_latest_version() on whatever it gets. +""" + +from __future__ import annotations + +import importlib +import logging +import os +from pathlib import Path + +from projects.core.library import config + +logger = logging.getLogger(__name__) + + +def run(): + """Entrypoint for the receive-image phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + module = importlib.import_module(f"projects.{project}.resolvers") + receiver = module.get_receiver() + + logger.info("Using receiver: %s", type(receiver).__name__) + version = receiver.get_latest_version() + logger.info("Resolved version: %s", version) + + artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) + version_file = artifact_base / "resolved_version.txt" + version_file.write_text(version) + logger.info("Wrote version to %s", version_file) + + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "resolved_version.txt").write_text(version) + + print(f"RESOLVED_VERSION={version}") diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index b73f7da9d..d592c8bcf 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,3 +1,10 @@ +nightly: + cluster: "smoke-cluster" + version_key: "infrastructure.mcp_gateway_version" + pipeline: "fournos-full" + preset: "demo" + owner: "ashtakb" + vaults: all: [] all-optional: diff --git a/projects/mcp_gateway/resolvers/__init__.py b/projects/mcp_gateway/resolvers/__init__.py new file mode 100644 index 000000000..a6bf3463b --- /dev/null +++ b/projects/mcp_gateway/resolvers/__init__.py @@ -0,0 +1,66 @@ +"""MCP Gateway nightly resolvers. + +Auto-discovers ImageReceiver and NightlyVerifier subclasses in this package. +To add a new receiver or verifier, just create a new file with a class that: + - Inherits from ImageReceiver or NightlyVerifier + - Defines a NAME class attribute (used for selection) + +No modifications to this file are needed. +""" + +from __future__ import annotations + +import importlib +import os +import pkgutil + +from projects.core.nightly.base_receiver import ImageReceiver +from projects.core.nightly.base_verifier import NightlyVerifier + +_receivers: dict[str, type[ImageReceiver]] = {} +_verifiers: dict[str, type[NightlyVerifier]] = {} + + +def _discover(): + """Import all modules in this package and register subclasses.""" + if _receivers or _verifiers: + return + + for _finder, module_name, _ in pkgutil.iter_modules(__path__): + module = importlib.import_module(f"{__name__}.{module_name}") + for attr in vars(module).values(): + if not isinstance(attr, type): + continue + if issubclass(attr, ImageReceiver) and attr is not ImageReceiver: + name = getattr(attr, "NAME", None) + if name: + _receivers[name] = attr + if issubclass(attr, NightlyVerifier) and attr is not NightlyVerifier: + name = getattr(attr, "NAME", None) + if name: + _verifiers[name] = attr + + +def get_receiver() -> ImageReceiver: + """Return the image receiver based on NIGHTLY_SOURCE env var.""" + _discover() + source = os.environ.get("NIGHTLY_SOURCE", "ghcr") + + cls = _receivers.get(source) + if cls is None: + raise ValueError( + f"Unknown source '{source}' for mcp_gateway. " + f"Available: {list(_receivers.keys())}" + ) + return cls() + + +def get_verifier() -> NightlyVerifier: + """Return the nightly verifier (first discovered NightlyVerifier subclass).""" + _discover() + + if not _verifiers: + raise RuntimeError("No NightlyVerifier subclass found in resolvers package") + + cls = next(iter(_verifiers.values())) + return cls() diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml new file mode 100644 index 000000000..c18e444a3 --- /dev/null +++ b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml @@ -0,0 +1,50 @@ +# CronJob to trigger the nightly pipeline for mcp_gateway (GHCR source). +# Runs daily at 2:00 AM UTC. Creates a clusterless FournosJob that +# uses the "nightly" pipeline to check for new versions on ghcr.io. +# +# Apply manually: +# oc apply -f projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nightly-mcp-gateway-ghcr + namespace: fournos-workloads + labels: + fournos.dev/nightly: "true" + fournos.dev/project: mcp-gateway + fournos.dev/source: ghcr +spec: + schedule: "0 2 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: fournos + restartPolicy: Never + containers: + - name: submit + image: bitnami/kubectl:latest + command: ["sh", "-c"] + args: + - | + cat <<'EOF' | oc apply -f - + apiVersion: fournos.dev/v1 + kind: FournosJob + metadata: + generateName: nightly-mcp-gateway- + namespace: fournos-workloads + spec: + clusterless: true + exclusive: false + pipeline: "nightly" + env: + NIGHTLY_SOURCE: "ghcr" + executionEngine: + forge: + project: "mcp_gateway" + args: [] + EOF + echo "Nightly FournosJob created for mcp_gateway (ghcr)" diff --git a/projects/mcp_gateway/resolvers/ghcr_receiver.py b/projects/mcp_gateway/resolvers/ghcr_receiver.py new file mode 100644 index 000000000..b07db825d --- /dev/null +++ b/projects/mcp_gateway/resolvers/ghcr_receiver.py @@ -0,0 +1,70 @@ +"""GHCR image receiver for mcp_gateway. + +Resolves the latest mcp-gateway commit SHA that has a published image +on ghcr.io. Checks recent commits from the repo and finds the first +one with a corresponding container image tag (sha-). + +No authentication needed for public packages (anonymous token). +""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request + +from projects.core.nightly.base_receiver import ImageReceiver + +REPO = "Kuadrant/mcp-gateway" +IMAGE = "kuadrant/mcp-gateway" +COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" +TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" +MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" + + +class GHCRReceiver(ImageReceiver): + """Find the latest mcp-gateway commit with a published ghcr.io image.""" + + NAME = "ghcr" + + def _fetch_version(self) -> str: + token = self._get_ghcr_token() + shas = self._get_recent_commits() + + for sha in shas: + if self._image_exists(sha, token): + return sha + + raise RuntimeError( + f"No commit found with a published image on ghcr.io " + f"(checked {len(shas)} recent commits)" + ) + + def _get_ghcr_token(self) -> str: + with urllib.request.urlopen(TOKEN_URL, timeout=30) as resp: + data = json.loads(resp.read()) + return data["token"] + + def _get_recent_commits(self) -> list[str]: + req = urllib.request.Request(COMMITS_URL) + with urllib.request.urlopen(req, timeout=30) as resp: + commits = json.loads(resp.read()) + return [c["sha"] for c in commits] + + def _image_exists(self, sha: str, token: str) -> bool: + tag = f"sha-{sha}" + req = urllib.request.Request( + MANIFEST_URL + tag, + headers={ + "Accept": "application/vnd.oci.image.index.v1+json, " + "application/vnd.docker.distribution.manifest.v2+json", + "Authorization": f"Bearer {token}", + }, + ) + try: + urllib.request.urlopen(req, timeout=30) + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise diff --git a/projects/mcp_gateway/resolvers/mlflow_verifier.py b/projects/mcp_gateway/resolvers/mlflow_verifier.py new file mode 100644 index 000000000..a956854cb --- /dev/null +++ b/projects/mcp_gateway/resolvers/mlflow_verifier.py @@ -0,0 +1,121 @@ +"""MLflow nightly verifier for mcp_gateway. + +Connects to the MLflow tracking server, queries runs in the project's +experiment, and extracts the version from the most recent run name. + +Uses the same secret loading mechanism as caliper's mlflow export. +""" + +from __future__ import annotations + +import logging +import os +import re +from pathlib import Path + +from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + mlflow_connection_env, + validate_mlflow_secrets, +) +from projects.core.nightly.base_verifier import NightlyVerifier + +logger = logging.getLogger(__name__) + +VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") + + +class MLflowVerifier(NightlyVerifier): + """Check MLflow experiment runs for the last tested version. + + Reads its configuration from the project's config.yaml under: + caliper.export.backend.mlflow.config.experiment + caliper.export.backend.mlflow.config.workspace + caliper.export.backend.mlflow.secrets.vault.name + caliper.export.backend.mlflow.secrets.vault.mlflow_secret + """ + + NAME = "mlflow" + + def __init__(self): + from projects.core.library import config + + self.experiment = config.project.get_config( + "caliper.export.backend.mlflow.config.experiment" + ) + self.workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace" + ) + vault_cfg = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault" + ) + self.secret_name = vault_cfg["name"] + self.secret_file = vault_cfg["mlflow_secret"] + + def _fetch_last_version(self) -> str: + connection = self._load_connection() + + with mlflow_connection_env(connection): + return self._query_latest_run() + + def _load_connection(self) -> dict: + secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) + secret_path = secrets_dir / self.secret_name / self.secret_file + + if not secret_path.exists(): + logger.warning( + "MLflow secret not found at expected path, assuming no previous run" + ) + return {} + + data = load_mlflow_secrets_yaml(secret_path) + validate_mlflow_secrets(data) + return data + + def _query_latest_run(self) -> str: + import mlflow + + if self.workspace: + os.environ["MLFLOW_WORKSPACE"] = self.workspace + + client = mlflow.tracking.MlflowClient() + + experiment = client.get_experiment_by_name(self.experiment) + if experiment is None: + logger.warning("Experiment '%s' not found in MLflow", self.experiment) + return "" + + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + order_by=["start_time DESC"], + max_results=1, + ) + + if not runs: + logger.info("No runs found in experiment '%s'", self.experiment) + return "" + + latest_run = runs[0] + run_name = latest_run.info.run_name or "" + logger.info("Latest MLflow run: %s", run_name) + + return self._extract_version(run_name) + + def _extract_version(self, run_name: str) -> str: + """Extract the version from a run name like: + forge-mcp-gateway-s150-u500-vsha--20260714-224646 + + Returns the raw SHA (without the 'sha-' prefix) to match + what the receiver returns. + """ + match = VERSION_PATTERN.search(run_name) + if not match: + logger.warning("Could not extract version from run name: %s", run_name) + return "" + + version = match.group(1) + + if version.startswith("sha-"): + return version[4:] + + return version From 625cf603d38450190ae09d3c431060cd9561cd1c Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 13:18:52 +0300 Subject: [PATCH 136/344] fix: use openshift origin-cli image for oc in cronjob Co-authored-by: Cursor --- projects/mcp_gateway/resolvers/cronjob-ghcr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml index c18e444a3..2f6a9b472 100644 --- a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml @@ -26,7 +26,7 @@ spec: restartPolicy: Never containers: - name: submit - image: bitnami/kubectl:latest + image: quay.io/openshift/origin-cli:latest command: ["sh", "-c"] args: - | From e16dc568413eb088a4cfcaa9e4b999cb321ae544 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 15:43:42 +0300 Subject: [PATCH 137/344] refactor: collapse nightly pipeline into single task with notifications Address PR review comments: - Merge two pipeline tasks (receive-image, confirm-nightly) into single "nightly" task for simplicity; error visibility via notification files - Remove retry from base ABCs, keep it in concrete implementations - Reuse fournos_launcher submit_and_wait with wait=False + EarlyReturn - Rename resolvers/ to nightly_resolvers/ - Use env module instead of raw os.environ for artifact paths - Make should_init_vaults configurable via vaults.skip_phases in config - Use quay.io/openshift/origin-cli image, fix namespace to psap-automation - Fix list_vaults() to skip the skip_phases key Co-authored-by: Cursor --- .../base/workflows/pipeline-nightly.yaml | 32 +---- projects/agentic_tools/ci_base.py | 21 +-- projects/core/nightly/__init__.py | 14 +- projects/core/nightly/base_receiver.py | 27 +--- projects/core/nightly/base_verifier.py | 27 +--- projects/core/nightly/confirm.py | 131 ------------------ projects/core/nightly/handler.py | 117 ++++++++++++++++ projects/core/nightly/receive_image.py | 42 ------ .../toolbox/submit_and_wait/main.py | 5 + .../__init__.py | 3 +- .../cronjob-ghcr.yaml | 4 +- .../ghcr_receiver.py | 27 +++- .../mlflow_verifier.py | 39 ++++-- .../mcp_gateway/orchestration/config.yaml | 2 + 14 files changed, 205 insertions(+), 286 deletions(-) delete mode 100644 projects/core/nightly/confirm.py create mode 100644 projects/core/nightly/handler.py delete mode 100644 projects/core/nightly/receive_image.py rename projects/mcp_gateway/{resolvers => nightly_resolvers}/__init__.py (94%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/cronjob-ghcr.yaml (93%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/ghcr_receiver.py (70%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/mlflow_verifier.py (73%) diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml index 18f729de1..a3e7a38ab 100644 --- a/fournos/gitops/base/workflows/pipeline-nightly.yaml +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -1,11 +1,11 @@ # Nightly pipeline: clusterless job that checks for new image versions # and conditionally triggers a full test FournosJob. # -# Flow: receive-image (resolve latest version) → confirm-nightly (compare with S3, create FournosJob if new) +# Single step that resolves the latest version, compares against the last +# tested version, and creates a FournosJob if a new version is detected. +# Error visibility is handled via notification files in the artifact directory. # # This pipeline runs as a clusterless FournosJob (no target cluster needed). -# Both tasks use the standard forge-step, dispatching to phases registered -# in the project's ci.py (backed by shared handlers in projects/core/nightly/). apiVersion: tekton.dev/v1 kind: Pipeline metadata: @@ -23,7 +23,7 @@ spec: - name: kubeconfig-secret type: string tasks: - - name: 00-receive-image + - name: 00-nightly taskRef: name: forge-step kind: Task @@ -38,26 +38,6 @@ spec: - name: kubeconfig-secret value: "$(params.kubeconfig-secret)" - name: job-step - value: receive-image + value: nightly - name: job-step-name - value: 00__receive-image - - - name: 01-confirm-nightly - runAfter: [00-receive-image] - taskRef: - name: forge-step - kind: Task - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: confirm-nightly - - name: job-step-name - value: 01__confirm-nightly + value: 00__nightly diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index 853848660..3c078ab70 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -127,14 +127,19 @@ def list_vaults(self) -> list[str]: if isinstance(vault_config, list): return vault_config all_vaults: list[str] = [] - for _category, vaults in vault_config.items(): + for category, vaults in vault_config.items(): + if category == "skip_phases": + continue if isinstance(vaults, list): all_vaults.extend(vaults) return list(dict.fromkeys(all_vaults)) def should_init_vaults(self, phase: str) -> bool: - """Return False to skip vault init for certain phases.""" - return phase not in ("resolve-fournos-config", "receive-image", "confirm-nightly") + """Return False to skip vault init for resolve and phases listed in vaults.skip_phases.""" + if phase == "resolve-fournos-config": + return False + skip_phases = config.project.get_config("vaults.skip_phases", [], print=False) + return phase not in skip_phases def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. @@ -177,13 +182,9 @@ class path like ``projects.foo.notifications.MyProvider``) and # ------------------------------------------------------------------ _NIGHTLY_PHASES: dict[str, PhaseSpec] = { - "receive-image": PhaseSpec( - func="projects.core.nightly.receive_image:run", - help="Nightly: resolve latest available image version.", - ), - "confirm-nightly": PhaseSpec( - func="projects.core.nightly.confirm:run", - help="Nightly: check S3 for last run, create FournosJob if new version.", + "nightly": PhaseSpec( + func="projects.core.nightly.handler:run", + help="Nightly: check for new image version and trigger test if needed.", ), } diff --git a/projects/core/nightly/__init__.py b/projects/core/nightly/__init__.py index 183c607a2..c0cdba66f 100644 --- a/projects/core/nightly/__init__.py +++ b/projects/core/nightly/__init__.py @@ -2,21 +2,19 @@ Architecture: Core (this package) defines abstract interfaces and the orchestration logic. - Each project implements concrete receivers/verifiers in its resolvers/ package + Each project implements concrete receivers/verifiers in its nightly_resolvers/ package and exposes them via get_receiver() and get_verifier() factory functions. - The core handlers import the project's resolvers module at runtime and call + The core handler imports the project's nightly_resolvers module at runtime and calls those factories — pure polymorphism, no knowledge of concrete implementations. Interfaces: - ImageReceiver: fetches the latest version from a registry - NightlyVerifier: checks if a version was already tested -Handlers: - - receive_image.run(): calls project's get_receiver().get_latest_version() - - confirm.run(): calls project's get_verifier().get_last_tested_version() +Handler: + - handler.run(): single phase that resolves version, compares, and creates FournosJob -Projects register these as phases in ci_base: - "receive-image" -> projects.core.nightly.receive_image:run - "confirm-nightly" -> projects.core.nightly.confirm:run +Projects register this as a phase in ci_base: + "nightly" -> projects.core.nightly.handler:run """ diff --git a/projects/core/nightly/base_receiver.py b/projects/core/nightly/base_receiver.py index ba93c82c0..2e6e70ab7 100644 --- a/projects/core/nightly/base_receiver.py +++ b/projects/core/nightly/base_receiver.py @@ -6,41 +6,16 @@ from __future__ import annotations -import logging -import time from abc import ABC, abstractmethod -logger = logging.getLogger(__name__) - -MAX_ATTEMPTS = 3 -RETRY_DELAY_SECONDS = 10 - class ImageReceiver(ABC): """Fetches the latest available version from a container registry.""" @abstractmethod - def _fetch_version(self) -> str: + def get_latest_version(self) -> str: """Query the registry and return the latest version string. Raises on failure (network, auth, no tags found, etc.). """ ... - - def get_latest_version(self) -> str: - """Call _fetch_version with retry on transient failures.""" - last_error = None - for attempt in range(1, MAX_ATTEMPTS + 1): - try: - return self._fetch_version() - except Exception as e: - last_error = e - if attempt < MAX_ATTEMPTS: - logger.warning( - "Receiver attempt %d/%d failed: %s. Retrying in %ds...", - attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, - ) - time.sleep(RETRY_DELAY_SECONDS) - raise RuntimeError( - f"Receiver failed after {MAX_ATTEMPTS} attempts" - ) from last_error diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py index 8d1212aa1..a0c644d6d 100644 --- a/projects/core/nightly/base_verifier.py +++ b/projects/core/nightly/base_verifier.py @@ -6,41 +6,16 @@ from __future__ import annotations -import logging -import time from abc import ABC, abstractmethod -logger = logging.getLogger(__name__) - -MAX_ATTEMPTS = 3 -RETRY_DELAY_SECONDS = 10 - class NightlyVerifier(ABC): """Checks historical run data to determine if a version was already tested.""" @abstractmethod - def _fetch_last_version(self) -> str: + def get_last_tested_version(self) -> str: """Return the version string of the last successful run. Returns empty string if no previous run exists. """ ... - - def get_last_tested_version(self) -> str: - """Call _fetch_last_version with retry on transient failures.""" - last_error = None - for attempt in range(1, MAX_ATTEMPTS + 1): - try: - return self._fetch_last_version() - except Exception as e: - last_error = e - if attempt < MAX_ATTEMPTS: - logger.warning( - "Verifier attempt %d/%d failed: %s. Retrying in %ds...", - attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, - ) - time.sleep(RETRY_DELAY_SECONDS) - raise RuntimeError( - f"Verifier failed after {MAX_ATTEMPTS} attempts" - ) from last_error diff --git a/projects/core/nightly/confirm.py b/projects/core/nightly/confirm.py deleted file mode 100644 index b677b0176..000000000 --- a/projects/core/nightly/confirm.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Nightly confirm handler. - -Loads the project's resolvers module to get a NightlyVerifier instance. -Compares the resolved version (from receive-image step) against what -the verifier reports as the last tested version. Creates a FournosJob -if a new version is detected. - -The core never knows how or where history is stored — it just calls -get_last_tested_version() on whatever the project provides. -""" - -from __future__ import annotations - -import importlib -import json -import logging -import os -import subprocess -from pathlib import Path - -from projects.core.library import config - -logger = logging.getLogger(__name__) - - -def _read_resolved_version() -> str: - """Read the version written by the receive-image step.""" - artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) - version_file = artifact_base / "resolved_version.txt" - - if not version_file.exists(): - raise FileNotFoundError( - f"resolved_version.txt not found at {version_file}. " - "Did the receive-image step run successfully?" - ) - - version = version_file.read_text().strip() - if not version: - raise RuntimeError("resolved_version.txt is empty") - return version - - -def _create_fournos_job(project: str, version: str) -> None: - """Create a FournosJob for the full test run.""" - namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") - cluster = config.project.get_config("nightly.cluster") - version_key = config.project.get_config("nightly.version_key") - pipeline = config.project.get_config("nightly.pipeline") - preset = config.project.get_config("nightly.preset", "") - owner = config.project.get_config("nightly.owner", "nightly-pipeline") - - args_list = [preset] if preset else [] - - fjob_manifest = { - "apiVersion": "fournos.dev/v1", - "kind": "FournosJob", - "metadata": { - "generateName": f"nightly-{project.replace('_', '-')}-", - "namespace": namespace, - }, - "spec": { - "cluster": cluster, - "displayName": f"nightly {project} {version}", - "owner": owner, - "pipeline": pipeline, - "exclusive": True, - "executionEngine": { - "forge": { - "project": project, - "args": args_list, - "configOverrides": { - version_key: version, - }, - }, - }, - }, - } - - manifest_json = json.dumps(fjob_manifest) - logger.info( - "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", - project, version, cluster, pipeline, - ) - - result = subprocess.run( - ["oc", "apply", "-f", "-"], - input=manifest_json, - capture_output=True, - text=True, - ) - - if result.returncode != 0: - raise RuntimeError(f"Failed to create FournosJob: {result.stderr}") - - logger.info("FournosJob created: %s", result.stdout.strip()) - - -def run(): - """Entrypoint for the confirm-nightly phase.""" - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - project = config.project.get_config("project.name") - logger.info("Project: %s", project) - - image_version = _read_resolved_version() - logger.info("New version available: %s", image_version) - - module = importlib.import_module(f"projects.{project}.resolvers") - verifier = module.get_verifier() - - logger.info("Using verifier: %s", type(verifier).__name__) - last_version = verifier.get_last_tested_version() - logger.info("Last tested version: '%s'", last_version or "") - - if image_version == last_version: - logger.info("Version %s already tested. No new run needed.", image_version) - print("RESULT: NO-OP") - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "result.txt").write_text("NO-OP") - return - - logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") - - _create_fournos_job(project, image_version) - - result_msg = f"FournosJob created for {project} version {image_version}" - logger.info("RESULT: %s", result_msg) - print(f"RESULT: {result_msg}") - - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "result.txt").write_text(result_msg) diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py new file mode 100644 index 000000000..2ea3e3286 --- /dev/null +++ b/projects/core/nightly/handler.py @@ -0,0 +1,117 @@ +"""Nightly pipeline handler — single phase that orchestrates: + +1. Resolve the latest available image version (via project's ImageReceiver) +2. Compare against last tested version (via project's NightlyVerifier) +3. Create a FournosJob if a new version is detected + +Error visibility is provided via notification files in the CI metadata directory. +""" + +from __future__ import annotations + +import importlib +import logging +import os + +from projects.core.library import ci as ci_lib +from projects.core.library import config, env + +logger = logging.getLogger(__name__) + + +def _receive_image(project: str) -> str: + """Resolve the latest available version from the project's receiver.""" + module = importlib.import_module(f"projects.{project}.nightly_resolvers") + receiver = module.get_receiver() + + logger.info("Using receiver: %s", type(receiver).__name__) + version = receiver.get_latest_version() + logger.info("Resolved version: %s", version) + + version_file = env.BASE_ARTIFACT_DIR / "resolved_version.txt" + version_file.write_text(version) + + (env.ARTIFACT_DIR / "resolved_version.txt").write_text(version) + return version + + +def _confirm_nightly(project: str, image_version: str) -> None: + """Compare resolved version against last tested, create FournosJob if new.""" + module = importlib.import_module(f"projects.{project}.nightly_resolvers") + verifier = module.get_verifier() + + logger.info("Using verifier: %s", type(verifier).__name__) + last_version = verifier.get_last_tested_version() + logger.info("Last tested version: '%s'", last_version or "") + + if image_version == last_version: + logger.info("Version %s already tested. No new run needed.", image_version) + ci_lib.add_notification_file( + "nightly-noop", f"Version {image_version} already tested — no-op." + ) + (env.ARTIFACT_DIR / "result.txt").write_text("NO-OP") + return + + logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") + _create_fournos_job(project, image_version) + + result_msg = f"FournosJob created for {project} version {image_version}" + ci_lib.add_notification_file("nightly-triggered", result_msg) + (env.ARTIFACT_DIR / "result.txt").write_text(result_msg) + + +def _create_fournos_job(project: str, version: str) -> None: + """Create a FournosJob for the full test run using the fournos_launcher toolbox.""" + from projects.fournos_launcher.toolbox.submit_and_wait.main import run as submit_and_wait + + namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") + cluster = config.project.get_config("nightly.cluster") + version_key = config.project.get_config("nightly.version_key") + pipeline = config.project.get_config("nightly.pipeline") + preset = config.project.get_config("nightly.preset", "") + owner = config.project.get_config("nightly.owner", "nightly-pipeline") + + args_list = [preset] if preset else [] + + logger.info( + "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", + project, + version, + cluster, + pipeline, + ) + + submit_and_wait( + project, + cluster_name=cluster, + args=args_list, + variables_overrides={version_key: version}, + namespace=namespace, + owner=owner, + display_name=f"nightly {project} {version}", + pipeline_name=pipeline, + exclusive=True, + wait=False, + ) + + +def run(): + """Entrypoint for the nightly phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + try: + image_version = _receive_image(project) + except Exception as e: + logger.error("receive-image failed: %s", e) + ci_lib.add_notification_file("nightly-receive-image-failed", f"FATAL: {e}") + raise + + try: + _confirm_nightly(project, image_version) + except Exception as e: + logger.error("confirm-nightly failed: %s", e) + ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") + raise diff --git a/projects/core/nightly/receive_image.py b/projects/core/nightly/receive_image.py deleted file mode 100644 index 0bc2d3979..000000000 --- a/projects/core/nightly/receive_image.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Nightly receive-image handler. - -Loads the project's resolvers module and calls get_receiver() to obtain -an ImageReceiver instance. The core never knows what registry or strategy -the project uses — it just calls get_latest_version() on whatever it gets. -""" - -from __future__ import annotations - -import importlib -import logging -import os -from pathlib import Path - -from projects.core.library import config - -logger = logging.getLogger(__name__) - - -def run(): - """Entrypoint for the receive-image phase.""" - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - project = config.project.get_config("project.name") - logger.info("Project: %s", project) - - module = importlib.import_module(f"projects.{project}.resolvers") - receiver = module.get_receiver() - - logger.info("Using receiver: %s", type(receiver).__name__) - version = receiver.get_latest_version() - logger.info("Resolved version: %s", version) - - artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) - version_file = artifact_base / "resolved_version.txt" - version_file.write_text(version) - logger.info("Wrote version to %s", version_file) - - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "resolved_version.txt").write_text(version) - - print(f"RESOLVED_VERSION={version}") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 40e4917b6..7272fae7f 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -20,6 +20,7 @@ task, template, ) +from projects.core.dsl.control_flow import EarlyReturn from projects.core.dsl.utils.k8s import is_valid_k8s_name, sanitize_k8s_name from projects.core.library import env as env_mod @@ -64,6 +65,7 @@ def run( gpu_count: int = None, gpu_type: str = None, clusterless: bool = False, + wait: bool = True, ): """ Submit a FOURNOS job and wait for completion @@ -208,6 +210,9 @@ def submit_fournos_job(args, ctx): # Apply the job manifest (will raise CalledProcessError with full details on failure) shell.run(f"oc apply -f {ctx.manifest_file}") + if not args.wait: + return EarlyReturn(f"Submitted FournosJob: {ctx.final_job_name} (wait=False)") + return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" diff --git a/projects/mcp_gateway/resolvers/__init__.py b/projects/mcp_gateway/nightly_resolvers/__init__.py similarity index 94% rename from projects/mcp_gateway/resolvers/__init__.py rename to projects/mcp_gateway/nightly_resolvers/__init__.py index a6bf3463b..27c4f1e56 100644 --- a/projects/mcp_gateway/resolvers/__init__.py +++ b/projects/mcp_gateway/nightly_resolvers/__init__.py @@ -49,8 +49,7 @@ def get_receiver() -> ImageReceiver: cls = _receivers.get(source) if cls is None: raise ValueError( - f"Unknown source '{source}' for mcp_gateway. " - f"Available: {list(_receivers.keys())}" + f"Unknown source '{source}' for mcp_gateway. Available: {list(_receivers.keys())}" ) return cls() diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml similarity index 93% rename from projects/mcp_gateway/resolvers/cronjob-ghcr.yaml rename to projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml index 2f6a9b472..bf053c29a 100644 --- a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml @@ -3,12 +3,12 @@ # uses the "nightly" pipeline to check for new versions on ghcr.io. # # Apply manually: -# oc apply -f projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +# oc apply -f projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml apiVersion: batch/v1 kind: CronJob metadata: name: nightly-mcp-gateway-ghcr - namespace: fournos-workloads + namespace: psap-automation labels: fournos.dev/nightly: "true" fournos.dev/project: mcp-gateway diff --git a/projects/mcp_gateway/resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py similarity index 70% rename from projects/mcp_gateway/resolvers/ghcr_receiver.py rename to projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index b07db825d..949de372f 100644 --- a/projects/mcp_gateway/resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -10,23 +10,48 @@ from __future__ import annotations import json +import logging +import time import urllib.error import urllib.request from projects.core.nightly.base_receiver import ImageReceiver +logger = logging.getLogger(__name__) + REPO = "Kuadrant/mcp-gateway" IMAGE = "kuadrant/mcp-gateway" COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + class GHCRReceiver(ImageReceiver): """Find the latest mcp-gateway commit with a published ghcr.io image.""" NAME = "ghcr" + def get_latest_version(self) -> str: + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_version() + except urllib.error.URLError as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "GHCR attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"GHCR receiver failed after {MAX_ATTEMPTS} attempts") from last_error + def _fetch_version(self) -> str: token = self._get_ghcr_token() shas = self._get_recent_commits() @@ -57,7 +82,7 @@ def _image_exists(self, sha: str, token: str) -> bool: MANIFEST_URL + tag, headers={ "Accept": "application/vnd.oci.image.index.v1+json, " - "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", "Authorization": f"Bearer {token}", }, ) diff --git a/projects/mcp_gateway/resolvers/mlflow_verifier.py b/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py similarity index 73% rename from projects/mcp_gateway/resolvers/mlflow_verifier.py rename to projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py index a956854cb..2e9628a77 100644 --- a/projects/mcp_gateway/resolvers/mlflow_verifier.py +++ b/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py @@ -11,6 +11,7 @@ import logging import os import re +import time from pathlib import Path from projects.caliper.engine.file_export.mlflow_secrets import ( @@ -24,6 +25,9 @@ VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 15 + class MLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. @@ -43,29 +47,40 @@ def __init__(self): self.experiment = config.project.get_config( "caliper.export.backend.mlflow.config.experiment" ) - self.workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace" - ) - vault_cfg = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault" - ) + self.workspace = config.project.get_config("caliper.export.backend.mlflow.config.workspace") + vault_cfg = config.project.get_config("caliper.export.backend.mlflow.secrets.vault") self.secret_name = vault_cfg["name"] self.secret_file = vault_cfg["mlflow_secret"] - def _fetch_last_version(self) -> str: + def get_last_tested_version(self) -> str: connection = self._load_connection() + if not connection: + return "" - with mlflow_connection_env(connection): - return self._query_latest_run() + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + with mlflow_connection_env(connection): + return self._query_latest_run() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "MLflow attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"MLflow verifier failed after {MAX_ATTEMPTS} attempts") from last_error def _load_connection(self) -> dict: secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) secret_path = secrets_dir / self.secret_name / self.secret_file if not secret_path.exists(): - logger.warning( - "MLflow secret not found at expected path, assuming no previous run" - ) + logger.warning("MLflow secret not found at expected path, assuming no previous run") return {} data = load_mlflow_secrets_yaml(secret_path) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index d592c8bcf..ee211ebf1 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -12,6 +12,8 @@ vaults: - psap-forge-mlflow-export prepare: [] test: [] + skip_phases: + - nightly notifications: slack: From 318657bf00a492ed0397b730be016fc7b75ec154 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 18:47:56 +0300 Subject: [PATCH 138/344] fix: use correct cluster name for nightly job target Co-authored-by: Cursor --- projects/mcp_gateway/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index ee211ebf1..2f469520d 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,5 +1,5 @@ nightly: - cluster: "smoke-cluster" + cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" pipeline: "fournos-full" preset: "demo" From 9370ef2f1ff4b92afb022cd3add1798000a3c40e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 18:54:16 +0300 Subject: [PATCH 139/344] fix: correct pipeline name to forge-full Co-authored-by: Cursor --- projects/mcp_gateway/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 2f469520d..8abbba249 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,7 +1,7 @@ nightly: cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" - pipeline: "fournos-full" + pipeline: "forge-full" preset: "demo" owner: "ashtakb" From fdc49f1c13aa49e2171e57adcf0d249308857a73 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 19:33:45 +0300 Subject: [PATCH 140/344] fix: rename _mesh_ready_spec to propagate across DSL tasks The DSL runtime skips underscore-prefixed context attributes when propagating between tasks, causing wait_service_mesh_ready to never see the readiness spec set by install_service_mesh_instance. Co-authored-by: Cursor --- projects/mcp_gateway/toolbox/install_platform/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index fcf5c5d5c..6fdc75046 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -115,7 +115,7 @@ def install_service_mesh_instance(args, ctx): if "wait_for_ready" in step: spec = step["wait_for_ready"] - ctx._mesh_ready_spec = spec + ctx.mesh_ready_spec = spec return (False, "Service Mesh instance applied, readiness checked in next task") return f"Service Mesh instance applied from {kustomize_path}" @@ -125,7 +125,7 @@ def install_service_mesh_instance(args, ctx): @task def wait_service_mesh_ready(args, ctx): """Wait for Istio instance to report Ready condition""" - spec = getattr(ctx, "_mesh_ready_spec", None) + spec = getattr(ctx, "mesh_ready_spec", None) if not spec: return "No readiness spec, skipping" From b983a1d290dff06192b481146b1b0a1fecc2af7b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 14:01:50 -0400 Subject: [PATCH 141/344] Add Slack failure alert when RHAIIS pipeline crashes Sends a webhook notification with job metadata (model, TP/DP, version, cluster, workloads, error) so failures are visible without checking the Fournos dashboard. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 58 ++++++++++-- projects/rhaiis/postprocess/regression.py | 99 +++++++++++++++++++++ 2 files changed, 150 insertions(+), 7 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 180cd46e7..942d127de 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import os +import traceback import uuid as _uuid_mod from projects.core.library import env @@ -12,6 +14,44 @@ _K8S_NAME_MAX = 63 +def _send_pipeline_failure_alert( + exc: Exception, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert when the pipeline fails.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=traceback.format_exception_only(type(exc), exc)[-1].strip(), + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline failure alert", exc_info=True) + + def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -43,13 +83,17 @@ def do_test( namespace: str, deployment_name: str | None = None, ) -> int: - with env.NextArtifactDir("testing"): - return _run_test( - model_key=model_key, - workload_keys=workload_keys, - namespace=namespace, - deployment_name=deployment_name, - ) + try: + with env.NextArtifactDir("testing"): + return _run_test( + model_key=model_key, + workload_keys=workload_keys, + namespace=namespace, + deployment_name=deployment_name, + ) + except Exception as exc: + _send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) + raise def _run_test( diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index d992e859d..87273f854 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -222,3 +222,102 @@ def send_regression_notification( dry_run=dry_run, notification_vault=notification_vault, ) + + +def send_failure_notification( + *, + error: str, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, + tp: str = "", + dp: str = "", + version: str = "", + workload_keys: list[str] | None = None, + cluster: str = "", +) -> bool: + """Send a Slack alert when the RHAIIS pipeline fails. + + Args: + error: Error message or traceback summary + model: Model name for display + accelerator: Accelerator name for display + job_id: FournosJob name + slack_user: Slack user ID to @-mention + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + tp: Tensor parallelism size + dp: Data parallelism size + version: vLLM / RHAIIS version string + workload_keys: List of workload profile keys + cluster: Cluster name the job ran on + + Returns: + True if notification sent successfully + """ + from projects.core.notifications.send import ( + _get_webhook_url, + _send_via_webhook, + send_notification, + ) + + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + + profiles_line = "" + if workload_keys: + profiles_line = f"*Workloads:* {', '.join(workload_keys)}\n" + + cluster_line = f"*Cluster:* {cluster}\n" if cluster else "" + version_line = f"*Version:* {version}\n" if version else "" + + error_text = error if len(error) <= 500 else error[:500] + "..." + + message = ( + f":x: *RHAIIS Pipeline Failed*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"{version_line}" + f"{cluster_line}" + f"{profiles_line}" + f"*Error:*\n```{error_text}```" + ) + + if dry_run: + logger.info("DRY RUN failure notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) From 3948bd1c6b29980e781546c592c79ad039559ef4 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 15:17:55 -0400 Subject: [PATCH 142/344] Append FJOB suffix to deployment name for parallel job isolation Each FournosJob now gets a unique vLLM deployment name derived from the FJOB_NAME suffix, allowing multiple jobs to run in parallel on the same namespace without colliding on the same InferenceService. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 942d127de..df526a651 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -113,7 +113,10 @@ def _run_test( benchmark_cfg = runtime_config.get_benchmark_config() if not deployment_name: - deployment_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) + base_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) + fjob = os.environ.get("FJOB_NAME", "") + suffix = fjob.rsplit("-", 1)[-1] if fjob else "" + deployment_name = f"{base_name}-{suffix}" if suffix else base_name vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() From d4c7600c80a31676097079625c85a70b00165718 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 16:43:43 -0400 Subject: [PATCH 143/344] Switch RHAIIS Slack notifications to topsail bot on psap-rhaiis-alerts channel - Use topsail bot token instead of webhook for sending Slack notifications - Route RHAIIS notifications to dedicated psap-rhaiis-alerts channel - Add pipeline-level failure detection in post_cleanup for infra errors (e.g. image pull timeout) - Fix UnboundLocalError for 'os' caused by redundant import in _run_test Co-authored-by: Cursor --- projects/rhaiis/orchestration/ci.py | 67 +++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 4 +- projects/rhaiis/postprocess/regression.py | 74 +++++++++------------ 3 files changed, 100 insertions(+), 45 deletions(-) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 6b50a96e2..25ea42404 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import logging +import os import types +from pathlib import Path import click import prepare_rhaiis @@ -16,6 +18,70 @@ logger = logging.getLogger(__name__) +def _check_pipeline_failure_and_notify() -> None: + """Detect early pipeline failures (e.g. image pull errors) and send a Slack alert. + + Runs in the post-cleanup finally step. Checks whether prior steps + produced FAILURE artifacts or the test step was skipped entirely. + """ + try: + base_dir = Path(os.environ.get("ARTIFACT_BASE_DIR", "")) + if not base_dir.is_dir(): + return + + test_dir = base_dir / "03__test" + test_ran = test_dir.exists() + + # Skip steps already handled by do_test's exception handler + failure_files = sorted( + f for f in base_dir.glob("*/FAILURE") + if f.parent.name != "03__test" + ) + + if not failure_files and test_ran: + return + + errors = [] + for f in failure_files: + step_name = f.parent.name + content = f.read_text().strip() + summary = content[:300] if content else "unknown error" + errors.append(f"[{step_name}] {summary}") + + if not test_ran and not errors: + errors.append("Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)") + + error_text = "\n".join(errors) + + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_key = _cfg.project.get_config("tests.rhaiis.model_key", "unknown") + try: + model_cfg = runtime_config.get_model(model_key) + model_name = model_cfg.get("hf_model_id", model_key) + except Exception: + model_name = model_key + + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() + + send_failure_notification( + error=error_text, + model=model_name, + accelerator=accelerator_key, + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-notifications", + version=_cfg.project.get_config("tests.rhaiis.version", ""), + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to check/send pipeline failure notification", exc_info=True) + + def list_vaults() -> list[str]: test_rhaiis.init() return runtime_config.get_vaults() @@ -85,6 +151,7 @@ def pre_cleanup(ctx): @ci_lib.safe_ci_entrypoint def post_cleanup(ctx): """Post-cleanup phase - Clean up resources after test.""" + _check_pipeline_failure_and_notify() return prepare_rhaiis.cleanup() diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index df526a651..ce17b2984 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -136,7 +136,7 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) - import os, subprocess + import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: @@ -618,7 +618,6 @@ def _run_standalone_analysis( except Exception: logger.warning("Standalone analysis failed", exc_info=True) finally: - import os if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) if current_csv_path and os.path.exists(current_csv_path): @@ -705,7 +704,6 @@ def _run_regression_check( except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) finally: - import os if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 87273f854..2c70abc5b 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -5,8 +5,38 @@ import logging import re +import projects.core.notifications.slack.api as slack_api +from projects.core.library import vault + logger = logging.getLogger(__name__) + +RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" + + +def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None, channel_id: str | None = None) -> bool: + """Send a Slack message using the topsail bot token from the forge notifications vault.""" + vault_name = notification_vault or "psap-forge-notifications" + try: + token_path = vault.get_vault_content_path(vault_name, "topsail-bot.slack-token") + except Exception: + logger.warning("Cannot resolve topsail bot token from vault %s", vault_name) + return False + + if not token_path or not token_path.exists(): + logger.warning("topsail-bot.slack-token not found in vault %s", vault_name) + return False + + token = token_path.read_text().strip() + client = slack_api.init_client(token) + if not client: + logger.error("Failed to init Slack client with topsail bot token") + return False + + _, ok = slack_api.send_message(client, message=message, channel_id=channel_id) + return ok + + PROFILE_MAP = { (1000, 1000): "profile1", (512, 2048): "profile2", @@ -105,12 +135,6 @@ def send_regression_notification( Returns: True if notification sent successfully """ - from projects.core.notifications.send import ( - _get_webhook_url, - _send_via_webhook, - send_notification, - ) - status = analysis_result.get("status") if status == "skipped": reason = analysis_result.get("reason", "unknown") @@ -207,21 +231,7 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) + return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) def send_failure_notification( @@ -260,12 +270,6 @@ def send_failure_notification( Returns: True if notification sent successfully """ - from projects.core.notifications.send import ( - _get_webhook_url, - _send_via_webhook, - send_notification, - ) - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): user_line = f"*Triggered by:* <@{slack_user}>\n" elif slack_user: @@ -306,18 +310,4 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) + return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) From 7850ad42a34612c43efbaa9b31272139fadc6d12 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 22:46:39 -0400 Subject: [PATCH 144/344] Truncate deployment name to respect KServe 63-char hostname limit Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ce17b2984..a1e264258 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -116,7 +116,14 @@ def _run_test( base_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) fjob = os.environ.get("FJOB_NAME", "") suffix = fjob.rsplit("-", 1)[-1] if fjob else "" - deployment_name = f"{base_name}-{suffix}" if suffix else base_name + if suffix: + # KServe hostname: {name}-predictor-{namespace} must be <= 63 chars + ns = runtime_config.get_namespace() + max_name_len = 63 - len(f"-predictor-{ns}") + full_name = f"{base_name}-{suffix}" + deployment_name = full_name[:max_name_len].rstrip("-") + else: + deployment_name = base_name vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() From e30e0ebf1266d1e0a1bdbfc9a7d344239ebf1b81 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 23:10:18 -0400 Subject: [PATCH 145/344] Fix deployment name truncation to preserve unique FJOB suffix Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index a1e264258..709a03f91 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -119,9 +119,9 @@ def _run_test( if suffix: # KServe hostname: {name}-predictor-{namespace} must be <= 63 chars ns = runtime_config.get_namespace() - max_name_len = 63 - len(f"-predictor-{ns}") - full_name = f"{base_name}-{suffix}" - deployment_name = full_name[:max_name_len].rstrip("-") + max_base = 63 - len(f"-predictor-{ns}") - len(suffix) - 1 + truncated_base = base_name[:max_base].rstrip("-") + deployment_name = f"{truncated_base}-{suffix}" else: deployment_name = base_name From 0119b54960964252ad867d646efcc35d88735ed8 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Tue, 21 Jul 2026 10:42:17 +0300 Subject: [PATCH 146/344] Add AGENTS.md with security guidelines for secret handling --- AGENTS.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..6da05ab1d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,73 @@ +# FORGE CI Framework — Agent Instructions + +## Project Overview + +FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on OpenShift clusters. +It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging. + +Key paths: +- `projects/` — individual project toolboxes (install scripts, test harnesses) +- `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers) +- `fournos/gitops/` — Tekton pipelines and GitOps base workflows +- `docs/` — project documentation + +## Security: Secret Handling (CRITICAL) + +**NEVER write secrets or sensitive data to artifact directories, logs, or files.** + +### Rules + +1. **Never persist secrets to `ARTIFACT_DIR`.** + The `oc_apply(path, manifest)` helper writes the manifest YAML to disk before applying it. + If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys), + it MUST NOT be written to any path under `env.ARTIFACT_DIR`. + +2. **Use `oc_apply_secret()` or `oc("apply", "-f", "-", input_text=...)` for secrets.** + Apply secrets via stdin or a dedicated helper that does not persist the payload. + Never use the standard `oc_apply()` for Secret-kind manifests. + +3. **Never log secret values.** + Do not print, log, or include in error messages: passwords, tokens, bearer credentials, + TLS keys, pull-secret `.dockerconfigjson`, or any `data:`/`stringData:` field from a Secret. + +4. **Check the `kind:` field.** + Before calling `oc_apply(artifact_path, manifest)`, verify that `manifest.get("kind") != "Secret"`. + If it is a Secret, use a non-persisting apply path. + +5. **Artifact directories are public.** + Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. + Only write non-sensitive operational data (CRs, logs, pod descriptions, events). + +### Safe Patterns + +```python +# GOOD: Apply a Secret without persisting to disk +def apply_secret(manifest: dict) -> None: + """Apply a Secret manifest via stdin — nothing written to disk.""" + import yaml + from projects.core.dsl.utils.k8s import oc + oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False)) + +# GOOD: Guard before using oc_apply +def safe_oc_apply(artifact_path, manifest): + if manifest.get("kind") == "Secret": + raise ValueError( + f"Refusing to write Secret '{manifest['metadata']['name']}' to artifacts. " + "Use apply_secret() instead." + ) + oc_apply(artifact_path, manifest) +``` + +### Forbidden Patterns + +```python +# BAD: Writing a pull-secret to artifacts +src_dir = env.ARTIFACT_DIR / "src" +oc_apply(src_dir / "image-pull-secret.yaml", secret_manifest) # LEAKS CREDENTIALS + +# BAD: Logging secret content +logger.info(f"Applying secret: {manifest}") # LEAKS if manifest has sensitive data + +# BAD: Including tokens in error messages +raise RuntimeError(f"Failed to auth with token {token}") # LEAKS TOKEN +``` From dca57ba777803b4dcc4afd7bd67c210e26d2f33f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 10:09:06 +0200 Subject: [PATCH 147/344] Apply suggestion from @kpouget --- AGENTS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 6da05ab1d..35943c693 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,10 @@ FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on Ope It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging. Key paths: -- `projects/` — individual project toolboxes (install scripts, test harnesses) +- `projects/` — individual project test harnesses +- `projects/NAME/orchestration` — upper layer of the test harness (config, prepare, test, cleanup) +- `projects/NAME/toolbox` — lower layer of the test harness (toolbox of scripts altering the state of the cluster, using the DSL language) +- `projects/NAME/test` — unit testing of the test harness - `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers) - `fournos/gitops/` — Tekton pipelines and GitOps base workflows - `docs/` — project documentation From b328d550a650d3630f519a5d5afdfb2c4aa6f3b8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 10:14:18 +0200 Subject: [PATCH 148/344] Apply suggestion from @kpouget --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 35943c693..4ba19d7a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,12 @@ Key paths: Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. Only write non-sensitive operational data (CRs, logs, pod descriptions, events). +6. **Use the vault module.** + Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret. + +7. **Never pass secrets via environment variables** + For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care. + ### Safe Patterns ```python From cee42b3bb73a1f04f3472d80f7c2f96beffa9420 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 17:09:36 -0400 Subject: [PATCH 149/344] Remove slack-webhook-url, use topsail bot exclusively for notifications Drop webhook_vault parameter from notification functions and callers, remove slack-webhook-url from psap-forge-notifications vault definition. Update default workload rates and durations. Co-authored-by: Cursor --- projects/rhaiis/orchestration/ci.py | 2 +- .../rhaiis/orchestration/config.d/workloads.yaml | 12 ++++++------ projects/rhaiis/orchestration/test_phase.py | 3 +-- projects/rhaiis/postprocess/regression.py | 15 ++++----------- vaults/psap-forge-notifications.yaml | 5 ----- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 25ea42404..763bca95c 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -74,7 +74,7 @@ def _check_pipeline_failure_and_notify() -> None: accelerator=accelerator_key, job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", + notification_vault="psap-forge-notifications", version=_cfg.project.get_config("tests.rhaiis.version", ""), cluster=cluster_tag, ) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index c413b505e..1cee1e4c6 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,13 +5,13 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [50] - max_seconds: 150 + rates: [1,50,100,200,300] + max_seconds: 450 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" - rates: [1] - max_seconds: 45 + rates: [1,50,100,200,300] + max_seconds: 450 profile3: data: "prompt_tokens=2048,output_tokens=128" @@ -20,7 +20,7 @@ profile3: profile4: data: "prompt_tokens=8000,output_tokens=1000" - rates: [1] - max_seconds: 45 + rates: [1,25,50,75,100] + max_seconds: 450 samples: 50 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 709a03f91..b47d9a176 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -41,7 +41,7 @@ def _send_pipeline_failure_alert( accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", + notification_vault="psap-forge-notifications", tp=str(vllm_args.get("tensor-parallel-size", "")), dp=str(vllm_args.get("data-parallel-size", "")), version=_cfg.project.get_config("tests.rhaiis.version", ""), @@ -702,7 +702,6 @@ def _run_regression_check( accelerator=accelerator, job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", notification_vault="psap-forge-notifications", report_url=report_url, tp=str(_vllm.get("tensor-parallel-size", "")), diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 2c70abc5b..1b524f7fa 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -107,7 +107,6 @@ def send_regression_notification( accelerator: str = "", job_id: str = "", slack_user: str = "", - webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, report_url: str = "", @@ -116,17 +115,13 @@ def send_regression_notification( ) -> bool: """Send regression/improvement Slack notification from analysis results. - Uses a webhook URL (from *webhook_vault*) when available, falling - back to the Bot-Token path (from *notification_vault*). - Args: analysis_result: Output from run_regression_analysis() model: Model name for display accelerator: Accelerator name for display job_id: Job identifier slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials + notification_vault: Vault containing topsail-bot.slack-token dry_run: Log only, don't send report_url: Optional presigned URL to an agent analysis report tp: Tensor parallelism size for display @@ -231,7 +226,7 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) def send_failure_notification( @@ -241,7 +236,6 @@ def send_failure_notification( accelerator: str = "", job_id: str = "", slack_user: str = "", - webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, tp: str = "", @@ -258,8 +252,7 @@ def send_failure_notification( accelerator: Accelerator name for display job_id: FournosJob name slack_user: Slack user ID to @-mention - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials + notification_vault: Vault containing topsail-bot.slack-token dry_run: Log only, don't send tp: Tensor parallelism size dp: Data parallelism size @@ -310,4 +303,4 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 5646df832..12ae01cc8 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -12,8 +12,3 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot - slack-webhook-url: - description: | - Plain-text file containing a Slack Incoming Webhook URL - for regression/comparison alerts. - Example: https://hooks.slack.com/services/T.../B.../... From 57f08489ce85efa3f5e5b9c0489ca2b03c522e74 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 19:24:10 -0400 Subject: [PATCH 150/344] Refactor: extract analysis, notifications; configurable ISVC labels and S3 settings - Move analyze functions to orchestration/analysis.py - Move pipeline failure alert to orchestration/notifications.py - Make ISVC labels configurable via labels dict parameter - Move S3 bucket/vault/credentials to rhaiis.s3 config section - Remove hardcoded defaults from s3_dashboard.py functions - Drop redundant mode="w+b" from NamedTemporaryFile calls Co-authored-by: Cursor --- projects/rhaiis/orchestration/analysis.py | 278 ++++++++++++++ projects/rhaiis/orchestration/config.yaml | 11 +- .../rhaiis/orchestration/notifications.py | 49 +++ projects/rhaiis/orchestration/test_phase.py | 346 ++---------------- projects/rhaiis/postprocess/s3_dashboard.py | 24 +- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 1 + .../templates/inferenceservice.yaml.j2 | 7 +- 7 files changed, 379 insertions(+), 337 deletions(-) create mode 100644 projects/rhaiis/orchestration/analysis.py create mode 100644 projects/rhaiis/orchestration/notifications.py diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py new file mode 100644 index 000000000..5f22c36ac --- /dev/null +++ b/projects/rhaiis/orchestration/analysis.py @@ -0,0 +1,278 @@ +"""RHAIIS regression analysis and agent analysis helpers.""" + +from __future__ import annotations + +import logging +import os +import tempfile +from pathlib import Path + +from projects.core.library import env +from projects.rhaiis.orchestration import runtime_config + +logger = logging.getLogger(__name__) + + +def run_standalone_analysis( + model_cfg: dict, + accelerator_key: str, + vllm_args: dict, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, +) -> None: + """Run regression check + agent analysis using existing S3 data (no new benchmark).""" + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if not agent_cfg.get("enabled", False): + logger.info("Standalone analysis skipped: agent_analysis not enabled") + return + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not version or not compare_version: + logger.info("Standalone analysis skipped: version or compare_version not configured") + return + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping standalone analysis") + return + + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + + consolidated_path = None + current_csv_path = None + try: + import pandas as pd + + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + + df = pd.read_csv(consolidated_path, on_bad_lines="warn") + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].str.strip() + + model_id = model_cfg.get("hf_model_id", "") + tp = str(vllm_args.get("tensor-parallel-size", 1)) + + current_rows = df[ + (df["version"] == version) + & (df["model"] == model_id) + & (df["accelerator"] == accelerator) + & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) + ] + + if current_rows.empty: + logger.warning( + "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", + version, model_id, accelerator, tp, + ) + return + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + current_csv_path = tmp.name + current_rows.to_csv(tmp, index=False) + + logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) + run_regression_check( + current_csv_path, compare_version, version, model_cfg, accelerator, + run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + if current_csv_path and os.path.exists(current_csv_path): + os.unlink(current_csv_path) + + +def run_regression_check( + csv_path, + compare_version: str, + current_version: str, + model_cfg: dict, + accelerator: str, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, + vllm_args: dict | None = None, +) -> None: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.caliper.engine.kpi.analyze import run_regression_analysis + from projects.core.library import config + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping regression analysis") + return + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + except Exception as e: + logger.warning("Could not download consolidated CSV for regression check: %s", e) + return + + try: + output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" + analysis = run_regression_analysis( + current_csv_path=Path(str(csv_path)), + consolidated_csv_path=Path(consolidated_path), + compare_version=compare_version, + current_version=current_version, + output_file=output_file, + profile_map=PROFILE_MAP, + metrics=METRICS, + restrict_profiles=restrict_profiles, + ) + + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + report_url = "" + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if agent_cfg.get("enabled", False): + report_url = run_agent_analysis( + analysis, model_cfg, accelerator, + current_version, compare_version, run_uuid, + severity_threshold=agent_cfg.get("severity_threshold", 10), + ) + + from projects.rhaiis.postprocess.regression import send_regression_notification + + _vllm = vllm_args or {} + send_regression_notification( + analysis, + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=run_uuid, + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + report_url=report_url, + tp=str(_vllm.get("tensor-parallel-size", "")), + dp=str(_vllm.get("data-parallel-size", "")), + ) + except Exception: + logger.warning("Regression analysis failed; continuing", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def run_agent_analysis( + analysis: dict, + model_cfg: dict, + accelerator: str, + current_version: str, + compare_version: str, + run_uuid: str, + *, + severity_threshold: int = 10, +) -> str: + """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" + from projects.core.library import config + from projects.rhaiis.postprocess.agent import ( + AGENT_SEVERITY_THRESHOLD, + build_pr_followup_prompt, + check_agent_connectivity, + markdown_to_html, + request_agent_analysis, + send_followup, + ) + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + agent_url = agent_cfg.get("url", "") + if not agent_url: + logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") + return "" + + threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD + severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] + if not severe: + logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) + return "" + + ok, detail = check_agent_connectivity(agent_url) + if not ok: + logger.warning("Agent not reachable, skipping analysis: %s", detail) + return "" + + tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + model = model_cfg.get("hf_model_id", "") + improvements = analysis.get("improvements", []) + + agent_response = request_agent_analysis( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe, + job_id=run_uuid, + improvements=improvements if improvements else None, + agent_url=agent_url, + ) + if not agent_response: + return "" + + pr_prompt = build_pr_followup_prompt(current_version, compare_version) + pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) + if pr_analysis: + agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" + + html_content = markdown_to_html( + agent_response, run_uuid, model, current_version, compare_version, + ) + html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" + html_path.write_text(html_content, encoding="utf-8") + logger.info("Agent analysis saved to %s", html_path) + + try: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + credentials_path = get_aws_credentials(vault_name, credentials_file) + if credentials_path: + s3 = create_s3_client(credentials_path) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" + s3.upload_file( + str(html_path), s3_bucket, s3_key, + ExtraArgs={"ContentType": "text/html"}, + ) + report_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": s3_bucket, "Key": s3_key}, + ExpiresIn=2592000, + ) + logger.info("Agent analysis uploaded to S3, presigned URL generated") + return report_url + except Exception: + logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) + + return "" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index c0477ff2f..f1d53b638 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -29,6 +29,15 @@ tests: compare_version: "" slack_user: "" +rhaiis: + s3: + bucket: psap-dashboard-data + vault: psap-forge-dashboard-s3 + credentials_file: aws.credentials + profiler: + enabled: false + s3_prefix: pytorch-profiles/rhaiis + caliper: postprocess: enabled: true @@ -45,9 +54,7 @@ caliper: output: kpis.json csv_dashboard: enabled: false - s3_bucket: psap-dashboard-data s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv - vault: psap-forge-dashboard-s3 analyze: enabled: false historical_kpis: historical_kpis diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py new file mode 100644 index 000000000..486c6c340 --- /dev/null +++ b/projects/rhaiis/orchestration/notifications.py @@ -0,0 +1,49 @@ +"""RHAIIS pipeline notification helpers.""" + +from __future__ import annotations + +import logging +import os +import traceback + +from projects.rhaiis.orchestration import runtime_config + +logger = logging.getLogger(__name__) + + +def send_pipeline_failure_alert( + exc: Exception, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert when the pipeline fails.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=traceback.format_exception_only(type(exc), exc)[-1].strip(), + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline failure alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b47d9a176..0cad2b784 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -2,7 +2,6 @@ import logging import os -import traceback import uuid as _uuid_mod from projects.core.library import env @@ -14,43 +13,6 @@ _K8S_NAME_MAX = 63 -def _send_pipeline_failure_alert( - exc: Exception, - *, - model_key: str, - workload_keys: list[str], -) -> None: - """Best-effort Slack alert when the pipeline fails.""" - try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - - send_failure_notification( - error=traceback.format_exception_only(type(exc), exc)[-1].strip(), - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) - except Exception: - logger.warning("Failed to send pipeline failure alert", exc_info=True) - def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" @@ -92,7 +54,8 @@ def do_test( deployment_name=deployment_name, ) except Exception as exc: - _send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) + from projects.rhaiis.orchestration.notifications import send_pipeline_failure_alert + send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) raise @@ -168,7 +131,8 @@ def _run_test( if not run_benchmark and not profiler_enabled: logger.info("run_benchmark=false and profiler=false, running standalone analysis only") try: - _run_standalone_analysis( + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis( model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) @@ -180,6 +144,10 @@ def _run_test( wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) try: + isvc_labels = {"opendatahub.io/dashboard": "true"} + if profiler_enabled: + isvc_labels["vllm-profiler/enabled"] = "true" + logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) deploy_kserve_isvc( deployment_name=deployment_name, @@ -196,6 +164,7 @@ def _run_test( storage_pvc=deploy_cfg.get("storage_pvc", ""), image_pull_secret=deploy_cfg.get("image_pull_secret", ""), service_account_name=deploy_cfg.get("service_account_name", ""), + labels=isvc_labels, ) logger.info("Waiting for InferenceService to be ready") @@ -336,7 +305,8 @@ def _run_workload_benchmark( if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") try: - _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Standalone analysis failed; continuing", exc_info=True) else: @@ -526,11 +496,13 @@ def _generate_and_sync_dashboard_csv( from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 + s3_cfg = config.project.get_config("rhaiis.s3", {}) sync_result = sync_csv_to_s3( csv_path, - s3_bucket=csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data"), - s3_key=csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv"), - vault_name=csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3"), + s3_bucket=s3_cfg.get("bucket", ""), + s3_key=csv_dashboard_cfg.get("s3_key", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Dashboard CSV sync result: %s", sync_result) @@ -539,277 +511,10 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) + from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) -def _run_standalone_analysis( - model_cfg: dict, - accelerator_key: str, - vllm_args: dict, - *, - run_uuid: str = "", - restrict_profiles: list[str] | None = None, -) -> None: - """Run regression check + agent analysis using existing S3 data (no new benchmark).""" - import tempfile - from pathlib import Path - - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.core.library import config - - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - if not agent_cfg.get("enabled", False): - logger.info("Standalone analysis skipped: agent_analysis not enabled") - return - - version = config.project.get_config("tests.rhaiis.version", "") - compare_version = config.project.get_config("tests.rhaiis.compare_version", "") - if not version or not compare_version: - logger.info("Standalone analysis skipped: version or compare_version not configured") - return - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") - s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if not credentials_path: - logger.warning("AWS credentials not available, skipping standalone analysis") - return - - # CSV stores just the chip family (e.g. "H200"), not the full accelerator_key with cluster tag - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() - - consolidated_path = None - current_csv_path = None - try: - import pandas as pd - - s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: - consolidated_path = tmp.name - s3.download_file(s3_bucket, s3_key, consolidated_path) - - df = pd.read_csv(consolidated_path, on_bad_lines="warn") - for col in df.columns: - if df[col].dtype == object: - df[col] = df[col].str.strip() - - model_id = model_cfg.get("hf_model_id", "") - tp = str(vllm_args.get("tensor-parallel-size", 1)) - - current_rows = df[ - (df["version"] == version) - & (df["model"] == model_id) - & (df["accelerator"] == accelerator) - & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) - ] - - if current_rows.empty: - logger.warning( - "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", - version, model_id, accelerator, tp, - ) - return - - with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: - current_csv_path = tmp.name - current_rows.to_csv(tmp, index=False) - - logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) - _run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, - run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, - ) - except Exception: - logger.warning("Standalone analysis failed", exc_info=True) - finally: - if consolidated_path and os.path.exists(consolidated_path): - os.unlink(consolidated_path) - if current_csv_path and os.path.exists(current_csv_path): - os.unlink(current_csv_path) - - -def _run_regression_check( - csv_path, - compare_version: str, - current_version: str, - model_cfg: dict, - accelerator: str, - *, - run_uuid: str = "", - restrict_profiles: list[str] | None = None, - vllm_args: dict | None = None, -) -> None: - import tempfile - from pathlib import Path - - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.caliper.engine.kpi.analyze import run_regression_analysis - from projects.core.library import config - from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") - s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if not credentials_path: - logger.warning("AWS credentials not available, skipping regression analysis") - return - - consolidated_path = None - try: - s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: - consolidated_path = tmp.name - s3.download_file(s3_bucket, s3_key, consolidated_path) - except Exception as e: - logger.warning("Could not download consolidated CSV for regression check: %s", e) - return - - try: - output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" - analysis = run_regression_analysis( - current_csv_path=Path(str(csv_path)), - consolidated_csv_path=Path(consolidated_path), - compare_version=compare_version, - current_version=current_version, - output_file=output_file, - profile_map=PROFILE_MAP, - metrics=METRICS, - restrict_profiles=restrict_profiles, - ) - - if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: - report_url = "" - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - if agent_cfg.get("enabled", False): - report_url = _run_agent_analysis( - analysis, model_cfg, accelerator, - current_version, compare_version, run_uuid, - severity_threshold=agent_cfg.get("severity_threshold", 10), - ) - - from projects.rhaiis.postprocess.regression import send_regression_notification - - _vllm = vllm_args or {} - send_regression_notification( - analysis, - model=model_cfg.get("hf_model_id", ""), - accelerator=accelerator, - job_id=run_uuid, - slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - report_url=report_url, - tp=str(_vllm.get("tensor-parallel-size", "")), - dp=str(_vllm.get("data-parallel-size", "")), - ) - except Exception: - logger.warning("Regression analysis failed; continuing", exc_info=True) - finally: - if consolidated_path and os.path.exists(consolidated_path): - os.unlink(consolidated_path) - - -def _run_agent_analysis( - analysis: dict, - model_cfg: dict, - accelerator: str, - current_version: str, - compare_version: str, - run_uuid: str, - *, - severity_threshold: int = 10, -) -> str: - """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" - from pathlib import Path - - from projects.core.library import config - from projects.rhaiis.postprocess.agent import ( - AGENT_SEVERITY_THRESHOLD, - build_pr_followup_prompt, - check_agent_connectivity, - markdown_to_html, - request_agent_analysis, - send_followup, - ) - - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - agent_url = agent_cfg.get("url", "") - if not agent_url: - logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") - return "" - - threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD - severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] - if not severe: - logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) - return "" - - ok, detail = check_agent_connectivity(agent_url) - if not ok: - logger.warning("Agent not reachable, skipping analysis: %s", detail) - return "" - - tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) - model = model_cfg.get("hf_model_id", "") - improvements = analysis.get("improvements", []) - - agent_response = request_agent_analysis( - model=model, - accelerator=accelerator, - current_version=current_version, - compare_version=compare_version, - tp=tp, - severe_regressions=severe, - job_id=run_uuid, - improvements=improvements if improvements else None, - agent_url=agent_url, - ) - if not agent_response: - return "" - - pr_prompt = build_pr_followup_prompt(current_version, compare_version) - pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) - if pr_analysis: - agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" - - html_content = markdown_to_html( - agent_response, run_uuid, model, current_version, compare_version, - ) - html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" - html_path.write_text(html_content, encoding="utf-8") - logger.info("Agent analysis saved to %s", html_path) - - try: - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if credentials_path: - s3 = create_s3_client(credentials_path) - s3_bucket = "psap-dashboard-data" - s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" - s3.upload_file( - str(html_path), s3_bucket, s3_key, - ExtraArgs={"ContentType": "text/html"}, - ) - report_url = s3.generate_presigned_url( - "get_object", - Params={"Bucket": s3_bucket, "Key": s3_key}, - ExpiresIn=2592000, - ) - logger.info("Agent analysis uploaded to S3, presigned URL generated") - return report_url - except Exception: - logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) - - return "" - def _upload_predictor_log(run_uuid: str) -> None: """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" @@ -824,13 +529,14 @@ def _upload_predictor_log(run_uuid: str) -> None: logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) return - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + s3_cfg = config.project.get_config("rhaiis.s3", {}) result = upload_predictor_log_to_s3( log_path, run_uuid=run_uuid, - vault_name=vault_name, + s3_bucket=s3_cfg.get("bucket", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Predictor log upload result: %s", result) @@ -1020,6 +726,7 @@ def _upload_profiler_traces( profile_labels = _infer_profile_labels_from_traces(trace_files) logger.info("Inferred profile labels from trace filenames: %s", profile_labels) + s3_cfg = config.project.get_config("rhaiis.s3", {}) result = upload_profiler_traces_to_s3( traces_dir, model_name=model_cfg.get("hf_model_id", ""), @@ -1027,9 +734,10 @@ def _upload_profiler_traces( tp_size=int(vllm_args.get("tensor-parallel-size", 1)), version=version, profile_labels=profile_labels, - s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), - s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), - vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), + s3_bucket=s3_cfg.get("bucket", ""), + s3_prefix=profiler_cfg.get("s3_prefix", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Profiler trace upload result: %s", result) diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index d8b3705a0..c648ce9e7 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -13,16 +13,13 @@ def sync_csv_to_s3( csv_path: Path, *, - s3_bucket: str = "psap-dashboard-data", - s3_key: str = "staging/rhaiis-dashboard/consolidated_dashboard.csv", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + s3_key: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: - """Download consolidated CSV from S3, append new rows, re-upload. - - Ported from model-furnace's ModelValidator._sync_csv_to_s3(). - """ + """Download consolidated CSV from S3, append new rows, re-upload.""" try: import pandas as pd except ImportError: @@ -42,7 +39,7 @@ def sync_csv_to_s3( try: s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: consolidated_path = tmp.name try: @@ -87,15 +84,14 @@ def upload_profiler_traces_to_s3( tp_size: int, version: str, profile_labels: list[str], - s3_bucket: str = "psap-dashboard-data", - s3_prefix: str = "pytorch-profiles/rhaiis", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + s3_prefix: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: """Upload profiler traces to S3. - Ported from model-furnace's ModelValidator._upload_profiler_traces_to_s3(). Path: s3://{bucket}/{prefix}/{accel}/{model}/tp{N}/{version}/{profile_label}/ """ if not traces_dir.exists(): @@ -168,8 +164,8 @@ def upload_predictor_log_to_s3( log_path: Path, *, run_uuid: str, - s3_bucket: str = "psap-dashboard-data", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index c8cd621d9..e8908595a 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -26,6 +26,7 @@ def run( storage_pvc: str = "", image_pull_secret: str = "", service_account_name: str = "", + labels: dict | None = None, ): return execute_tasks(locals()) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index 3d04aa276..5ca599383 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -8,9 +8,12 @@ metadata: prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "8080" +{%- if args.labels %} labels: - opendatahub.io/dashboard: "true" - vllm-profiler/enabled: "true" +{%- for key, val in args.labels.items() %} + {{ key }}: "{{ val }}" +{%- endfor %} +{%- endif %} name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: From 069859a7710daae539a4b99c6cc91a7f9e29d296 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 21:39:18 -0400 Subject: [PATCH 151/344] Fix config collision: move rhaiis.s3 to config.d/rhaiis.yaml Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 5 +++++ projects/rhaiis/orchestration/config.yaml | 9 --------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 12da18a24..90faeab5b 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,6 +36,11 @@ accelerator_env_vars: amd: VLLM_ROCM_USE_AITER: "1" +s3: + bucket: psap-dashboard-data + vault: psap-forge-dashboard-s3 + credentials_file: aws.credentials + warmup: rate: 200 max_seconds: 60 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index f1d53b638..9c6bf6f99 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -29,15 +29,6 @@ tests: compare_version: "" slack_user: "" -rhaiis: - s3: - bucket: psap-dashboard-data - vault: psap-forge-dashboard-s3 - credentials_file: aws.credentials - profiler: - enabled: false - s3_prefix: pytorch-profiles/rhaiis - caliper: postprocess: enabled: true From c3a89abb6a2c37cfaae8f38df4547513769b3784 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 11:51:51 +0300 Subject: [PATCH 152/344] refactor: config-driven nightly pipeline with reusable MLflow verifier Address PR #133 review feedback: - Remove skip_phases mechanism, let vault system handle nightly normally - Move MLflow verifier to caliper.orchestration for cross-project reuse - Use @requires config annotation for handler and verifier - Make receiver/verifier selection config-driven (dotted class paths) - Move GHCR repo/image from hardcoded constants to nightly.sources config - Replace NIGHTLY_SOURCE env var with configOverrides in CronJob - Add per-source mlflow_experiment for routing results to correct experiment - Add /pipeline nightly PR directive for source selection - Use vault.get_vault_content_path() instead of manual secret path construction --- projects/agentic_tools/ci_base.py | 11 +-- .../orchestration}/mlflow_verifier.py | 69 +++++++++--------- projects/core/nightly/handler.py | 70 ++++++++++++------- .../mcp_gateway/nightly_resolvers/__init__.py | 64 +---------------- .../nightly_resolvers/cronjob-ghcr.yaml | 4 +- .../nightly_resolvers/ghcr_receiver.py | 37 ++++++---- .../mcp_gateway/orchestration/config.yaml | 12 +++- projects/mcp_gateway/orchestration/pr_args.py | 12 ++++ 8 files changed, 133 insertions(+), 146 deletions(-) rename projects/{mcp_gateway/nightly_resolvers => caliper/orchestration}/mlflow_verifier.py (59%) diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index 3c078ab70..22247beb3 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -127,19 +127,14 @@ def list_vaults(self) -> list[str]: if isinstance(vault_config, list): return vault_config all_vaults: list[str] = [] - for category, vaults in vault_config.items(): - if category == "skip_phases": - continue + for _category, vaults in vault_config.items(): if isinstance(vaults, list): all_vaults.extend(vaults) return list(dict.fromkeys(all_vaults)) def should_init_vaults(self, phase: str) -> bool: - """Return False to skip vault init for resolve and phases listed in vaults.skip_phases.""" - if phase == "resolve-fournos-config": - return False - skip_phases = config.project.get_config("vaults.skip_phases", [], print=False) - return phase not in skip_phases + """Return False to skip vault init for the resolve phase.""" + return phase != "resolve-fournos-config" def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. diff --git a/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py similarity index 59% rename from projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py rename to projects/caliper/orchestration/mlflow_verifier.py index 2e9628a77..6a4b50bf8 100644 --- a/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -1,9 +1,15 @@ -"""MLflow nightly verifier for mcp_gateway. +"""Reusable MLflow nightly verifier. Connects to the MLflow tracking server, queries runs in the project's experiment, and extracts the version from the most recent run name. -Uses the same secret loading mechanism as caliper's mlflow export. +Projects reuse this by subclassing in their nightly_resolvers/ package +and setting NAME = "mlflow". The version extraction pattern is read from +``nightly.mlflow_verifier.version_pattern`` (regex with one capture group) +and an optional ``nightly.mlflow_verifier.strip_prefix``. + +All MLflow connection settings come from the caliper export config that +every FORGE project already has. """ from __future__ import annotations @@ -12,45 +18,47 @@ import os import re import time -from pathlib import Path from projects.caliper.engine.file_export.mlflow_secrets import ( load_mlflow_secrets_yaml, mlflow_connection_env, validate_mlflow_secrets, ) +from projects.core.library import vault +from projects.core.library.config import requires from projects.core.nightly.base_verifier import NightlyVerifier logger = logging.getLogger(__name__) -VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") - MAX_ATTEMPTS = 3 RETRY_DELAY_SECONDS = 15 -class MLflowVerifier(NightlyVerifier): +class CaliperMLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. - Reads its configuration from the project's config.yaml under: + Config requirements (via @requires on __init__): caliper.export.backend.mlflow.config.experiment caliper.export.backend.mlflow.config.workspace - caliper.export.backend.mlflow.secrets.vault.name - caliper.export.backend.mlflow.secrets.vault.mlflow_secret + caliper.export.backend.mlflow.secrets.vault (dict with 'name' and 'mlflow_secret') + nightly.mlflow_verifier.version_pattern (regex with one capture group) + nightly.mlflow_verifier.strip_prefix (prefix to remove from captured version) """ - NAME = "mlflow" - - def __init__(self): - from projects.core.library import config - - self.experiment = config.project.get_config( - "caliper.export.backend.mlflow.config.experiment" - ) - self.workspace = config.project.get_config("caliper.export.backend.mlflow.config.workspace") - vault_cfg = config.project.get_config("caliper.export.backend.mlflow.secrets.vault") - self.secret_name = vault_cfg["name"] - self.secret_file = vault_cfg["mlflow_secret"] + @requires( + experiment="caliper.export.backend.mlflow.config.experiment", + workspace="caliper.export.backend.mlflow.config.workspace", + vault_cfg="caliper.export.backend.mlflow.secrets.vault", + version_pattern="nightly.mlflow_verifier.version_pattern", + strip_prefix="nightly.mlflow_verifier.strip_prefix", + ) + def __init__(self, _cfg): + self.experiment = _cfg.experiment + self.workspace = _cfg.workspace + self.secret_name = _cfg.vault_cfg["name"] + self.secret_file = _cfg.vault_cfg["mlflow_secret"] + self._version_re = re.compile(_cfg.version_pattern) + self._strip_prefix = _cfg.strip_prefix or "" def get_last_tested_version(self) -> str: connection = self._load_connection() @@ -76,11 +84,10 @@ def get_last_tested_version(self) -> str: raise RuntimeError(f"MLflow verifier failed after {MAX_ATTEMPTS} attempts") from last_error def _load_connection(self) -> dict: - secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) - secret_path = secrets_dir / self.secret_name / self.secret_file + secret_path = vault.get_vault_content_path(self.secret_name, self.secret_file) - if not secret_path.exists(): - logger.warning("MLflow secret not found at expected path, assuming no previous run") + if secret_path is None or not secret_path.exists(): + logger.warning("MLflow secret not found in vault, assuming no previous run") return {} data = load_mlflow_secrets_yaml(secret_path) @@ -117,20 +124,14 @@ def _query_latest_run(self) -> str: return self._extract_version(run_name) def _extract_version(self, run_name: str) -> str: - """Extract the version from a run name like: - forge-mcp-gateway-s150-u500-vsha--20260714-224646 - - Returns the raw SHA (without the 'sha-' prefix) to match - what the receiver returns. - """ - match = VERSION_PATTERN.search(run_name) + match = self._version_re.search(run_name) if not match: logger.warning("Could not extract version from run name: %s", run_name) return "" version = match.group(1) - if version.startswith("sha-"): - return version[4:] + if self._strip_prefix and version.startswith(self._strip_prefix): + return version[len(self._strip_prefix):] return version diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 2ea3e3286..02a536efe 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -15,14 +15,28 @@ from projects.core.library import ci as ci_lib from projects.core.library import config, env +from projects.core.library.config import requires logger = logging.getLogger(__name__) -def _receive_image(project: str) -> str: - """Resolve the latest available version from the project's receiver.""" - module = importlib.import_module(f"projects.{project}.nightly_resolvers") - receiver = module.get_receiver() +def _get_source_config() -> dict: + """Read the source-specific config from nightly.sources..""" + source = config.project.get_config("nightly.source") + return config.project.get_config(f"nightly.sources.{source}") + + +def _resolve_class(dotted_path: str): + """Import a class from a dotted path like 'projects.foo.bar.MyClass'.""" + module_path, class_name = dotted_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + return getattr(mod, class_name) + + +def _receive_image(source_cfg: dict) -> str: + """Resolve the latest available version from the source's receiver.""" + receiver_cls = _resolve_class(source_cfg["receiver"]) + receiver = receiver_cls() logger.info("Using receiver: %s", type(receiver).__name__) version = receiver.get_latest_version() @@ -35,10 +49,10 @@ def _receive_image(project: str) -> str: return version -def _confirm_nightly(project: str, image_version: str) -> None: +def _confirm_nightly(project: str, image_version: str, source_cfg: dict) -> None: """Compare resolved version against last tested, create FournosJob if new.""" - module = importlib.import_module(f"projects.{project}.nightly_resolvers") - verifier = module.get_verifier() + verifier_cls = _resolve_class(config.project.get_config("nightly.verifier")) + verifier = verifier_cls() logger.info("Using verifier: %s", type(verifier).__name__) last_version = verifier.get_last_tested_version() @@ -53,43 +67,50 @@ def _confirm_nightly(project: str, image_version: str) -> None: return logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") - _create_fournos_job(project, image_version) + _create_fournos_job(project, image_version, source_cfg) result_msg = f"FournosJob created for {project} version {image_version}" ci_lib.add_notification_file("nightly-triggered", result_msg) (env.ARTIFACT_DIR / "result.txt").write_text(result_msg) -def _create_fournos_job(project: str, version: str) -> None: +@requires( + cluster="nightly.cluster", + version_key="nightly.version_key", + pipeline="nightly.pipeline", + preset="nightly.preset", + owner="nightly.owner", +) +def _create_fournos_job(project: str, version: str, source_cfg: dict, _cfg) -> None: """Create a FournosJob for the full test run using the fournos_launcher toolbox.""" from projects.fournos_launcher.toolbox.submit_and_wait.main import run as submit_and_wait namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") - cluster = config.project.get_config("nightly.cluster") - version_key = config.project.get_config("nightly.version_key") - pipeline = config.project.get_config("nightly.pipeline") - preset = config.project.get_config("nightly.preset", "") - owner = config.project.get_config("nightly.owner", "nightly-pipeline") - args_list = [preset] if preset else [] + args_list = [_cfg.preset] if _cfg.preset else [] + + overrides = {_cfg.version_key: version} + if mlflow_experiment := source_cfg.get("mlflow_experiment"): + overrides["caliper.export.backend.mlflow.config.experiment"] = mlflow_experiment + logger.info("MLflow experiment override: %s", mlflow_experiment) logger.info( "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", project, version, - cluster, - pipeline, + _cfg.cluster, + _cfg.pipeline, ) submit_and_wait( project, - cluster_name=cluster, + cluster_name=_cfg.cluster, args=args_list, - variables_overrides={version_key: version}, + variables_overrides=overrides, namespace=namespace, - owner=owner, + owner=_cfg.owner, display_name=f"nightly {project} {version}", - pipeline_name=pipeline, + pipeline_name=_cfg.pipeline, exclusive=True, wait=False, ) @@ -100,17 +121,18 @@ def run(): logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") project = config.project.get_config("project.name") - logger.info("Project: %s", project) + source_cfg = _get_source_config() + logger.info("Project: %s, source config: %s", project, source_cfg) try: - image_version = _receive_image(project) + image_version = _receive_image(source_cfg) except Exception as e: logger.error("receive-image failed: %s", e) ci_lib.add_notification_file("nightly-receive-image-failed", f"FATAL: {e}") raise try: - _confirm_nightly(project, image_version) + _confirm_nightly(project, image_version, source_cfg) except Exception as e: logger.error("confirm-nightly failed: %s", e) ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") diff --git a/projects/mcp_gateway/nightly_resolvers/__init__.py b/projects/mcp_gateway/nightly_resolvers/__init__.py index 27c4f1e56..73ebd8af5 100644 --- a/projects/mcp_gateway/nightly_resolvers/__init__.py +++ b/projects/mcp_gateway/nightly_resolvers/__init__.py @@ -1,65 +1,5 @@ """MCP Gateway nightly resolvers. -Auto-discovers ImageReceiver and NightlyVerifier subclasses in this package. -To add a new receiver or verifier, just create a new file with a class that: - - Inherits from ImageReceiver or NightlyVerifier - - Defines a NAME class attribute (used for selection) - -No modifications to this file are needed. +Contains ImageReceiver implementations for different image sources. +Receiver selection is config-driven via nightly.sources..receiver. """ - -from __future__ import annotations - -import importlib -import os -import pkgutil - -from projects.core.nightly.base_receiver import ImageReceiver -from projects.core.nightly.base_verifier import NightlyVerifier - -_receivers: dict[str, type[ImageReceiver]] = {} -_verifiers: dict[str, type[NightlyVerifier]] = {} - - -def _discover(): - """Import all modules in this package and register subclasses.""" - if _receivers or _verifiers: - return - - for _finder, module_name, _ in pkgutil.iter_modules(__path__): - module = importlib.import_module(f"{__name__}.{module_name}") - for attr in vars(module).values(): - if not isinstance(attr, type): - continue - if issubclass(attr, ImageReceiver) and attr is not ImageReceiver: - name = getattr(attr, "NAME", None) - if name: - _receivers[name] = attr - if issubclass(attr, NightlyVerifier) and attr is not NightlyVerifier: - name = getattr(attr, "NAME", None) - if name: - _verifiers[name] = attr - - -def get_receiver() -> ImageReceiver: - """Return the image receiver based on NIGHTLY_SOURCE env var.""" - _discover() - source = os.environ.get("NIGHTLY_SOURCE", "ghcr") - - cls = _receivers.get(source) - if cls is None: - raise ValueError( - f"Unknown source '{source}' for mcp_gateway. Available: {list(_receivers.keys())}" - ) - return cls() - - -def get_verifier() -> NightlyVerifier: - """Return the nightly verifier (first discovered NightlyVerifier subclass).""" - _discover() - - if not _verifiers: - raise RuntimeError("No NightlyVerifier subclass found in resolvers package") - - cls = next(iter(_verifiers.values())) - return cls() diff --git a/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml index bf053c29a..1eb501472 100644 --- a/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml @@ -40,8 +40,8 @@ spec: clusterless: true exclusive: false pipeline: "nightly" - env: - NIGHTLY_SOURCE: "ghcr" + configOverrides: + nightly.source: "ghcr" executionEngine: forge: project: "mcp_gateway" diff --git a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index 949de372f..eac5a8026 100644 --- a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -1,10 +1,14 @@ """GHCR image receiver for mcp_gateway. -Resolves the latest mcp-gateway commit SHA that has a published image -on ghcr.io. Checks recent commits from the repo and finds the first +Resolves the latest commit SHA that has a published image on ghcr.io. +Checks recent commits from the configured repo and finds the first one with a corresponding container image tag (sha-). No authentication needed for public packages (anonymous token). + +Configuration (in config.yaml under nightly.sources.ghcr): + repo: GitHub repo in "owner/name" format (e.g. "Kuadrant/mcp-gateway") + image: GHCR image path without registry (e.g. "kuadrant/mcp-gateway") """ from __future__ import annotations @@ -19,21 +23,26 @@ logger = logging.getLogger(__name__) -REPO = "Kuadrant/mcp-gateway" -IMAGE = "kuadrant/mcp-gateway" -COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" -TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" -MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" - MAX_ATTEMPTS = 3 RETRY_DELAY_SECONDS = 10 class GHCRReceiver(ImageReceiver): - """Find the latest mcp-gateway commit with a published ghcr.io image.""" + """Find the latest commit with a published ghcr.io image.""" NAME = "ghcr" + def __init__(self): + from projects.core.library import config + + source_cfg = config.project.get_config("nightly.sources.ghcr") + self.repo = source_cfg["repo"] + self.image = source_cfg["image"] + + self._commits_url = f"https://api.github.com/repos/{self.repo}/commits?per_page=20" + self._token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + self._manifest_url = f"https://ghcr.io/v2/{self.image}/manifests/" + def get_latest_version(self) -> str: last_error = None for attempt in range(1, MAX_ATTEMPTS + 1): @@ -66,12 +75,12 @@ def _fetch_version(self) -> str: ) def _get_ghcr_token(self) -> str: - with urllib.request.urlopen(TOKEN_URL, timeout=30) as resp: + with urllib.request.urlopen(self._token_url, timeout=30) as resp: data = json.loads(resp.read()) return data["token"] def _get_recent_commits(self) -> list[str]: - req = urllib.request.Request(COMMITS_URL) + req = urllib.request.Request(self._commits_url) with urllib.request.urlopen(req, timeout=30) as resp: commits = json.loads(resp.read()) return [c["sha"] for c in commits] @@ -79,7 +88,7 @@ def _get_recent_commits(self) -> list[str]: def _image_exists(self, sha: str, token: str) -> bool: tag = f"sha-{sha}" req = urllib.request.Request( - MANIFEST_URL + tag, + self._manifest_url + tag, headers={ "Accept": "application/vnd.oci.image.index.v1+json, " "application/vnd.docker.distribution.manifest.v2+json", @@ -87,8 +96,8 @@ def _image_exists(self, sha: str, token: str) -> bool: }, ) try: - urllib.request.urlopen(req, timeout=30) - return True + with urllib.request.urlopen(req, timeout=30): + return True except urllib.error.HTTPError as e: if e.code == 404: return False diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 8abbba249..7b33d74c9 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,4 +1,14 @@ nightly: + sources: + ghcr: + receiver: "projects.mcp_gateway.nightly_resolvers.ghcr_receiver.GHCRReceiver" + repo: "Kuadrant/mcp-gateway" + image: "kuadrant/mcp-gateway" + mlflow_experiment: "mcp-gw-cpt-ghcr" + verifier: "projects.caliper.orchestration.mlflow_verifier.CaliperMLflowVerifier" + mlflow_verifier: + version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" + strip_prefix: "sha-" cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" pipeline: "forge-full" @@ -12,8 +22,6 @@ vaults: - psap-forge-mlflow-export prepare: [] test: [] - skip_phases: - - nightly notifications: slack: diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 82cffdb40..1b11aade6 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -9,6 +9,8 @@ /test fournos mcp_gateway smoke /test fournos mcp_gateway smoke collect_cluster_info /version 0.7.0 + /pipeline nightly ghcr + /pipeline nightly quay """ from __future__ import annotations @@ -77,4 +79,14 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st parsed_directives.append(line) logger.info("Parsed mcp_gateway version: %s", parts[0]) + # Parse /pipeline nightly — extract source from pipeline directive + if line.startswith("/pipeline nightly "): + parts = line.split() + if len(parts) == 3: + source = parts[2] + config_overrides["nightly.source"] = source + config_overrides["fournos.job.pipeline_name"] = "nightly" + parsed_directives.append(line) + logger.info("Parsed nightly source from pipeline directive: %s", source) + return config_overrides, parsed_directives From cf1ff57a877bb7e2830d86a323cde39e29d2de07 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 11:55:18 +0300 Subject: [PATCH 153/344] style: apply ruff formatting --- projects/caliper/orchestration/mlflow_verifier.py | 2 +- projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 6a4b50bf8..8d7d00adb 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -132,6 +132,6 @@ def _extract_version(self, run_name: str) -> str: version = match.group(1) if self._strip_prefix and version.startswith(self._strip_prefix): - return version[len(self._strip_prefix):] + return version[len(self._strip_prefix) :] return version diff --git a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index eac5a8026..36853fc71 100644 --- a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -40,7 +40,9 @@ def __init__(self): self.image = source_cfg["image"] self._commits_url = f"https://api.github.com/repos/{self.repo}/commits?per_page=20" - self._token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + self._token_url = ( + f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + ) self._manifest_url = f"https://ghcr.io/v2/{self.image}/manifests/" def get_latest_version(self) -> str: From abfe64aaa2b8d2f833f2e71a1ef64a6af2e97513 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 22 Jul 2026 10:52:26 +0100 Subject: [PATCH 154/344] fix: Increase number of attempts Signed-off-by: Alberto Perdomo --- projects/fournos_launcher/toolbox/submit_and_wait/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 40e4917b6..9d205c410 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -211,7 +211,7 @@ def submit_fournos_job(args, ctx): return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" -@retry(attempts=360, delay=10, backoff=1.0) +@retry(attempts=3000, delay=10, backoff=1.0) @task def wait_for_job_completion(args, ctx): """Wait for FOURNOS job to complete""" From a7cea8ac07df78e2bbf18a2fe6da6476dc5ed004 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 13:53:43 +0300 Subject: [PATCH 155/344] small fix to run in the smoke cluster --- projects/mcp_gateway/toolbox/install_platform/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index 6fdc75046..1bbd97fca 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -116,7 +116,7 @@ def install_service_mesh_instance(args, ctx): if "wait_for_ready" in step: spec = step["wait_for_ready"] ctx.mesh_ready_spec = spec - return (False, "Service Mesh instance applied, readiness checked in next task") + return "Service Mesh instance applied, readiness checked in next task" return f"Service Mesh instance applied from {kustomize_path}" From 6196820bd0d04bc400e3a9890f7f9e5c251d97bf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:16:40 +0200 Subject: [PATCH 156/344] [core] dsl: show the function duration in the completion banner --- projects/core/dsl/log.py | 22 +++++++++++++++++++++- projects/core/dsl/runtime.py | 13 +++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/projects/core/dsl/log.py b/projects/core/dsl/log.py index 0f3c2d18d..16f619975 100644 --- a/projects/core/dsl/log.py +++ b/projects/core/dsl/log.py @@ -104,7 +104,9 @@ def log_execution_banner(function_args: dict = None, log_file: str = None): logger.info("") -def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): +def log_completion_banner( + function_args: dict = None, status: str = "SUCCESS", start_time: float = None +): """Log the completion banner with function info and completion status""" frame = inspect.currentframe() caller_frame = frame.f_back.f_back @@ -118,6 +120,24 @@ def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): logger.info(f"| {rel_filename}") logger.info(f"| STATUS: {status}") logger.info(f"| COMMAND: {function_name}") + + # Add duration if start_time is provided + if start_time is not None: + end_time = time.time() + duration = end_time - start_time + minutes, seconds = divmod(duration, 60) + hours, minutes = divmod(minutes, 60) + + # Format duration nicely + if hours > 0: + duration_str = f"{int(hours)}h {int(minutes)}m {int(seconds)}s" + elif minutes > 0: + duration_str = f"{int(minutes)}m {int(seconds)}s" + else: + duration_str = f"{duration:.1f}s" + + logger.info(f"| DURATION: {duration_str}") + logger.info(f"| ARTIFACTS: {env.ARTIFACT_DIR}") logger.info("===============================================================================") logger.info("") diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index 024555f64..78ff145d6 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -174,7 +174,7 @@ def execute_tasks(function_args: dict = None): if not file_tasks: logger.error(f"No tasks found for file: {rel_filename}") - log_completion_banner(function_args, status="NO_TASKS") + log_completion_banner(function_args, status="NO_TASKS", start_time=start_time) raise RuntimeError(f"No tasks found for file: {rel_filename}") execution_error = None @@ -193,7 +193,7 @@ def execute_tasks(function_args: dict = None): logger.fatal("==> INTERRUPTED: Received KeyboardInterrupt (Ctrl+C)") logger.info("==> Exiting...") # Show completion banner with interrupted status - log_completion_banner(function_args, status="INTERRUPTED") + log_completion_banner(function_args, status="INTERRUPTED", start_time=start_time) raise except EarlyReturnException as e: @@ -213,7 +213,11 @@ def execute_tasks(function_args: dict = None): # Log error but continue to always tasks logger.info("") logger.fatal(f"==> {e.__class__.__name__}: {e}") - log_completion_banner(function_args, status=f"EXCEPTION ({e.__class__.__name__})") + log_completion_banner( + function_args, + status=f"EXCEPTION ({e.__class__.__name__})", + start_time=start_time, + ) # Save error to re-raise after always tasks execute execution_error = e @@ -278,6 +282,7 @@ def execute_tasks(function_args: dict = None): log_completion_banner( function_args, status=f"FAILED ({len(all_exceptions)} exception(s))", + start_time=start_time, ) if len(all_exceptions) == 1: raise all_exceptions[0] @@ -287,7 +292,7 @@ def execute_tasks(function_args: dict = None): ) # Log completion banner if execution was successful - log_completion_banner(function_args) + log_completion_banner(function_args, start_time=start_time) # for the time being, return the shared context. In the # future we'll return more info about what has been From 71c91ae450fd8ef6e6ef023e329f7f46ffac66fc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:38:19 +0200 Subject: [PATCH 157/344] [core] library: postprocess: skip in the postprocessing on keyboard interrupt --- projects/core/library/postprocess.py | 128 +++++++++++++++++---------- 1 file changed, 82 insertions(+), 46 deletions(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 893c9e7f9..fa2c8ddb0 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -143,17 +143,29 @@ def run_and_postprocess(test_func, *args, **kwargs): exc_msg: str | None = None ret: int | None = None original_exc: BaseException | None = None # Store the test exception + interrupted: bool = False try: ret = test_func(*args, **kwargs) return ret - except BaseException as e: + except (KeyboardInterrupt, BaseException) as e: + # Import here to avoid circular dependencies + from projects.core.library.run import SignalInterrupt + + # Check if this is an interrupt + if isinstance(e, KeyboardInterrupt | SignalInterrupt): + interrupted = True + logger.info("==> Test interrupted - skipping Caliper post-processing") + exc_msg = str(e) original_exc = e # Capture before the name is cleared raise finally: - # Determine test outcome based on exception/return code - if exc_msg is not None: + # Skip post-processing when interrupted to avoid blocking on finalizers + if interrupted: + logger.info("==> Skipping Caliper post-processing due to interrupt") + outcome = None + elif exc_msg is not None: outcome = TestPhaseOutcome("FAILED", exc_msg) elif ret == 0: outcome = TestPhaseOutcome("SUCCESS") @@ -164,56 +176,80 @@ def run_and_postprocess(test_func, *args, **kwargs): # Run postprocessing and check status for failures try: - status = run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) - - # Check if postprocessing failed - success = status.get("success", False) - if not success: - final_status = status.get("final_status", "unknown") - - # Check if failure is only due to warnings - if _is_warnings_only_failure(status): - if original_exc is not None: - # Test failed, postprocess has warnings: still fail due to test - logger.error( - "Test failed and postprocessing completed with warnings (final_status: %s)", - final_status, - ) - raise # Re-raise the original test exception - else: - # Test succeeded, postprocess has warnings only: treat as success - logger.warning( - "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", - final_status, - ) - return 0 + status = ( + run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) + if not interrupted + else dict(success=False, final_status="Test interrupted") + ) + + # Handle None status when postprocessing is disabled - treat as unsuccessful + if status is None or not status.get("success", False): + if status is None: + final_status = "postprocessing disabled" + else: + final_status = status.get("final_status", "unknown") + result = _handle_postprocess_failure(status, original_exc, final_status) + if result is None: + # Let original test exception propagate through outer flow + pass else: - # Actual postprocessing failures (not just warnings) - if original_exc is not None: - # Both test and postprocess failed: log both issues - logger.error( - "Both test and postprocessing failed (final_status: %s)", final_status - ) - raise # Re-raise the original test exception - else: - # Only postprocess failed: return failure code - logger.error( - "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", - final_status, - ) - return 1 + return result except Exception as postprocess_exc: logger.exception("Caliper postprocess after test failed with exception") if original_exc is not None: # Both test and postprocess failed: chain so both are visible in the traceback raise postprocess_exc from original_exc - else: - # Only postprocess failed: return failure code instead of raising - logger.error( - "Test succeeded but postprocessing failed with exception - returning exit code 1" - ) - return 1 + + # Only postprocess failed: return failure code instead of raising + logger.error( + "Test succeeded but postprocessing failed with exception - returning exit code 1" + ) + return 1 + + +def _handle_postprocess_failure( + status: dict, original_exc: BaseException | None, final_status: str +) -> int | None: + """Handle postprocessing failure logic. + + Args: + status: Postprocessing status dictionary + original_exc: Original test exception if any + final_status: Final status from postprocessing + + Returns: + Exit code to return, or None to re-raise original exception + """ + # Check if failure is only due to warnings + if _is_warnings_only_failure(status): + if original_exc is not None: + # Test failed, postprocess has warnings: still fail due to test + logger.error( + "Test failed and postprocessing completed with warnings (final_status: %s)", + final_status, + ) + return None # Signal to re-raise original exception + else: + # Test succeeded, postprocess has warnings only: treat as success + logger.warning( + "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", + final_status, + ) + return 0 + else: + # Actual postprocessing failures (not just warnings) + if original_exc is not None: + # Both test and postprocess failed: log both issues + logger.error("Both test and postprocessing failed (final_status: %s)", final_status) + return None # Signal to re-raise original exception + else: + # Only postprocess failed: return failure code + logger.error( + "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", + final_status, + ) + return 1 def _is_warnings_only_failure(status: dict) -> bool: From aaed126b9dd26c6625aba7cb80f0e5d1c9e9c93a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:52:39 +0200 Subject: [PATCH 158/344] [core] library: ci: better handle early exception before the env module is initialized --- projects/core/library/ci.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 475da31eb..894e5c7d4 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -108,11 +108,16 @@ def _display_error_summary(e: Exception) -> None: for line in summary_lines: logger.error(line) - if env.ARTIFACT_DIR is None: - logging.error("env.ARTIFACT_DIR not set, cannot generate the error summary file") - else: - # Write to FAILURES file - _write_error_summary_to_file(summary_lines) + # Attempt to write to FAILURES file if environment is initialized + try: + if env.ARTIFACT_DIR is not None: + _write_error_summary_to_file(summary_lines) + else: + logger.warning( + "Environment not fully initialized - error summary displayed above but not saved to file" + ) + except Exception as write_error: + logger.warning(f"Could not save error summary to file: {write_error}") logger.error("=" * 80) From 69062f65bb3cd34f40355d3108b0d8e8cad2ad75 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 13:58:29 +0200 Subject: [PATCH 159/344] [core] library: config: more more reliable if 'extends: None' --- projects/core/library/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index d0d0437cb..9dd5d51f2 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -210,7 +210,7 @@ def apply_preset(self, name): for key, value in values.items(): if key == "extends": - for extend_name in value: + for extend_name in value or []: self.apply_preset(extend_name) continue From 7affb8215557c1a949205132a6c6c1d09615b046 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:17:11 +0200 Subject: [PATCH 160/344] [core] dsl: utils: k8s: allow oc to save to file --- projects/core/dsl/utils/k8s.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 460ff0755..62a7f0985 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -23,6 +23,7 @@ def oc( timeout_seconds: float | None = 300, log_stdout: bool = True, log_stderr: bool = True, + stdout_dest: Path | None = None, ) -> shell.CommandResult: """Run an oc command. @@ -33,6 +34,7 @@ def oc( timeout_seconds: Command timeout (not supported) log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) + stdout_dest: Optional file path to write stdout to Returns: CommandResult with execution details @@ -45,6 +47,7 @@ def oc( shell=False, log_stdout=log_stdout, log_stderr=log_stderr, + stdout_dest=stdout_dest, ) From c9d19f4a5893684f6ab72705f80449286c4759df Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:22:51 +0200 Subject: [PATCH 161/344] [core] dsl: utils: k8s: use stdout_dest --- projects/core/dsl/utils/k8s.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 62a7f0985..82a283384 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -269,6 +269,7 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: logger.info("Capturing logs from %d pods in %s", len(pod_names), namespace) for pod_name in pod_names: + log_file = output_dir / f"{pod_name}.log" log_result = oc( "logs", pod_name, @@ -277,11 +278,10 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: "--all-containers=true", check=False, log_stdout=False, + stdout_dest=log_file, ) - log_file = output_dir / f"{pod_name}.log" - if log_result.returncode == 0 and log_result.stdout: - log_file.write_text(log_result.stdout, encoding="utf-8") - else: + # If command failed and no output was written, write error info + if log_result.returncode != 0 and (not log_file.exists() or log_file.stat().st_size == 0): log_file.write_text( f"(failed to collect logs: exit_code={log_result.returncode})\n" f"{log_result.stderr or ''}", From 5caaa6ca1312c4ac1943c56ee09dc4ef2f4f10be Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:23:46 +0200 Subject: [PATCH 162/344] [agentic_tools] mcp: toolbox: deploy_mock_servers/main: use stdout_dest --- .../agentic_tools/mcp/toolbox/deploy_mock_servers/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py index 6b65ecb11..19baedb65 100644 --- a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py +++ b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py @@ -292,6 +292,4 @@ def _generate_server_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) From 5a637299aad9ea1697d9785a53864f7acaa75ade Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:24:05 +0200 Subject: [PATCH 163/344] [guidellm] toolbox: run_guidellm_benchmark: use stdout_dest --- .../toolbox/run_guidellm_benchmark/main.py | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 91cbcc445..61f6cabc4 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -393,19 +393,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name ) # Capture job logs - result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job.logs", ) - if result.returncode == 0 and result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job.logs", result.stdout) # Capture additional debugging info - pods_result = oc( + oc( "get", "pods", "-n", @@ -415,11 +414,10 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_pods.yaml", ) - if pods_result.returncode == 0 and pods_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_pods.yaml", pods_result.stdout) - job_result = oc( + oc( "get", "job", benchmark_name, @@ -428,20 +426,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_detailed.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_detailed.yaml", job_result.stdout) - logs_result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_logs.txt", ) - if logs_result.returncode == 0 and logs_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_logs.txt", logs_result.stdout) def capture_get( @@ -460,9 +456,7 @@ def capture_get( if selector: args.extend(["-l", selector]) args.extend(["-o", output]) - result = oc(*args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + oc(*args, check=False, log_stdout=False, stdout_dest=destination) if __name__ == "__main__": From 9cbfac0208e97f77aa893cb502788da484198635 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:24:39 +0200 Subject: [PATCH 164/344] [agentic_tools] locust: toolbox: generate_prompts/main: use stdout_dest --- .../agentic_tools/locust/toolbox/generate_prompts/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py index c70ed336b..78504f9be 100644 --- a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py +++ b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py @@ -237,16 +237,15 @@ def capture_logs(args, ctx): artifacts_dir = env.ARTIFACT_DIR / "artifacts" / "prompts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc( + oc( "logs", f"job/{GENERATOR_JOB_NAME}", "-n", args.namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "generator.log", ) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "generator.log").write_text(result.stdout, encoding="utf-8") return "Logs captured" From 9790aa06663cc44f28a386bca39fd96bccda66c4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:25:10 +0200 Subject: [PATCH 165/344] [kserve] toolbox: prepare_hf_model_cache: use oc stdout_dest --- .../toolbox/prepare_hf_model_cache/main.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index 93c747a80..572ec43dc 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -11,7 +11,6 @@ slugify_identifier, truncate_k8s_name, write_json, - write_text, write_yaml, ) from projects.core.dsl.utils.k8s import ( @@ -365,7 +364,7 @@ def capture_download_artifacts(args, ctx): ) # Capture PVC YAML to file (not logs) - pvc_result = oc( + oc( "get", "persistentvolumeclaim", cache_spec["pvc_name"], @@ -376,12 +375,11 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "pvc.yaml", ) - if pvc_result.returncode == 0 and pvc_result.stdout: - write_text(artifact_dir / "pvc.yaml", pvc_result.stdout) # Capture Job YAML to file (not logs) - job_result = oc( + oc( "get", "job", cache_spec["download_job_name"], @@ -392,14 +390,13 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "job.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifact_dir / "job.yaml", job_result.stdout) # Capture pod logs and YAML to files (not console) for pod_name in job_pod_names(cache_spec["download_job_name"], cache_spec["namespace"]): # Get pod YAML to file - pod_result = oc( + oc( "get", "pod", pod_name, @@ -410,21 +407,19 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.yaml", ) - if pod_result.returncode == 0 and pod_result.stdout: - write_text(artifact_dir / f"{pod_name}.yaml", pod_result.stdout) # Get pod logs to file - log_result = oc( + oc( "logs", pod_name, "-n", cache_spec["namespace"], check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.log", ) - if log_result.returncode == 0 and log_result.stdout: - write_text(artifact_dir / f"{pod_name}.log", log_result.stdout) return f"Artifacts captured for {cache_spec['pvc_name']}" From 101b4b88b3f80939db5d6aa134b673d51f9afcc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:25:42 +0200 Subject: [PATCH 166/344] [llm_d] toolbox: capture_prepare_state: use stdout_dest --- .../llm_d/toolbox/capture_prepare_state/main.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/toolbox/capture_prepare_state/main.py b/projects/llm_d/toolbox/capture_prepare_state/main.py index 8d38c8783..f1e7a1248 100644 --- a/projects/llm_d/toolbox/capture_prepare_state/main.py +++ b/projects/llm_d/toolbox/capture_prepare_state/main.py @@ -6,7 +6,6 @@ from pathlib import Path from projects.core.dsl import entrypoint, execute_tasks, task -from projects.core.dsl.utils import write_text from projects.core.dsl.utils.k8s import oc logger = logging.getLogger(__name__) @@ -65,10 +64,10 @@ def capture_datasciencecluster_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured DataScienceCluster: {destination}" else: logger.warning("Failed to capture DataScienceCluster: %s", result.stderr) @@ -91,10 +90,10 @@ def capture_gateway_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway: {destination}" else: logger.warning("Failed to capture Gateway: %s", result.stderr) @@ -117,10 +116,10 @@ def capture_gateway_service_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway service: {destination}" else: logger.warning("Failed to capture Gateway service: %s", result.stderr) @@ -144,10 +143,10 @@ def capture_namespace_events(args, ctx): "--sort-by=.metadata.creationTimestamp", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured namespace events: {destination}" else: logger.warning("Failed to capture namespace events: %s", result.stderr) From d032774cfbc33d8df9eac057c4f1bdd00f92b13d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:26:07 +0200 Subject: [PATCH 167/344] [mcp_gateway] toolbox: use stdout_dest --- .../toolbox/apply_infrastructure/main.py | 4 +-- .../toolbox/cleanup_platform/main.py | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py index 353298467..062efe707 100644 --- a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py +++ b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py @@ -253,6 +253,4 @@ def _generate_infrastructure_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) diff --git a/projects/mcp_gateway/toolbox/cleanup_platform/main.py b/projects/mcp_gateway/toolbox/cleanup_platform/main.py index bbfe8ced2..f28860a0a 100644 --- a/projects/mcp_gateway/toolbox/cleanup_platform/main.py +++ b/projects/mcp_gateway/toolbox/cleanup_platform/main.py @@ -200,13 +200,26 @@ def capture_cleanup_state(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc("get", "csv", "-A", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-csvs.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "csv", + "-A", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-csvs.txt", + ) - result = oc("get", "namespaces", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-namespaces.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "namespaces", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-namespaces.txt", + ) return "Captured post-cleanup state" From 83cdf21ef40a1dbcbc5b1d134397c79e9b13d0af Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:26:31 +0200 Subject: [PATCH 168/344] [mcp_gateway] toolbox: install_platform: use stdout_dest --- projects/mcp_gateway/toolbox/install_platform/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index fcf5c5d5c..623b01fc3 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -524,9 +524,7 @@ def _get_mcp_host() -> str: def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) def _parse_version(version: str) -> tuple[int, ...]: From 885f6185656403b225d90ef9808a40506fa3062f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 17:40:00 +0200 Subject: [PATCH 169/344] [core] Add the config changelog --- projects/core/CHANGELOG-config.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 projects/core/CHANGELOG-config.md diff --git a/projects/core/CHANGELOG-config.md b/projects/core/CHANGELOG-config.md new file mode 100644 index 000000000..3fbf66f62 --- /dev/null +++ b/projects/core/CHANGELOG-config.md @@ -0,0 +1,17 @@ +# Configuration Framework Changelog + +## 2026-07-16 - Variables Override Helper + +### New Features +- **Variables Override Helper**: Added `write_variables_override()` function for programmatic configuration override generation + - **Behavior**: Creates variables_override.yaml files without requiring full project config initialization + - **Usage**: Supports both preset configuration (`project.args`) and additional variable overrides + - **Error Handling**: Validates `env.ARTIFACT_DIR` initialization with clear error messages + +### Files Modified +- `projects/core/library/config.py` - Added `write_variables_override()` helper function + +### Benefits +- **Early Configuration**: Enables configuration override before project initialization +- **Flexible Override Structure**: Supports both preset lists and arbitrary configuration variables +- **Robust Error Handling**: Fails fast with clear messages when environment not properly initialized From 80b3ac2faec268d2267b069a0e63e5cc2a314285 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:24:34 +0200 Subject: [PATCH 170/344] [llm_d] start working on P/D testing --- projects/llm_d/docs/PD_TESTING.md | 381 ++++++++++++++++++ .../orchestration/config.d/deployments.yaml | 42 ++ .../manifests/deployments/pd-scheduler.yaml | 86 ++++ .../orchestration/presets.d/presets.yaml | 99 ++++- .../llm_d/orchestration/runtime_config.py | 29 ++ 5 files changed, 633 insertions(+), 4 deletions(-) create mode 100644 projects/llm_d/docs/PD_TESTING.md create mode 100644 projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml diff --git a/projects/llm_d/docs/PD_TESTING.md b/projects/llm_d/docs/PD_TESTING.md new file mode 100644 index 000000000..b62c752d5 --- /dev/null +++ b/projects/llm_d/docs/PD_TESTING.md @@ -0,0 +1,381 @@ +# P/D (Prefill/Decode) Testing in llm-d + +This document describes the P/D testing capabilities in the llm-d project, which allows testing various Prefill/Decode configurations with flexible pod counts, tensor parallelism, and scheduler configurations using profile name-based parameter extraction. + +## Overview + +P/D testing enables you to: +- Configure the number of prefill and decode pods via profile names +- Set different tensor parallelism for prefill vs decode pods +- Test various scheduler configurations +- Run matrix tests across multiple P/D configurations +- Use template-based profiles with automatic parameter extraction + +## P/D Configuration Architecture + +### Profile Name-Based Configuration + +P/D deployments use profile names to encode configuration parameters: + +**Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + +**Examples**: +- `pd-d.x2-p.tp1-d.tp4-p.x8` - 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-d.x1-p.tp2-d.tp8-p.x4` - 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +### Global P/D Configuration + +P/D-specific settings are centralized in `deployments.yaml`: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Template-Based Profiles + +Template profiles use `FROM_NAME` placeholders for automatic parameter extraction: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME # extracted from profile name + pd_config: + prefill_pods: FROM_NAME # extracted from p.x{N} pattern + decode_pods: FROM_NAME # extracted from d.x{N} pattern + scheduler_config: "high-throughput" +``` + +## P/D Profile Examples + +### Small Scale P/D +```yaml +# Profile: pd-d.x1-p.tp1-d.tp2-p.x2 +# 1 decode pod with 2 GPUs, 2 prefill pods with 1 GPU each +deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 +``` + +### Medium Scale P/D +```yaml +# Profile: pd-d.x2-p.tp2-d.tp4-p.x4 +# 2 decode pods with 4 GPUs each, 4 prefill pods with 2 GPUs each +deployment_profile: pd-d.x2-p.tp2-d.tp4-p.x4 +``` + +### Large Scale P/D +```yaml +# Profile: pd-d.x4-p.tp4-d.tp8-p.x8 +# 4 decode pods with 8 GPUs each, 8 prefill pods with 4 GPUs each +deployment_profile: pd-d.x4-p.tp4-d.tp8-p.x8 +``` + +## P/D Testing with Presets + +P/D testing integrates with the existing preset system. Use presets that include P/D profile configurations: + +### Custom P/D Testing +```bash +llm_d ci --preset your-pd-preset +``` + +Where your preset contains P/D profile configurations in `presets.yaml`: + +```yaml +your-pd-preset: + runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 + benchmark_key: short +``` + +## Custom P/D Testing via PR Comments + +You can override P/D configurations using PR comment directives: + +### Test Specific P/D Configurations +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +/var runtime.model_name: meta-llama/Llama-3.1-8B-Instruct +/var runtime.benchmark_key: short +``` + +### Test Different P/D Scales +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp4-p.x1, pd-d.x4-p.tp4-d.tp8-p.x8] +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.benchmark_key: [short, concurrent-1k-1k] +``` + +### Mix P/D and Traditional Profiles +``` +/var runtime.deployment_profile: [pd-d.x2-p.tp1-d.tp4-p.x8, approximate-prefix-cache, precise-prefix-cache] +/var runtime.model_name: Qwen/Qwen3-0.6B +``` + +## Parameter Extraction Rules + +The system extracts parameters from P/D profile names using these patterns: + +### Tensor Parallelism +- **Main container (decode)**: `d.tp{N}` or `tp{N}` → N GPUs per decode pod +- **Prefill container**: `p.tp{N}` → N GPUs per prefill pod + +### Pod Replicas +- **Main container (decode)**: `d.x{N}` or `x{N}` → N decode pods +- **Prefill container**: `p.x{N}` → N prefill pods + +### Examples +- `pd-d.x2-p.tp1-d.tp4-p.x8`: 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-tp8-x1-p.tp2-p.x4`: 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +## Runtime Configuration Functions + +New runtime configuration functions are available for P/D testing: + +```python +from projects.llm_d.orchestration import runtime_config + +# Get P/D configuration +pd_config = runtime_config.get_pd_config() +# Returns: {'prefill_pods': 2, 'decode_pods': 4, 'scheduler_config': 'high-throughput'} + +# Get individual P/D parameters +prefill_pods = runtime_config.get_prefill_pod_count() # Returns: 2 +decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 +scheduler_config = runtime_config.get_scheduler_config() # Returns: 'high-throughput' + +# Check if current deployment is P/D +is_pd = runtime_config.is_pd_deployment() # Returns: True for P/D profiles +``` + +## Test Output and Artifacts + +P/D tests generate the same artifacts as standard llm-d tests, with additional P/D-specific metadata: + +### Test Labels +```yaml +model_name: meta-llama/Llama-3.1-8B-Instruct +deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 +guidellm_loadshape: short +pd_prefill_pods: 8 # Extracted from p.x8 +pd_decode_pods: 2 # Extracted from d.x2 +pd_prefill_tensor_parallelism: 1 # Extracted from p.tp1 +pd_decode_tensor_parallelism: 4 # Extracted from d.tp4 +pd_scheduler_config: high-throughput +``` + +### Artifact Directory Structure +``` +ARTIFACT_DIR/ +├── llmd_test/ +│ ├── __test_labels__.yaml +│ ├── artifacts/ +│ │ ├── llminferenceservice.yaml +│ │ ├── endpoint.url +│ │ └── results/ +│ │ └── benchmark_short/ +├── prepare_llmd_run_pd_d_x2_p_tp1_d_tp4_p_x8_meta_llama_instruct/ +└── ... +``` + +### LLMISVC Manifest Annotations +P/D deployments include profile metadata in annotations: + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + namespace: forge-llm-d + annotations: + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 +spec: + replicas: 2 # decode pods + prefill: + replicas: 8 # prefill pods + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' # prefill tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + template: # decode template + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' # decode tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' +``` + +## Implementation Details + +### Template Profile Configuration +P/D template profiles are defined in `orchestration/config.d/deployments.yaml`: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME + pd_config: + prefill_pods: FROM_NAME + decode_pods: FROM_NAME + scheduler_config: "high-throughput" +``` + +### Global P/D Configuration +P/D-specific VLLM args, environment variables, and resources: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Parameter Extraction +The `_extract_value_from_profile_name()` function extracts values using regex patterns: + +```python +# For tensor_parallelism (main container) +match = re.search(r"d\.tp(\d+)", profile_name) # d.tp4 → 4 +if not match: + match = re.search(r"(? my-pd-preset.yaml << EOF +runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 + benchmark_key: short +EOF + +llm_d ci --preset my-pd-preset +``` + +### Matrix P/D Testing +```bash +# Use PR comment for matrix testing: +# /var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +# /var runtime.model_name: [Qwen/Qwen3-0.6B, meta-llama/Llama-3.1-8B-Instruct] +# /var runtime.benchmark_key: short + +llm_d ci # Will use PR directive overrides +``` + +### Custom P/D with Different GPU Allocation +```bash +# Test asymmetric GPU allocation: many small prefill pods, few large decode pods +# Profile: pd-d.x1-p.tp1-d.tp8-p.x16 +# 1 decode pod with 8 GPUs, 16 prefill pods with 1 GPU each + +cat > asymmetric-pd.yaml << EOF +runtime: + model_name: meta-llama/Llama-3.1-8B-Instruct + deployment_profile: pd-d.x1-p.tp1-d.tp8-p.x16 + benchmark_key: concurrent-1k-1k +EOF + +llm_d ci --preset asymmetric-pd +``` + +### Validating P/D Profile Names +```bash +# Test profile name parsing +python3 -c " +from projects.llm_d.orchestration.runtime_config import _extract_value_from_profile_name + +profile = 'pd-d.x2-p.tp1-d.tp4-p.x8' +print(f'Decode pods: {_extract_value_from_profile_name(profile, \"decode_pods\")}') +print(f'Prefill pods: {_extract_value_from_profile_name(profile, \"prefill_pods\")}') +print(f'Decode TP: {_extract_value_from_profile_name(profile, \"tensor_parallelism\")}') +print(f'Prefill TP: {_extract_value_from_profile_name(profile, \"prefill_tensor_parallelism\")}') +" +``` + +This P/D testing system provides flexible, template-based configuration with automatic parameter extraction from profile names, enabling sophisticated testing of Prefill/Decode configurations at scale. diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index cc74273d8..9c8d1b8ff 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -16,3 +16,45 @@ precise-prefix-cache: distributed-default: scheduler: {} + +# P/D (Prefill/Decode) deployment profiles +pd-small: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 2 + decode_pods: 4 + scheduler_config: "high-throughput" + +pd-medium: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 4 + decode_pods: 8 + scheduler_config: "balanced" + +pd-large: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 8 + decode_pods: 16 + scheduler_config: "low-latency" + +# New flavor configurations +pd-x2-ptp4-px1-dtp4: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + tensor_parallelism: 4 + pd_config: + prefill_pods: 1 # px1 + decode_pods: 4 # dtp4 + scheduler_config: "high-throughput" + +intelligentrouting-tp4-x4: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml + tensor_parallelism: 4 + replicas: 4 + scheduler: {} # scheduler enabled with default config + +simple-tp4-x4: + tensor_parallelism: 4 + replicas: 4 + # No scheduler_manifest means scheduler disabled diff --git a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml new file mode 100644 index 000000000..8d6c7e2cd --- /dev/null +++ b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml @@ -0,0 +1,86 @@ +scheduler: + template: + containers: + - name: main + env: + - name: TOKENIZER_CACHE_DIR + value: /tmp/tokenizer-cache + - name: HF_HOME + value: /tmp/tokenizer-cache + - name: TRANSFORMERS_CACHE + value: /tmp/tokenizer-cache + - name: XDG_CACHE_HOME + value: /tmp + # P/D specific environment variables + - name: PD_PREFILL_PODS + value: "{{ deployment_profile.pd_config.prefill_pods }}" + - name: PD_DECODE_PODS + value: "{{ deployment_profile.pd_config.decode_pods }}" + - name: PD_SCHEDULER_CONFIG + value: "{{ deployment_profile.pd_config.scheduler_config }}" + args: + - --cert-path + - /var/run/kserve/tls + - --pool-group + - inference.networking.x-k8s.io + - --pool-name + - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" + - --pool-namespace + - "{{ .ObjectMeta.Namespace }}" + - --zap-encoder + - json + - --grpc-port + - "9002" + - --grpc-health-port + - "9003" + - --secure-serving + - --model-server-metrics-scheme + - https + - --config-text + - | + apiVersion: inference.networking.x-k8s.io/v1alpha1 + kind: EndpointPickerConfig + plugins: + - type: pd-prefill-scheduler + config: + prefillPods: {{ deployment_profile.pd_config.prefill_pods }} + schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" + - type: pd-decode-scheduler + config: + decodePods: {{ deployment_profile.pd_config.decode_pods }} + schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" + - type: queue-scorer + - type: kv-cache-utilization-scorer + schedulingProfiles: + - name: prefill-profile + plugins: + - pluginRef: pd-prefill-scheduler + weight: 5 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - name: decode-profile + plugins: + - pluginRef: pd-decode-scheduler + weight: 5 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - name: default + plugins: + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + volumeMounts: + - name: tokenizer-cache + mountPath: /tmp/tokenizer-cache + - name: cachi2-cache + mountPath: /cachi2 + volumes: + - name: tokenizer-cache + emptyDir: {} + - name: cachi2-cache + emptyDir: {} diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 34d0609e2..169664efc 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -45,19 +45,110 @@ benchmark-short: runtime.smoke_request_key: default runtime.benchmark_key: short -gpt-oss-120b-inference-scheduling-release: - runtime.selected_preset: gpt-oss-120b-inference-scheduling-release +# Base gpt-oss configuration that can be extended +gpt-oss: runtime.model_name: openai/gpt-oss-120b runtime.deployment_profile: distributed-default runtime.smoke_request_key: default + model_cache.pvc.size: 300Gi + workloads.args: + request_type: text_completions + +gpt-oss-120b-inference-scheduling-release: + extends: + - gpt-oss + runtime.selected_preset: gpt-oss-120b-inference-scheduling-release runtime.benchmark_key: - concurrent-1k-1k - heavy-heterogeneous - multi-turn + +cks: + extends: + - smoke + +# P/D deployment profile presets +deployment-pd-small: + runtime.selected_preset: deployment-pd-small + runtime.deployment_profile: pd-small + +deployment-pd-medium: + runtime.selected_preset: deployment-pd-medium + runtime.deployment_profile: pd-medium + +deployment-pd-large: + runtime.selected_preset: deployment-pd-large + runtime.deployment_profile: pd-large + +# P/D testing presets +pd-smoke: + extends: + - smoke-base + - deployment-pd-small + runtime.selected_preset: pd-smoke + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-smoke-medium: + extends: + - smoke-base + - deployment-pd-medium + runtime.selected_preset: pd-smoke-medium + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-smoke-large: + extends: + - smoke-base + - deployment-pd-large + runtime.selected_preset: pd-smoke-large + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-matrix: + runtime.selected_preset: pd-matrix + runtime.model_name: + - meta-llama/Llama-3.1-8B-Instruct + - Qwen/Qwen3-0.6B + runtime.deployment_profile: + - pd-small + - pd-medium + - pd-large + runtime.benchmark_key: + - short + - concurrent-1k-1k + runtime.smoke_request_key: default + platform.gateway.status_address_name: gateway-internal + +pd-scale-test: + runtime.selected_preset: pd-scale-test + runtime.model_name: openai/gpt-oss-120b + runtime.deployment_profile: + - pd-small + - pd-medium + - pd-large + runtime.benchmark_key: heavy-heterogeneous + runtime.smoke_request_key: default model_cache.pvc.size: 300Gi workloads.args: request_type: text_completions -cks: +# New presets ported from other tool +pd-flavors: extends: - - smoke + - gpt-oss + runtime.selected_preset: pd-flavors + runtime.deployment_profile: + - pd-x2-ptp4-px1-dtp4 + - intelligentrouting-tp4-x4 + - simple-tp4-x4 + +guidellm_multiturn_eval: + runtime.selected_preset: guidellm_multiturn_eval + runtime.benchmark_key: guidellm_multiturn_eval + runtime.smoke_request_key: default + +guidellm_heterogeneous_eval: + runtime.selected_preset: guidellm_heterogeneous_eval + runtime.benchmark_key: guidellm_heterogeneous_eval + runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index dabbe22af..c9f1c5513 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -299,6 +299,35 @@ def get_deployment_profile() -> dict[str, Any]: return resolved_profile +def get_pd_config() -> dict[str, Any]: + """Get P/D configuration from active deployment profile.""" + profile = get_deployment_profile() + return profile.get("pd_config", {}) + + +def get_prefill_pod_count() -> int: + """Get number of prefill pods for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("prefill_pods", 1) + + +def get_decode_pod_count() -> int: + """Get number of decode pods for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("decode_pods", 1) + + +def get_scheduler_config() -> str: + """Get scheduler configuration for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("scheduler_config", "default") + + +def is_pd_deployment() -> bool: + """Check if current deployment profile is a P/D deployment.""" + return bool(get_pd_config()) + + def get_smoke_request() -> dict[str, Any]: """Get the smoke request configuration""" smoke_request_key = config.project.get_config("runtime.smoke_request_key") From 545b173e61900ab6c7fde23cdeb7507afc4eb4da Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 17:36:33 +0200 Subject: [PATCH 171/344] [llm_d] orchestration: render_inference_service: add support for P/D --- .../orchestration/render_inference_service.py | 186 +++++++++++++++--- projects/llm_d/orchestration/test_phase.py | 2 + 2 files changed, 165 insertions(+), 23 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index e823a867d..6a522b7f6 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -1,6 +1,5 @@ from __future__ import annotations -import copy import hashlib from pathlib import Path from typing import Any @@ -24,11 +23,15 @@ def render_inference_service_from_parts( model_slug: str, deployment_profile: dict[str, Any], model_cache: dict[str, Any], + deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render an llm_d-owned LLMInferenceService manifest from concrete runtime inputs.""" template_path = Path(config_dir) / inference_service["template"] manifest = _load_yaml(template_path) + # Check if this is a P/D deployment + is_pd_deployment = "pd_config" in deployment_profile + name = inference_service["name"] manifest["metadata"]["name"] = name manifest["metadata"]["namespace"] = namespace @@ -78,32 +81,13 @@ def render_inference_service_from_parts( "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), } - manifest["spec"]["replicas"] = deployment_profile["replicas"] manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug - serving_container = manifest["spec"]["template"]["containers"][0] - serving_container["resources"] = _build_serving_resources(deployment_profile) - if deployment_profile.get("serving_image"): - serving_container["image"] = deployment_profile["serving_image"] - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args - - scheduler = deployment_profile.get("scheduler", {}) - if scheduler is None: - manifest["spec"]["router"].pop("scheduler", None) + if is_pd_deployment: + return _render_pd_deployment(manifest, deployment_profile, deployment_profile_name) else: - manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) - if deployment_profile.get("router_image"): - manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( - deployment_profile["router_image"] - ) - - return manifest + return _render_standard_deployment(manifest, deployment_profile) def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: @@ -142,3 +126,159 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: prefix = f"--{option_name}=" bare = f"--{option_name}" return any(arg == bare or arg.startswith(prefix) for arg in args) + + +def _render_standard_deployment( + manifest: dict[str, Any], deployment_profile: dict[str, Any] +) -> dict[str, Any]: + """Render standard (non-P/D) deployment configuration.""" + manifest["spec"]["replicas"] = deployment_profile["replicas"] + + serving_container = manifest["spec"]["template"]["containers"][0] + serving_container["resources"] = _build_serving_resources(deployment_profile) + if deployment_profile.get("serving_image"): + serving_container["image"] = deployment_profile["serving_image"] + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + if vllm_args: + serving_container["args"] = vllm_args + + scheduler = deployment_profile.get("scheduler", {}) + if scheduler is None: + manifest["spec"]["router"].pop("scheduler", None) + else: + manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) + if deployment_profile.get("router_image"): + manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( + deployment_profile["router_image"] + ) + + return manifest + + +def _render_pd_deployment( + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, +) -> dict[str, Any]: + """Render P/D (Prefill/Decode) deployment configuration.""" + pd_config = deployment_profile["pd_config"] + + # Update service name to include deployment profile for P/D + current_name = manifest["metadata"]["name"] + if current_name == "llm-d" and deployment_profile_name: # Only modify if it's the default name + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + + # Set main replicas to 2 for P/D + manifest["spec"]["replicas"] = 2 + + # Configure prefill section + manifest["spec"]["prefill"] = { + "replicas": 2, + "template": _build_pd_pod_template(deployment_profile, is_prefill=True), + } + + # Configure main template (decode) + manifest["spec"]["template"] = _build_pd_pod_template(deployment_profile, is_prefill=False) + + # Simplified router for P/D + manifest["spec"]["router"] = { + "scheduler": {"template": {"containers": [{"name": "main"}]}}, + "route": {}, + "gateway": {}, + } + + return manifest + + +def _build_pd_pod_template( + deployment_profile: dict[str, Any], is_prefill: bool = False +) -> dict[str, Any]: + """Build pod template for P/D deployment.""" + tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + + # Build VLLM environment variable configuration specific to prefill/decode + if is_prefill: + # Prefill configuration + vllm_args_list = [ + "--disable-uvicorn-access-log", + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", # Appears twice in prefill + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len=40960", + ] + else: + # Decode configuration + vllm_args_list = [ + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", # Space instead of equals for decode + ] + + vllm_additional_args = " ".join(vllm_args_list) + + container = { + "name": "main", + "env": [ + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, + { + "name": "VLLM_NIXL_SIDE_CHANNEL_HOST", + "valueFrom": {"fieldRef": {"apiVersion": "v1", "fieldPath": "status.podIP"}}, + }, + {"name": "UCX_IB_GID_INDEX", "value": "3"}, + ], + "livenessProbe": { + "failureThreshold": 1000, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "initialDelaySeconds": 900, + "periodSeconds": 60, + "timeoutSeconds": 60, + }, + "readinessProbe": { + "failureThreshold": 10000, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "initialDelaySeconds": 60, + "periodSeconds": 30, + "successThreshold": 1, + "timeoutSeconds": 30, + }, + "resources": { + "limits": {"dra.llm-d.io/gpu-nic-pair": tensor_parallelism}, + "requests": { + "cpu": "4", + "dra.llm-d.io/gpu-nic-pair": tensor_parallelism, + "memory": "64Gi", + }, + }, + "securityContext": {"capabilities": {"add": ["IPC_LOCK", "SYS_RAWIO"]}}, + "startupProbe": { + "failureThreshold": 150, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1, + }, + } + + template = {"serviceAccountName": "llm-d-privileged", "containers": [container]} + + # Add tolerations only to main template (decode), not prefill + if not is_prefill: + template["tolerations"] = [ + {"effect": "NoSchedule", "key": "nvidia.com/gpu", "operator": "Exists"} + ] + + return template diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2a68c2070..0fa44b436 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -249,6 +249,7 @@ def _build_inference_service_manifest() -> Path: deployment_profile = runtime_config.deep_merge(deployment_profile, benchmark_overrides) # Build the InferenceService manifest + deployment_profile_name = runtime_config.get_deployment_profile_name() manifest = render_inference_service_from_parts( config_dir=config_dir, namespace=namespace, @@ -257,6 +258,7 @@ def _build_inference_service_manifest() -> Path: model_slug=model_slug, deployment_profile=deployment_profile, model_cache=model_cache, + deployment_profile_name=deployment_profile_name, ) # Write the manifest to artifacts From b6cf2c87ea6d2156669b6e552e49ce273a566e78 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 17 Jul 2026 12:46:46 +0200 Subject: [PATCH 172/344] [llm_d] orchestration: work on the P/D deployment --- .../orchestration/config.d/deployments.yaml | 3 + .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../orchestration/presets.d/presets.yaml | 38 +++ .../orchestration/render_inference_service.py | 270 +++++++++++++++--- .../llm_d/orchestration/runtime_config.py | 18 +- 5 files changed, 288 insertions(+), 42 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 9c8d1b8ff..85cb9100c 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -43,6 +43,7 @@ pd-large: pd-x2-ptp4-px1-dtp4: scheduler_manifest: manifests/deployments/pd-scheduler.yaml tensor_parallelism: 4 + use_defaults: true pd_config: prefill_pods: 1 # px1 decode_pods: 4 # dtp4 @@ -52,9 +53,11 @@ intelligentrouting-tp4-x4: scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml tensor_parallelism: 4 replicas: 4 + use_defaults: true scheduler: {} # scheduler enabled with default config simple-tp4-x4: tensor_parallelism: 4 replicas: 4 + use_defaults: true # No scheduler_manifest means scheduler disabled diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 95ac92ebb..d125a4bd9 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -6,3 +6,4 @@ benchmark_key: null job_name: null namespace_override: null kserve_dry_run: false +kserve_use_kueue: false diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 169664efc..d8af640ac 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -133,6 +133,44 @@ pd-scale-test: workloads.args: request_type: text_completions +# New deployment profiles +deployment-intelligentrouting-tp4-x4: + runtime.selected_preset: deployment-intelligentrouting-tp4-x4 + runtime.deployment_profile: intelligentrouting-tp4-x4 + +deployment-simple-tp4-x4: + runtime.selected_preset: deployment-simple-tp4-x4 + runtime.deployment_profile: simple-tp4-x4 + +deployment-pd-x2-ptp4-px1-dtp4: + runtime.selected_preset: deployment-pd-x2-ptp4-px1-dtp4 + runtime.deployment_profile: pd-x2-ptp4-px1-dtp4 + +# Smoke presets for new profiles +smoke-intelligentrouting: + extends: + - smoke-base + - deployment-intelligentrouting-tp4-x4 + runtime.selected_preset: smoke-intelligentrouting + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +smoke-simple-tp4: + extends: + - smoke-base + - deployment-simple-tp4-x4 + runtime.selected_preset: smoke-simple-tp4 + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +smoke-pd-tp4: + extends: + - smoke-base + - deployment-pd-x2-ptp4-px1-dtp4 + runtime.selected_preset: smoke-pd-tp4 + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + # New presets ported from other tool pd-flavors: extends: diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 6a522b7f6..a31b4fdb5 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import hashlib from pathlib import Path from typing import Any @@ -7,6 +8,7 @@ import yaml from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.library import config def _load_yaml(path: Path) -> Any: @@ -85,9 +87,18 @@ def render_inference_service_from_parts( manifest["spec"]["model"]["name"] = model_slug if is_pd_deployment: - return _render_pd_deployment(manifest, deployment_profile, deployment_profile_name) + rendered_manifest = _render_pd_deployment( + manifest, deployment_profile, deployment_profile_name + ) else: - return _render_standard_deployment(manifest, deployment_profile) + rendered_manifest = _render_standard_deployment( + manifest, deployment_profile, deployment_profile_name + ) + + # Apply Kueue configuration if enabled + _apply_kueue_configuration(rendered_manifest) + + return rendered_manifest def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: @@ -129,32 +140,114 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: def _render_standard_deployment( - manifest: dict[str, Any], deployment_profile: dict[str, Any] + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render standard (non-P/D) deployment configuration.""" + # Check if this is intelligent routing (scheduler_manifest exists) + scheduler = deployment_profile.get("scheduler") + has_scheduler_manifest = "scheduler_manifest" in deployment_profile + is_intelligent_routing = scheduler is not None and has_scheduler_manifest + + # Update service name for intelligent routing or use_defaults + current_name = manifest["metadata"]["name"] + use_defaults = deployment_profile.get("use_defaults", False) + if ( + current_name == "llm-d" + and deployment_profile_name + and (is_intelligent_routing or use_defaults) + ): + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + manifest["spec"]["replicas"] = deployment_profile["replicas"] serving_container = manifest["spec"]["template"]["containers"][0] serving_container["resources"] = _build_serving_resources(deployment_profile) if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + + # Configure VLLM args based on deployment type and use_defaults flag tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args + use_defaults = deployment_profile.get("use_defaults", False) - scheduler = deployment_profile.get("scheduler", {}) - if scheduler is None: + if use_defaults: + # Use VLLM_ADDITIONAL_ARGS environment variable when use_defaults is set + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + + vllm_additional_args = " ".join(vllm_args) + + # Add environment variable (don't set generic env vars or args) + if "env" not in serving_container: + serving_container["env"] = [] + serving_container["env"].append( + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} + ) + elif is_intelligent_routing: + # Use environment variables for intelligent routing + vllm_args_list = [ + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", + ] + vllm_additional_args = " ".join(vllm_args_list) + + # Add or update environment variables + if "env" not in serving_container: + serving_container["env"] = [] + serving_container["env"].append( + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} + ) + else: + # Use CLI args for basic deployments + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + if vllm_args: + serving_container["args"] = vllm_args + + # Configure router/scheduler + has_scheduler_key = "scheduler" in deployment_profile + is_simple_deployment = not has_scheduler_key and not has_scheduler_manifest + + if is_simple_deployment: + # Simple deployments (no scheduler key, no scheduler_manifest) have no router section at all + manifest["spec"].pop("router", None) + elif scheduler is None: + # Some deployments might have router but no scheduler manifest["spec"]["router"].pop("scheduler", None) else: + # Configure scheduler for intelligent routing manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) if deployment_profile.get("router_image"): manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( deployment_profile["router_image"] ) + # Enhance scheduler for intelligent routing + if is_intelligent_routing: + # Ensure template structure exists + if "template" not in manifest["spec"]["router"]["scheduler"]: + manifest["spec"]["router"]["scheduler"]["template"] = { + "containers": [{"name": "main"}] + } + + scheduler_template = manifest["spec"]["router"]["scheduler"]["template"] + + # Add nodeSelector and serviceAccountName (if not already set by runtime_config.py) + if "nodeSelector" not in scheduler_template: + scheduler_template["nodeSelector"] = { + "nvidia.com/gpu.deploy.container-toolkit": "true" + } + if "serviceAccountName" not in scheduler_template: + scheduler_template["serviceAccountName"] = "llm-d-privileged" + return manifest @@ -164,7 +257,6 @@ def _render_pd_deployment( deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" - pd_config = deployment_profile["pd_config"] # Update service name to include deployment profile for P/D current_name = manifest["metadata"]["name"] @@ -198,37 +290,45 @@ def _build_pd_pod_template( ) -> dict[str, Any]: """Build pod template for P/D deployment.""" tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - - # Build VLLM environment variable configuration specific to prefill/decode - if is_prefill: - # Prefill configuration - vllm_args_list = [ - "--disable-uvicorn-access-log", - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", # Appears twice in prefill - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len=40960", - ] + use_defaults = deployment_profile.get("use_defaults", False) + + # Build VLLM environment variable configuration + if use_defaults: + # Use deployment profile's vllm_args when use_defaults is set + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + vllm_additional_args = " ".join(vllm_args) else: - # Decode configuration - vllm_args_list = [ - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", # Space instead of equals for decode - ] - - vllm_additional_args = " ".join(vllm_args_list) + # Use P/D-specific configuration when use_defaults is false + if is_prefill: + # Prefill configuration + vllm_args_list = [ + "--disable-uvicorn-access-log", + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", # Appears twice in prefill + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len=40960", + ] + else: + # Decode configuration + vllm_args_list = [ + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", # Space instead of equals for decode + ] + vllm_additional_args = " ".join(vllm_args_list) container = { "name": "main", @@ -282,3 +382,91 @@ def _build_pd_pod_template( ] return template + + +def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: + """Apply Kueue annotations and labels to the ISVC manifest. + + Based on the implementation from topsail's test_llmd.py. + Can be enabled by setting runtime.kserve_use_kueue config. + """ + # Check if kueue annotations should be enabled + enable_kueue = config.project.get_config("runtime.kserve_use_kueue", False) + + if not enable_kueue: + return + + # Default kueue configuration (can be made configurable later) + kueue_config = { + "enabled": True, + "prefix": "kueue.x-k8s.io/", + "labels": {"queue-name": "default-queue"}, + "annotations": {"queue-name": "default-queue"}, + } + + # Get prefix for kueue labels/annotations + kueue_prefix = kueue_config.get("prefix", "kueue.x-k8s.io/") + + # Ensure metadata sections exist + if "metadata" not in manifest: + manifest["metadata"] = {} + if "labels" not in manifest["metadata"]: + manifest["metadata"]["labels"] = {} + if "annotations" not in manifest["metadata"]: + manifest["metadata"]["annotations"] = {} + + # Apply Kueue labels + kueue_labels = kueue_config.get("labels", {}) + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + manifest["metadata"]["labels"][full_label_key] = label_value + + # Apply Kueue annotations + kueue_annotations = kueue_config.get("annotations", {}) + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + manifest["metadata"]["annotations"][full_annotation_key] = annotation_value + + # Apply Kueue annotations to router scheduler pod template if it exists + if ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ): + scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + + # Ensure metadata exists in scheduler template + if "metadata" not in scheduler_template: + scheduler_template["metadata"] = {} + if "annotations" not in scheduler_template["metadata"]: + scheduler_template["metadata"]["annotations"] = {} + + # Apply the same Kueue annotations to the scheduler pod template + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + + # Update the scheduler template back to the data structure + manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template + + # Calculate pod group total count: 1 scheduler + number of replicas + replicas = manifest.get("spec", {}).get("replicas", 1) + + # For P/D deployments, we need to account for prefill replicas too + prefill_replicas = 0 + if "spec" in manifest and "prefill" in manifest["spec"]: + prefill_replicas = manifest["spec"]["prefill"].get("replicas", 0) + + # Total: main replicas + prefill replicas + (1 scheduler if router exists) + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) + + scheduler_count = 1 if has_scheduler else 0 + pod_group_total_count = replicas + prefill_replicas + scheduler_count + + manifest["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index c9f1c5513..f261f514e 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -294,7 +294,23 @@ def get_deployment_profile() -> dict[str, Any]: resolved_profile = deep_merge(defaults, profile) if scheduler_manifest: scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - resolved_profile = deep_merge(resolved_profile, scheduler_data) + + # When use_defaults is true, use simplified scheduler template + use_defaults = resolved_profile.get("use_defaults", False) + if use_defaults: + # Create simplified scheduler with just basic container structure + simplified_scheduler = { + "scheduler": { + "template": { + "containers": [{"name": "main"}], + "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + "serviceAccountName": "llm-d-privileged", + } + } + } + resolved_profile = deep_merge(resolved_profile, simplified_scheduler) + else: + resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile From fc629cd74e9179f569adc87cd611325476488b3b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 17 Jul 2026 14:18:19 +0200 Subject: [PATCH 173/344] [llm_d] tests: add reference profiles --- .../llmisvc.yaml | 62 ++++++++ .../llmisvc.yaml | 140 ++++++++++++++++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 52 +++++++ .../llm_d/tests/test_deployment_profiles.py | 137 +++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml create mode 100644 projects/llm_d/tests/test_deployment_profiles.py diff --git a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..bf220304b --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -0,0 +1,62 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-intelligentrouting-tp4-x4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + router: + scheduler: + template: + containers: + - name: main + nodeSelector: + nvidia.com/gpu.deploy.container-toolkit: 'true' + serviceAccountName: llm-d-privileged + route: {} + gateway: {} + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 diff --git a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml new file mode 100644 index 000000000..c88a7140f --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml @@ -0,0 +1,140 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-x2-ptp4-px1-dtp4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 2 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + router: + scheduler: + template: + containers: + - name: main + route: {} + gateway: {} + template: + serviceAccountName: llm-d-privileged + containers: + - name: main + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: UCX_IB_GID_INDEX + value: '3' + livenessProbe: + failureThreshold: 1000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + resources: + limits: + dra.llm-d.io/gpu-nic-pair: '4' + requests: + cpu: '4' + dra.llm-d.io/gpu-nic-pair: '4' + memory: 64Gi + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RAWIO + startupProbe: + failureThreshold: 150 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + prefill: + replicas: 2 + template: + serviceAccountName: llm-d-privileged + containers: + - name: main + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: UCX_IB_GID_INDEX + value: '3' + livenessProbe: + failureThreshold: 1000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + resources: + limits: + dra.llm-d.io/gpu-nic-pair: '4' + requests: + cpu: '4' + dra.llm-d.io/gpu-nic-pair: '4' + memory: 64Gi + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RAWIO + startupProbe: + failureThreshold: 150 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..752c71d66 --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml @@ -0,0 +1,52 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-simple-tp4-x4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py new file mode 100644 index 000000000..3e0232375 --- /dev/null +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +from projects.core.library import config as core_config +from projects.core.library import env + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" +REFERENCE_DIR = Path(__file__).resolve().parent / "reference_deployments" + +# Deployment presets to test +DEPLOYMENT_PRESETS = [ + "deployment-simple-tp4-x4", + "deployment-intelligentrouting-tp4-x4", + "deployment-pd-x2-ptp4-px1-dtp4", +] + + +@pytest.fixture(autouse=True) +def _reset_project_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARTIFACT_DIR", str(tmp_path / "artifacts")) + # Unset KUBECONFIG as requested + monkeypatch.delenv("KUBECONFIG", raising=False) + env.init() + core_config.project = None + yield + core_config.project = None + + +@pytest.mark.parametrize("preset", DEPLOYMENT_PRESETS) +def test_deployment_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Test that deployment presets generate the expected LLMISVC manifests.""" + _test_preset_generates_expected_llmisvc(preset, tmp_path) + + +def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Helper function to test a preset generates the expected LLMISVC manifest.""" + # Set up environment + artifact_dir = tmp_path / "artifacts" / preset + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Create variable overrides to ensure dry-run mode + variable_overrides_path = artifact_dir / "000__ci_metadata" / "variable_overrides.yaml" + variable_overrides_path.parent.mkdir(parents=True, exist_ok=True) + variable_overrides_path.write_text( + yaml.safe_dump( + { + "runtime.kserve_dry_run": True, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + env_vars = os.environ.copy() + env_vars["ARTIFACT_DIR"] = str(artifact_dir) + env_vars.pop("KUBECONFIG", None) + + # Run the CI script with the preset + ci_script = PROJECT_ROOT / "projects" / "llm_d" / "orchestration" / "ci.py" + cmd = [str(ci_script), "--preset", preset, "test"] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + env=env_vars, + ) + + # Check that the command succeeded + if result.returncode != 0: + pytest.fail( + f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + + # Find the generated LLMISVC manifest + generated_manifest = _find_generated_llmisvc(artifact_dir) + if not generated_manifest.exists(): + pytest.fail(f"Generated LLMISVC manifest not found at {generated_manifest}") + + # Load the generated manifest + with generated_manifest.open("r", encoding="utf-8") as f: + generated_content = yaml.safe_load(f) + + # Load the reference manifest + reference_manifest = REFERENCE_DIR / preset / "llmisvc.yaml" + if not reference_manifest.exists(): + pytest.fail(f"Reference manifest not found at {reference_manifest}") + + with reference_manifest.open("r", encoding="utf-8") as f: + reference_content = yaml.safe_load(f) + + # Compare the manifests + if generated_content != reference_content: + # Generate a diff for better error reporting + import difflib + + generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) + reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) + + diff = list( + difflib.unified_diff( + reference_yaml.splitlines(keepends=True), + generated_yaml.splitlines(keepends=True), + fromfile=f"reference/{preset}/llmisvc.yaml", + tofile=f"generated/{preset}/llmisvc.yaml", + lineterm="", + ) + ) + + pytest.fail( + f"Generated LLMISVC does not match reference for preset {preset}:\n{''.join(diff)}" + ) + + +def _find_generated_llmisvc(artifact_dir: Path) -> Path: + """Find the generated LLMISVC manifest in the artifact directory.""" + # Look for the pattern: $ARTIFACT_DIR/**/deploy_llmisvc/src/llminferenceservice.yaml + deploy_llmisvc_dirs = list(artifact_dir.glob("**/*__deploy_llmisvc")) + + if not deploy_llmisvc_dirs: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") + + if len(deploy_llmisvc_dirs) > 1: + raise ValueError( + f"Multiple deploy_llmisvc directories found in {artifact_dir}: {deploy_llmisvc_dirs}" + ) + + llmisvc_path = deploy_llmisvc_dirs[0] / "src" / "llminferenceservice.yaml" + return llmisvc_path From 4badfcc16d43bcebdccfc952ca8cccfc285bce35 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 20 Jul 2026 13:47:13 +0200 Subject: [PATCH 174/344] [llm_d] orchestration: keep working on P/D --- .../orchestration/config.d/deployments.yaml | 52 ++-- .../llm_d/orchestration/config.d/runtime.yaml | 11 +- .../orchestration/presets.d/presets.yaml | 132 +---------- .../orchestration/render_inference_service.py | 108 +++++---- .../llm_d/orchestration/runtime_config.py | 222 +++++++++++++++++- .../llmisvc.yaml | 2 +- .../llmisvc.yaml | 4 +- .../deployment-simple-tp4-x4/llmisvc.yaml | 5 +- .../llm_d/tests/test_deployment_profiles.py | 206 ++++++++++++---- 9 files changed, 471 insertions(+), 271 deletions(-) rename projects/llm_d/tests/reference_deployments/{deployment-intelligentrouting-tp4-x4 => cpt-reference-flavors/deployment-intelligentrouting-tp4-x}/llmisvc.yaml (96%) rename projects/llm_d/tests/reference_deployments/{deployment-pd-x2-ptp4-px1-dtp4 => cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1}/llmisvc.yaml (97%) rename projects/llm_d/tests/reference_deployments/{ => cpt-reference-flavors}/deployment-simple-tp4-x4/llmisvc.yaml (93%) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 85cb9100c..e643c56c1 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -17,47 +17,25 @@ precise-prefix-cache: distributed-default: scheduler: {} -# P/D (Prefill/Decode) deployment profiles -pd-small: +# Template flavor configurations with name-based extraction +template__pd: scheduler_manifest: manifests/deployments/pd-scheduler.yaml + tensor_parallelism: FROM_NAME + use_kserve_defaults: true pd_config: - prefill_pods: 2 - decode_pods: 4 + prefill_pods: FROM_NAME + decode_pods: FROM_NAME scheduler_config: "high-throughput" -pd-medium: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - pd_config: - prefill_pods: 4 - decode_pods: 8 - scheduler_config: "balanced" - -pd-large: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - pd_config: - prefill_pods: 8 - decode_pods: 16 - scheduler_config: "low-latency" - -# New flavor configurations -pd-x2-ptp4-px1-dtp4: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - tensor_parallelism: 4 - use_defaults: true - pd_config: - prefill_pods: 1 # px1 - decode_pods: 4 # dtp4 - scheduler_config: "high-throughput" - -intelligentrouting-tp4-x4: +template__intelligentrouting: + use_kserve_defaults: true scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml - tensor_parallelism: 4 - replicas: 4 - use_defaults: true + tensor_parallelism: FROM_NAME + replicas: FROM_NAME scheduler: {} # scheduler enabled with default config -simple-tp4-x4: - tensor_parallelism: 4 - replicas: 4 - use_defaults: true - # No scheduler_manifest means scheduler disabled +template__simple: + tensor_parallelism: FROM_NAME + replicas: FROM_NAME + use_kserve_defaults: true + scheduler: null # scheduler disabled diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index d125a4bd9..73b527cf8 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,5 +5,12 @@ smoke_request_key: default benchmark_key: null job_name: null namespace_override: null -kserve_dry_run: false -kserve_use_kueue: false +kserve_dry_run: true +kueue: + enable: true + queue_name: default-queue +router: + template: + file: manifests/scheduler_config/__router_template__.yaml + placeholder: __ENDPOINT_PICKER_CONFIG__ + epp: manifests/scheduler_config diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index d8af640ac..d8ef284ee 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -1,15 +1,12 @@ __multiple: true deployment-approximate-prefix-cache: - runtime.selected_preset: deployment-approximate-prefix-cache runtime.deployment_profile: approximate-prefix-cache deployment-precise-prefix-cache: - runtime.selected_preset: deployment-precise-prefix-cache runtime.deployment_profile: precise-prefix-cache deployment-distributed-default: - runtime.selected_preset: deployment-distributed-default runtime.deployment_profile: distributed-default smoke-base: @@ -21,7 +18,6 @@ smoke: extends: - smoke-base - deployment-approximate-prefix-cache - runtime.selected_preset: smoke runtime.benchmark_key: short platform.gateway.status_address_name: gateway-internal @@ -29,23 +25,19 @@ smoke-precise: extends: - smoke-base - deployment-precise-prefix-cache - runtime.selected_preset: smoke-precise smoke-default-scheduler: extends: - smoke-base - deployment-distributed-default - runtime.selected_preset: smoke-default-scheduler benchmark-short: extends: - deployment-approximate-prefix-cache - runtime.selected_preset: benchmark-short runtime.model_name: meta-llama/Llama-3.1-8B-Instruct runtime.smoke_request_key: default runtime.benchmark_key: short -# Base gpt-oss configuration that can be extended gpt-oss: runtime.model_name: openai/gpt-oss-120b runtime.deployment_profile: distributed-default @@ -57,136 +49,26 @@ gpt-oss: gpt-oss-120b-inference-scheduling-release: extends: - gpt-oss - runtime.selected_preset: gpt-oss-120b-inference-scheduling-release runtime.benchmark_key: - concurrent-1k-1k - heavy-heterogeneous - multi-turn -cks: - extends: - - smoke - -# P/D deployment profile presets -deployment-pd-small: - runtime.selected_preset: deployment-pd-small - runtime.deployment_profile: pd-small - -deployment-pd-medium: - runtime.selected_preset: deployment-pd-medium - runtime.deployment_profile: pd-medium - -deployment-pd-large: - runtime.selected_preset: deployment-pd-large - runtime.deployment_profile: pd-large - -# P/D testing presets -pd-smoke: - extends: - - smoke-base - - deployment-pd-small - runtime.selected_preset: pd-smoke - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-smoke-medium: - extends: - - smoke-base - - deployment-pd-medium - runtime.selected_preset: pd-smoke-medium - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-smoke-large: - extends: - - smoke-base - - deployment-pd-large - runtime.selected_preset: pd-smoke-large - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-matrix: - runtime.selected_preset: pd-matrix - runtime.model_name: - - meta-llama/Llama-3.1-8B-Instruct - - Qwen/Qwen3-0.6B - runtime.deployment_profile: - - pd-small - - pd-medium - - pd-large - runtime.benchmark_key: - - short - - concurrent-1k-1k - runtime.smoke_request_key: default - platform.gateway.status_address_name: gateway-internal - -pd-scale-test: - runtime.selected_preset: pd-scale-test - runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: - - pd-small - - pd-medium - - pd-large - runtime.benchmark_key: heavy-heterogeneous - runtime.smoke_request_key: default - model_cache.pvc.size: 300Gi - workloads.args: - request_type: text_completions - -# New deployment profiles -deployment-intelligentrouting-tp4-x4: - runtime.selected_preset: deployment-intelligentrouting-tp4-x4 +# Test deployment profiles +test-intelligentrouting: runtime.deployment_profile: intelligentrouting-tp4-x4 -deployment-simple-tp4-x4: - runtime.selected_preset: deployment-simple-tp4-x4 +test-simple: runtime.deployment_profile: simple-tp4-x4 -deployment-pd-x2-ptp4-px1-dtp4: - runtime.selected_preset: deployment-pd-x2-ptp4-px1-dtp4 - runtime.deployment_profile: pd-x2-ptp4-px1-dtp4 - -# Smoke presets for new profiles -smoke-intelligentrouting: - extends: - - smoke-base - - deployment-intelligentrouting-tp4-x4 - runtime.selected_preset: smoke-intelligentrouting - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -smoke-simple-tp4: - extends: - - smoke-base - - deployment-simple-tp4-x4 - runtime.selected_preset: smoke-simple-tp4 - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -smoke-pd-tp4: - extends: - - smoke-base - - deployment-pd-x2-ptp4-px1-dtp4 - runtime.selected_preset: smoke-pd-tp4 - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal +test-pd: + runtime.deployment_profile: pd-d.x2-p.tp4-d.tp4-p.x1 # New presets ported from other tool -pd-flavors: +cpt-reference-flavors: extends: - gpt-oss - runtime.selected_preset: pd-flavors runtime.deployment_profile: - - pd-x2-ptp4-px1-dtp4 + - pd-d.x2-p.tp4-d.tp4-p.x1 - intelligentrouting-tp4-x4 - simple-tp4-x4 - -guidellm_multiturn_eval: - runtime.selected_preset: guidellm_multiturn_eval - runtime.benchmark_key: guidellm_multiturn_eval - runtime.smoke_request_key: default - -guidellm_heterogeneous_eval: - runtime.selected_preset: guidellm_heterogeneous_eval - runtime.benchmark_key: guidellm_heterogeneous_eval - runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index a31b4fdb5..ba510a273 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -16,6 +16,44 @@ def _load_yaml(path: Path) -> Any: return yaml.safe_load(handle) +def _create_model_cache_spec( + model_cache: dict[str, Any], + source_uri: str, + source_scheme: str, + model_slug: str, + namespace: str, +) -> dict[str, Any] | None: + """Create model cache specification if caching is enabled and applicable.""" + if not model_cache.get("enabled", False) or source_uri.startswith(("pvc://", "pvc+hf://")): + return None + + pvc_defaults = model_cache["pvc"] + pvc_prefix = pvc_defaults["name_prefix"] + cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] + pvc_name = truncate_k8s_name( + f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" + ) + model_path = pvc_defaults["model_directory_name"] + + return { + "source_uri": source_uri, + "source_scheme": source_scheme, + "cache_key": cache_key, + "namespace": namespace, + "pvc_name": pvc_name, + "pvc_size": pvc_defaults["size"], + "access_mode": pvc_defaults["access_mode"], + "storage_class_name": pvc_defaults.get("storage_class_name"), + "model_path": model_path, + "model_uri": f"pvc://{pvc_name}/{model_path}", + "marker_filename": model_cache["marker_filename"], + "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", + "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), + "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), + "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), + } + + def render_inference_service_from_parts( *, config_dir: str | Path, @@ -55,33 +93,13 @@ def render_inference_service_from_parts( source_uri = f"hf://{model_name}" source_scheme = "hf" - cache_spec = None - if model_cache.get("enabled", False) and not source_uri.startswith(("pvc://", "pvc+hf://")): - pvc_defaults = model_cache["pvc"] - pvc_prefix = pvc_defaults["name_prefix"] - cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] - pvc_name = truncate_k8s_name( - f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" - ) - model_path = pvc_defaults["model_directory_name"] - - cache_spec = { - "source_uri": source_uri, - "source_scheme": source_scheme, - "cache_key": cache_key, - "namespace": namespace, - "pvc_name": pvc_name, - "pvc_size": pvc_defaults["size"], - "access_mode": pvc_defaults["access_mode"], - "storage_class_name": pvc_defaults.get("storage_class_name"), - "model_path": model_path, - "model_uri": f"pvc://{pvc_name}/{model_path}", - "marker_filename": model_cache["marker_filename"], - "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", - "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), - "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), - "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), - } + cache_spec = _create_model_cache_spec( + model_cache=model_cache, + source_uri=source_uri, + source_scheme=source_scheme, + model_slug=model_slug, + namespace=namespace, + ) manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug @@ -150,15 +168,10 @@ def _render_standard_deployment( has_scheduler_manifest = "scheduler_manifest" in deployment_profile is_intelligent_routing = scheduler is not None and has_scheduler_manifest - # Update service name for intelligent routing or use_defaults + # Update service name for intelligent routing or use_kserve_defaults current_name = manifest["metadata"]["name"] - use_defaults = deployment_profile.get("use_defaults", False) - if ( - current_name == "llm-d" - and deployment_profile_name - and (is_intelligent_routing or use_defaults) - ): - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" manifest["spec"]["replicas"] = deployment_profile["replicas"] @@ -167,12 +180,12 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - # Configure VLLM args based on deployment type and use_defaults flag + # Configure VLLM args based on deployment type and use_kserve_defaults flag tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_defaults = deployment_profile.get("use_defaults", False) + use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) - if use_defaults: - # Use VLLM_ADDITIONAL_ARGS environment variable when use_defaults is set + if use_kserve_defaults: + # Use VLLM_ADDITIONAL_ARGS environment variable when use_kserve_defaults is set vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) if not _has_cli_arg(vllm_args, "tensor-parallel-size"): vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") @@ -290,17 +303,17 @@ def _build_pd_pod_template( ) -> dict[str, Any]: """Build pod template for P/D deployment.""" tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_defaults = deployment_profile.get("use_defaults", False) + use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) # Build VLLM environment variable configuration - if use_defaults: - # Use deployment profile's vllm_args when use_defaults is set + if use_kserve_defaults: + # Use deployment profile's vllm_args when use_kserve_defaults is set vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) if not _has_cli_arg(vllm_args, "tensor-parallel-size"): vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") vllm_additional_args = " ".join(vllm_args) else: - # Use P/D-specific configuration when use_defaults is false + # Use P/D-specific configuration when use_kserve_defaults is false if is_prefill: # Prefill configuration vllm_args_list = [ @@ -391,17 +404,18 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: Can be enabled by setting runtime.kserve_use_kueue config. """ # Check if kueue annotations should be enabled - enable_kueue = config.project.get_config("runtime.kserve_use_kueue", False) + enable_kueue = config.project.get_config("runtime.kueue.enabled") if not enable_kueue: return - # Default kueue configuration (can be made configurable later) + # Configure kueue settings + queue_name = config.project.get_config("runtime.kueue.queue_name") kueue_config = { "enabled": True, "prefix": "kueue.x-k8s.io/", - "labels": {"queue-name": "default-queue"}, - "annotations": {"queue-name": "default-queue"}, + "labels": {"queue-name": queue_name}, + "annotations": {"queue-name": queue_name}, } # Get prefix for kueue labels/annotations diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f261f514e..f2cb4939a 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -114,11 +114,166 @@ def deep_merge(base: Any, override: Any) -> Any: return merged +def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: + """Extract configuration values from profile names. + + Supports patterns: + - simple-tp4-x4: tensor_parallelism=4, replicas=4 + - intelligentrouting-tp4-x4: tensor_parallelism=4, replicas=4 + - pd-d.x2-p.tp4-d.tp4-p.x1: prefill_pods=1, decode_pods=2, tensor_parallelism=4 + + Args: + profile_name: The deployment profile name + field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods) + + Returns: + Extracted integer value + + Raises: + ValueError: If the pattern is not recognized or value cannot be extracted + """ + # P/D pattern: pd-d.x2-p.tp4-d.tp4-p.x1 + if profile_name.startswith("pd-"): + # P/D naming: pd-d.x{decode_pods}-p.tp{tensor_parallelism}-d.tp{decode_tensor_parallelism}-p.x{prefill_pods} + # Extract different components + if field_name == "tensor_parallelism": + # Look for p.tp4 pattern (prefill tensor_parallelism) + match = re.search(r"p\.tp(\d+)", profile_name) + if match: + return int(match.group(1)) + # Fallback: look for any tp pattern + match = re.search(r"tp(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "prefill_pods": + # Look for p.x1 pattern + match = re.search(r"p\.x(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "decode_pods": + # Look for d.x2 pattern + match = re.search(r"d\.x(\d+)", profile_name) + if match: + return int(match.group(1)) + else: + # Standard patterns: simple-tp4-x4, intelligentrouting-tp4-x4 + if field_name == "tensor_parallelism": + # Look for tp4 pattern + match = re.search(r"tp(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "replicas": + # Look for x4 pattern + match = re.search(r"x(\d+)", profile_name) + if match: + return int(match.group(1)) + + raise ValueError(f"Could not extract {field_name} from profile name: {profile_name}") + + +def _resolve_from_name_values(profile_name: str, profile_data: dict[str, Any]) -> dict[str, Any]: + """Resolve FROM_NAME placeholders in profile configuration with values from profile name. + + Args: + profile_name: The deployment profile name to extract values from + profile_data: The profile configuration that may contain FROM_NAME placeholders + + Returns: + Profile configuration with FROM_NAME placeholders resolved to actual values + """ + resolved = copy.deepcopy(profile_data) + + def resolve_value(obj: Any, path: str = "") -> Any: + if isinstance(obj, dict): + result = {} + for key, value in obj.items(): + current_path = f"{path}.{key}" if path else key + result[key] = resolve_value(value, current_path) + return result + elif isinstance(obj, list): + return [resolve_value(item, path) for item in obj] + elif obj == "FROM_NAME": + # Determine which field to extract based on the path + field_name = path.split(".")[-1] # Get the last part of the path + try: + return _extract_value_from_profile_name(profile_name, field_name) + except ValueError as e: + logger.warning(f"Failed to extract {field_name} from {profile_name}: {e}") + return obj # Return the original FROM_NAME if extraction fails + else: + return obj + + return resolve_value(resolved) + + def _load_yaml(path: Path) -> Any: with path.open(encoding="utf-8") as handle: return yaml.safe_load(handle) +def _load_scheduler_with_epp_config( + config_dir: Path, scheduler_manifest: str, deployment_profile: dict[str, Any] +) -> dict[str, Any]: + """Load scheduler configuration using router template with EPP config replacement. + + Uses configurable paths from runtime.router config. + + Args: + config_dir: Configuration directory path + scheduler_manifest: Path to EPP config file (e.g., "manifests/deployments/approximate-prefix-cache.yaml") + deployment_profile: Resolved deployment profile for templating + + Returns: + Scheduler data with EPP config content replacing placeholder + """ + + # Get configurable paths and placeholder + router_template_file = template_config.get("runtime.router.template.file") + router_placeholder = template_config.get("runtime.router.template.placeholder") + epp_config_dir = router_config.get("runtime.router.template.epp") + + # Load the router template + router_template_path = config_dir / router_template_file + if not router_template_path.exists(): + # Fallback to legacy behavior if template doesn't exist + raise FileNotFoundError( + f"Router template not found at {router_template_path}" + ) + + # Load the EPP config content + # Convert scheduler_manifest path to EPP config directory + # e.g., "manifests/deployments/approximate-prefix-cache.yaml" -> "manifests/scheduler_config/approximate-prefix-cache.yaml" + manifest_path = Path(scheduler_manifest) + epp_config_filename = manifest_path.name + epp_config_path = config_dir / epp_config_dir / epp_config_filename + + if not epp_config_path.exists(): + raise FileNotFoundError( + f"EPP config not found at {epp_config_path}" + ) + + # Load both files + router_template_data = _load_yaml(router_template_path) + epp_config_data = _load_yaml(epp_config_path) + + # Replace placeholder in the parsed data structure (avoids YAML formatting issues) + def replace_placeholder_in_structure(obj): + if isinstance(obj, dict): + return {key: replace_placeholder_in_structure(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [replace_placeholder_in_structure(item) for item in obj] + elif isinstance(obj, str) and obj == router_placeholder: + # Convert EPP config to YAML string for the --config-text argument + return yaml.dump(epp_config_data, default_flow_style=False, sort_keys=False).strip() + else: + return obj + + # Perform the replacement in the structure + replaced_data = replace_placeholder_in_structure(router_template_data) + + return replaced_data + + def get_config_dir() -> Path: """Get the LLM-D configuration directory""" return ORCHESTRATION_DIR @@ -130,8 +285,8 @@ def get_job_name() -> str: if job_name: return job_name - preset_name = config.project.get_config("runtime.selected_preset") - return f"local-{preset_name}" + deployment_profile = config.project.get_config("runtime.deployment_profile") + return f"local-{deployment_profile}" def get_platform_config() -> dict[str, Any]: @@ -284,32 +439,78 @@ def get_deployment_profile_name() -> str: return deployment_profiles[0] +def _resolve_template_profile( + profile_name: str, deployment_config: dict[str, Any] +) -> dict[str, Any]: + """Resolve a profile name to a template-based configuration if no direct match exists. + + Args: + profile_name: The deployment profile name (e.g., "simple-tp4-x4") + deployment_config: The full deployments configuration + + Returns: + Template configuration to use, or raises ValueError if no template matches + + Raises: + ValueError: If no template pattern matches the profile name + """ + # Try to match profile name patterns to templates + if profile_name.startswith("pd-"): + template_name = "template__pd" + elif "intelligentrouting" in profile_name: + template_name = "template__intelligentrouting" + elif profile_name.startswith("simple-") or re.search(r"-tp\d+-x\d+$", profile_name): + template_name = "template__simple" + else: + raise ValueError(f"No template found for profile name pattern: {profile_name}") + + if template_name not in deployment_config: + raise ValueError(f"Template {template_name} not found in deployment config") + + logger.info(f"Using template {template_name} for profile {profile_name}") + return copy.deepcopy(deployment_config[template_name]) + + def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() deployment_config = copy.deepcopy(config.project.get_config("deployments")) - profile = copy.deepcopy(deployment_config[profile_name]) + + # First try direct profile lookup + if profile_name in deployment_config: + profile = copy.deepcopy(deployment_config[profile_name]) + else: + # Fall back to template-based resolution + profile = _resolve_template_profile(profile_name, deployment_config) + defaults = copy.deepcopy(deployment_config.get("defaults", {})) scheduler_manifest = profile.pop("scheduler_manifest", None) resolved_profile = deep_merge(defaults, profile) - if scheduler_manifest: - scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - # When use_defaults is true, use simplified scheduler template - use_defaults = resolved_profile.get("use_defaults", False) - if use_defaults: + # Resolve FROM_NAME placeholders with values extracted from profile name + resolved_profile = _resolve_from_name_values(profile_name, resolved_profile) + + if scheduler_manifest: + RENAME THIS TO use_rhoai_defaults + # When use_kserve_defaults is true, use simplified scheduler template + use_kserve_defaults = resolved_profile.get("use_kserve_defaults", False) + if use_kserve_defaults: # Create simplified scheduler with just basic container structure simplified_scheduler = { "scheduler": { "template": { "containers": [{"name": "main"}], + # force the scheduler to run on a GPU node. 'gpu.present: true' not working on AKS/CKS "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, - "serviceAccountName": "llm-d-privileged", } } } resolved_profile = deep_merge(resolved_profile, simplified_scheduler) else: + # When use_kserve_defaults is false, use __router_template__.yaml with EPP config replacement + scheduler_data = _load_scheduler_with_epp_config( + get_config_dir(), scheduler_manifest, resolved_profile + ) resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile @@ -341,7 +542,8 @@ def get_scheduler_config() -> str: def is_pd_deployment() -> bool: """Check if current deployment profile is a P/D deployment.""" - return bool(get_pd_config()) + profile = get_deployment_profile() + return "pd_config" in profile def get_smoke_request() -> dict[str, Any]: diff --git a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml similarity index 96% rename from projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml index bf220304b..7761ef972 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-intelligentrouting-tp4-x4 - namespace: forge-llm-d + namespace: forge-llm-d-qwen-qwen3-0-6b-intelligentrouting-tp4-x annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics diff --git a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml similarity index 97% rename from projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml index c88a7140f..1618f4d63 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml @@ -1,8 +1,8 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-pd-x2-ptp4-px1-dtp4 - namespace: forge-llm-d + name: llm-d-pd-d.x2-p.tp4-d.tp4-p.x1 + namespace: forge-llm-d-qwen-qwen3-0-6b-pd-d-x2-p-tp4-d-tp4-p-x1 annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics diff --git a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml similarity index 93% rename from projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 752c71d66..0c1c9bbfd 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-simple-tp4-x4 - namespace: forge-llm-d + namespace: forge-llm-d-qwen-qwen3-0-6b-simple-tp4-x4 annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics @@ -15,6 +15,9 @@ spec: model: uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model name: qwen-qwen3-0-6b + router: + route: {} + gateway: {} template: tolerations: - key: nvidia.com/gpu diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 3e0232375..f17952695 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -16,11 +16,12 @@ # Deployment presets to test DEPLOYMENT_PRESETS = [ - "deployment-simple-tp4-x4", - "deployment-intelligentrouting-tp4-x4", - "deployment-pd-x2-ptp4-px1-dtp4", + "cpt-reference-flavors", ] +# Check for save deployments mode via environment variable +SAVE_DEPLOYMENTS = os.environ.get("SAVE_DEPLOYMENTS", "false").lower() in ("true", "1", "yes") + @pytest.fixture(autouse=True) def _reset_project_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): @@ -77,61 +78,174 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): # Check that the command succeeded if result.returncode != 0: pytest.fail( - f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" ) - # Find the generated LLMISVC manifest - generated_manifest = _find_generated_llmisvc(artifact_dir) - if not generated_manifest.exists(): - pytest.fail(f"Generated LLMISVC manifest not found at {generated_manifest}") - - # Load the generated manifest - with generated_manifest.open("r", encoding="utf-8") as f: - generated_content = yaml.safe_load(f) - - # Load the reference manifest - reference_manifest = REFERENCE_DIR / preset / "llmisvc.yaml" - if not reference_manifest.exists(): - pytest.fail(f"Reference manifest not found at {reference_manifest}") - - with reference_manifest.open("r", encoding="utf-8") as f: - reference_content = yaml.safe_load(f) - - # Compare the manifests - if generated_content != reference_content: - # Generate a diff for better error reporting - import difflib - - generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) - reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) - - diff = list( - difflib.unified_diff( - reference_yaml.splitlines(keepends=True), - generated_yaml.splitlines(keepends=True), - fromfile=f"reference/{preset}/llmisvc.yaml", - tofile=f"generated/{preset}/llmisvc.yaml", - lineterm="", - ) - ) + # Find all generated LLMISVC manifests + deployments = _find_generated_llmisvcs(artifact_dir) + if not deployments: pytest.fail( - f"Generated LLMISVC does not match reference for preset {preset}:\n{''.join(diff)}" + f"No generated LLMISVC manifests found in {artifact_dir}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" ) + # If saving deployments, save them and skip comparison + if SAVE_DEPLOYMENTS: + _save_deployments(preset, deployments) + pytest.skip(f"Saved {len(deployments)} deployments for preset {preset}") + return -def _find_generated_llmisvc(artifact_dir: Path) -> Path: - """Find the generated LLMISVC manifest in the artifact directory.""" + # Test each deployment against its reference + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + print(f"Testing deployment for profile: {profile_name}") + + if not generated_manifest.exists(): + pytest.fail( + f"Generated LLMISVC manifest not found at {generated_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + # Load the generated manifest + with generated_manifest.open("r", encoding="utf-8") as f: + generated_content = yaml.safe_load(f) + + # Determine reference manifest path + # Look in preset-specific directory structure: preset/deployment-{profile_name}/llmisvc.yaml + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + if not reference_manifest.exists(): + pytest.fail( + f"Reference manifest not found at {reference_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + with reference_manifest.open("r", encoding="utf-8") as f: + reference_content = yaml.safe_load(f) + + # Compare the manifests + if generated_content != reference_content: + # Generate a diff for better error reporting + import difflib + + generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) + reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) + + diff = list( + difflib.unified_diff( + reference_yaml.splitlines(keepends=True), + generated_yaml.splitlines(keepends=True), + fromfile=f"reference/{preset}/deployment-{profile_name}/llmisvc.yaml", + tofile=f"generated/{preset}/{profile_name}/llmisvc.yaml", + lineterm="", + ) + ) + + pytest.fail( + f"Generated LLMISVC does not match reference for profile {profile_name} in preset {preset}:\n{''.join(diff)}\n\n" + f"💡 To update reference deployments with new output, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + +def _save_deployments(preset: str, deployments: list[dict[str, Path]]): + """Save generated deployments as reference manifests.""" + print(f"\nSaving {len(deployments)} deployments for preset {preset}:") + + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + # Create reference directory for this preset and deployment profile + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_deployment_dir.mkdir(parents=True, exist_ok=True) + + # Copy the generated manifest to the reference location + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + with generated_manifest.open("r", encoding="utf-8") as f: + content = f.read() + + with reference_manifest.open("w", encoding="utf-8") as f: + f.write(content) + + print(f" Saved {profile_name} → {reference_manifest}") + + print(f"Reference deployments saved to: {reference_preset_dir}") + + +def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: + """Find all generated LLMISVC manifests in the artifact directory. + + Returns: + List of dictionaries with 'profile_name' and 'manifest_path' keys + """ # Look for the pattern: $ARTIFACT_DIR/**/deploy_llmisvc/src/llminferenceservice.yaml deploy_llmisvc_dirs = list(artifact_dir.glob("**/*__deploy_llmisvc")) if not deploy_llmisvc_dirs: raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") - if len(deploy_llmisvc_dirs) > 1: + deployments = [] + for deploy_dir in deploy_llmisvc_dirs: + llmisvc_path = deploy_dir / "src" / "llminferenceservice.yaml" + if llmisvc_path.exists(): + # Extract profile name from the directory structure + # Pattern: .../__llmd_run_{profile_name}_{model_slug}/.../__deploy_llmisvc + parent_path = ( + deploy_dir.parent.parent.name + ) # e.g., "001__llmd_run_pd-d-x2-p-tp4-d-tp4-p-x1_qwen-qwen3-0-6b" + if "__llmd_run_" in parent_path: + # Extract the part after __llmd_run_ + run_part = parent_path.split("__llmd_run_", 1)[1] + + # Split by underscore to separate profile from model + parts = run_part.split("_") + + # Find where the model name starts (models typically start with specific patterns) + model_start_idx = len(parts) + for i, part in enumerate(parts): + if part.startswith(("qwen", "meta", "openai", "llama")): + model_start_idx = i + break + + # Profile name is everything before the model + if model_start_idx > 0: + profile_parts = parts[:model_start_idx] + profile_name = "-".join(profile_parts) + else: + # Fallback: use entire run part minus likely model + profile_name = "-".join(parts[:-1]) if len(parts) > 1 else parts[0] + + deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) + else: + # Fallback: use directory name + deployments.append({"profile_name": "unknown", "manifest_path": llmisvc_path}) + + return deployments + + +def _find_generated_llmisvc(artifact_dir: Path) -> Path: + """Find the generated LLMISVC manifest in the artifact directory (legacy single deployment).""" + deployments = _find_generated_llmisvcs(artifact_dir) + + if len(deployments) == 1: + return deployments[0]["manifest_path"] + elif len(deployments) > 1: raise ValueError( - f"Multiple deploy_llmisvc directories found in {artifact_dir}: {deploy_llmisvc_dirs}" + f"Multiple deploy_llmisvc directories found in {artifact_dir}. " + f"Use _find_generated_llmisvcs() instead for multi-deployment presets." ) - - llmisvc_path = deploy_llmisvc_dirs[0] / "src" / "llminferenceservice.yaml" - return llmisvc_path + else: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") From 75274092f8696e8509e20531ab643bcb966d1a3a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 20 Jul 2026 14:51:39 +0200 Subject: [PATCH 175/344] [llm_d] orchestration: manifests: scheduler_config: unify --- .../manifests/deployments/pd-scheduler.yaml | 86 ------------------- .../deployments/precise-prefix-cache.yaml | 73 ---------------- .../__router_template__.yaml} | 19 +--- .../approximate-prefix-cache.yaml | 15 ++++ .../precise-prefix-cache.yaml | 30 +++++++ 5 files changed, 48 insertions(+), 175 deletions(-) delete mode 100644 projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml delete mode 100644 projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml rename projects/llm_d/orchestration/manifests/{deployments/approximate-prefix-cache.yaml => scheduler_config/__router_template__.yaml} (66%) create mode 100644 projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml create mode 100644 projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml diff --git a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml deleted file mode 100644 index 8d6c7e2cd..000000000 --- a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml +++ /dev/null @@ -1,86 +0,0 @@ -scheduler: - template: - containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - # P/D specific environment variables - - name: PD_PREFILL_PODS - value: "{{ deployment_profile.pd_config.prefill_pods }}" - - name: PD_DECODE_PODS - value: "{{ deployment_profile.pd_config.decode_pods }}" - - name: PD_SCHEDULER_CONFIG - value: "{{ deployment_profile.pd_config.scheduler_config }}" - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: pd-prefill-scheduler - config: - prefillPods: {{ deployment_profile.pd_config.prefill_pods }} - schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" - - type: pd-decode-scheduler - config: - decodePods: {{ deployment_profile.pd_config.decode_pods }} - schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" - - type: queue-scorer - - type: kv-cache-utilization-scorer - schedulingProfiles: - - name: prefill-profile - plugins: - - pluginRef: pd-prefill-scheduler - weight: 5 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - name: decode-profile - plugins: - - pluginRef: pd-decode-scheduler - weight: 5 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} diff --git a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml deleted file mode 100644 index 7e4e4ff1a..000000000 --- a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml +++ /dev/null @@ -1,73 +0,0 @@ -scheduler: - template: - containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. - # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with - # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead - # of the older precise-prefix-cache-scorer-only configuration kept here. - - type: single-profile-handler - - type: precise-prefix-cache-scorer - parameters: - indexerConfig: - tokenProcessorConfig: - blockSize: 64 - hashSeed: "42" - tokenizersPoolConfig: - hf: - tokenizersCacheDir: /tmp/tokenizers - - type: kv-cache-utilization-scorer - - type: queue-scorer - - type: max-score-picker - schedulingProfiles: - - name: default - plugins: - - pluginRef: precise-prefix-cache-scorer - weight: 3.0 - - pluginRef: kv-cache-utilization-scorer - weight: 2.0 - - pluginRef: queue-scorer - weight: 2.0 - - pluginRef: max-score-picker - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} diff --git a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml similarity index 66% rename from projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml rename to projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml index ae62e49d4..8c33cbd30 100644 --- a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml +++ b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml @@ -30,22 +30,9 @@ scheduler: - --model-server-metrics-scheme - https - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: prefix-cache-scorer - schedulingProfiles: - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 + + - __ENDPOINT_PICKER_CONFIG__ + volumeMounts: - name: tokenizer-cache mountPath: /tmp/tokenizer-cache diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml new file mode 100644 index 000000000..ea0cf650c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml @@ -0,0 +1,15 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: prefix-cache-scorer +schedulingProfiles: + - name: default + plugins: + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: prefix-cache-scorer + weight: 3.0 diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml new file mode 100644 index 000000000..20e8895bf --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml @@ -0,0 +1,30 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. + # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with + # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead + # of the older precise-prefix-cache-scorer-only configuration kept here. + - type: single-profile-handler + - type: precise-prefix-cache-scorer + parameters: + indexerConfig: + tokenProcessorConfig: + blockSize: 64 + hashSeed: "42" + tokenizersPoolConfig: + hf: + tokenizersCacheDir: /tmp/tokenizers + - type: kv-cache-utilization-scorer + - type: queue-scorer + - type: max-score-picker +schedulingProfiles: + - name: default + plugins: + - pluginRef: precise-prefix-cache-scorer + weight: 3.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: max-score-picker From c76f03871de5008a5b2597a57b84ee976df9a81a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:27:40 +0200 Subject: [PATCH 176/344] [core] library: env: create link from /tmp/forge_last to the current ARTIFACT_DIR --- projects/core/library/env.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 7c07ca732..05ea5eed1 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -103,6 +103,12 @@ def init(daily_artifact_dir=False): artifact_dir.mkdir(parents=True, exist_ok=True) os.environ["ARTIFACT_DIR"] = str(artifact_dir) + # Create or update forge_last symlink + forge_last_link = env_forge_base_dir / "forge_last" + if forge_last_link.exists() or forge_last_link.is_symlink(): + forge_last_link.unlink() + forge_last_link.symlink_to(artifact_dir.name) + artifact_dir.mkdir(parents=True, exist_ok=True) # Set BASE_ARTIFACT_DIR to the initial value (immutable) From a4cbb648f6d0b1ab89b1d067a21763c14db661fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:29:00 +0200 Subject: [PATCH 177/344] [core] library: env: use '%Y%m%d-%H%M-%S' if '%Y%m%d-%H%M' already exists --- projects/core/library/env.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 05ea5eed1..4def14e1e 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -98,7 +98,14 @@ def init(daily_artifact_dir=False): else: env_forge_base_dir = pathlib.Path(os.environ.get("FORGE_BASE_DIR", "/tmp")) - artifact_dir = env_forge_base_dir / f"forge_{time.strftime('%Y%m%d-%H%M')}" + # Try base format first, add seconds if directory exists + base_name = f"forge_{time.strftime('%Y%m%d-%H%M')}" + artifact_dir = env_forge_base_dir / base_name + + if artifact_dir.exists(): + # Directory exists, add seconds to make it unique + unique_name = f"forge_{time.strftime('%Y%m%d-%H%M-%S')}" + artifact_dir = env_forge_base_dir / unique_name artifact_dir.mkdir(parents=True, exist_ok=True) os.environ["ARTIFACT_DIR"] = str(artifact_dir) From 50dd143cb5ad4f6c1fea806687823e1243e14e23 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:56:52 +0200 Subject: [PATCH 178/344] [llm_d] work on P/D --- .../orchestration/config.d/deployments.yaml | 77 +++-- .../orchestration/config.d/platform.yaml | 4 - .../llm_d/orchestration/config.d/project.yaml | 3 - .../llm_d/orchestration/config.d/runtime.yaml | 6 +- .../manifests/llminferenceservice.yaml | 40 +-- .../llm_d/orchestration/presets.d/cpt.yaml | 10 + .../orchestration/presets.d/presets.yaml | 19 -- .../orchestration/render_inference_service.py | 293 +++++++----------- .../llm_d/orchestration/runtime_config.py | 216 +++++-------- .../llmisvc.yaml | 10 +- .../llmisvc.yaml | 140 --------- .../llmisvc.yaml | 69 +++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 12 +- .../llm_d/tests/test_deployment_profiles.py | 43 +-- 14 files changed, 341 insertions(+), 601 deletions(-) delete mode 100644 projects/llm_d/orchestration/config.d/project.yaml create mode 100644 projects/llm_d/orchestration/presets.d/cpt.yaml rename projects/llm_d/tests/reference_deployments/cpt-reference-flavors/{deployment-intelligentrouting-tp4-x => deployment-intelligentrouting-tp4-x4}/llmisvc.yaml (83%) delete mode 100644 projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index e643c56c1..f25bdfdf8 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -1,41 +1,56 @@ defaults: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d replicas: 1 tensor_parallelism: 1 - vllm_args: - - --max-model-len=4096 - - --gpu-memory-utilization=0.92 - - --trust-remote-code - - --no-enable-log-requests - - --enable-prefix-caching + vllm_extra: + args: + - --max-model-len=4096 + - --gpu-memory-utilization=0.92 + - --trust-remote-code + - --no-enable-log-requests + - --enable-prefix-caching -approximate-prefix-cache: - scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' -precise-prefix-cache: - scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml +profiles: + approximate-prefix-cache: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml -distributed-default: - scheduler: {} + precise-prefix-cache: + scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml + + distributed-default: + scheduler: {} # Template flavor configurations with name-based extraction -template__pd: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - tensor_parallelism: FROM_NAME - use_kserve_defaults: true - pd_config: - prefill_pods: FROM_NAME - decode_pods: FROM_NAME - scheduler_config: "high-throughput" +templates: + pd: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME + pd_config: + prefill_pods: FROM_NAME + decode_pods: FROM_NAME -template__intelligentrouting: - use_kserve_defaults: true - scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml - tensor_parallelism: FROM_NAME - replicas: FROM_NAME - scheduler: {} # scheduler enabled with default config + intelligentrouting: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME + replicas: FROM_NAME -template__simple: - tensor_parallelism: FROM_NAME - replicas: FROM_NAME - use_kserve_defaults: true - scheduler: null # scheduler disabled + simple: + scheduler_manifest: null # scheduler disabled + tensor_parallelism: FROM_NAME + replicas: FROM_NAME diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 32325769a..8f592e46f 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -1,9 +1,5 @@ cluster: minimum_openshift_version: "4.19.9" - namespace: - name: forge-llm-d - prefix: llm-d - max_length: 63 gpu_node_label_selector: nvidia.com/gpu.present=true nfd_gpu_detection_labels: - feature.node.kubernetes.io/pci-10de.present diff --git a/projects/llm_d/orchestration/config.d/project.yaml b/projects/llm_d/orchestration/config.d/project.yaml deleted file mode 100644 index 4d6b0b88f..000000000 --- a/projects/llm_d/orchestration/config.d/project.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: llm_d -args: - - smoke diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 73b527cf8..c2f2dd14e 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -4,10 +4,10 @@ deployment_profile: approximate-prefix-cache smoke_request_key: default benchmark_key: null job_name: null -namespace_override: null -kserve_dry_run: true +namespace: forge-llm-d +kserve_dry_run: false kueue: - enable: true + enabled: false queue_name: default-queue router: template: diff --git a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml index cff616f84..c3043bf31 100644 --- a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml +++ b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml @@ -16,45 +16,7 @@ spec: scheduler: template: containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} + - name: main nodeSelector: nvidia.com/gpu.present: "true" route: {} diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml new file mode 100644 index 000000000..dfb731711 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -0,0 +1,10 @@ +__multiple: true + +# CPT reference profiles +cpt-reference-flavors: + extends: + - gpt-oss + runtime.deployment_profile: + - pd-d.x2-p.tp1-d.tp4-p.x8 + - intelligentrouting-tp4-x4 + - simple-tp4-x4 diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index d8ef284ee..491296ed8 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -53,22 +53,3 @@ gpt-oss-120b-inference-scheduling-release: - concurrent-1k-1k - heavy-heterogeneous - multi-turn - -# Test deployment profiles -test-intelligentrouting: - runtime.deployment_profile: intelligentrouting-tp4-x4 - -test-simple: - runtime.deployment_profile: simple-tp4-x4 - -test-pd: - runtime.deployment_profile: pd-d.x2-p.tp4-d.tp4-p.x1 - -# New presets ported from other tool -cpt-reference-flavors: - extends: - - gpt-oss - runtime.deployment_profile: - - pd-d.x2-p.tp4-d.tp4-p.x1 - - intelligentrouting-tp4-x4 - - simple-tp4-x4 diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index ba510a273..79e433ee0 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -77,12 +77,16 @@ def render_inference_service_from_parts( manifest["metadata"]["namespace"] = namespace manifest["metadata"].setdefault("labels", {}) manifest["metadata"]["labels"].update( - { - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - } + config.project.get_config("deployments.defaults.labels") or {} ) + # Add deployment profile name as annotation for testing + manifest["metadata"].setdefault("annotations", {}) + if deployment_profile_name: + manifest["metadata"]["annotations"]["forge.openshift.io/deployment-profile"] = ( + deployment_profile_name + ) + if model_name.startswith("oci://"): source_uri = model_name source_scheme = "oci" @@ -120,13 +124,13 @@ def render_inference_service_from_parts( def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + tensor_parallelism = deployment_profile["tensor_parallelism"] profile_resources = deployment_profile.get("resources", {}) rendered_resources: dict[str, Any] = {} for bound in ("requests", "limits"): source = profile_resources.get(bound, {}) - rendered_bound = {"nvidia.com/gpu": tensor_parallelism} + rendered_bound = {"nvidia.com/gpu": str(tensor_parallelism)} for resource_name in ("cpu", "memory"): value = source.get(resource_name) if value not in (None, ""): @@ -157,6 +161,26 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: return any(arg == bare or arg.startswith(prefix) for arg in args) +def _build_vllm_additional_args( + deployment_profile: dict[str, Any], +) -> str: + """Build VLLM_ADDITIONAL_ARGS string based on deployment profile configuration. + + Args: + deployment_profile: The deployment profile configuration + + Returns: + String suitable for VLLM_ADDITIONAL_ARGS environment variable + """ + + vllm_default_args = _build_vllm_args( + config.project.get_config("deployments.defaults.vllm_extra.args") + ) + vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra.args", {})) + + return " ".join(vllm_default_args + vllm_deploy_args) + + def _render_standard_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], @@ -166,10 +190,6 @@ def _render_standard_deployment( # Check if this is intelligent routing (scheduler_manifest exists) scheduler = deployment_profile.get("scheduler") has_scheduler_manifest = "scheduler_manifest" in deployment_profile - is_intelligent_routing = scheduler is not None and has_scheduler_manifest - - # Update service name for intelligent routing or use_kserve_defaults - current_name = manifest["metadata"]["name"] manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" @@ -180,50 +200,13 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - # Configure VLLM args based on deployment type and use_kserve_defaults flag - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) + vllm_additional_args = _build_vllm_additional_args(deployment_profile) - if use_kserve_defaults: - # Use VLLM_ADDITIONAL_ARGS environment variable when use_kserve_defaults is set - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + # Add environment variable (don't set generic env vars or args) + if "env" not in serving_container: + serving_container["env"] = [] - vllm_additional_args = " ".join(vllm_args) - - # Add environment variable (don't set generic env vars or args) - if "env" not in serving_container: - serving_container["env"] = [] - serving_container["env"].append( - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} - ) - elif is_intelligent_routing: - # Use environment variables for intelligent routing - vllm_args_list = [ - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", - ] - vllm_additional_args = " ".join(vllm_args_list) - - # Add or update environment variables - if "env" not in serving_container: - serving_container["env"] = [] - serving_container["env"].append( - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} - ) - else: - # Use CLI args for basic deployments - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args + serving_container["env"].append({"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}) # Configure router/scheduler has_scheduler_key = "scheduler" in deployment_profile @@ -243,24 +226,6 @@ def _render_standard_deployment( deployment_profile["router_image"] ) - # Enhance scheduler for intelligent routing - if is_intelligent_routing: - # Ensure template structure exists - if "template" not in manifest["spec"]["router"]["scheduler"]: - manifest["spec"]["router"]["scheduler"]["template"] = { - "containers": [{"name": "main"}] - } - - scheduler_template = manifest["spec"]["router"]["scheduler"]["template"] - - # Add nodeSelector and serviceAccountName (if not already set by runtime_config.py) - if "nodeSelector" not in scheduler_template: - scheduler_template["nodeSelector"] = { - "nvidia.com/gpu.deploy.container-toolkit": "true" - } - if "serviceAccountName" not in scheduler_template: - scheduler_template["serviceAccountName"] = "llm-d-privileged" - return manifest @@ -270,131 +235,111 @@ def _render_pd_deployment( deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" + from .runtime_config import get_decode_pod_count, get_prefill_pod_count - # Update service name to include deployment profile for P/D - current_name = manifest["metadata"]["name"] - if current_name == "llm-d" and deployment_profile_name: # Only modify if it's the default name - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" - - # Set main replicas to 2 for P/D - manifest["spec"]["replicas"] = 2 + # Set manifest name with deployment profile + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" # Configure prefill section manifest["spec"]["prefill"] = { - "replicas": 2, - "template": _build_pd_pod_template(deployment_profile, is_prefill=True), + "replicas": get_prefill_pod_count(), + "template": _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=True + ), } # Configure main template (decode) - manifest["spec"]["template"] = _build_pd_pod_template(deployment_profile, is_prefill=False) - - # Simplified router for P/D - manifest["spec"]["router"] = { - "scheduler": {"template": {"containers": [{"name": "main"}]}}, - "route": {}, - "gateway": {}, - } + manifest["spec"]["replicas"] = get_decode_pod_count() + manifest["spec"]["template"] = _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=False + ) return manifest def _build_pd_pod_template( - deployment_profile: dict[str, Any], is_prefill: bool = False + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, + is_prefill: bool = False, ) -> dict[str, Any]: """Build pod template for P/D deployment.""" - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) - - # Build VLLM environment variable configuration - if use_kserve_defaults: - # Use deployment profile's vllm_args when use_kserve_defaults is set - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - vllm_additional_args = " ".join(vllm_args) + + # Get P/D extra configuration from deployments config + + pd_vllm_extra = config.project.get_config("deployments.pd.vllm_extra") + + # Build VLLM args with correct tensor parallelism + + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_vllm_args = _build_vllm_additional_args(prefill_profile) + else: + # For decode pods, use main tensor parallelism + base_vllm_args = _build_vllm_additional_args(deployment_profile) + + # Add P/D extra args + + pd_extra_args = pd_vllm_extra.get("args", []) + all_vllm_args = base_vllm_args.split() + pd_extra_args + vllm_additional_args = " ".join(all_vllm_args) + + # Build base environment variables + base_env = [ + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, + ] + + # Add P/D extra environment variables + pd_extra_env = pd_vllm_extra.get("env", []) + all_env = base_env + copy.deepcopy(pd_extra_env) + + # Build base resources with correct tensor parallelism + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_resources = _build_serving_resources(prefill_profile) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_resources = _build_serving_resources(deployment_profile) else: - # Use P/D-specific configuration when use_kserve_defaults is false - if is_prefill: - # Prefill configuration - vllm_args_list = [ - "--disable-uvicorn-access-log", - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", # Appears twice in prefill - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len=40960", - ] - else: - # Decode configuration - vllm_args_list = [ - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", # Space instead of equals for decode - ] - vllm_additional_args = " ".join(vllm_args_list) + # For decode pods, use main tensor parallelism + base_resources = _build_serving_resources(deployment_profile) + # Add P/D extra resources to both requests and limits + pd_resources = config.project.get_config("deployments.pd.resources") + + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + base_resources[bound].update(pd_resources) + + # Build container configuration container = { "name": "main", - "env": [ - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, - { - "name": "VLLM_NIXL_SIDE_CHANNEL_HOST", - "valueFrom": {"fieldRef": {"apiVersion": "v1", "fieldPath": "status.podIP"}}, - }, - {"name": "UCX_IB_GID_INDEX", "value": "3"}, - ], - "livenessProbe": { - "failureThreshold": 1000, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "initialDelaySeconds": 900, - "periodSeconds": 60, - "timeoutSeconds": 60, - }, - "readinessProbe": { - "failureThreshold": 10000, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "initialDelaySeconds": 60, - "periodSeconds": 30, - "successThreshold": 1, - "timeoutSeconds": 30, - }, - "resources": { - "limits": {"dra.llm-d.io/gpu-nic-pair": tensor_parallelism}, - "requests": { - "cpu": "4", - "dra.llm-d.io/gpu-nic-pair": tensor_parallelism, - "memory": "64Gi", - }, - }, - "securityContext": {"capabilities": {"add": ["IPC_LOCK", "SYS_RAWIO"]}}, - "startupProbe": { - "failureThreshold": 150, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1, - }, + "resources": base_resources, + "env": all_env, } - template = {"serviceAccountName": "llm-d-privileged", "containers": [container]} - - # Add tolerations only to main template (decode), not prefill - if not is_prefill: - template["tolerations"] = [ - {"effect": "NoSchedule", "key": "nvidia.com/gpu", "operator": "Exists"} - ] + # Add serving image if specified + if deployment_profile.get("serving_image"): + container["image"] = deployment_profile["serving_image"] - return template + return {"containers": [container]} def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f2cb4939a..19d3ba066 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -12,7 +12,7 @@ import yaml -from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.dsl.utils import slugify_identifier from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -32,7 +32,6 @@ class RunSpec: benchmark_slug: str | None namespace: str artifact_dirname: str - namespace_is_managed: bool def init() -> Path: @@ -124,7 +123,7 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: Args: profile_name: The deployment profile name - field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods) + field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods, prefill_tensor_parallelism) Returns: Extracted integer value @@ -137,12 +136,11 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: # P/D naming: pd-d.x{decode_pods}-p.tp{tensor_parallelism}-d.tp{decode_tensor_parallelism}-p.x{prefill_pods} # Extract different components if field_name == "tensor_parallelism": - # Look for p.tp4 pattern (prefill tensor_parallelism) - match = re.search(r"p\.tp(\d+)", profile_name) + # For main container: d.tp4 or tp4 + match = re.search(r"d\.tp(\d+)", profile_name) if match: return int(match.group(1)) - # Fallback: look for any tp pattern - match = re.search(r"tp(\d+)", profile_name) + match = re.search(r"(? int: if match: return int(match.group(1)) elif field_name == "decode_pods": - # Look for d.x2 pattern + # For main container replicas: d.x2 or x2 match = re.search(r"d\.x(\d+)", profile_name) if match: return int(match.group(1)) + match = re.search(r"(? int: if match: return int(match.group(1)) elif field_name == "replicas": - # Look for x4 pattern - match = re.search(r"x(\d+)", profile_name) + # For standard deployments: x4 (but not d.x4 or p.x4) + match = re.search(r"(? dict[str, Any]: return normalize_platform_config(copy.deepcopy(config.project.get_config("platform"))) -def _derive_base_namespace() -> str: - platform_data = get_platform_config() - namespace_override = _get_runtime_value("namespace_override") - namespace_config = platform_data["cluster"]["namespace"] - default_namespace = namespace_config.get("name") - - if namespace_override: - return namespace_override - if default_namespace: - return default_namespace - - return derive_namespace( - get_job_name(), - namespace_config["prefix"], - namespace_config["max_length"], - ) - - def get_namespace() -> str: - """Get the resolved namespace for this execution""" - return _derive_base_namespace() - - -_namespace_is_managed_override: bool | None = None - - -def get_namespace_is_managed() -> bool: - """Check if namespace is managed (auto-derived) vs explicitly configured. - - Inside activate_run_spec(), returns the base managed state captured before - the run-spec override was applied (otherwise the namespace_override set by - activate_run_spec would always flip this to False). - """ - if _namespace_is_managed_override is not None: - return _namespace_is_managed_override - - namespace_override = _get_runtime_value("namespace_override") - platform_data = get_platform_config() - default_namespace = platform_data["cluster"]["namespace"].get("name") - - return namespace_override is None and default_namespace is None + """Get the namespace for this execution""" + return _get_runtime_value("namespace") def get_model_name() -> str: @@ -454,30 +418,27 @@ def _resolve_template_profile( Raises: ValueError: If no template pattern matches the profile name """ - # Try to match profile name patterns to templates - if profile_name.startswith("pd-"): - template_name = "template__pd" - elif "intelligentrouting" in profile_name: - template_name = "template__intelligentrouting" - elif profile_name.startswith("simple-") or re.search(r"-tp\d+-x\d+$", profile_name): - template_name = "template__simple" - else: - raise ValueError(f"No template found for profile name pattern: {profile_name}") + templates = deployment_config.get("templates", {}) + if not templates: + raise ValueError("No templates section found in deployment config") - if template_name not in deployment_config: - raise ValueError(f"Template {template_name} not found in deployment config") + # Extract first component and check if it matches a template + template_name = profile_name.partition("-")[0] + if template_name not in templates: + raise ValueError(f"No template found for profile name pattern: {profile_name}") logger.info(f"Using template {template_name} for profile {profile_name}") - return copy.deepcopy(deployment_config[template_name]) + return copy.deepcopy(templates[template_name]) def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() deployment_config = copy.deepcopy(config.project.get_config("deployments")) - # First try direct profile lookup - if profile_name in deployment_config: - profile = copy.deepcopy(deployment_config[profile_name]) + # First try direct profile lookup in profiles section + profiles = deployment_config.get("profiles", {}) + if profile_name in profiles: + profile = copy.deepcopy(profiles[profile_name]) else: # Fall back to template-based resolution profile = _resolve_template_profile(profile_name, deployment_config) @@ -490,28 +451,41 @@ def get_deployment_profile() -> dict[str, Any]: # Resolve FROM_NAME placeholders with values extracted from profile name resolved_profile = _resolve_from_name_values(profile_name, resolved_profile) - if scheduler_manifest: - RENAME THIS TO use_rhoai_defaults - # When use_kserve_defaults is true, use simplified scheduler template - use_kserve_defaults = resolved_profile.get("use_kserve_defaults", False) - if use_kserve_defaults: - # Create simplified scheduler with just basic container structure - simplified_scheduler = { - "scheduler": { - "template": { - "containers": [{"name": "main"}], - # force the scheduler to run on a GPU node. 'gpu.present: true' not working on AKS/CKS - "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, - } + scheduler_content = None + if scheduler_manifest is None: + # scheduler disabled, nothing to do + return resolved_profile + + if isinstance(scheduler_manifest, dict) and len(scheduler_manifest) == 0: # {} + # Create simplified scheduler with just basic container structure - override completely + scheduler_content = { + "scheduler": { + "template": { + "containers": [{"name": "main"}], } } - resolved_profile = deep_merge(resolved_profile, simplified_scheduler) - else: - # When use_kserve_defaults is false, use __router_template__.yaml with EPP config replacement - scheduler_data = _load_scheduler_with_epp_config( - get_config_dir(), scheduler_manifest, resolved_profile - ) - resolved_profile = deep_merge(resolved_profile, scheduler_data) + } + # Don't merge - directly assign to override template defaults + resolved_profile["scheduler"] = scheduler_content["scheduler"] + else: + # use __router_template__.yaml with EPP config replacement + scheduler_content = _load_scheduler_with_epp_config( + get_config_dir(), scheduler_manifest, resolved_profile + ) + + resolved_profile = deep_merge(resolved_profile, scheduler_content) + + # force the scheduler to run on a GPU node. + # 'gpu.present: true' doesn't work on AKS/CKS + scheduler_node_selector = { + "scheduler": { + "template": { + "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + } + } + } + + resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) return resolved_profile @@ -579,29 +553,13 @@ def get_run_specs() -> list[RunSpec]: benchmark_entries = [(None, None)] combinations = list(product(model_names, profile_names, benchmark_entries)) - base_namespace = _derive_base_namespace() - namespace_max_length = get_platform_config()["cluster"]["namespace"]["max_length"] - # Compute base managed state once (ignores per-spec overrides) - namespace_is_managed = get_namespace_is_managed() + namespace = get_namespace() + run_specs: list[RunSpec] = [] for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) - if len(combinations) == 1: - namespace = base_namespace - artifact_dirname = "llmd_run" - else: - ns_parts = [base_namespace, model_slug, profile_slug] - dir_parts = ["llmd_run", profile_slug, model_slug] - if bench_slug is not None: - ns_parts.append(bench_slug) - dir_parts.append(bench_slug) - namespace = truncate_k8s_name( - "-".join(ns_parts), - max_length=namespace_max_length, - ) - artifact_dirname = "_".join(dir_parts) run_specs.append( RunSpec( @@ -612,8 +570,7 @@ def get_run_specs() -> list[RunSpec]: benchmark_key=bench_key, benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=artifact_dirname, - namespace_is_managed=namespace_is_managed, + artifact_dirname=f"llmd__{bench_key or 'default'}", ) ) @@ -623,7 +580,6 @@ def get_run_specs() -> list[RunSpec]: @contextmanager def activate_run_spec(run_spec: RunSpec): """Temporarily activate a run spec by setting its runtime config values.""" - global _namespace_is_managed_override saved = { "model_name": _get_runtime_value("model_name"), @@ -631,32 +587,19 @@ def activate_run_spec(run_spec: RunSpec): "namespace_override": _get_runtime_value("namespace_override"), "benchmark_key": _get_runtime_value("benchmark_key"), } - prev_managed_override = _namespace_is_managed_override config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) - config.project.set_config("runtime.namespace_override", run_spec.namespace) config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) - _namespace_is_managed_override = run_spec.namespace_is_managed try: yield finally: config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) - config.project.set_config("runtime.namespace_override", saved["namespace_override"]) config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) - _namespace_is_managed_override = prev_managed_override def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: - cluster = platform_data["cluster"] - if "namespace" not in cluster: - cluster["namespace"] = { - "name": cluster.pop("namespace_name", None), - "prefix": cluster.pop("namespace_prefix"), - "max_length": cluster.pop("namespace_max_length"), - } - operators = platform_data["operators"] if isinstance(operators, list): platform_data["operators"] = { @@ -669,23 +612,6 @@ def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: return platform_data -def derive_namespace(job_name: str, prefix: str, max_length: int) -> str: - slug = re.sub(r"[^a-z0-9-]+", "-", job_name.lower()) - slug = re.sub(r"-{2,}", "-", slug).strip("-") - if not slug: - slug = "run" - - if slug.startswith(f"{prefix}-"): - namespace = slug - else: - namespace = f"{prefix}-{slug}" - - namespace = namespace[:max_length].rstrip("-") - if not namespace: - raise ValueError(f"Could not derive a valid namespace from job name: {job_name}") - return namespace - - def version_tuple(value: str) -> tuple[int, ...]: numbers = re.findall(r"\d+", value) return tuple(int(number) for number in numbers[:3]) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml similarity index 83% rename from projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 7761ef972..6265d7c82 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -2,19 +2,20 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-intelligentrouting-tp4-x4 - namespace: forge-llm-d-qwen-qwen3-0-6b-intelligentrouting-tp4-x + namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: intelligentrouting-tp4-x4 labels: app.kubernetes.io/managed-by: forge forge.openshift.io/project: llm_d spec: replicas: 4 model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b router: scheduler: template: @@ -22,7 +23,6 @@ spec: - name: main nodeSelector: nvidia.com/gpu.deploy.container-toolkit: 'true' - serviceAccountName: llm-d-privileged route: {} gateway: {} template: @@ -59,4 +59,4 @@ spec: env: - name: VLLM_ADDITIONAL_ARGS value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + --no-enable-log-requests --enable-prefix-caching diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml deleted file mode 100644 index 1618f4d63..000000000 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: LLMInferenceService -metadata: - name: llm-d-pd-d.x2-p.tp4-d.tp4-p.x1 - namespace: forge-llm-d-qwen-qwen3-0-6b-pd-d-x2-p-tp4-d-tp4-p-x1 - annotations: - security.opendatahub.io/enable-auth: 'false' - prometheus.io/path: /metrics - prometheus.io/port: '8000' - labels: - app.kubernetes.io/managed-by: forge - forge.openshift.io/project: llm_d -spec: - replicas: 2 - model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b - router: - scheduler: - template: - containers: - - name: main - route: {} - gateway: {} - template: - serviceAccountName: llm-d-privileged - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: UCX_IB_GID_INDEX - value: '3' - livenessProbe: - failureThreshold: 1000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 - resources: - limits: - dra.llm-d.io/gpu-nic-pair: '4' - requests: - cpu: '4' - dra.llm-d.io/gpu-nic-pair: '4' - memory: 64Gi - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RAWIO - startupProbe: - failureThreshold: 150 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Exists - prefill: - replicas: 2 - template: - serviceAccountName: llm-d-privileged - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: UCX_IB_GID_INDEX - value: '3' - livenessProbe: - failureThreshold: 1000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 - resources: - limits: - dra.llm-d.io/gpu-nic-pair: '4' - requests: - cpu: '4' - dra.llm-d.io/gpu-nic-pair: '4' - memory: 64Gi - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RAWIO - startupProbe: - failureThreshold: 150 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml new file mode 100644 index 000000000..bc538b5ce --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -0,0 +1,69 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 2 + model: + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b + router: + scheduler: + template: + containers: + - name: main + nodeSelector: + nvidia.com/gpu.present: 'true' + route: {} + gateway: {} + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + rdma/ib: '1' + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + prefill: + replicas: 8 + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' + rdma/ib: '1' + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 0c1c9bbfd..6e3edb0be 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -2,22 +2,20 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-simple-tp4-x4 - namespace: forge-llm-d-qwen-qwen3-0-6b-simple-tp4-x4 + namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: simple-tp4-x4 labels: app.kubernetes.io/managed-by: forge forge.openshift.io/project: llm_d spec: replicas: 4 model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b - router: - route: {} - gateway: {} + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b template: tolerations: - key: nvidia.com/gpu @@ -52,4 +50,4 @@ spec: env: - name: VLLM_ADDITIONAL_ARGS value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + --no-enable-log-requests --enable-prefix-caching diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index f17952695..9c99c8d52 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -53,6 +53,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): yaml.safe_dump( { "runtime.kserve_dry_run": True, + "runtime.namespace_override": "forge-llm-d", }, sort_keys=True, ), @@ -201,37 +202,17 @@ def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: for deploy_dir in deploy_llmisvc_dirs: llmisvc_path = deploy_dir / "src" / "llminferenceservice.yaml" if llmisvc_path.exists(): - # Extract profile name from the directory structure - # Pattern: .../__llmd_run_{profile_name}_{model_slug}/.../__deploy_llmisvc - parent_path = ( - deploy_dir.parent.parent.name - ) # e.g., "001__llmd_run_pd-d-x2-p-tp4-d-tp4-p-x1_qwen-qwen3-0-6b" - if "__llmd_run_" in parent_path: - # Extract the part after __llmd_run_ - run_part = parent_path.split("__llmd_run_", 1)[1] - - # Split by underscore to separate profile from model - parts = run_part.split("_") - - # Find where the model name starts (models typically start with specific patterns) - model_start_idx = len(parts) - for i, part in enumerate(parts): - if part.startswith(("qwen", "meta", "openai", "llama")): - model_start_idx = i - break - - # Profile name is everything before the model - if model_start_idx > 0: - profile_parts = parts[:model_start_idx] - profile_name = "-".join(profile_parts) - else: - # Fallback: use entire run part minus likely model - profile_name = "-".join(parts[:-1]) if len(parts) > 1 else parts[0] - - deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) - else: - # Fallback: use directory name - deployments.append({"profile_name": "unknown", "manifest_path": llmisvc_path}) + # Read the manifest and extract profile name from annotations + with llmisvc_path.open("r", encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + ANNOTATION = "forge.openshift.io/deployment-profile" + profile_name = manifest.get("metadata", {}).get("annotations", {}).get(ANNOTATION) + + if not profile_name: + raise ValueError(f"Annotation '{ANNOTATION}' missing in {llmisvc_path}") + + deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) return deployments From 0c0f6a2022838fe154b9201e3a55823727e7307b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 13:57:38 +0200 Subject: [PATCH 179/344] [llm_d] CHANGELOG-orchestration: add a changelog --- projects/llm_d/CHANGELOG-orchestration.md | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/projects/llm_d/CHANGELOG-orchestration.md b/projects/llm_d/CHANGELOG-orchestration.md index 46fdeddcd..c471d93c2 100644 --- a/projects/llm_d/CHANGELOG-orchestration.md +++ b/projects/llm_d/CHANGELOG-orchestration.md @@ -1,5 +1,66 @@ # LLM-D Orchestration Changelog +## 2026-07-21 - Enhanced P/D Configuration System + +### P/D Profile Name-Based Configuration +- **FROM_NAME Parameter Extraction**: Implemented automatic parameter extraction from P/D profile names + - **Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + - **Tensor Parallelism**: Separate extraction for prefill (`p.tp4`) vs decode (`d.tp4` or `tp4`) containers + - **Pod Replicas**: Separate extraction for prefill (`p.x8`) vs decode (`d.x2` or `x2`) pods + - **Strict Parsing**: Removed fallback logic for precise parameter extraction + +### Template-Based Deployment Profiles +- **Template System**: Added `template__pd` with `FROM_NAME` placeholders for dynamic configuration + - **Auto-Resolution**: Profile names automatically populate `tensor_parallelism`, `prefill_pods`, `decode_pods` + - **Scheduler Integration**: Template profiles support simplified scheduler configuration (`scheduler_manifest: {}`) + +### Global P/D Configuration Architecture +- **Centralized P/D Config**: Moved P/D-specific settings to global `deployments.pd` configuration + - **VLLM Extra Args**: Global P/D VLLM arguments (e.g., `--block-size`, `--kv-transfer-config`) + - **Environment Variables**: Global P/D environment variables (e.g., `VLLM_NIXL_SIDE_CHANNEL_HOST`) + - **Resource Requirements**: Global P/D resource specifications (e.g., `rdma/ib: '1'`) + +### Enhanced VLLM Arguments System +- **Configuration Rename**: Changed `vllm_args` → `vllm_extra_args` for clarity +- **Conditional Behavior**: Different VLLM arg handling based on `use_kserve_defaults` flag + - **KServe Mode**: Only extra args when `use_kserve_defaults: true` + - **Standard Mode**: Extra args + automatic `--tensor-parallel-size` when `use_kserve_defaults: false` +- **P/D Integration**: P/D deployments receive both profile args and global P/D args with correct tensor parallelism + +### Resource Allocation Improvements +- **Per-Pod-Type Configuration**: Different resource allocation for prefill vs decode pods + - **Prefill Pods**: Use prefill tensor parallelism (extracted from `p.tp{N}`) + - **Decode Pods**: Use main tensor parallelism (extracted from `d.tp{N}` or `tp{N}`) +- **Resource Merging**: Automatic merging of base resources with P/D-specific resources + +### Testing Infrastructure Enhancements +- **Annotation-Based Profile Detection**: Added `forge.openshift.io/deployment-profile` annotation for reliable profile identification + - **Replaced Heuristics**: Eliminated complex directory name parsing in test system + - **Reliable Testing**: Tests now read profile names directly from manifest annotations +- **Namespace Configuration**: Fixed namespace override behavior for consistent testing + +### Environment and Artifact Management +- **Artifact Directory Naming**: Enhanced artifact directory creation with conflict resolution + - **Base Format**: `forge_YYYYMMDD-HHMM` without seconds when possible + - **Conflict Resolution**: Add seconds (`forge_YYYYMMDD-HHMM-SS`) when directory exists +- **Symlink Management**: Automatic `/tmp/forge_last` symlink to latest artifact directory + +### Files Modified +- `projects/llm_d/orchestration/runtime_config.py` - FROM_NAME extraction system, template resolution, parameter parsing +- `projects/llm_d/orchestration/render_inference_service.py` - P/D rendering, VLLM args system, resource allocation, annotations +- `projects/llm_d/orchestration/config.d/deployments.yaml` - Template profiles, global P/D configuration +- `projects/llm_d/orchestration/config.d/runtime.yaml` - Namespace override, kueue configuration updates +- `projects/llm_d/tests/test_deployment_profiles.py` - Annotation-based profile detection, namespace override +- `projects/core/library/env.py` - Artifact directory conflict resolution, symlink management +- `projects/llm_d/docs/PD_TESTING.md` - Updated documentation for new P/D configuration system + +### Benefits +- **Flexible Configuration**: Profile names encode all P/D parameters for easy testing +- **Template Reuse**: Single template profile supports multiple P/D configurations +- **Resource Optimization**: Different GPU allocation for prefill vs decode workloads +- **Simplified Testing**: Annotation-based profile detection eliminates complex heuristics +- **Clean Architecture**: Global P/D configuration reduces duplication across profiles + ## 2026-07-16 - Preset Configuration Support ### Configuration Management From a6099dfc8babf5caa2f2812a0d022be0c3a9f298 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:45:56 +0200 Subject: [PATCH 180/344] [core] library: postprocess: return the status dict if postprocessing is disabled --- projects/core/library/postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index fa2c8ddb0..5fe2c100a 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -309,7 +309,7 @@ def run_postprocess_after_test( if not postprocess_config.enabled: logger.info("Caliper post-processing disabled (caliper.postprocess.enabled: false).") - return + return dict(success=True, final_status="Post-processing disabled") artifact_root_path = Path(artifact_root).resolve() if artifact_root is not None else None From 103e33cb00510ad2b4e337c3f0c5451554a40423 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:46:41 +0200 Subject: [PATCH 181/344] [llm_d] orchestration: use the config file to define the labels --- projects/llm_d/orchestration/config.d/platform.yaml | 4 ++++ projects/llm_d/orchestration/test_phase.py | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 8f592e46f..a8bf91b73 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -5,6 +5,10 @@ cluster: - feature.node.kubernetes.io/pci-10de.present - feature.node.kubernetes.io/pci-0302_10de.present - feature.node.kubernetes.io/pci-0300_10de.present + namespace: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d" operators: openshift-cert-manager-operator: diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 0fa44b436..9e22bc7c8 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -116,11 +116,7 @@ def do_test() -> int: if not dry_run: # Ensure namespace exists before starting any deployments ensure_namespace( - namespace, - labels={ - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - }, + namespace, labels=config.project.get_config("platform.cluster.namespace.labels") ) endpoint_url: str | None = None From bdf4935c4ee7337b456c8a4f1acd863056f324a4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:53:24 +0200 Subject: [PATCH 182/344] [llm_d] finish removing the managed namespace --- projects/llm_d/orchestration/phase_inputs.py | 11 -- projects/llm_d/orchestration/prepare_phase.py | 1 - projects/llm_d/tests/test_profiles.py | 112 +----------------- 3 files changed, 5 insertions(+), 119 deletions(-) diff --git a/projects/llm_d/orchestration/phase_inputs.py b/projects/llm_d/orchestration/phase_inputs.py index 70f8b4387..2fbcc62ca 100644 --- a/projects/llm_d/orchestration/phase_inputs.py +++ b/projects/llm_d/orchestration/phase_inputs.py @@ -21,7 +21,6 @@ class PrepareModelCacheInputs: artifact_dir: Path preset_name: str namespace: str - namespace_is_managed: bool model_key: str model: dict[str, Any] model_cache: dict[str, Any] @@ -33,7 +32,6 @@ class PrepareInputs: config_dir: Path preset_name: str namespace: str - namespace_is_managed: bool platform: dict[str, Any] model_key: str model: dict[str, Any] @@ -83,7 +81,6 @@ def write_prepare_inputs_from_prepare(inputs: PrepareInputs) -> Path: "config_dir": str(inputs.config_dir), "preset_name": inputs.preset_name, "namespace": inputs.namespace, - "namespace_is_managed": inputs.namespace_is_managed, "platform": inputs.platform, "model_key": inputs.model_key, "model": inputs.model, @@ -125,7 +122,6 @@ def load_prepare_model_cache_inputs(path: str | Path) -> PrepareModelCacheInputs artifact_dir=Path(payload["artifact_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], model_key=payload["model_key"], model=payload["model"], model_cache=payload["model_cache"], @@ -137,7 +133,6 @@ def build_prepare_model_cache_inputs( artifact_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, model_key: str, model: dict[str, Any], model_cache: dict[str, Any], @@ -146,7 +141,6 @@ def build_prepare_model_cache_inputs( artifact_dir=Path(artifact_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, model_key=model_key, model=model, model_cache=model_cache, @@ -160,7 +154,6 @@ def load_prepare_inputs(path: str | Path) -> PrepareInputs: config_dir=Path(payload["config_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], platform=payload["platform"], model_key=payload["model_key"], model=payload["model"], @@ -175,7 +168,6 @@ def build_prepare_inputs( config_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, platform: dict[str, Any], model_key: str, model: dict[str, Any], @@ -187,7 +179,6 @@ def build_prepare_inputs( config_dir=Path(config_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, platform=platform, model_key=model_key, model=model, @@ -274,7 +265,6 @@ def prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> PrepareMod artifact_dir=inputs.artifact_dir, preset_name=inputs.preset_name, namespace=inputs.namespace, - namespace_is_managed=inputs.namespace_is_managed, model_key=inputs.model_key, model=inputs.model, model_cache=inputs.model_cache, @@ -290,7 +280,6 @@ def write_prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> Path "artifact_dir": str(cache_inputs.artifact_dir), "preset_name": cache_inputs.preset_name, "namespace": cache_inputs.namespace, - "namespace_is_managed": cache_inputs.namespace_is_managed, "model_key": cache_inputs.model_key, "model": cache_inputs.model, "model_cache": cache_inputs.model_cache, diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 930f35e6a..533596f69 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -283,7 +283,6 @@ def prepare_model_cache() -> None: common_args = { "namespace": namespace, - "namespace_is_managed": runtime_config.get_namespace_is_managed(), "model_key": model_slug, "model_uri": model_uri, "pvc_size": model_cache["pvc"]["size"], diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index c4df70be7..54180e8dc 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -58,13 +58,6 @@ def test_release_deployment_profiles_have_expected_shape() -> None: for profile in (approximate, precise, distributed): assert profile["replicas"] == 1 assert profile["tensor_parallelism"] == 1 - assert profile["vllm_args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - ] assert isinstance(approximate["scheduler"], dict) assert isinstance(precise["scheduler"], dict) @@ -188,9 +181,6 @@ def test_release_preset_produces_3_run_specs() -> None: assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) - namespaces = [spec.namespace for spec in run_specs] - assert len(namespaces) == len(set(namespaces)) - def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: variable_overrides_path = env.ARTIFACT_DIR / "000__ci_metadata" / "variable_overrides.yaml" @@ -213,15 +203,6 @@ def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> assert runtime_config.get_benchmark_keys() == ["multi-turn"] -def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided() -> None: - llmd_ci.init() - - assert runtime_config.get_model_name() == "Qwen/Qwen3-0.6B" - assert runtime_config.get_deployment_profile_name() == "approximate-prefix-cache" - # Default preset "smoke" enables the short benchmark - assert runtime_config.get_benchmark_keys() == ["short"] - - def test_list_vaults_only_includes_rhoai_custom_catalog_vaults_for_custom_catalog_runs() -> None: _init_project_config() @@ -574,15 +555,15 @@ def test_3d_matrix_with_multiple_benchmark_keys() -> None: core_config.project.set_config( "runtime.model_name", - "[openai/gpt-oss-120b, Qwen/Qwen3-0.6B]", + ["openai/gpt-oss-120b", "Qwen/Qwen3-0.6B"], ) core_config.project.set_config( "runtime.deployment_profile", - "[distributed-default, precise-prefix-cache]", + ["distributed-default", "precise-prefix-cache"], ) core_config.project.set_config( "runtime.benchmark_key", - "[concurrent-1k-1k, multi-turn]", + ["concurrent-1k-1k", "multi-turn"], ) run_specs = runtime_config.get_run_specs() @@ -596,14 +577,6 @@ def test_3d_matrix_with_multiple_benchmark_keys() -> None: assert run_specs[0].deployment_profile_name == "distributed-default" assert run_specs[0].benchmark_key == "concurrent-1k-1k" - # Verify all namespaces are unique - namespaces = [spec.namespace for spec in run_specs] - assert len(namespaces) == len(set(namespaces)) - - # Verify all artifact dirnames are unique - dirnames = [spec.artifact_dirname for spec in run_specs] - assert len(dirnames) == len(set(dirnames)) - def test_null_benchmark_key_produces_smoke_only_specs() -> None: _init_project_config() @@ -627,7 +600,7 @@ def test_single_benchmark_key_backward_compatible() -> None: assert len(run_specs) == 1 assert run_specs[0].benchmark_key == "short" assert run_specs[0].benchmark_slug == "short" - assert run_specs[0].artifact_dirname == "llmd_run" + assert run_specs[0].artifact_dirname == "llmd__short" def test_activate_run_spec_sets_benchmark_key() -> None: @@ -708,48 +681,9 @@ def test_render_uses_sanitized_model_name_and_profile_resources() -> None: "requests": {"nvidia.com/gpu": "1"}, "limits": {"nvidia.com/gpu": "1"}, } - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - "--tensor-parallel-size=1", - ] assert manifest["spec"]["router"]["scheduler"] == {} -def test_render_preserves_explicit_tensor_parallel_arg() -> None: - _init_project_config() - core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["explicit-tp"] = { - "replicas": 1, - "tensor_parallelism": 2, - "scheduler": {}, - "vllm_args": [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ], - } - core_config.project.set_config("runtime.model_name", "openai/gpt-oss-120b") - core_config.project.set_config("runtime.deployment_profile", "explicit-tp") - - manifest = render_inference_service_from_parts( - config_dir=str(PROJECT_ORCHESTRATION_DIR), - namespace="forge-llm-d", - inference_service=runtime_config.get_platform_config()["inference_service"], - model_name=runtime_config.get_model_name(), - model_slug=runtime_config.get_model_slug(), - deployment_profile=runtime_config.get_deployment_profile(), - model_cache=runtime_config.get_model_cache_config(), - ) - - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ] - - def test_render_uses_embedded_scheduler_config() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) @@ -774,7 +708,7 @@ def test_render_uses_embedded_scheduler_config() -> None: def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["no-scheduler"] = { + core_config.project.config["deployments"]["profiles"]["no-scheduler"] = { "replicas": 1, "tensor_parallelism": 1, "scheduler": None, @@ -818,42 +752,6 @@ def test_smoke_preset_benchmark_behavior() -> None: assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" -def test_run_spec_preserves_namespace_managed_flag() -> None: - _init_project_config() - - core_config.project.apply_preset("smoke") - base_managed = runtime_config.get_namespace_is_managed() - - run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed == base_managed - - with runtime_config.activate_run_spec(run_spec): - # Inside the active spec the override is applied, but managed-ness must - # still reflect the base value, not flip to False. - assert runtime_config.get_namespace_is_managed() == base_managed - # And it is restored on exit. - assert runtime_config.get_namespace_is_managed() == base_managed - - -def test_run_spec_preserves_managed_flag_when_namespace_is_auto_derived() -> None: - """Verify managed flag stays True even when activate_run_spec sets namespace_override.""" - _init_project_config() - - # Force auto-derived namespace by clearing the hardcoded name - core_config.project.config["platform"]["cluster"]["namespace"]["name"] = None - - core_config.project.apply_preset("smoke") - assert runtime_config.get_namespace_is_managed() is True - - run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed is True - - with runtime_config.activate_run_spec(run_spec): - # activate_run_spec sets namespace_override, but managed flag must stay True - assert runtime_config.get_namespace_is_managed() is True - assert runtime_config.get_namespace_is_managed() is True - - @pytest.mark.parametrize( "raw,expected", [ From 20f1854dbc07530c1cb653157fbddc8264e94b5b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:07:41 +0200 Subject: [PATCH 183/344] [kserve] toolbox: prepare_hf_model_cache: remove the useless namespace_is_managed flag --- projects/kserve/toolbox/prepare_hf_model_cache/main.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index 572ec43dc..a25544e87 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -35,7 +35,6 @@ def run( *, namespace: str, - namespace_is_managed: bool, model_key: str, model_uri: str, pvc_size: str, @@ -54,7 +53,6 @@ def run( Args: namespace: Namespace where the workload runs - namespace_is_managed: Whether namespace lifecycle is managed by llm_d model_key: Selected model key model_uri: HuggingFace model URI (hf://...) pvc_size: Size of the PVC to create @@ -112,13 +110,6 @@ def build_cache_spec(args, ctx): "oci_registry_auth_secret_key": None, } - if args.namespace_is_managed: - logger.warning( - "Model cache PVC %s lives in managed namespace %s. Namespace cleanup will remove it; cache reuse requires a stable namespace override.", - cache_spec["pvc_name"], - cache_spec["namespace"], - ) - ctx.cache_spec = cache_spec ctx.hf_secret_created = False ctx.hf_secret_name = None From e133a3a122842e44ea57c4475abbdddd7dc2f25c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:17:21 +0200 Subject: [PATCH 184/344] [llm_d] relocatethe runtime.kserve.dry_run flag --- projects/llm_d/orchestration/config.d/runtime.yaml | 3 ++- projects/llm_d/orchestration/test_phase.py | 8 ++++---- projects/llm_d/tests/test_deployment_profiles.py | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index c2f2dd14e..81abc4371 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,7 +5,8 @@ smoke_request_key: default benchmark_key: null job_name: null namespace: forge-llm-d -kserve_dry_run: false +kserve: + dry_run: false kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 9e22bc7c8..8402815d8 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -50,7 +50,7 @@ def create_test_labels() -> None: def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if dry_run: ret = do_test() logger.info("Kserve dry-run mode enabled - Skipping caliper post-processing") @@ -111,7 +111,7 @@ def do_test() -> int: # Load minimal config needed for orchestration flow namespace = runtime_config.get_namespace() - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if not dry_run: # Ensure namespace exists before starting any deployments @@ -181,7 +181,7 @@ def deploy_inference_service() -> str: gateway = platform["gateway"] # Check if dry-run mode is enabled early - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: @@ -339,7 +339,7 @@ def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" # Skip cleanup when in dry-run mode - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if dry_run: logger.info("Skipping cleanup_test_resources - dry-run mode enabled") return diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 9c99c8d52..631ab48a0 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -52,8 +52,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): variable_overrides_path.write_text( yaml.safe_dump( { - "runtime.kserve_dry_run": True, - "runtime.namespace_override": "forge-llm-d", + "runtime.kserve.dry_run": True, }, sort_keys=True, ), From d52fc2534b4fb57dbe6dc0257a4fcf3b8eec72c1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 16:12:33 -0400 Subject: [PATCH 185/344] Fix result extraction truncation and PVC cleanup for parallel jobs - Switch GuideLLM result extraction from oc exec cat to oc cp to avoid truncation of large JSON files (~73MB+) streamed through K8s API - Scope post-cleanup to current job's FJOB suffix for parallel safety - Add PVC cleanup alongside job/pod/isvc cleanup - Remove unused webhook functions from core notifications Co-authored-by: Cursor --- projects/core/notifications/send.py | 30 ------------------- .../toolbox/run_guidellm_benchmark/main.py | 16 +++++----- .../rhaiis/orchestration/prepare_rhaiis.py | 27 ++++++++++++++--- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index dfc494765..8f2d55b2e 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -16,7 +16,6 @@ GITHUB_APP_PEM_FILE = "topsail-bot.2024-09-18.private-key.pem" GITHUB_APP_CLIENT_ID_FILE = "topsail-bot.clientid" SLACK_TOKEN_FILE = "topsail-bot.slack-token" -SLACK_WEBHOOK_URL_FILE = "slack-webhook-url" DEFAULT_REPO_OWNER = "openshift-psap" DEFAULT_REPO_NAME = "forge" @@ -715,35 +714,6 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok -def _send_via_webhook(webhook_url: str, blocks: list[dict]) -> bool: - """Post a Slack message via Incoming Webhook URL.""" - from urllib.error import URLError - from urllib.request import Request, urlopen - - body = json.dumps({"blocks": blocks}) - try: - req = Request( - webhook_url, - data=body.encode("utf-8"), - headers={"Content-Type": "application/json"}, - ) - urlopen(req, timeout=15) # noqa: S310 - return True - except (URLError, OSError) as e: - logger.warning("Slack webhook POST failed: %s", e) - return False - - -def _get_webhook_url(vault_name: str) -> str | None: - """Read the webhook URL from a vault file.""" - from projects.core.library import vault as vault_lib - - path = vault_lib.get_vault_content_path(vault_name, SLACK_WEBHOOK_URL_FILE) - if path is None or not path.exists(): - logger.warning("Webhook URL file not found in vault %s", vault_name) - return None - return path.read_text().strip() - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 1d301988b..7095eb7db 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -291,6 +291,8 @@ def extract_results(args, ctx): """Extract GuideLLM results from copy pod""" results_dir = args.artifact_dir / "artifacts" / "results" + results_dir.mkdir(parents=True, exist_ok=True) + copy_pod = f"{ctx.benchmark_name}-copy" extracted_files: list[dict[str, str | None]] = [] for run in ctx.guidellm_runs: if run.rate is None: @@ -301,20 +303,16 @@ def extract_results(args, ctx): local_path = results_dir / f"benchmarks-{run.label}.json" result = oc( - "exec", - "-n", - ctx.target_namespace, - f"{ctx.benchmark_name}-copy", - "--", - "cat", - remote_path, + "cp", + f"{ctx.target_namespace}/{copy_pod}:{remote_path}", + str(local_path), + "-c", "copy-helper", check=False, log_stdout=False, ) - if result.returncode != 0 or not result.stdout: + if result.returncode != 0 or not local_path.exists(): raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") - write_text(local_path, result.stdout) extracted_files.append( { "label": run.label, diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index 5f7421035..29d111ff7 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -1,4 +1,5 @@ import logging +import os from projects.core.dsl.utils.k8s import oc from projects.rhaiis.orchestration import runtime_config @@ -44,12 +45,30 @@ def prepare(): ) +def _delete_resources_by_suffix(resource_type: str, ns: str, suffix: str) -> None: + """Delete resources whose names contain the FJOB suffix.""" + result = oc("get", resource_type, "-n", ns, "-o", "name", check=False, log_stdout=False) + if result.returncode != 0 or not result.stdout: + return + for line in result.stdout.strip().splitlines(): + name = line.strip() + if suffix in name: + oc("delete", name, "-n", ns, "--ignore-not-found", check=False) + + def cleanup(): ns = runtime_config.get_namespace() logger.info(f"Cleaning up rhaiis benchmark resources in {ns}") - oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + fjob = os.environ.get("FJOB_NAME", "") + if fjob: + suffix = fjob.rsplit("-", 1)[-1] + for resource_type in ("inferenceservice", "servingruntime", "job", "pod", "pvc"): + _delete_resources_by_suffix(resource_type, ns, suffix) + else: + oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + logger.info("Cleanup complete") From fdf73c3e244b19b82f3b9141f217674d398960bf Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:15:48 -0400 Subject: [PATCH 186/344] Refactor dashboard CSV to use Caliper postprocessing pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace manual benchmarks.json lookup and PSAP payload generation with Caliper plugin's parse → KPI → CSV export workflow - Implement export_kpis_to_csv in RhaiisPlugin to produce dashboard CSV format by pivoting per-rate-point KPIs - Modify RhaiisKpiHandler to emit per-rate-point KPIs from performance_curves instead of aggregate scalars - Enrich test labels with hf_model_id, version, image_tag, cluster_tag, and runtime_args for CSV metadata - Enable kpis_to_csv step in caliper postprocess config - Add _sync_postprocessed_dashboard_csv to sync the generated CSV to S3 and run regression after Caliper postprocessing - Remove _generate_psap_payload and _generate_and_sync_dashboard_csv from per-workload loop - Remove unused generate_psap_payload/write_psap_payload from parser Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.yaml | 5 +- .../rhaiis/orchestration/notifications.py | 39 ++ projects/rhaiis/orchestration/test_phase.py | 187 +++++----- projects/rhaiis/postprocess/kpis.py | 340 +++++++++--------- projects/rhaiis/postprocess/parser.py | 74 ---- projects/rhaiis/postprocess/plugin.py | 89 +++++ 6 files changed, 375 insertions(+), 359 deletions(-) diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 9c6bf6f99..c158eb8eb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -43,8 +43,11 @@ caliper: artifacts_to_kpis: enabled: true output: kpis.json + kpis_to_csv: + enabled: true + output: dashboard.csv csv_dashboard: - enabled: false + enabled: true s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv analyze: enabled: false diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index 486c6c340..98b35a931 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -47,3 +47,42 @@ def send_pipeline_failure_alert( ) except Exception: logger.warning("Failed to send pipeline failure alert", exc_info=True) + + +def send_pipeline_warning( + *, + warnings: list[str], + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert for non-critical pipeline warnings.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + warning_text = "\n".join(f"• {w}" for w in warnings) + send_failure_notification( + error=f"Pipeline completed with warnings:\n{warning_text}", + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline warning alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0cad2b784..f8a6dd487 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) _K8S_NAME_MAX = 63 +_warnings: list[str] = [] @@ -29,7 +30,7 @@ def run( namespace: str, deployment_name: str | None = None, ) -> int: - return run_and_postprocess( + ret = run_and_postprocess( do_test, model_key=model_key, workload_keys=workload_keys, @@ -37,6 +38,13 @@ def run( deployment_name=deployment_name, ) + try: + _sync_postprocessed_dashboard_csv(model_key, workload_keys) + except Exception: + logger.warning("Dashboard CSV S3 sync after postprocessing failed", exc_info=True) + + return ret + def do_test( *, @@ -94,7 +102,16 @@ def _run_test( vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) - _create_test_labels(model_key, workload_keys[0], accelerator, vllm_args) + from projects.core.library import config as _cfg + version = _cfg.project.get_config("tests.rhaiis.version", "") + + _create_test_labels( + model_key, workload_keys[0], accelerator, vllm_args, + hf_model_id=model_cfg["hf_model_id"], + version=version, + vllm_image=vllm_image, + cluster_tag=cluster_tag, + ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, @@ -102,7 +119,6 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - from projects.core.library import config as _cfg run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) @@ -137,8 +153,9 @@ def _run_test( run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) - return 0 + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append("Standalone analysis failed") + return 1 if _warnings else 0 benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -265,7 +282,19 @@ def _run_test( try: _upload_predictor_log(run_uuid) except Exception: - logger.warning("Predictor log upload failed; continuing", exc_info=True) + logger.warning("Predictor log upload failed", exc_info=True) + _warnings.append("Predictor log upload failed") + + if _warnings: + logger.warning("Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings)) + from projects.rhaiis.orchestration.notifications import send_pipeline_warning + send_pipeline_warning( + warnings=list(_warnings), + model_key=model_key, + workload_keys=workload_keys, + ) + _warnings.clear() + return 1 return 0 @@ -301,14 +330,14 @@ def _run_workload_benchmark( run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append(f"Standalone analysis failed for {workload_key}") else: logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) @@ -322,7 +351,6 @@ def _run_workload_benchmark( max_seconds=max_seconds, ) - pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), @@ -333,34 +361,34 @@ def _run_workload_benchmark( guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) - from pathlib import Path - candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) - if candidates: - main_benchmark_dir = candidates[0] - if main_benchmark_dir: - try: - _generate_psap_payload( - model_cfg, accelerator_key, vllm_image, vllm_args, workload_key, - run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, - ) - except Exception: - logger.warning("PSAP payload generation failed; continuing", exc_info=True) - - try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator_key, workload_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + # Dashboard CSV generation and S3 sync is handled by the Caliper + # postprocessing pipeline (kpis_to_csv step) + _sync_postprocessed_dashboard_csv def _create_test_labels( - model_key: str, workload_key: str, accelerator: str, vllm_args: dict + model_key: str, + workload_key: str, + accelerator: str, + vllm_args: dict, + *, + hf_model_id: str = "", + version: str = "", + vllm_image: str = "", + cluster_tag: str = "", ) -> None: + _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") + runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) labels = { "model_key": model_key, "workload_key": workload_key, "accelerator": accelerator, "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), + "hf_model_id": hf_model_id, + "version": version, + "image_tag": image_tag, + "cluster_tag": cluster_tag, + "runtime_args": runtime_args, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) @@ -407,92 +435,30 @@ def _set_mlflow_metadata( logger.info("Set MLflow tags: %s", list(tags.keys())) -def _generate_psap_payload( - model_cfg: dict, - accelerator: str, - vllm_image: str, - vllm_args: dict, - workload_key: str, - *, - run_uuid: str = "", - benchmark_dir: Path | None = None, -) -> None: - from pathlib import Path - - from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - - if benchmark_dir: - benchmarks_json = benchmark_dir / "artifacts" / "results" / "benchmarks.json" - else: - matches = list( - Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") - ) - benchmarks_json = matches[-1] if matches else None - - if not benchmarks_json or not benchmarks_json.exists(): - logger.warning( - "benchmarks.json not found, skipping PSAP payload" - ) - return - - payload = generate_psap_payload( - benchmarks_json_path=benchmarks_json, - model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, - vllm_args=vllm_args, - accelerator=accelerator, - workload_key=workload_key, - run_uuid=run_uuid, - ) - output_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "results" - write_psap_payload( - payload=payload, - output_dir=output_dir, - accelerator=accelerator, - model_id=model_cfg["hf_model_id"], - workload_key=workload_key, - ) - - -def _generate_and_sync_dashboard_csv( - model_cfg: dict, - accelerator_key: str, - workload_key: str, - vllm_args: dict, - *, - run_uuid: str = "", -) -> None: +def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) -> None: + """Find the dashboard CSV generated by Caliper kpis_to_csv and sync to S3.""" from pathlib import Path from projects.core.library import config - from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv - # Use chip family only (e.g. "H200") — cluster_tag is only for run ID in the CSV - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() - - psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) - if not psap_files: - logger.warning("No PSAP JSON files found, skipping dashboard CSV") + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): + logger.info("Dashboard CSV S3 sync not enabled") return - version = config.project.get_config("tests.rhaiis.version", "") - if not version: - logger.info("No version configured, skipping dashboard CSV generation") + # Find the CSV produced by the kpis_to_csv step in the postprocessing directory + postprocess_dirs = sorted(Path(env.ARTIFACT_DIR).glob("*__postprocessing")) + if not postprocess_dirs: + logger.info("No postprocessing directory found, skipping dashboard CSV sync") return - csv_path = None - for psap_file in psap_files: - output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" - csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path, run_uuid=run_uuid) - logger.info("Generated dashboard CSV: %s", csv_path) - - if not csv_path: + kpis_to_csv_output = config.project.get_config("caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv") + csv_path = postprocess_dirs[-1] / kpis_to_csv_output + if not csv_path.exists(): + logger.info("Dashboard CSV not found at %s, skipping sync", csv_path) return - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): - logger.info("Dashboard CSV S3 sync not enabled") - return + logger.info("Found postprocessed dashboard CSV: %s", csv_path) from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 @@ -507,13 +473,22 @@ def _generate_and_sync_dashboard_csv( ) logger.info("Dashboard CSV sync result: %s", sync_result) + version = config.project.get_config("tests.rhaiis.version", "") compare_version = config.project.get_config("tests.rhaiis.compare_version", "") - if not compare_version: + if not compare_version or not version: return - from projects.rhaiis.orchestration.analysis import run_regression_check - run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + vllm_defaults = runtime_config.get_vllm_defaults() + first_workload = runtime_config.get_workload(workload_keys[0]) + vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check( + csv_path, compare_version, version, model_cfg, accelerator, + run_uuid="", vllm_args=vllm_args, + ) def _upload_predictor_log(run_uuid: str) -> None: diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 7704bd50c..426432e8b 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -10,138 +10,25 @@ class RhaiisKpiHandler: @staticmethod def get_catalog() -> list[dict[str, Any]]: return [ - { - "kpi_id": "guidellm_request_rate", - "name": "Request Rate", - "unit": "req/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_tokens_per_second", - "name": "Total Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_input_tokens_per_second", - "name": "Input Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_output_tokens_per_second", - "name": "Output Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_ttft_median", - "name": "Time to First Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p95", - "name": "Time to First Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p99", - "name": "Time to First Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_median", - "name": "Inter Token Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p95", - "name": "Inter Token Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p99", - "name": "Inter Token Latency (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_median", - "name": "Time Per Output Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p95", - "name": "Time Per Output Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p99", - "name": "Time Per Output Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_median", - "name": "End-to-End Request Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_p95", - "name": "End-to-End Request Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_input_tokens_per_request", - "name": "Input Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_output_tokens_per_request", - "name": "Output Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_concurrency", - "name": "Measured Request Concurrency", - "unit": "count", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_completed_requests", - "name": "Completed Requests", - "unit": "count", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_failed_requests", - "name": "Failed Requests", - "unit": "count", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_duration", - "name": "Benchmark Duration", - "unit": "s", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_prompt_token_count_mean", - "name": "Prompt Token Count (Mean)", - "unit": "tokens", - "higher_is_better": None, - }, + {"kpi_id": "rhaiis_output_tok_per_sec", "name": "Output Token Throughput", "unit": "tokens/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_total_tok_per_sec", "name": "Total Token Throughput", "unit": "tokens/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_measured_rps", "name": "Measured Request Rate", "unit": "req/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_measured_concurrency", "name": "Measured Concurrency", "unit": "count", "higher_is_better": None}, + {"kpi_id": "rhaiis_completed_requests", "name": "Completed Requests", "unit": "count", "higher_is_better": True}, + {"kpi_id": "rhaiis_failed_requests", "name": "Failed Requests", "unit": "count", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_median", "name": "TTFT (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_p95", "name": "TTFT (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_p99", "name": "TTFT (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_median", "name": "TPOT (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_p95", "name": "TPOT (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_p99", "name": "TPOT (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_median", "name": "ITL (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_p95", "name": "ITL (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_p99", "name": "ITL (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_request_latency_median", "name": "Request Latency (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_request_latency_p95", "name": "Request Latency (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_prompt_token_count_mean", "name": "Prompt Token Count (Mean)", "unit": "tokens", "higher_is_better": None}, + {"kpi_id": "rhaiis_duration", "name": "Benchmark Duration", "unit": "s", "higher_is_better": None}, ] @staticmethod @@ -149,6 +36,26 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, Any]] = [] + # Mapping from performance_curves keys to KPI IDs + curve_kpi_mappings = [ + ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_completed_requests", "completed_requests", "count", True), + ("rhaiis_failed_requests", "failed_requests", "count", False), + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), + ] + for r in model.unified_result_records: if not r.run_identity.get("guidellm"): continue @@ -156,54 +63,39 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: continue base_labels = {**r.distinguishing_labels} + curves = r.metrics.get("performance_curves", {}) + request_rates = r.metrics.get("request_rate", []) - kpi_mappings = [ - ("guidellm_request_rate", "request_rate", "req/s", True), - ("guidellm_tokens_per_second", "tokens_per_second", "tokens/s", True), - ("guidellm_input_tokens_per_second", "input_tokens_per_second", "tokens/s", True), - ("guidellm_output_tokens_per_second", "output_tokens_per_second", "tokens/s", True), - ("guidellm_ttft_median", "ttft_median", "s", False), - ("guidellm_ttft_p95", "ttft_p95", "s", False), - ("guidellm_ttft_p99", "ttft_p99", "s", False), - ("guidellm_itl_median", "itl_median", "s", False), - ("guidellm_itl_p95", "itl_p95", "s", False), - ("guidellm_itl_p99", "itl_p99", "s", False), - ("guidellm_tpot_median", "tpot_median", "s", False), - ("guidellm_tpot_p95", "tpot_p95", "s", False), - ("guidellm_tpot_p99", "tpot_p99", "s", False), - ("guidellm_request_latency_median", "request_latency_median", "s", False), - ("guidellm_request_latency_p95", "request_latency_p95", "s", False), - ("guidellm_input_tokens_per_request", "input_tokens_per_request", "tokens", False), - ( - "guidellm_output_tokens_per_request", - "output_tokens_per_request", - "tokens", - False, - ), - ("guidellm_request_concurrency", "request_concurrency", "count", None), - ("guidellm_completed_requests", "completed_requests", "count", True), - ("guidellm_failed_requests", "failed_requests", "count", False), - ("guidellm_duration", "duration", "s", None), - ("guidellm_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ] - - for kpi_id, metric_key, unit, higher_is_better in kpi_mappings: - raw_value = r.metrics.get(metric_key) - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None + if not curves or not request_rates: + # Fallback: emit scalar KPIs from top-level metrics + _emit_scalar_kpis(out, r, base_labels, ts, model.plugin_module) + continue - # Skip KPIs with null values - if value is None: - continue + # Emit per-rate-point KPIs from performance curves + for idx, rate in enumerate(request_rates): + rate_labels = { + **base_labels, + "rate_index": str(idx), + "intended_concurrency": str(rate), + } - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better + for kpi_id, curve_key, unit, higher_is_better in curve_kpi_mappings: + curve_values = curves.get(curve_key, []) + if idx >= len(curve_values): + continue + raw_value = curve_values[idx] + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**rate_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better - out.append( - { + out.append({ "schema_version": "1", "kpi_id": kpi_id, "value": value, @@ -215,7 +107,99 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: "test_base_path": r.test_base_path, "plugin_module": model.plugin_module, }, - } - ) + }) + + # Also emit scalar KPIs that don't come from curves + for kpi_id, metric_key, unit, higher_is_better in [ + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_duration", "duration", "s", None), + ("rhaiis_measured_rps", "request_rate", "req/s", True), + ]: + raw_value = r.metrics.get(metric_key) + # request_rate is a list; skip it for scalar emission + if isinstance(raw_value, list): + continue + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**base_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better + + out.append({ + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": model.plugin_module, + }, + }) return out + + +def _emit_scalar_kpis( + out: list[dict[str, Any]], + r, + base_labels: dict[str, Any], + ts: str, + plugin_module: str, +) -> None: + """Fallback: emit KPIs from top-level scalar metrics when no curves are available.""" + scalar_mappings = [ + ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_completed_requests", "completed_requests", "count", True), + ("rhaiis_failed_requests", "failed_requests", "count", False), + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_duration", "duration", "s", None), + ] + for kpi_id, metric_key, unit, higher_is_better in scalar_mappings: + raw_value = r.metrics.get(metric_key) + if isinstance(raw_value, list): + continue + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**base_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better + + out.append({ + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": plugin_module, + }, + }) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index c0151a298..92b971442 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -2,9 +2,6 @@ import json import logging -import uuid -from datetime import UTC, datetime -from pathlib import Path from typing import Any from projects.caliper.engine.model import ( @@ -109,74 +106,3 @@ def _stat(metric_name: str, stat: str, default: float = 0.0) -> float: return extra -def generate_psap_payload( - *, - benchmarks_json_path: Path, - model_id: str, - vllm_image: str, - vllm_args: dict[str, Any], - accelerator: str, - workload_key: str, - run_uuid: str = "", -) -> dict[str, Any]: - report = json.loads(benchmarks_json_path.read_text(encoding="utf-8")) - - image, tag = _split_image_tag(vllm_image) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) - - guidellm_start = None - guidellm_end = None - benchmarks = report.get("benchmarks", []) - if benchmarks: - starts = [b.get("start_time", 0) for b in benchmarks if b.get("start_time")] - ends = [b.get("end_time", 0) for b in benchmarks if b.get("end_time")] - if starts: - guidellm_start = int(min(starts) * 1000) - if ends: - guidellm_end = int(max(ends) * 1000) - - return { - "experiment_id": run_uuid or str(uuid.uuid4()).upper(), - "experiment_type": "perf", - "model": model_id, - "inference_server": "vllm", - "inference_server_version": tag, - "container_image": image, - "container_image_tag": tag, - "container_entrypoint": None, - "inference_server_args": dict(vllm_args), - "accelerator_type": accelerator.upper(), - "accelerator_count": tp_size, - "accelerator_memory_gb": 0, - "machine_type": None, - "provider": "redhat", - "report": report, - "timestamp": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S%z"), - "guidellm_start_time_ms": guidellm_start, - "guidellm_end_time_ms": guidellm_end, - } - - -def write_psap_payload( - *, - payload: dict[str, Any], - output_dir: Path, - accelerator: str, - model_id: str, - workload_key: str, -) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - model_short = model_id.rsplit("/", 1)[-1].lower().replace(".", "-") - date_str = datetime.now(UTC).strftime("%Y%m%d") - filename = f"PSAP_perf_{workload_key}_{accelerator}_{model_short}_{date_str}.json" - path = output_dir / filename - path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - logger.info("PSAP payload written to %s", path) - return path - - -def _split_image_tag(full_image: str) -> tuple[str, str]: - if ":" in full_image: - parts = full_image.rsplit(":", 1) - return parts[0], parts[1] - return full_image, "latest" diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 3b2809d91..60de3a313 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -1,6 +1,8 @@ from __future__ import annotations +import csv import logging +from collections import defaultdict from pathlib import Path from typing import Any @@ -17,6 +19,28 @@ logger = logging.getLogger(__name__) +# KPI ID → dashboard CSV column name mapping +_KPI_TO_CSV_COLUMN = { + "rhaiis_output_tok_per_sec": "output_tok/sec", + "rhaiis_total_tok_per_sec": "total_tok/sec", + "rhaiis_measured_concurrency": "measured concurrency", + "rhaiis_completed_requests": "successful_requests", + "rhaiis_failed_requests": "errored_requests", + "rhaiis_ttft_median": "ttft_median", + "rhaiis_ttft_p95": "ttft_p95", + "rhaiis_ttft_p99": "ttft_p99", + "rhaiis_tpot_median": "tpot_median", + "rhaiis_tpot_p95": "tpot_p95", + "rhaiis_tpot_p99": "tpot_p99", + "rhaiis_itl_median": "itl_median", + "rhaiis_itl_p95": "itl_p95", + "rhaiis_itl_p99": "itl_p99", + "rhaiis_request_latency_median": "request_latency_median", + "rhaiis_request_latency_p95": "request_latency_max", + "rhaiis_prompt_token_count_mean": "prompt_token_count_mean", +} + + class RhaiisPlugin(PostProcessingPlugin): def __init__(self) -> None: self.parser = RhaiisParser() @@ -53,6 +77,71 @@ def kpi_catalog(self) -> list[dict[str, Any]]: def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to dashboard-format CSV. + + Groups per-rate-point KPIs by (run_path, rate_index) and pivots them into + rows matching the RHAIIS dashboard CSV schema. + """ + from projects.rhaiis.postprocess.csv_export import FIELDNAMES + + # Group KPIs by (run_path, rate_index) + groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(dict) + group_labels: dict[tuple[str, str], dict[str, Any]] = {} + + for kpi in kpi_records: + labels = kpi.get("labels", {}) + run_path = kpi.get("run_path", "") + rate_index = labels.get("rate_index", "0") + key = (run_path, rate_index) + + csv_col = _KPI_TO_CSV_COLUMN.get(kpi.get("kpi_id", "")) + if csv_col: + groups[key][csv_col] = kpi.get("value") + + if key not in group_labels: + group_labels[key] = labels + + # Build CSV rows + rows = [] + for key in sorted(groups): + metrics = groups[key] + labels = group_labels.get(key, {}) + + acc = labels.get("accelerator", "") + cluster_tag = labels.get("cluster_tag", "") + model_id = labels.get("hf_model_id", "") + tp = labels.get("tensor_parallel_size", "1") + version = labels.get("version", "") + run_name = f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" + + row = {col: "" for col in FIELDNAMES} + row.update(metrics) + row["run"] = run_name + row["accelerator"] = acc + row["model"] = model_id + row["version"] = version + row["TP"] = tp + row["intended concurrency"] = labels.get("intended_concurrency", "") + row["image_tag"] = labels.get("image_tag", "") + row["runtime_args"] = labels.get("runtime_args", "") + row["uuid"] = labels.get("run_uuid", "") + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + logger.info("Exported %d dashboard CSV rows to %s", len(rows), output_path) + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: return {} From 4d6e58d4339aab900d02b6222725e02c7c8a511c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:25:07 -0400 Subject: [PATCH 187/344] Populate all dashboard CSV fields from Caliper KPIs - Extend RHAIIS parser to extract ALL per-benchmark metrics from raw benchmarks.json: ttft/tpot/itl p1/p999/mean, request_latency min/max, measured_rps, token count percentiles, output/total tok/sec, intended_concurrency, successful/errored requests - Add report-level metadata: guidellm_version, prompt_toks, output_toks, guidellm_start/end_time_ms - Store extra per-benchmark metrics as additional performance_curves entries merged into the parsed record - Add comprehensive KPI mappings covering every dashboard CSV column - Update plugin CSV export to populate all metadata fields from KPI labels (version, image_tag, runtime_args, timestamps, etc.) Co-authored-by: Cursor --- projects/rhaiis/postprocess/kpis.py | 204 ++++++++------------------ projects/rhaiis/postprocess/parser.py | 160 ++++++++++++++++---- projects/rhaiis/postprocess/plugin.py | 49 +++++-- 3 files changed, 234 insertions(+), 179 deletions(-) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 426432e8b..6453ee7a8 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -5,30 +5,57 @@ from projects.caliper.engine.model import UnifiedRunModel +# All per-rate-point metrics we emit as KPIs. +# (kpi_id, performance_curves key, unit, higher_is_better) +_CURVE_KPI_MAPPINGS: list[tuple[str, str, str, bool | None]] = [ + # Throughput + ("rhaiis_output_tok_per_sec", "output_tok_per_sec", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "total_tok_per_sec", "tokens/s", True), + # Concurrency / rate + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_measured_rps", "measured_rps", "req/s", True), + ("rhaiis_intended_concurrency", "intended_concurrency", "count", None), + # Request counts + ("rhaiis_completed_requests", "successful_requests", "count", True), + ("rhaiis_failed_requests", "errored_requests", "count", False), + # TTFT + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_ttft_p1", "ttft_p1", "s", False), + ("rhaiis_ttft_p999", "ttft_p999", "s", False), + ("rhaiis_ttft_mean", "ttft_mean", "s", False), + # TPOT + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_tpot_p1", "tpot_p1", "s", False), + ("rhaiis_tpot_p999", "tpot_p999", "s", False), + # ITL + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_itl_p1", "itl_p1", "s", False), + ("rhaiis_itl_p999", "itl_p999", "s", False), + ("rhaiis_itl_mean", "itl_mean", "s", False), + # Request latency + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_min", "request_latency_min", "s", False), + ("rhaiis_request_latency_max", "request_latency_max", "s", False), + # Token counts + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_prompt_token_count_p99", "prompt_token_count_p99", "tokens", None), + ("rhaiis_output_token_count_mean", "output_token_count_mean", "tokens", None), + ("rhaiis_output_token_count_p99", "output_token_count_p99", "tokens", None), +] + class RhaiisKpiHandler: @staticmethod def get_catalog() -> list[dict[str, Any]]: return [ - {"kpi_id": "rhaiis_output_tok_per_sec", "name": "Output Token Throughput", "unit": "tokens/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_total_tok_per_sec", "name": "Total Token Throughput", "unit": "tokens/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_measured_rps", "name": "Measured Request Rate", "unit": "req/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_measured_concurrency", "name": "Measured Concurrency", "unit": "count", "higher_is_better": None}, - {"kpi_id": "rhaiis_completed_requests", "name": "Completed Requests", "unit": "count", "higher_is_better": True}, - {"kpi_id": "rhaiis_failed_requests", "name": "Failed Requests", "unit": "count", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_median", "name": "TTFT (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_p95", "name": "TTFT (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_p99", "name": "TTFT (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_median", "name": "TPOT (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_p95", "name": "TPOT (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_p99", "name": "TPOT (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_median", "name": "ITL (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_p95", "name": "ITL (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_p99", "name": "ITL (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_request_latency_median", "name": "Request Latency (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_request_latency_p95", "name": "Request Latency (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_prompt_token_count_mean", "name": "Prompt Token Count (Mean)", "unit": "tokens", "higher_is_better": None}, - {"kpi_id": "rhaiis_duration", "name": "Benchmark Duration", "unit": "s", "higher_is_better": None}, + {"kpi_id": kpi_id, "name": kpi_id, "unit": unit, "higher_is_better": hib} + for kpi_id, _, unit, hib in _CURVE_KPI_MAPPINGS ] @staticmethod @@ -36,26 +63,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, Any]] = [] - # Mapping from performance_curves keys to KPI IDs - curve_kpi_mappings = [ - ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), - ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), - ("rhaiis_measured_concurrency", "request_concurrency", "count", None), - ("rhaiis_completed_requests", "completed_requests", "count", True), - ("rhaiis_failed_requests", "failed_requests", "count", False), - ("rhaiis_ttft_median", "ttft_median", "s", False), - ("rhaiis_ttft_p95", "ttft_p95", "s", False), - ("rhaiis_ttft_p99", "ttft_p99", "s", False), - ("rhaiis_tpot_median", "tpot_median", "s", False), - ("rhaiis_tpot_p95", "tpot_p95", "s", False), - ("rhaiis_tpot_p99", "tpot_p99", "s", False), - ("rhaiis_itl_median", "itl_median", "s", False), - ("rhaiis_itl_p95", "itl_p95", "s", False), - ("rhaiis_itl_p99", "itl_p99", "s", False), - ("rhaiis_request_latency_median", "request_latency_median", "s", False), - ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), - ] - for r in model.unified_result_records: if not r.run_identity.get("guidellm"): continue @@ -66,29 +73,37 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: curves = r.metrics.get("performance_curves", {}) request_rates = r.metrics.get("request_rate", []) + # Report-level metadata — same for every rate point in this record + meta_labels = { + "guidellm_version": str(r.metrics.get("guidellm_version", "")), + "prompt_toks": str(r.metrics.get("prompt_toks", "")), + "output_toks": str(r.metrics.get("output_toks", "")), + "guidellm_start_time_ms": str(r.metrics.get("guidellm_start_time_ms", "")), + "guidellm_end_time_ms": str(r.metrics.get("guidellm_end_time_ms", "")), + } + if not curves or not request_rates: - # Fallback: emit scalar KPIs from top-level metrics - _emit_scalar_kpis(out, r, base_labels, ts, model.plugin_module) continue - # Emit per-rate-point KPIs from performance curves - for idx, rate in enumerate(request_rates): + n_rates = len(request_rates) + + for idx in range(n_rates): rate_labels = { **base_labels, + **meta_labels, "rate_index": str(idx), - "intended_concurrency": str(rate), } - for kpi_id, curve_key, unit, higher_is_better in curve_kpi_mappings: + for kpi_id, curve_key, unit, higher_is_better in _CURVE_KPI_MAPPINGS: curve_values = curves.get(curve_key, []) if idx >= len(curve_values): continue raw_value = curve_values[idx] + if raw_value is None: + continue try: - value = float(raw_value) if raw_value is not None else None + value = float(raw_value) except (TypeError, ValueError): - value = None - if value is None: continue labels = {**rate_labels} @@ -109,97 +124,4 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: }, }) - # Also emit scalar KPIs that don't come from curves - for kpi_id, metric_key, unit, higher_is_better in [ - ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ("rhaiis_duration", "duration", "s", None), - ("rhaiis_measured_rps", "request_rate", "req/s", True), - ]: - raw_value = r.metrics.get(metric_key) - # request_rate is a list; skip it for scalar emission - if isinstance(raw_value, list): - continue - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None - if value is None: - continue - - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better - - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": model.plugin_module, - }, - }) - return out - - -def _emit_scalar_kpis( - out: list[dict[str, Any]], - r, - base_labels: dict[str, Any], - ts: str, - plugin_module: str, -) -> None: - """Fallback: emit KPIs from top-level scalar metrics when no curves are available.""" - scalar_mappings = [ - ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), - ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), - ("rhaiis_measured_concurrency", "request_concurrency", "count", None), - ("rhaiis_completed_requests", "completed_requests", "count", True), - ("rhaiis_failed_requests", "failed_requests", "count", False), - ("rhaiis_ttft_median", "ttft_median", "s", False), - ("rhaiis_ttft_p95", "ttft_p95", "s", False), - ("rhaiis_ttft_p99", "ttft_p99", "s", False), - ("rhaiis_tpot_median", "tpot_median", "s", False), - ("rhaiis_tpot_p95", "tpot_p95", "s", False), - ("rhaiis_tpot_p99", "tpot_p99", "s", False), - ("rhaiis_itl_median", "itl_median", "s", False), - ("rhaiis_itl_p95", "itl_p95", "s", False), - ("rhaiis_itl_p99", "itl_p99", "s", False), - ("rhaiis_request_latency_median", "request_latency_median", "s", False), - ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), - ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ("rhaiis_duration", "duration", "s", None), - ] - for kpi_id, metric_key, unit, higher_is_better in scalar_mappings: - raw_value = r.metrics.get(metric_key) - if isinstance(raw_value, list): - continue - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None - if value is None: - continue - - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better - - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": plugin_module, - }, - }) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 92b971442..1e2a3eccd 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -2,6 +2,7 @@ import json import logging +import re from typing import Any from projects.caliper.engine.model import ( @@ -35,6 +36,12 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: if node: extra = _extract_extra_metrics(node) merged_metrics = {**record.metrics, **extra} + # Merge extra curves into existing performance_curves + existing_curves = merged_metrics.get("performance_curves", {}) + extra_curves = extra.pop("_extra_curves", {}) + if extra_curves: + existing_curves.update(extra_curves) + merged_metrics["performance_curves"] = existing_curves enriched_records.append( UnifiedResultRecord( test_base_path=record.test_base_path, @@ -62,47 +69,142 @@ def _find_node_for_record( def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: extra: dict[str, Any] = {} - benchmarks_files = [p for p in node.artifact_paths if p.name == "benchmarks.json"] + benchmarks_files = sorted( + [p for p in node.artifact_paths if p.name == "benchmarks.json" + or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json")], + key=lambda p: p.name, + ) if not benchmarks_files: return extra - try: - data = json.loads(benchmarks_files[0].read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): + # Collect all benchmarks across all files + all_benchmarks: list[dict] = [] + report_metadata: dict[str, Any] = {} + report_args: dict[str, Any] = {} + + for bf in benchmarks_files: + try: + data = json.loads(bf.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + all_benchmarks.extend(data.get("benchmarks", [])) + if not report_metadata: + report_metadata = data.get("metadata", {}) + if not report_args: + report_args = data.get("args", {}) + + if not all_benchmarks: return extra - benchmarks = data.get("benchmarks", []) - if not benchmarks: - return extra - - bench = benchmarks[0] - metrics = bench.get("metrics", {}) - - def _percentile(metric_name: str, pct: str, default: float = 0.0) -> float: + # Extract report-level metadata + extra["guidellm_version"] = report_metadata.get("guidellm_version", "") + + data_str = "" + if isinstance(report_args, dict): + data_list = report_args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + extra["prompt_toks"] = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + extra["output_toks"] = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + # Compute global start/end timestamps + start_times = [] + end_times = [] + for bench in all_benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + extra["guidellm_start_time_ms"] = int(min(start_times) * 1000) if start_times else "" + extra["guidellm_end_time_ms"] = int(max(end_times) * 1000) if end_times else "" + + # Scalar extras from first benchmark (backward compat) + bench0 = all_benchmarks[0] + m0 = bench0.get("metrics", {}) + + def _percentile0(metric_name: str, pct: str, default: float = 0.0) -> float: return float( - metrics.get(metric_name, {}) - .get("successful", {}) - .get("percentiles", {}) - .get(pct, default) + m0.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct, default) ) - def _stat(metric_name: str, stat: str, default: float = 0.0) -> float: - return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, default)) + def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: + return float(m0.get(metric_name, {}).get("successful", {}).get(stat, default)) - extra["ttft_p99"] = _percentile("time_to_first_token_ms", "p99") / 1000.0 - extra["tpot_p99"] = _percentile("time_per_output_token_ms", "p99") / 1000.0 - extra["itl_p99"] = _percentile("inter_token_latency_ms", "p99") / 1000.0 + extra["ttft_p99"] = _percentile0("time_to_first_token_ms", "p99") / 1000.0 + extra["tpot_p99"] = _percentile0("time_per_output_token_ms", "p99") / 1000.0 + extra["itl_p99"] = _percentile0("inter_token_latency_ms", "p99") / 1000.0 - request_totals = metrics.get("request_totals", {}) - extra["completed_requests"] = int(request_totals.get("successful", 0)) - extra["failed_requests"] = int(request_totals.get("errored", 0)) + request_totals0 = m0.get("request_totals", {}) + extra["completed_requests"] = int(request_totals0.get("successful", 0)) + extra["failed_requests"] = int(request_totals0.get("errored", 0)) + extra["prompt_token_count_mean"] = _stat0("prompt_token_count", "mean") - extra["prompt_token_count_mean"] = _stat("prompt_token_count", "mean") - - concurrency = _stat("request_concurrency", "mean") + concurrency = _stat0("request_concurrency", "mean") if concurrency > 0: extra["request_concurrency"] = concurrency + # Build per-benchmark extra curves for ALL dashboard metrics + # These will be merged into the existing performance_curves dict + extra_curves: dict[str, list] = { + "ttft_p1": [], + "ttft_p999": [], + "ttft_mean": [], + "tpot_p1": [], + "tpot_p999": [], + "itl_p1": [], + "itl_p999": [], + "itl_mean": [], + "request_latency_min": [], + "request_latency_max": [], + "measured_rps": [], + "prompt_token_count_mean": [], + "prompt_token_count_p99": [], + "output_token_count_mean": [], + "output_token_count_p99": [], + "output_tok_per_sec": [], + "total_tok_per_sec": [], + "successful_requests": [], + "errored_requests": [], + "intended_concurrency": [], + } + + for bench in all_benchmarks: + metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + + def _pct(metric_name: str, pct: str): + return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + + def _stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("successful", {}).get(stat) + + def _total_stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("total", {}).get(stat) + + request_totals = metrics.get("request_totals", {}) + + extra_curves["ttft_p1"].append(_pct("time_to_first_token_ms", "p01")) + extra_curves["ttft_p999"].append(_pct("time_to_first_token_ms", "p999")) + extra_curves["ttft_mean"].append(_stat("time_to_first_token_ms", "mean")) + extra_curves["tpot_p1"].append(_pct("time_per_output_token_ms", "p01")) + extra_curves["tpot_p999"].append(_pct("time_per_output_token_ms", "p999")) + extra_curves["itl_p1"].append(_pct("inter_token_latency_ms", "p01")) + extra_curves["itl_p999"].append(_pct("inter_token_latency_ms", "p999")) + extra_curves["itl_mean"].append(_stat("inter_token_latency_ms", "mean")) + extra_curves["request_latency_min"].append(_stat("request_latency", "min")) + extra_curves["request_latency_max"].append(_stat("request_latency", "max")) + extra_curves["measured_rps"].append(_stat("requests_per_second", "mean")) + extra_curves["prompt_token_count_mean"].append(_stat("prompt_token_count", "mean")) + extra_curves["prompt_token_count_p99"].append(_pct("prompt_token_count", "p99")) + extra_curves["output_token_count_mean"].append(_stat("output_token_count", "mean")) + extra_curves["output_token_count_p99"].append(_pct("output_token_count", "p99")) + extra_curves["output_tok_per_sec"].append(_total_stat("output_tokens_per_second", "mean")) + extra_curves["total_tok_per_sec"].append(_total_stat("tokens_per_second", "mean")) + extra_curves["successful_requests"].append(request_totals.get("successful", 0)) + extra_curves["errored_requests"].append(request_totals.get("errored", 0)) + extra_curves["intended_concurrency"].append(strategy.get("streams")) + + extra["_extra_curves"] = extra_curves return extra - - diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 60de3a313..4d67db20d 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -19,25 +19,39 @@ logger = logging.getLogger(__name__) -# KPI ID → dashboard CSV column name mapping +# KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { "rhaiis_output_tok_per_sec": "output_tok/sec", "rhaiis_total_tok_per_sec": "total_tok/sec", "rhaiis_measured_concurrency": "measured concurrency", + "rhaiis_measured_rps": "measured rps", + "rhaiis_intended_concurrency": "intended concurrency", "rhaiis_completed_requests": "successful_requests", "rhaiis_failed_requests": "errored_requests", "rhaiis_ttft_median": "ttft_median", "rhaiis_ttft_p95": "ttft_p95", "rhaiis_ttft_p99": "ttft_p99", + "rhaiis_ttft_p1": "ttft_p1", + "rhaiis_ttft_p999": "ttft_p999", + "rhaiis_ttft_mean": "ttft_mean", "rhaiis_tpot_median": "tpot_median", "rhaiis_tpot_p95": "tpot_p95", "rhaiis_tpot_p99": "tpot_p99", + "rhaiis_tpot_p1": "tpot_p1", + "rhaiis_tpot_p999": "tpot_p999", "rhaiis_itl_median": "itl_median", "rhaiis_itl_p95": "itl_p95", "rhaiis_itl_p99": "itl_p99", + "rhaiis_itl_p1": "itl_p1", + "rhaiis_itl_p999": "itl_p999", + "rhaiis_itl_mean": "itl_mean", "rhaiis_request_latency_median": "request_latency_median", - "rhaiis_request_latency_p95": "request_latency_max", + "rhaiis_request_latency_min": "request_latency_min", + "rhaiis_request_latency_max": "request_latency_max", "rhaiis_prompt_token_count_mean": "prompt_token_count_mean", + "rhaiis_prompt_token_count_p99": "prompt_token_count_p99", + "rhaiis_output_token_count_mean": "output_token_count_mean", + "rhaiis_output_token_count_p99": "output_token_count_p99", } @@ -85,8 +99,9 @@ def export_kpis_to_csv( ) -> str: """Export KPI records to dashboard-format CSV. - Groups per-rate-point KPIs by (run_path, rate_index) and pivots them into - rows matching the RHAIIS dashboard CSV schema. + Groups per-rate-point KPIs by (run_path, rate_index) and pivots them + into rows matching the RHAIIS dashboard CSV schema with all fields + populated. """ from projects.rhaiis.postprocess.csv_export import FIELDNAMES @@ -104,7 +119,8 @@ def export_kpis_to_csv( if csv_col: groups[key][csv_col] = kpi.get("value") - if key not in group_labels: + # Capture the richest set of labels for this group + if key not in group_labels or len(labels) > len(group_labels[key]): group_labels[key] = labels # Build CSV rows @@ -113,24 +129,39 @@ def export_kpis_to_csv( metrics = groups[key] labels = group_labels.get(key, {}) - acc = labels.get("accelerator", "") + acc = labels.get("accelerator", "").upper() cluster_tag = labels.get("cluster_tag", "") model_id = labels.get("hf_model_id", "") tp = labels.get("tensor_parallel_size", "1") version = labels.get("version", "") - run_name = f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" - row = {col: "" for col in FIELDNAMES} + run_name = ( + f"{acc}-{cluster_tag}-{model_id}-{tp}" + if cluster_tag + else f"{acc}-{model_id}-{tp}" + ) + + row: dict[str, Any] = {col: "" for col in FIELDNAMES} + # Populate all metric columns from KPI values row.update(metrics) + # Populate metadata columns from labels row["run"] = run_name row["accelerator"] = acc row["model"] = model_id row["version"] = version row["TP"] = tp - row["intended concurrency"] = labels.get("intended_concurrency", "") + row["prompt toks"] = labels.get("prompt_toks", "") + row["output toks"] = labels.get("output_toks", "") + # intended concurrency is populated from the rhaiis_intended_concurrency KPI; + # only fall back to labels if the KPI didn't set it + if not row.get("intended concurrency"): + row["intended concurrency"] = labels.get("intended_concurrency", "") row["image_tag"] = labels.get("image_tag", "") row["runtime_args"] = labels.get("runtime_args", "") row["uuid"] = labels.get("run_uuid", "") + row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") + row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") + row["guidellm_version"] = labels.get("guidellm_version", "") rows.append(row) output_path.parent.mkdir(parents=True, exist_ok=True) From d15938f7a319b970455a86104815ae05a852fbeb Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:32:59 -0400 Subject: [PATCH 188/344] Handle large benchmark files: compress before oc cp oc cp fails with 'unexpected EOF' on large benchmarks.json files (~70MB+). Fix by compressing the file inside the copy pod with gzip first, copying the compressed version, then decompressing locally. Falls back to direct oc cp for small files that succeed on the first attempt. Co-authored-by: Cursor --- .../toolbox/run_guidellm_benchmark/main.py | 70 ++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 7095eb7db..632c96efb 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -302,15 +302,8 @@ def extract_results(args, ctx): remote_path = f"/results/benchmarks-{run.label}.json" local_path = results_dir / f"benchmarks-{run.label}.json" - result = oc( - "cp", - f"{ctx.target_namespace}/{copy_pod}:{remote_path}", - str(local_path), - "-c", "copy-helper", - check=False, - log_stdout=False, - ) - if result.returncode != 0 or not local_path.exists(): + local_path = _copy_result_file(ctx.target_namespace, copy_pod, remote_path, local_path) + if local_path is None: raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") extracted_files.append( @@ -333,6 +326,65 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" +def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "Path | None": + """Copy a result file from a pod, compressing first to handle large files. + + Falls back to direct ``oc cp`` for small files. + """ + import gzip + import shutil + + # Try direct oc cp first + result = oc( + "cp", + f"{namespace}/{pod}:{remote_path}", + str(local_path), + "-c", "copy-helper", + check=False, + log_stdout=False, + ) + if result.returncode == 0 and local_path.exists() and local_path.stat().st_size > 0: + return local_path + + logging.getLogger(__name__).info( + "Direct oc cp failed (rc=%d), retrying with gzip compression", result.returncode, + ) + + # Compress inside the pod, copy the gz, decompress locally + gz_remote = f"{remote_path}.gz" + compress = oc( + "exec", pod, "-n", namespace, "-c", "copy-helper", "--", + "gzip", "-kf", remote_path, + check=False, + ) + if compress.returncode != 0: + logging.getLogger(__name__).warning("gzip inside pod failed (rc=%d)", compress.returncode) + return None + + local_gz = local_path.parent / f"{local_path.name}.gz" + cp_result = oc( + "cp", + f"{namespace}/{pod}:{gz_remote}", + str(local_gz), + "-c", "copy-helper", + check=False, + log_stdout=False, + ) + if cp_result.returncode != 0 or not local_gz.exists() or local_gz.stat().st_size == 0: + logging.getLogger(__name__).warning("oc cp of gzipped file failed") + local_gz.unlink(missing_ok=True) + return None + + with gzip.open(local_gz, "rb") as f_in, open(local_path, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + local_gz.unlink(missing_ok=True) + + if local_path.exists() and local_path.stat().st_size > 0: + return local_path + + return None + + @task def cleanup_copy_pod(args, ctx): """Delete the copy pod after results extraction""" From 2b55199c45421819c5109ee5f348caa5c4a8b86c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:08:48 -0400 Subject: [PATCH 189/344] Fix KPI validation: add required run_id field Co-authored-by: Cursor --- projects/rhaiis/postprocess/kpis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 6453ee7a8..a262bb41d 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -115,6 +115,7 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: "kpi_id": kpi_id, "value": value, "unit": unit, + "run_id": r.test_base_path, "run_path": r.test_base_path, "timestamp": ts, "labels": labels, From af3b991b247d29a82e7f1cb9a5ecfc3c5f919848 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:10:53 -0400 Subject: [PATCH 190/344] Fix dashboard CSV: use GPU chip name for accelerator, include run_uuid - Pass accelerator_chip (e.g. H200) to test labels instead of vendor name (nvidia) so dashboard shows correct accelerator - Move run_uuid generation before _create_test_labels so it propagates into KPI records and dashboard CSV uuid column Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f8a6dd487..01793a977 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -104,6 +104,8 @@ def _run_test( from projects.core.library import config as _cfg version = _cfg.project.get_config("tests.rhaiis.version", "") + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) + logger.info("Run UUID for this job: %s", run_uuid) _create_test_labels( model_key, workload_keys[0], accelerator, vllm_args, @@ -111,6 +113,8 @@ def _run_test( version=version, vllm_image=vllm_image, cluster_tag=cluster_tag, + accelerator_chip=gpu_type.upper(), + run_uuid=run_uuid, ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -119,9 +123,6 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) - logger.info("Run UUID for this job: %s", run_uuid) - import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") @@ -376,19 +377,22 @@ def _create_test_labels( version: str = "", vllm_image: str = "", cluster_tag: str = "", + accelerator_chip: str = "", + run_uuid: str = "", ) -> None: _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) labels = { "model_key": model_key, "workload_key": workload_key, - "accelerator": accelerator, + "accelerator": accelerator_chip or accelerator, "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), "hf_model_id": hf_model_id, "version": version, "image_tag": image_tag, "cluster_tag": cluster_tag, "runtime_args": runtime_args, + "run_uuid": run_uuid, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) From c77bd806d66df780dc5f2ee840df4d5048e33268 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:22:26 -0400 Subject: [PATCH 191/344] fix: sort extra curves by request_rate and convert latency units to ms - Sort benchmarks in RHAIIS parser by request_rate to match GuideLLM parser ordering, preventing curve index misalignment - Convert latency extra curves (TTFT/TPOT/ITL) from ms to seconds in parser to match GuideLLM base curves - Convert latency values back to ms in CSV export for dashboard compat - Return extra_curves separately from scalar metrics to avoid _extra_curves key leaking into merged_metrics Co-authored-by: Cursor --- projects/rhaiis/postprocess/parser.py | 64 +++++++++++++++++---------- projects/rhaiis/postprocess/plugin.py | 15 +++++++ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 1e2a3eccd..752c49297 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -34,12 +34,10 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: node = _find_node_for_record(record, nodes) if node: - extra = _extract_extra_metrics(node) - merged_metrics = {**record.metrics, **extra} - # Merge extra curves into existing performance_curves - existing_curves = merged_metrics.get("performance_curves", {}) - extra_curves = extra.pop("_extra_curves", {}) + extra_metrics, extra_curves = _extract_extra_metrics(node) + merged_metrics = {**record.metrics, **extra_metrics} if extra_curves: + existing_curves = merged_metrics.get("performance_curves", {}) existing_curves.update(extra_curves) merged_metrics["performance_curves"] = existing_curves enriched_records.append( @@ -67,7 +65,18 @@ def _find_node_for_record( return None -def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: +def _bench_sort_key(bench: dict) -> float: + """Sort key matching GuideLLMParser: requests_per_second.successful.mean.""" + return float( + bench.get("metrics", {}) + .get("requests_per_second", {}) + .get("successful", {}) + .get("mean", 0) + ) + + +def _extract_extra_metrics(node: TestBaseNode) -> tuple[dict[str, Any], dict[str, list]]: + """Return (scalar_metrics, extra_curves) extracted from raw benchmarks.json files.""" extra: dict[str, Any] = {} benchmarks_files = sorted( [p for p in node.artifact_paths if p.name == "benchmarks.json" @@ -75,9 +84,8 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: key=lambda p: p.name, ) if not benchmarks_files: - return extra + return extra, {} - # Collect all benchmarks across all files all_benchmarks: list[dict] = [] report_metadata: dict[str, Any] = {} report_args: dict[str, Any] = {} @@ -94,9 +102,12 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: report_args = data.get("args", {}) if not all_benchmarks: - return extra + return extra, {} - # Extract report-level metadata + # Sort by request_rate to match GuideLLMParser curve ordering + all_benchmarks.sort(key=_bench_sort_key) + + # Report-level metadata extra["guidellm_version"] = report_metadata.get("guidellm_version", "") data_str = "" @@ -108,7 +119,6 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: extra["prompt_toks"] = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" extra["output_toks"] = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" - # Compute global start/end timestamps start_times = [] end_times = [] for bench in all_benchmarks: @@ -145,8 +155,7 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: if concurrency > 0: extra["request_concurrency"] = concurrency - # Build per-benchmark extra curves for ALL dashboard metrics - # These will be merged into the existing performance_curves dict + # Per-benchmark extra curves — indices align with GuideLLM's sorted curves extra_curves: dict[str, list] = { "ttft_p1": [], "ttft_p999": [], @@ -177,22 +186,32 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: def _pct(metric_name: str, pct: str): return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + def _pct_s(metric_name: str, pct: str): + """Percentile value converted from ms to seconds.""" + v = _pct(metric_name, pct) + return v / 1000.0 if v is not None else None + def _stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("successful", {}).get(stat) + def _stat_s(metric_name: str, stat: str): + """Stat value converted from ms to seconds.""" + v = _stat(metric_name, stat) + return v / 1000.0 if v is not None else None + def _total_stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("total", {}).get(stat) request_totals = metrics.get("request_totals", {}) - extra_curves["ttft_p1"].append(_pct("time_to_first_token_ms", "p01")) - extra_curves["ttft_p999"].append(_pct("time_to_first_token_ms", "p999")) - extra_curves["ttft_mean"].append(_stat("time_to_first_token_ms", "mean")) - extra_curves["tpot_p1"].append(_pct("time_per_output_token_ms", "p01")) - extra_curves["tpot_p999"].append(_pct("time_per_output_token_ms", "p999")) - extra_curves["itl_p1"].append(_pct("inter_token_latency_ms", "p01")) - extra_curves["itl_p999"].append(_pct("inter_token_latency_ms", "p999")) - extra_curves["itl_mean"].append(_stat("inter_token_latency_ms", "mean")) + extra_curves["ttft_p1"].append(_pct_s("time_to_first_token_ms", "p01")) + extra_curves["ttft_p999"].append(_pct_s("time_to_first_token_ms", "p999")) + extra_curves["ttft_mean"].append(_stat_s("time_to_first_token_ms", "mean")) + extra_curves["tpot_p1"].append(_pct_s("time_per_output_token_ms", "p01")) + extra_curves["tpot_p999"].append(_pct_s("time_per_output_token_ms", "p999")) + extra_curves["itl_p1"].append(_pct_s("inter_token_latency_ms", "p01")) + extra_curves["itl_p999"].append(_pct_s("inter_token_latency_ms", "p999")) + extra_curves["itl_mean"].append(_stat_s("inter_token_latency_ms", "mean")) extra_curves["request_latency_min"].append(_stat("request_latency", "min")) extra_curves["request_latency_max"].append(_stat("request_latency", "max")) extra_curves["measured_rps"].append(_stat("requests_per_second", "mean")) @@ -206,5 +225,4 @@ def _total_stat(metric_name: str, stat: str): extra_curves["errored_requests"].append(request_totals.get("errored", 0)) extra_curves["intended_concurrency"].append(strategy.get("streams")) - extra["_extra_curves"] = extra_curves - return extra + return extra, extra_curves diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 4d67db20d..658462638 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -18,6 +18,13 @@ logger = logging.getLogger(__name__) +# CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. +# GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. +_SECONDS_TO_MS_COLUMNS = frozenset({ + "ttft_median", "ttft_p95", "ttft_p99", "ttft_p1", "ttft_p999", "ttft_mean", + "tpot_median", "tpot_p95", "tpot_p99", "tpot_p1", "tpot_p999", + "itl_median", "itl_p95", "itl_p99", "itl_p1", "itl_p999", "itl_mean", +}) # KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { @@ -144,6 +151,14 @@ def export_kpis_to_csv( row: dict[str, Any] = {col: "" for col in FIELDNAMES} # Populate all metric columns from KPI values row.update(metrics) + # Convert latency columns from seconds back to ms for dashboard compat + for col in _SECONDS_TO_MS_COLUMNS: + val = row.get(col) + if val not in ("", None): + try: + row[col] = float(val) * 1000.0 + except (TypeError, ValueError): + pass # Populate metadata columns from labels row["run"] = run_name row["accelerator"] = acc From 05cf21b98c654c673dfbbf9548ec940b6b855215 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 14:54:16 +0200 Subject: [PATCH 192/344] [llm_d] orchestration: config: platform: remove trailing " --- projects/llm_d/orchestration/config.d/platform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index a8bf91b73..4e666be30 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -8,7 +8,7 @@ cluster: namespace: labels: app.kubernetes.io/managed-by: forge - forge.openshift.io/project: llm_d" + forge.openshift.io/project: llm_d operators: openshift-cert-manager-operator: From f3f956c74d2c58cd04d810aeb7fc668370692ae2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 15:04:10 +0200 Subject: [PATCH 193/344] [llm_d] orchestration: render_inference_service: vllm_default_args are already applied in runtime_config.get_deployment_profile() --- projects/llm_d/orchestration/render_inference_service.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 79e433ee0..29c254333 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -173,12 +173,9 @@ def _build_vllm_additional_args( String suitable for VLLM_ADDITIONAL_ARGS environment variable """ - vllm_default_args = _build_vllm_args( - config.project.get_config("deployments.defaults.vllm_extra.args") - ) - vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra.args", {})) + vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra", {}).get("args")) - return " ".join(vllm_default_args + vllm_deploy_args) + return " ".join(vllm_deploy_args) def _render_standard_deployment( From 3c629b86edeb0c5e74dc5d27e6bce8cb4613c012 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 24 Jul 2026 10:35:02 +0200 Subject: [PATCH 194/344] [projects] Fix ruff in MD files --- AGENTS.md | 2 + docs/orchestration/vaults.md | 24 ++++---- docs/toolbox/dsl.md | 6 +- docs/toolbox/toolbox-command-guide.md | 58 +++++++++++-------- .../cluster/toolbox/rebuild_image/README.md | 6 +- projects/core/CHANGELOG-dsl.md | 1 + projects/core/notifications/README.md | 2 +- projects/llm_d/docs/PD_TESTING.md | 8 +-- projects/skeleton/README.md | 42 ++++---------- 9 files changed, 70 insertions(+), 79 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4ba19d7a0..ddeaaaa52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,8 +55,10 @@ def apply_secret(manifest: dict) -> None: """Apply a Secret manifest via stdin — nothing written to disk.""" import yaml from projects.core.dsl.utils.k8s import oc + oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False)) + # GOOD: Guard before using oc_apply def safe_oc_apply(artifact_path, manifest): if manifest.get("kind") == "Secret": diff --git a/docs/orchestration/vaults.md b/docs/orchestration/vaults.md index edef7ad06..44a8928ce 100644 --- a/docs/orchestration/vaults.md +++ b/docs/orchestration/vaults.md @@ -116,7 +116,7 @@ from projects.core.library import vault vault.disable_strict_validation() # All subsequent vault operations will use warning mode -vault.init(['psap-forge-ci']) # Uses non-strict validation +vault.init(["psap-forge-ci"]) # Uses non-strict validation ``` **When to use non-strict mode:** @@ -140,10 +140,10 @@ vault.disable_strict_validation() # Optional: disable strict mode vault_manager = get_vault_manager() # Validate a specific vault -is_valid = vault_manager.validate_vault('psap-forge-ci') +is_valid = vault_manager.validate_vault("psap-forge-ci") # Validate all project vaults -is_valid = vault_manager.validate_project_vaults('llm_d') +is_valid = vault_manager.validate_project_vaults("llm_d") # Validate all defined vaults is_valid = vault_manager.validate_all_vaults() @@ -155,12 +155,12 @@ is_valid = vault_manager.validate_all_vaults() from projects.core.library.vault import get_vault_content_path # Get full path to vault content -kubeconfig_path = get_vault_content_path('psap-forge-ci', 'kubeconfig') +kubeconfig_path = get_vault_content_path("psap-forge-ci", "kubeconfig") # Returns: Path('/secrets/ci/kubeconfig') # Access via vault object -vault = get_vault_manager().get_vault('psap-forge-ci') -kubeconfig_path = vault.content['kubeconfig'].file_path +vault = get_vault_manager().get_vault("psap-forge-ci") +kubeconfig_path = vault.content["kubeconfig"].file_path # Returns: Path('/secrets/ci/kubeconfig') ``` @@ -239,7 +239,7 @@ In `projects/my-project/orchestration/vaults.yaml`: from projects.core.library.vault import validate_project_vaults # Validate the new vault (uses global strict validation setting) -validate_project_vaults('my-project') +validate_project_vaults("my-project") ``` ### 6. Expose the env var to the `forge_launcher` @@ -265,7 +265,7 @@ from projects.core.library import vault vault.init() # Initialize with specific vaults only -vault.init(['psap-forge-ci', 'forge-llm-d']) +vault.init(["psap-forge-ci", "forge-llm-d"]) ``` ### Runtime Usage @@ -277,13 +277,13 @@ from projects.core.library.vault import get_vault_manager vault_manager = get_vault_manager() # Validate project vaults before execution -if not vault_manager.validate_project_vaults('llm_d'): +if not vault_manager.validate_project_vaults("llm_d"): raise RuntimeError("Vault validation failed") # Get paths to vault content -vault = vault_manager.get_vault('forge-llm-d') -hf_token_path = vault.content['hf_token'].file_path -pull_secret_path = vault.content['rhoai_pull_secret'].file_path +vault = vault_manager.get_vault("forge-llm-d") +hf_token_path = vault.content["hf_token"].file_path +pull_secret_path = vault.content["rhoai_pull_secret"].file_path ``` ## Error Handling diff --git a/docs/toolbox/dsl.md b/docs/toolbox/dsl.md index 101326c51..54246707c 100644 --- a/docs/toolbox/dsl.md +++ b/docs/toolbox/dsl.md @@ -37,8 +37,7 @@ def ensure_project(args, ctx): ```python @when(lambda: some_other_task.status.return_value is True) @task -def follow_up(args, ctx): - ... +def follow_up(args, ctx): ... ``` Because the condition is called with **no arguments**, anything dynamic must come from a **closure**, module-level state, or another task’s `.status.return_value` (see `TaskResult` in `script_manager.py`). @@ -69,8 +68,7 @@ def wait_until_ready(args, ctx): ```python @retry(attempts=5, delay=2, backoff=1.5, retry_on_exceptions=True) @task -def call_flaky_api(args, ctx): - ... +def call_flaky_api(args, ctx): ... ``` If all attempts fail, `RetryFailure` is raised (wrapped in `TaskExecutionError` during `execute_tasks`, with the underlying `RetryFailure` available as `TaskExecutionError.__cause__` when that applies). diff --git a/docs/toolbox/toolbox-command-guide.md b/docs/toolbox/toolbox-command-guide.md index 3a6f688ee..37c15b24c 100644 --- a/docs/toolbox/toolbox-command-guide.md +++ b/docs/toolbox/toolbox-command-guide.md @@ -174,6 +174,7 @@ def capture_resources(args, ctx): shell.run("oc delete pod mypod", check=False) return "Cleanup completed" + @always @task def cleanup_resources(args, ctx): @@ -195,10 +196,13 @@ shell.run("oc get pods") result = shell.run("oc get pods -o json") if result.success: import json + pods = json.loads(result.stdout) # Save output to file -shell.run("oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt") +shell.run( + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt" +) # Don't fail on error shell.run("oc delete pod optional-pod", check=False) @@ -278,21 +282,16 @@ def capture_pod_status(args, ctx): # Human readable shell.run( - "oc describe pod mypod", - stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" ) # Machine readable shell.run( - "oc get pod mypod -o yaml", - stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" + "oc get pod mypod -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" ) # For large scale tests, also capture JSON - shell.run( - "oc get pods -o json", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json" - ) + shell.run("oc get pods -o json", stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json") ``` #### Source Directory (`src/`) @@ -356,7 +355,7 @@ def run(password: str): # DON'T DO THIS! #### ❌ BAD: Secrets in environment variables ```python -os.environ['SECRET_TOKEN'] = token # DON'T DO THIS! +os.environ["SECRET_TOKEN"] = token # DON'T DO THIS! # Environment variables appear in logs and process lists ``` @@ -490,10 +489,12 @@ Use descriptive task names that explain the **purpose**, not just the action: def wait_for_operator_to_be_ready(args, ctx): """Wait for operator deployment to reach ready state""" + @task def capture_failed_pod_logs(args, ctx): """Collect logs from pods that failed to start""" + @task def verify_service_endpoints_are_available(args, ctx): """Check that service endpoints are responding to requests""" @@ -523,7 +524,7 @@ def wait_for_resource_ready(args, ctx): shell.run( f"oc describe myresource {args.resource_name}", stdout_dest=args.artifact_dir / "artifacts" / "failed-resource-describe.txt", - check=False + check=False, ) raise RuntimeError(f"Failed to query resource {args.resource_name}: {result.stderr}") @@ -541,13 +542,17 @@ def capture_cluster_state(args, ctx): """Capture cluster state for post-mortem analysis""" # Capture even if tests failed - shell.run("oc get pods --all-namespaces", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", - check=False) + shell.run( + "oc get pods --all-namespaces", + stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", + check=False, + ) - shell.run("oc get events --sort-by='.lastTimestamp'", - stdout_dest=args.artifact_dir / "artifacts" / "events.txt", - check=False) + shell.run( + "oc get events --sort-by='.lastTimestamp'", + stdout_dest=args.artifact_dir / "artifacts" / "events.txt", + check=False, + ) return "Cluster state captured" ``` @@ -560,9 +565,8 @@ def capture_cluster_state(args, ctx): ```python #!/usr/bin/env python3 -from projects.core.dsl import ( - always, entrypoint, execute_tasks, retry, shell, task, template -) +from projects.core.dsl import always, entrypoint, execute_tasks, retry, shell, task, template + @entrypoint def run( @@ -585,6 +589,7 @@ def run( """ return execute_tasks(locals()) + @task def validate_inputs(args, ctx): """Validate input parameters""" @@ -598,6 +603,7 @@ def validate_inputs(args, ctx): return "Inputs validated" + @task def setup_directories(args, ctx): """Create artifact directories""" @@ -606,6 +612,7 @@ def setup_directories(args, ctx): shell.mkdir("src") return "Directories created" + @task def verify_namespace_exists(args, ctx): """Ensure target namespace exists""" @@ -616,6 +623,7 @@ def verify_namespace_exists(args, ctx): return f"Namespace {args.namespace} verified" + @task def create_deployment_manifest(args, ctx): """Generate deployment manifest""" @@ -625,6 +633,7 @@ def create_deployment_manifest(args, ctx): return f"Deployment manifest created: {manifest_file}" + @task def apply_deployment(args, ctx): """Apply the deployment to cluster""" @@ -634,6 +643,7 @@ def apply_deployment(args, ctx): return f"Deployment {args.service_name} applied" + @retry(attempts=10, delay=5) @task def wait_for_deployment_ready(args, ctx): @@ -642,7 +652,7 @@ def wait_for_deployment_ready(args, ctx): result = shell.run( f"oc get deployment {args.service_name} -n {args.namespace} " f"-o jsonpath='{{.status.readyReplicas}}'", - check=False + check=False, ) if not result.success: @@ -654,6 +664,7 @@ def wait_for_deployment_ready(args, ctx): return False # Retry + @always @task def capture_deployment_status(args, ctx): @@ -663,18 +674,19 @@ def capture_deployment_status(args, ctx): shell.run( f"oc describe deployment {args.service_name} -n {args.namespace}", stdout_dest=args.artifact_dir / "artifacts" / "deployment-describe.txt", - check=False + check=False, ) # Capture pod status shell.run( f"oc get pods -l app={args.service_name} -n {args.namespace} -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pods-status.yaml", - check=False + check=False, ) return "Deployment status captured" + if __name__ == "__main__": run.main() ``` diff --git a/projects/cluster/toolbox/rebuild_image/README.md b/projects/cluster/toolbox/rebuild_image/README.md index 324f2ea26..062defce5 100644 --- a/projects/cluster/toolbox/rebuild_image/README.md +++ b/projects/cluster/toolbox/rebuild_image/README.md @@ -43,11 +43,7 @@ Can be imported and used programmatically: ```python from projects.cluster.toolbox.rebuild_image.main import run as rebuild_image -result = rebuild_image( - build_name="my-build", - namespace="my-namespace", - timeout_minutes=30 -) +result = rebuild_image(build_name="my-build", namespace="my-namespace", timeout_minutes=30) ``` ## Common Use Cases diff --git a/projects/core/CHANGELOG-dsl.md b/projects/core/CHANGELOG-dsl.md index b3f69351c..a3c7cd951 100644 --- a/projects/core/CHANGELOG-dsl.md +++ b/projects/core/CHANGELOG-dsl.md @@ -42,6 +42,7 @@ ```python from projects.core.dsl import EarlyReturn + @task def check_operator_deployed(args, ctx): if operator_exists(): diff --git a/projects/core/notifications/README.md b/projects/core/notifications/README.md index cc7442565..e3bf9831b 100644 --- a/projects/core/notifications/README.md +++ b/projects/core/notifications/README.md @@ -59,7 +59,7 @@ success = send_notification( message="🟢 Test completed successfully\nResults: https://example.com/results", github=True, slack=False, - dry_run=False + dry_run=False, ) ``` diff --git a/projects/llm_d/docs/PD_TESTING.md b/projects/llm_d/docs/PD_TESTING.md index b62c752d5..aeb2c3d8d 100644 --- a/projects/llm_d/docs/PD_TESTING.md +++ b/projects/llm_d/docs/PD_TESTING.md @@ -152,7 +152,7 @@ pd_config = runtime_config.get_pd_config() # Get individual P/D parameters prefill_pods = runtime_config.get_prefill_pod_count() # Returns: 2 -decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 +decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 scheduler_config = runtime_config.get_scheduler_config() # Returns: 'high-throughput' # Check if current deployment is P/D @@ -272,13 +272,13 @@ if not match: # For prefill_tensor_parallelism match = re.search(r"p\.tp(\d+)", profile_name) # p.tp1 → 1 -# For decode_pods (main container replicas) -match = re.search(r"d\.x(\d+)", profile_name) # d.x2 → 2 +# For decode_pods (main container replicas) +match = re.search(r"d\.x(\d+)", profile_name) # d.x2 → 2 if not match: match = re.search(r"(? Date: Fri, 24 Jul 2026 10:47:37 +0200 Subject: [PATCH 195/344] ruff: pin the version to 0.16.0 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcab8c11a..cc1dcab29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.16.0 hooks: - id: ruff args: [--fix] diff --git a/pyproject.toml b/pyproject.toml index 400f0eb8f..4faa72b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dev = [ "pytest>=7.4.0", "pytest-asyncio>=0.21.1", "black>=23.7.0", - "ruff>=0.0.287", + "ruff==0.16.0", "pre-commit>=4.0", ] docs = [ From ece24b94b0eb94d580b3fb583979e78090fce49e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 26 Jul 2026 15:22:10 +0300 Subject: [PATCH 196/344] cluster: add capture_prometheus toolbox for extracting time-bounded metrics Adds a new toolbox under projects/cluster/toolbox/capture_prometheus that extracts Prometheus metrics for a specific time window from the cluster's monitoring stack and saves them as a compressed OpenMetrics file. Uses a symlink trick to scan only WAL + chunks_head (~1-2 GB) instead of the full TSDB, making extraction fast (~2 min for 20 min of metrics). Currently limited to tests under 2 hours. Co-authored-by: Cursor --- .../toolbox/capture_prometheus/README.md | 80 +++++++ .../toolbox/capture_prometheus/main.py | 211 ++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 projects/cluster/toolbox/capture_prometheus/README.md create mode 100644 projects/cluster/toolbox/capture_prometheus/main.py diff --git a/projects/cluster/toolbox/capture_prometheus/README.md b/projects/cluster/toolbox/capture_prometheus/README.md new file mode 100644 index 000000000..c59eec15d --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/README.md @@ -0,0 +1,80 @@ +# capture_prometheus + +Extracts cluster Prometheus metrics for a specific time window and saves them as a compressed OpenMetrics formated file. The output can later be imported into a local Prometheus instance for offline querying of any metric. + +## How it works + +1. **Symlink trick** — Creates a temporary directory on the Prometheus PVC containing only symlinks to `wal/` and `chunks_head/`. This limits `promtool` to scanning the recent in-memory data (~1-2 GB) instead of the full TSDB (which can be 20+ GB of persistent blocks). +2. **Time-filtered dump** — Runs `promtool tsdb dump-openmetrics --min-time=X --max-time=Y` inside the Prometheus container against the temp directory. Only samples within the requested window are emitted. +3. **Compress and copy** — The output is gzipped on the pod, then copied to the specified output directory via `oc cp`. +4. **Cleanup** — Removes the temp directory and compressed file from the pod. + + + +## Limitations + +- **Max test duration: 2 hours.** The WAL + chunks_head holds approximately the last 2 hours of un-compacted data. Tests longer than 2 hours will fail with an error. Support for including persistent blocks that overlap with the test window can be added later. +- **Data granularity:** The dump includes all metrics scraped by cluster monitoring (typically every 30s). For a 20-minute test this produces ~24M data points / ~130MB compressed. + + + +## Usage + + + +### Standalone (CLI) + +```bash +cd forge-prom/forge +python3 -m projects.cluster.toolbox.capture_prometheus.main \ + "2026-07-26T10:00:00+00:00" \ + "2026-07-26T10:20:00+00:00" \ + /path/to/output +``` + + + +### From orchestration + +```python +from projects.cluster.toolbox.capture_prometheus import main as capture_prom + +prometheus_dir = env.ARTIFACT_DIR / "prometheus" +prometheus_dir.mkdir(parents=True, exist_ok=True) + +capture_prom.run( + start_time=test_start_time, + end_time=test_end_time, + output_dir=str(prometheus_dir), +) +``` + + + +## Querying the data locally + +To analyze the captured metrics offline: + +1. **Decompress** the archive to get a plain OpenMetrics text file. +2. **Import** the file into a Prometheus TSDB using `promtool tsdb create-blocks-from openmetrics`. +3. **Run a Prometheus container** (Docker, Podman, or any OCI runtime) with: + - The imported TSDB directory mounted at `/prometheus` + - A minimal config file (an empty `global: {}` is sufficient since no scraping is needed) + - Retention set high enough that the data won't be garbage-collected (e.g. `--storage.tsdb.retention.time=365d`) +4. **Open the Prometheus UI** and query any metric using the time range from `metadata.json`. + +The output file (`metrics.openmetrics.gz`) follows the standard [OpenMetrics](https://openmetrics.io/) format and is compatible with any tool that can ingest it (Prometheus, Grafana via Prometheus data source, etc.). + +## Parameters + + +| Parameter | Required | Default | Description | +| ------------- | -------- | ---------------------- | ------------------------------------------------ | +| `start_time` | yes | — | Start of capture window (ISO 8601, UTC) | +| `end_time` | yes | — | End of capture window (ISO 8601, UTC) | +| `output_dir` | yes | — | Directory to write `metrics.openmetrics.gz` into | +| `--pod-name` | no | `prometheus-k8s-0` | Prometheus pod name | +| `--namespace` | no | `openshift-monitoring` | Namespace of Prometheus | +| `--container` | no | `prometheus` | Container name in the pod | + + diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py new file mode 100644 index 000000000..e290883f9 --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +""" +Capture Prometheus TSDB Toolbox + +Extracts cluster Prometheus metrics for a specific time window and saves them +as a compressed OpenMetrics archive. The output can later be imported into a +local Prometheus instance for offline querying. + +Can be run standalone: + python -m projects.cluster.toolbox.capture_prometheus.main \\ + --start-time "2026-07-26T10:00:00+00:00" \\ + --end-time "2026-07-26T10:20:00+00:00" \\ + --output-dir /path/to/output +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from pathlib import Path + +from projects.core.dsl import always, entrypoint, execute_tasks, shell, task + +logger = logging.getLogger("DSL") + +PROMETHEUS_NAMESPACE = "openshift-monitoring" +PROMETHEUS_POD = "prometheus-k8s-0" +PROMETHEUS_CONTAINER = "prometheus" +TSDB_PATH = "/prometheus" + + +@entrypoint +def run( + start_time: datetime, + end_time: datetime, + output_dir: str | Path, + *, + pod_name: str = PROMETHEUS_POD, + namespace: str = PROMETHEUS_NAMESPACE, + container: str = PROMETHEUS_CONTAINER, +) -> int: + """ + Capture Prometheus metrics for a specific time window. + + Extracts all metrics between start_time and end_time from the cluster's + Prometheus TSDB and writes them as a compressed OpenMetrics file to output_dir. + + Args: + start_time: Start of the capture window (UTC) + end_time: End of the capture window (UTC) + output_dir: Directory to write the metrics archive into + pod_name: Prometheus pod to extract from + namespace: Namespace where Prometheus runs + container: Container name within the Prometheus pod + """ + execute_tasks(locals()) + return 0 + + +def _parse_time(value) -> datetime: + """Parse a datetime from string (ISO format) or pass through if already datetime.""" + if isinstance(value, datetime): + return value + dt = datetime.fromisoformat(str(value)) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + + +@task +def validate_parameters(args, ctx): + """Validate inputs and compute time bounds.""" + + ctx.start_time = _parse_time(args.start_time) + ctx.end_time = _parse_time(args.end_time) + + if ctx.end_time <= ctx.start_time: + raise ValueError("end_time must be after start_time") + + max_duration_seconds = 2 * 3600 + duration = (ctx.end_time - ctx.start_time).total_seconds() + if duration > max_duration_seconds: + raise ValueError( + f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " + f"Tests longer than 2 hours require persistent block handling (not yet implemented)." + ) + + ctx.output_dir = Path(args.output_dir) + ctx.output_dir.mkdir(parents=True, exist_ok=True) + ctx.start_ms = int(ctx.start_time.timestamp() * 1000) + ctx.end_ms = int(ctx.end_time.timestamp() * 1000) + ctx.duration_seconds = int((ctx.end_time - ctx.start_time).total_seconds()) + + ctx.namespace = args.namespace or PROMETHEUS_NAMESPACE + ctx.pod_name = args.pod_name or PROMETHEUS_POD + ctx.container = args.container or PROMETHEUS_CONTAINER + + return ( + f"Capture window: {ctx.start_time.isoformat()} → {ctx.end_time.isoformat()} " + f"({ctx.duration_seconds}s)" + ) + + +@task +def validate_prometheus_pod(args, ctx): + """Verify the Prometheus pod is running and accessible.""" + + result = shell.run( + f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", + ) + + phase = result.stdout.strip() + if phase != "Running": + raise RuntimeError( + f"Prometheus pod {ctx.pod_name} is in phase '{phase}', expected 'Running'" + ) + + return f"Prometheus pod {ctx.pod_name} is running" + + +@task +def create_temp_tsdb_dir(args, ctx): + """Create a temp directory on the PVC with symlinks to WAL and chunks_head only. + + This limits promtool to scanning only recent in-memory data (~1-2 GB) + instead of the full TSDB which can be tens of GB. + """ + + ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" + + shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"sh -c '" + f"mkdir -p {ctx.temp_dir} && " + f"ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal && " + f"ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" + f"'", + ) + + return f"Created temp TSDB dir at {ctx.temp_dir}" + + +@task +def dump_metrics(args, ctx): + """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" + + ctx.remote_output = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}.gz" + + shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"sh -c '" + f"promtool tsdb dump-openmetrics " + f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " + f"{ctx.temp_dir} | gzip > {ctx.remote_output}" + f"'", + ) + + size_result = shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"stat -c %s {ctx.remote_output}", + check=False, + ) + + ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 + + return f"Dumped metrics ({ctx.archive_size_bytes} bytes compressed)" + + +@task +def copy_to_output(args, ctx): + """Copy the compressed metrics file from the pod to the output directory.""" + + ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" + + shell.run( + f"oc cp {ctx.namespace}/{ctx.pod_name}:{ctx.remote_output} " + f"{ctx.output_file} -c {ctx.container}", + ) + + return f"Copied metrics to {ctx.output_file}" + + +@always +@task +def cleanup_pod(args, ctx): + """Remove temp files from the Prometheus pod.""" + + namespace = getattr(ctx, "namespace", None) or PROMETHEUS_NAMESPACE + pod_name = getattr(ctx, "pod_name", None) or PROMETHEUS_POD + container = getattr(ctx, "container", None) or PROMETHEUS_CONTAINER + temp_dir = getattr(ctx, "temp_dir", None) + remote_output = getattr(ctx, "remote_output", None) + + if temp_dir: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -rf {temp_dir}", + check=False, + ) + + if remote_output: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_output}", + check=False, + ) + + return "Cleaned up temp files on Prometheus pod" + + +if __name__ == "__main__": + run.main() From 6502de3883c8981e4086d37d230001c959ebbfd2 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 26 Jul 2026 18:49:57 +0300 Subject: [PATCH 197/344] addres few code rabbit issues --- .../toolbox/capture_prometheus/main.py | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index e290883f9..da18d5368 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -59,13 +59,18 @@ def run( def _parse_time(value) -> datetime: - """Parse a datetime from string (ISO format) or pass through if already datetime.""" + """Parse a datetime from string (ISO format) or pass through if already datetime. + + Naive datetimes are assumed UTC. Aware datetimes are normalized to UTC. + """ if isinstance(value, datetime): - return value + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) dt = datetime.fromisoformat(str(value)) if dt.tzinfo is None: dt = dt.replace(tzinfo=UTC) - return dt + return dt.astimezone(UTC) @task @@ -81,10 +86,13 @@ def validate_parameters(args, ctx): max_duration_seconds = 2 * 3600 duration = (ctx.end_time - ctx.start_time).total_seconds() if duration > max_duration_seconds: - raise ValueError( - f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " - f"Tests longer than 2 hours require persistent block handling (not yet implemented)." + logger.warning( + "Test duration (%ds) exceeds the 2-hour WAL window. " + "Skipping Prometheus capture (persistent block handling not yet implemented).", + int(duration), ) + ctx.skip_capture = True + return f"SKIPPED: duration {int(duration)}s exceeds 2-hour WAL window" ctx.output_dir = Path(args.output_dir) ctx.output_dir.mkdir(parents=True, exist_ok=True) @@ -106,6 +114,9 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + result = shell.run( f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", ) @@ -127,6 +138,9 @@ def create_temp_tsdb_dir(args, ctx): instead of the full TSDB which can be tens of GB. """ + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" shell.run( @@ -145,14 +159,18 @@ def create_temp_tsdb_dir(args, ctx): def dump_metrics(args, ctx): """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" - ctx.remote_output = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}.gz" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + + ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" + ctx.remote_output = f"{ctx.remote_raw}.gz" shell.run( f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " f"sh -c '" f"promtool tsdb dump-openmetrics " f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " - f"{ctx.temp_dir} | gzip > {ctx.remote_output}" + f"{ctx.temp_dir} > {ctx.remote_raw} && gzip -f {ctx.remote_raw}" f"'", ) @@ -171,6 +189,9 @@ def dump_metrics(args, ctx): def copy_to_output(args, ctx): """Copy the compressed metrics file from the pod to the output directory.""" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" shell.run( @@ -204,6 +225,13 @@ def cleanup_pod(args, ctx): check=False, ) + remote_raw = getattr(ctx, "remote_raw", None) + if remote_raw: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_raw}", + check=False, + ) + return "Cleaned up temp files on Prometheus pod" From 343b78c7dfc83b8041b90d8b057aa8a61215eea8 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 27 Jul 2026 09:31:22 +0300 Subject: [PATCH 198/344] address kevin's comments, as well as a small fix to the core cli entry point to correctly parse defualt values --- .../toolbox/capture_prometheus/README.md | 3 +- .../toolbox/capture_prometheus/main.py | 107 +++++++----------- projects/core/dsl/cli.py | 1 + 3 files changed, 44 insertions(+), 67 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/README.md b/projects/cluster/toolbox/capture_prometheus/README.md index c59eec15d..63c8d09ce 100644 --- a/projects/cluster/toolbox/capture_prometheus/README.md +++ b/projects/cluster/toolbox/capture_prometheus/README.md @@ -25,8 +25,7 @@ Extracts cluster Prometheus metrics for a specific time window and saves them as ### Standalone (CLI) ```bash -cd forge-prom/forge -python3 -m projects.cluster.toolbox.capture_prometheus.main \ +./bin/run_toolbox cluster capture_prometheus \ "2026-07-26T10:00:00+00:00" \ "2026-07-26T10:20:00+00:00" \ /path/to/output diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index da18d5368..f2c5469f7 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -8,10 +8,10 @@ local Prometheus instance for offline querying. Can be run standalone: - python -m projects.cluster.toolbox.capture_prometheus.main \\ - --start-time "2026-07-26T10:00:00+00:00" \\ - --end-time "2026-07-26T10:20:00+00:00" \\ - --output-dir /path/to/output + ./bin/run_toolbox cluster capture_prometheus \\ + "2026-07-26T10:00:00+00:00" \\ + "2026-07-26T10:20:00+00:00" \\ + /path/to/output """ from __future__ import annotations @@ -58,6 +58,21 @@ def run( return 0 +def _oc_exec(ctx, command: str, **kwargs): + """Run a command inside the Prometheus container via oc exec.""" + return shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- {command}", + **kwargs, + ) + + +def _oc_cp_from_pod(ctx, remote_path: str, local_path): + """Copy a file from the Prometheus container to a local path.""" + return shell.run( + f"oc cp {ctx.namespace}/{ctx.pod_name}:{remote_path} {local_path} -c {ctx.container}", + ) + + def _parse_time(value) -> datetime: """Parse a datetime from string (ISO format) or pass through if already datetime. @@ -86,13 +101,10 @@ def validate_parameters(args, ctx): max_duration_seconds = 2 * 3600 duration = (ctx.end_time - ctx.start_time).total_seconds() if duration > max_duration_seconds: - logger.warning( - "Test duration (%ds) exceeds the 2-hour WAL window. " - "Skipping Prometheus capture (persistent block handling not yet implemented).", - int(duration), + raise ValueError( + f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " + f"Tests longer than 2 hours require persistent block handling (not yet implemented)." ) - ctx.skip_capture = True - return f"SKIPPED: duration {int(duration)}s exceeds 2-hour WAL window" ctx.output_dir = Path(args.output_dir) ctx.output_dir.mkdir(parents=True, exist_ok=True) @@ -100,9 +112,9 @@ def validate_parameters(args, ctx): ctx.end_ms = int(ctx.end_time.timestamp() * 1000) ctx.duration_seconds = int((ctx.end_time - ctx.start_time).total_seconds()) - ctx.namespace = args.namespace or PROMETHEUS_NAMESPACE - ctx.pod_name = args.pod_name or PROMETHEUS_POD - ctx.container = args.container or PROMETHEUS_CONTAINER + ctx.namespace = args.namespace + ctx.pod_name = args.pod_name + ctx.container = args.container return ( f"Capture window: {ctx.start_time.isoformat()} → {ctx.end_time.isoformat()} " @@ -114,9 +126,6 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - result = shell.run( f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", ) @@ -138,17 +147,14 @@ def create_temp_tsdb_dir(args, ctx): instead of the full TSDB which can be tens of GB. """ - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" - shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + _oc_exec( + ctx, f"sh -c '" - f"mkdir -p {ctx.temp_dir} && " - f"ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal && " - f"ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" + f"mkdir -p {ctx.temp_dir}" + f" && ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal" + f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" f"'", ) @@ -159,26 +165,20 @@ def create_temp_tsdb_dir(args, ctx): def dump_metrics(args, ctx): """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" ctx.remote_output = f"{ctx.remote_raw}.gz" - shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + _oc_exec( + ctx, f"sh -c '" - f"promtool tsdb dump-openmetrics " - f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " - f"{ctx.temp_dir} > {ctx.remote_raw} && gzip -f {ctx.remote_raw}" + f"promtool tsdb dump-openmetrics" + f" --min-time={ctx.start_ms} --max-time={ctx.end_ms}" + f" {ctx.temp_dir} > {ctx.remote_raw}" + f" && gzip -f {ctx.remote_raw}" f"'", ) - size_result = shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " - f"stat -c %s {ctx.remote_output}", - check=False, - ) + size_result = _oc_exec(ctx, f"stat -c %s {ctx.remote_output}", check=False) ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 @@ -189,15 +189,9 @@ def dump_metrics(args, ctx): def copy_to_output(args, ctx): """Copy the compressed metrics file from the pod to the output directory.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" - shell.run( - f"oc cp {ctx.namespace}/{ctx.pod_name}:{ctx.remote_output} " - f"{ctx.output_file} -c {ctx.container}", - ) + _oc_cp_from_pod(ctx, ctx.remote_output, ctx.output_file) return f"Copied metrics to {ctx.output_file}" @@ -207,30 +201,13 @@ def copy_to_output(args, ctx): def cleanup_pod(args, ctx): """Remove temp files from the Prometheus pod.""" - namespace = getattr(ctx, "namespace", None) or PROMETHEUS_NAMESPACE - pod_name = getattr(ctx, "pod_name", None) or PROMETHEUS_POD - container = getattr(ctx, "container", None) or PROMETHEUS_CONTAINER temp_dir = getattr(ctx, "temp_dir", None) remote_output = getattr(ctx, "remote_output", None) - - if temp_dir: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -rf {temp_dir}", - check=False, - ) - - if remote_output: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_output}", - check=False, - ) - remote_raw = getattr(ctx, "remote_raw", None) - if remote_raw: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_raw}", - check=False, - ) + + for path in [temp_dir, remote_output, remote_raw]: + if path: + _oc_exec(ctx, f"rm -rf {path}", check=False) return "Cleaned up temp files on Prometheus pod" diff --git a/projects/core/dsl/cli.py b/projects/core/dsl/cli.py index cc1617a47..fc902637b 100644 --- a/projects/core/dsl/cli.py +++ b/projects/core/dsl/cli.py @@ -216,6 +216,7 @@ def resolve_annotation(annotation): cli_name, type=arg_type, required=not has_default, + default=param.default if has_default else None, dest=param_name, help=help_text, ) From d5355c5c968e61aa791279f26a2332724f3067f0 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 27 Jul 2026 10:21:02 +0300 Subject: [PATCH 199/344] refactor: move oc_exec/oc_cp_from_pod to shared k8s utils library Address reviewer feedback to make pod exec and copy helpers reusable across toolboxes instead of keeping them private to capture_prometheus. Co-authored-by: Cursor --- .../toolbox/capture_prometheus/main.py | 56 +++++++++---------- projects/core/dsl/utils/k8s.py | 53 ++++++++++++++++++ 2 files changed, 80 insertions(+), 29 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index f2c5469f7..8fcd7949d 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -20,7 +20,8 @@ from datetime import UTC, datetime from pathlib import Path -from projects.core.dsl import always, entrypoint, execute_tasks, shell, task +from projects.core.dsl import always, entrypoint, execute_tasks, task +from projects.core.dsl.utils.k8s import oc, oc_cp_from_pod, oc_exec logger = logging.getLogger("DSL") @@ -58,19 +59,9 @@ def run( return 0 -def _oc_exec(ctx, command: str, **kwargs): - """Run a command inside the Prometheus container via oc exec.""" - return shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- {command}", - **kwargs, - ) - - -def _oc_cp_from_pod(ctx, remote_path: str, local_path): - """Copy a file from the Prometheus container to a local path.""" - return shell.run( - f"oc cp {ctx.namespace}/{ctx.pod_name}:{remote_path} {local_path} -c {ctx.container}", - ) +def _pod_kwargs(ctx) -> dict: + """Return the common namespace/pod/container kwargs for oc_exec/oc_cp_from_pod.""" + return {"namespace": ctx.namespace, "pod": ctx.pod_name, "container": ctx.container} def _parse_time(value) -> datetime: @@ -126,8 +117,14 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" - result = shell.run( - f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", + result = oc( + "-n", + ctx.namespace, + "get", + "pod", + ctx.pod_name, + "-o", + "jsonpath={.status.phase}", ) phase = result.stdout.strip() @@ -149,13 +146,13 @@ def create_temp_tsdb_dir(args, ctx): ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" - _oc_exec( - ctx, - f"sh -c '" + oc_exec( + "sh", + "-c", f"mkdir -p {ctx.temp_dir}" f" && ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal" - f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" - f"'", + f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head", + **_pod_kwargs(ctx), ) return f"Created temp TSDB dir at {ctx.temp_dir}" @@ -168,17 +165,17 @@ def dump_metrics(args, ctx): ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" ctx.remote_output = f"{ctx.remote_raw}.gz" - _oc_exec( - ctx, - f"sh -c '" + oc_exec( + "sh", + "-c", f"promtool tsdb dump-openmetrics" f" --min-time={ctx.start_ms} --max-time={ctx.end_ms}" f" {ctx.temp_dir} > {ctx.remote_raw}" - f" && gzip -f {ctx.remote_raw}" - f"'", + f" && gzip -f {ctx.remote_raw}", + **_pod_kwargs(ctx), ) - size_result = _oc_exec(ctx, f"stat -c %s {ctx.remote_output}", check=False) + size_result = oc_exec("stat", "-c", "%s", ctx.remote_output, check=False, **_pod_kwargs(ctx)) ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 @@ -191,7 +188,7 @@ def copy_to_output(args, ctx): ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" - _oc_cp_from_pod(ctx, ctx.remote_output, ctx.output_file) + oc_cp_from_pod(ctx.remote_output, ctx.output_file, **_pod_kwargs(ctx)) return f"Copied metrics to {ctx.output_file}" @@ -204,10 +201,11 @@ def cleanup_pod(args, ctx): temp_dir = getattr(ctx, "temp_dir", None) remote_output = getattr(ctx, "remote_output", None) remote_raw = getattr(ctx, "remote_raw", None) + kwargs = _pod_kwargs(ctx) for path in [temp_dir, remote_output, remote_raw]: if path: - _oc_exec(ctx, f"rm -rf {path}", check=False) + oc_exec("rm", "-rf", path, check=False, **kwargs) return "Cleaned up temp files on Prometheus pod" diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 82a283384..381dd1bed 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -289,6 +289,59 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: ) +def oc_exec( + *command: str, + namespace: str, + pod: str, + container: str | None = None, + **kwargs, +) -> shell.CommandResult: + """Exec a command inside a pod container. + + Args: + *command: Command and arguments to run in the pod + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + **kwargs: Additional keyword arguments passed to oc() + + Returns: + CommandResult with execution details + """ + args = ["-n", namespace, "exec", pod] + if container: + args.extend(["-c", container]) + args.append("--") + args.extend(command) + return oc(*args, **kwargs) + + +def oc_cp_from_pod( + remote_path: str, + local_path: str | Path, + *, + namespace: str, + pod: str, + container: str | None = None, +) -> shell.CommandResult: + """Copy a file from a pod to a local path. + + Args: + remote_path: Absolute path inside the pod + local_path: Local destination path + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + + Returns: + CommandResult with execution details + """ + args = ["cp", f"{namespace}/{pod}:{remote_path}", str(local_path)] + if container: + args.extend(["-c", container]) + return oc(*args) + + def best_effort_oc(*oc_args: str, description: str | None = None) -> None: """Run an oc command, swallowing timeout and other errors. From 5ac1d5c637aacde292d9a1ea764be90362c246fc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 10:04:32 +0200 Subject: [PATCH 200/344] Update secret handling guidelines in AGENTS.md Clarified guidelines on handling secrets and environment variables in AGENTS.md. --- AGENTS.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ddeaaaa52..b2043d04f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,13 +40,18 @@ Key paths: 5. **Artifact directories are public.** Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. Only write non-sensitive operational data (CRs, logs, pod descriptions, events). + Pod/Deployment and other resources are safe to store, as they must not contain secrets. See point 9. + Kubernetes Secret objects should never be stored in the artifacts. Kubernetes Secret descriptions can be safely stored. -6. **Use the vault module.** +7. **Use the vault module.** Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret. -7. **Never pass secrets via environment variables** +8. **Never pass secrets via environment variables** For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care. +9. **Never pass secrets via Pod environment variables** + The Pod definition will be stored in public artifacts. Do no use plain text Pod environment variables to pass secrets to the Pod. Instead, use Secret resources and Pod environment variables that reference this secret. + ### Safe Patterns ```python From 2b61dceaaf145713d774c633ba38380f0c52b0f3 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 10:12:47 -0400 Subject: [PATCH 201/344] fix: resolve all ruff lint errors - Remove unused imports (csv, tempfile, runtime_config) - Replace Optional[X] with X | None (UP045) - Remove quotes from type annotation (UP037) - Use datetime.UTC alias (UP017) - Sort import blocks (I001) - Rewrite generator as set comprehension (C401) - Use dict.fromkeys instead of dict comprehension (C420) - Bind loop variable in nested function definitions (B023) Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 10 ++++------ .../toolbox/run_guidellm_benchmark/main.py | 2 +- projects/rhaiis/orchestration/analysis.py | 1 - projects/rhaiis/orchestration/test_phase.py | 9 ++++----- projects/rhaiis/postprocess/agent.py | 16 ++++++++-------- projects/rhaiis/postprocess/parser.py | 14 ++++++-------- projects/rhaiis/postprocess/plugin.py | 2 +- 7 files changed, 24 insertions(+), 30 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 847fbf159..2659446ec 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,14 +2,12 @@ from __future__ import annotations -import csv import json import logging -import tempfile import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional +from typing import Any import numpy as np @@ -28,7 +26,7 @@ class RegressionResult: is_improvement: bool = False -def geometric_mean(values) -> Optional[float]: +def geometric_mean(values) -> float | None: positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] if len(positive) == 0: return None @@ -42,7 +40,7 @@ def compare_runs( current_version: str, profile_map: dict[tuple[int, int], str], metrics: dict[str, dict[str, Any]], - restrict_profiles: Optional[list[str]] = None, + restrict_profiles: list[str] | None = None, ) -> tuple[list[RegressionResult], str]: """Compare current run metrics against a baseline version. @@ -171,7 +169,7 @@ def run_regression_analysis( output_file: Path, profile_map: dict[tuple[int, int], str], metrics: dict[str, dict[str, Any]], - restrict_profiles: Optional[list[str]] = None, + restrict_profiles: list[str] | None = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index e2f4309e6..befad6354 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -326,7 +326,7 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" -def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "Path | None": +def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> Path | None: """Copy a result file from a pod, compressing first to handle large files. Falls back to direct ``oc cp`` for small files. diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 5f22c36ac..71c049908 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -8,7 +8,6 @@ from pathlib import Path from projects.core.library import env -from projects.rhaiis.orchestration import runtime_config logger = logging.getLogger(__name__) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 01793a977..5a2658e57 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -117,13 +117,13 @@ def _run_test( run_uuid=run_uuid, ) + import subprocess + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - - import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: @@ -533,12 +533,11 @@ def _run_warmup_step( benchmark_timeout: int, ) -> None: """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" + from projects.core.library import config from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( run as run_guidellm_benchmark, ) - from projects.core.library import config - warmup_cfg = config.project.get_config("rhaiis.warmup", {}) warmup_rate = warmup_cfg.get("rate", 200) warmup_max_seconds = warmup_cfg.get("max_seconds", 60) @@ -687,7 +686,7 @@ def _upload_profiler_traces( return traces_dir = trace_files[0].parent - if len(set(f.parent for f in trace_files)) > 1: + if len({f.parent for f in trace_files}) > 1: traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces_combined" traces_dir.mkdir(parents=True, exist_ok=True) for f in trace_files: diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py index d08140310..51e5591a6 100644 --- a/projects/rhaiis/postprocess/agent.py +++ b/projects/rhaiis/postprocess/agent.py @@ -7,7 +7,7 @@ import json import logging import uuid -from typing import Optional +from datetime import UTC from urllib.error import URLError from urllib.request import Request, urlopen @@ -47,7 +47,7 @@ def _build_prompt( compare_version: str, tp: str, severe_regressions: list, - improvements: Optional[list] = None, + improvements: list | None = None, ) -> str: """Build a natural-language prompt for the agent from regression context.""" lines = [ @@ -108,8 +108,8 @@ def request_agent_analysis( severe_regressions: list, agent_url: str, job_id: str = "", - improvements: Optional[list] = None, -) -> Optional[str]: + improvements: list | None = None, +) -> str | None: """Send regression context to the PSAP agent and return its analysis. Returns the agent's response as a markdown string, or None on failure. @@ -165,7 +165,7 @@ def request_agent_analysis( return None -def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: +def send_followup(message: str, job_id: str, agent_url: str) -> str | None: """Send a followup message to the agent on an existing session. Reuses the same thread_id/session_id as request_agent_analysis so the @@ -214,7 +214,7 @@ def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: return None -def _collect_response(resp) -> Optional[str]: +def _collect_response(resp) -> str | None: """Read the NDJSON stream and extract the final AI message content.""" collected_tokens = [] final_message = None @@ -265,9 +265,9 @@ def markdown_to_html( body = f"
{html_lib.escape(md_text)}
" - from datetime import datetime, timezone + from datetime import datetime - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC") return f""" diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 752c49297..1de53a46c 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -183,24 +183,22 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: metrics = bench.get("metrics", {}) strategy = bench.get("config", {}).get("strategy", {}) - def _pct(metric_name: str, pct: str): - return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + def _pct(metric_name: str, pct: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) def _pct_s(metric_name: str, pct: str): - """Percentile value converted from ms to seconds.""" v = _pct(metric_name, pct) return v / 1000.0 if v is not None else None - def _stat(metric_name: str, stat: str): - return metrics.get(metric_name, {}).get("successful", {}).get(stat) + def _stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get(stat) def _stat_s(metric_name: str, stat: str): - """Stat value converted from ms to seconds.""" v = _stat(metric_name, stat) return v / 1000.0 if v is not None else None - def _total_stat(metric_name: str, stat: str): - return metrics.get(metric_name, {}).get("total", {}).get(stat) + def _total_stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("total", {}).get(stat) request_totals = metrics.get("request_totals", {}) diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 658462638..bf93ae8b7 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -148,7 +148,7 @@ def export_kpis_to_csv( else f"{acc}-{model_id}-{tp}" ) - row: dict[str, Any] = {col: "" for col in FIELDNAMES} + row: dict[str, Any] = dict.fromkeys(FIELDNAMES, "") # Populate all metric columns from KPI values row.update(metrics) # Convert latency columns from seconds back to ms for dashboard compat From fd84c536dd62e408edb6963c5bcee7760b185579 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 10:24:25 -0400 Subject: [PATCH 202/344] style: apply ruff formatting to all changed files Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 50 ++++++---- projects/core/notifications/send.py | 1 - .../toolbox/run_guidellm_benchmark/main.py | 21 ++++- projects/rhaiis/orchestration/analysis.py | 37 ++++++-- projects/rhaiis/orchestration/ci.py | 9 +- projects/rhaiis/orchestration/test_phase.py | 68 +++++++++++--- projects/rhaiis/postprocess/csv_export.py | 92 ++++++++++++++----- projects/rhaiis/postprocess/kpis.py | 30 +++--- projects/rhaiis/postprocess/parser.py | 13 +-- projects/rhaiis/postprocess/plugin.py | 30 ++++-- projects/rhaiis/postprocess/regression.py | 32 +++++-- projects/rhaiis/postprocess/s3_dashboard.py | 7 +- .../toolbox/enable_profiler_gate/main.py | 3 +- .../toolbox/verify_profiler_prereqs/main.py | 10 +- 14 files changed, 282 insertions(+), 121 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 2659446ec..59e624f57 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -27,7 +27,9 @@ class RegressionResult: def geometric_mean(values) -> float | None: - positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + positive = ( + values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + ) if len(positive) == 0: return None return float(np.exp(np.mean(np.log(positive)))) @@ -69,8 +71,14 @@ def compare_runs( accelerator = current_df["accelerator"].iloc[0] tp = current_df["TP"].iloc[0] - logger.info("Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", - current_version, compare_version, model, accelerator, tp) + logger.info( + "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, + compare_version, + model, + accelerator, + tp, + ) def _assign_profile(row): try: @@ -91,7 +99,10 @@ def _assign_profile(row): ] if baseline_filtered.empty: - return [], f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}" + return ( + [], + f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", + ) current_profiles = set(current_df["_profile"].dropna().unique()) baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) @@ -112,7 +123,9 @@ def _assign_profile(row): cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") - common_conc = set(cur["intended concurrency"].dropna().unique()) & set(base["intended concurrency"].dropna().unique()) + common_conc = set(cur["intended concurrency"].dropna().unique()) & set( + base["intended concurrency"].dropna().unique() + ) common_conc.discard(1) if not common_conc: @@ -144,20 +157,23 @@ def _assign_profile(row): is_regression = pct_diff > threshold is_improvement = pct_diff < -threshold - results.append(RegressionResult( - profile=profile, - metric_column=col, - metric_label=meta["label"], - current_value=cur_gm, - baseline_value=base_gm, - pct_diff=pct_diff, - is_regression=is_regression, - is_improvement=is_improvement, - )) + results.append( + RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + ) + ) regressions = [r for r in results if r.is_regression] - logger.info("Comparison complete: %d metric comparisons, %d regressions", - len(results), len(regressions)) + logger.info( + "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) + ) return results, "" diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 8f2d55b2e..ab6f64acc 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -714,7 +714,6 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index befad6354..fa82ff1ec 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -339,7 +339,8 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "cp", f"{namespace}/{pod}:{remote_path}", str(local_path), - "-c", "copy-helper", + "-c", + "copy-helper", check=False, log_stdout=False, ) @@ -347,14 +348,23 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> return local_path logging.getLogger(__name__).info( - "Direct oc cp failed (rc=%d), retrying with gzip compression", result.returncode, + "Direct oc cp failed (rc=%d), retrying with gzip compression", + result.returncode, ) # Compress inside the pod, copy the gz, decompress locally gz_remote = f"{remote_path}.gz" compress = oc( - "exec", pod, "-n", namespace, "-c", "copy-helper", "--", - "gzip", "-kf", remote_path, + "exec", + pod, + "-n", + namespace, + "-c", + "copy-helper", + "--", + "gzip", + "-kf", + remote_path, check=False, ) if compress.returncode != 0: @@ -366,7 +376,8 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "cp", f"{namespace}/{pod}:{gz_remote}", str(local_gz), - "-c", "copy-helper", + "-c", + "copy-helper", check=False, log_stdout=False, ) diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 71c049908..236e420a1 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -47,7 +47,9 @@ def run_standalone_analysis( logger.warning("AWS credentials not available, skipping standalone analysis") return - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + accelerator = ( + accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + ) consolidated_path = None current_csv_path = None @@ -77,7 +79,10 @@ def run_standalone_analysis( if current_rows.empty: logger.warning( "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", - version, model_id, accelerator, tp, + version, + model_id, + accelerator, + tp, ) return @@ -87,8 +92,14 @@ def run_standalone_analysis( logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, - run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, + current_csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid=run_uuid, + restrict_profiles=restrict_profiles, + vllm_args=vllm_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -155,8 +166,12 @@ def run_regression_check( agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) if agent_cfg.get("enabled", False): report_url = run_agent_analysis( - analysis, model_cfg, accelerator, - current_version, compare_version, run_uuid, + analysis, + model_cfg, + accelerator, + current_version, + compare_version, + run_uuid, severity_threshold=agent_cfg.get("severity_threshold", 10), ) @@ -243,7 +258,11 @@ def run_agent_analysis( agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" html_content = markdown_to_html( - agent_response, run_uuid, model, current_version, compare_version, + agent_response, + run_uuid, + model, + current_version, + compare_version, ) html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" html_path.write_text(html_content, encoding="utf-8") @@ -261,7 +280,9 @@ def run_agent_analysis( s3_bucket = s3_cfg.get("bucket", "") s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" s3.upload_file( - str(html_path), s3_bucket, s3_key, + str(html_path), + s3_bucket, + s3_key, ExtraArgs={"ContentType": "text/html"}, ) report_url = s3.generate_presigned_url( diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 763bca95c..9ba9c2e26 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -33,10 +33,7 @@ def _check_pipeline_failure_and_notify() -> None: test_ran = test_dir.exists() # Skip steps already handled by do_test's exception handler - failure_files = sorted( - f for f in base_dir.glob("*/FAILURE") - if f.parent.name != "03__test" - ) + failure_files = sorted(f for f in base_dir.glob("*/FAILURE") if f.parent.name != "03__test") if not failure_files and test_ran: return @@ -49,7 +46,9 @@ def _check_pipeline_failure_and_notify() -> None: errors.append(f"[{step_name}] {summary}") if not test_ran and not errors: - errors.append("Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)") + errors.append( + "Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)" + ) error_text = "\n".join(errors) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 5a2658e57..ceb138947 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -14,7 +14,6 @@ _warnings: list[str] = [] - def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -63,6 +62,7 @@ def do_test( ) except Exception as exc: from projects.rhaiis.orchestration.notifications import send_pipeline_failure_alert + send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) raise @@ -78,6 +78,7 @@ def _run_test( accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator from projects.core.library import config as _cfg + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() deploy_cfg = runtime_config.get_deploy_config() @@ -103,12 +104,16 @@ def _run_test( env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) from projects.core.library import config as _cfg + version = _cfg.project.get_config("tests.rhaiis.version", "") run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) _create_test_labels( - model_key, workload_keys[0], accelerator, vllm_args, + model_key, + workload_keys[0], + accelerator, + vllm_args, hf_model_id=model_cfg["hf_model_id"], version=version, vllm_image=vllm_image, @@ -124,15 +129,27 @@ def _run_test( ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready + fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: try: mgmt_env = {k: v for k, v in os.environ.items() if k != "KUBECONFIG"} subprocess.run( - ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, - f"rhaiis.run-uuid={run_uuid}", "--overwrite"], - check=False, capture_output=True, timeout=10, env=mgmt_env, + [ + "oc", + "annotate", + "fournosjob", + fjob_name, + "-n", + fjob_ns, + f"rhaiis.run-uuid={run_uuid}", + "--overwrite", + ], + check=False, + capture_output=True, + timeout=10, + env=mgmt_env, ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: @@ -149,9 +166,13 @@ def _run_test( logger.info("run_benchmark=false and profiler=false, running standalone analysis only") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis( - model_cfg, accelerator_key, vllm_args, - run_uuid=run_uuid, restrict_profiles=workload_keys or None, + model_cfg, + accelerator_key, + vllm_args, + run_uuid=run_uuid, + restrict_profiles=workload_keys or None, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -202,7 +223,9 @@ def _run_test( ) logger.info( - "Running %d workload(s): %s", len(workload_keys), workload_keys, + "Running %d workload(s): %s", + len(workload_keys), + workload_keys, ) # Phase 1: warmup or profiler for ALL workloads first @@ -287,8 +310,11 @@ def _run_test( _warnings.append("Predictor log upload failed") if _warnings: - logger.warning("Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings)) + logger.warning( + "Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings) + ) from projects.rhaiis.orchestration.notifications import send_pipeline_warning + send_pipeline_warning( warnings=list(_warnings), model_key=model_key, @@ -335,6 +361,7 @@ def _run_workload_benchmark( logger.info("run_benchmark=false, skipping main benchmark") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -456,7 +483,9 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) logger.info("No postprocessing directory found, skipping dashboard CSV sync") return - kpis_to_csv_output = config.project.get_config("caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv") + kpis_to_csv_output = config.project.get_config( + "caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv" + ) csv_path = postprocess_dirs[-1] / kpis_to_csv_output if not csv_path.exists(): logger.info("Dashboard CSV not found at %s, skipping sync", csv_path) @@ -489,9 +518,15 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check( - csv_path, compare_version, version, model_cfg, accelerator, - run_uuid="", vllm_args=vllm_args, + csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid="", + vllm_args=vllm_args, ) @@ -502,7 +537,9 @@ def _upload_predictor_log(run_uuid: str) -> None: from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 - matches = sorted(Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs")) + matches = sorted( + Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs") + ) log_path = matches[-1] if matches else None if not log_path or not log_path.exists(): logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) @@ -680,7 +717,9 @@ def _upload_profiler_traces( from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 - trace_files = sorted(Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*")) + trace_files = sorted( + Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*") + ) if not trace_files: logger.info("No profiler traces to upload") return @@ -691,6 +730,7 @@ def _upload_profiler_traces( traces_dir.mkdir(parents=True, exist_ok=True) for f in trace_files: import shutil + shutil.copy2(f, traces_dir / f.name) logger.info("Found %d profiler trace files in %s", len(trace_files), traces_dir) diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py index 93270ac27..5f86396dc 100644 --- a/projects/rhaiis/postprocess/csv_export.py +++ b/projects/rhaiis/postprocess/csv_export.py @@ -14,19 +14,58 @@ logger = logging.getLogger(__name__) FIELDNAMES = [ - "run", "accelerator", "model", "version", "prompt toks", "output toks", "TP", - "measured concurrency", "intended concurrency", "measured rps", - "output_tok/sec", "total_tok/sec", "prompt_token_count_mean", - "prompt_token_count_p99", "output_token_count_mean", - "output_token_count_p99", "ttft_median", "ttft_p95", "ttft_p1", "ttft_p999", - "tpot_median", "tpot_p95", "tpot_p99", "tpot_p999", "tpot_p1", - "itl_median", "itl_p95", "itl_p999", "itl_p1", - "request_latency_median", "request_latency_min", "request_latency_max", - "successful_requests", "errored_requests", "uuid", - "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", - "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", - "DP", "dataset", "spec_decoding", "prefix_caching", - "turns", "prefix_tokens", "prefix_count", "request_type", + "run", + "accelerator", + "model", + "version", + "prompt toks", + "output toks", + "TP", + "measured concurrency", + "intended concurrency", + "measured rps", + "output_tok/sec", + "total_tok/sec", + "prompt_token_count_mean", + "prompt_token_count_p99", + "output_token_count_mean", + "output_token_count_p99", + "ttft_median", + "ttft_p95", + "ttft_p1", + "ttft_p999", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p999", + "tpot_p1", + "itl_median", + "itl_p95", + "itl_p999", + "itl_p1", + "request_latency_median", + "request_latency_min", + "request_latency_max", + "successful_requests", + "errored_requests", + "uuid", + "ttft_mean", + "ttft_p99", + "itl_mean", + "itl_p99", + "runtime_args", + "guidellm_start_time_ms", + "guidellm_end_time_ms", + "image_tag", + "guidellm_version", + "DP", + "dataset", + "spec_decoding", + "prefix_caching", + "turns", + "prefix_tokens", + "prefix_count", + "request_type", ] @@ -51,11 +90,15 @@ def _format_runtime_args(inference_server_args: dict, trtllm_config: dict | None parts = [] for key, value in (inference_server_args or {}).items(): formatted_key = key.replace("_", "-") - formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) parts.append(f"{formatted_key}: {formatted_value}") for key, value in (trtllm_config or {}).items(): formatted_key = f"trtllm.{key.replace('_', '-')}" - formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) parts.append(f"{formatted_key}: {formatted_value}") return "; ".join(parts) @@ -65,7 +108,9 @@ def find_psap_files(artifact_dir: Path) -> list[Path]: return sorted(artifact_dir.rglob("PSAP_perf*.json")) -def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "") -> Path: +def generate_dashboard_csv( + psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "" +) -> Path: """Read a PSAP JSON file and produce a dashboard CSV.""" with open(psap_json_path, encoding="utf-8") as f: payload = json.load(f) @@ -108,8 +153,12 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path start_times.append(sched["start_time"]) if "end_time" in sched: end_times.append(sched["end_time"]) - guidellm_start_ms = int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") - guidellm_end_ms = int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") + guidellm_start_ms = ( + int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") + ) + guidellm_end_ms = ( + int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") + ) rows = [] for bench in benchmarks: @@ -165,12 +214,7 @@ def _extract_row( run_uuid: str = "", ) -> dict: def _pct(metric_name: str, pct: str): - return ( - metrics.get(metric_name, {}) - .get("successful", {}) - .get("percentiles", {}) - .get(pct) - ) + return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) def _stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("successful", {}).get(stat) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index a262bb41d..fc137feea 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -110,19 +110,21 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if higher_is_better is not None: labels["higher_is_better"] = higher_is_better - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_id": r.test_base_path, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": model.plugin_module, - }, - }) + out.append( + { + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_id": r.test_base_path, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": model.plugin_module, + }, + } + ) return out diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 1de53a46c..a607811f8 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -68,10 +68,7 @@ def _find_node_for_record( def _bench_sort_key(bench: dict) -> float: """Sort key matching GuideLLMParser: requests_per_second.successful.mean.""" return float( - bench.get("metrics", {}) - .get("requests_per_second", {}) - .get("successful", {}) - .get("mean", 0) + bench.get("metrics", {}).get("requests_per_second", {}).get("successful", {}).get("mean", 0) ) @@ -79,8 +76,12 @@ def _extract_extra_metrics(node: TestBaseNode) -> tuple[dict[str, Any], dict[str """Return (scalar_metrics, extra_curves) extracted from raw benchmarks.json files.""" extra: dict[str, Any] = {} benchmarks_files = sorted( - [p for p in node.artifact_paths if p.name == "benchmarks.json" - or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json")], + [ + p + for p in node.artifact_paths + if p.name == "benchmarks.json" + or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json") + ], key=lambda p: p.name, ) if not benchmarks_files: diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index bf93ae8b7..3c6765053 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -20,11 +20,27 @@ # CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. # GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. -_SECONDS_TO_MS_COLUMNS = frozenset({ - "ttft_median", "ttft_p95", "ttft_p99", "ttft_p1", "ttft_p999", "ttft_mean", - "tpot_median", "tpot_p95", "tpot_p99", "tpot_p1", "tpot_p999", - "itl_median", "itl_p95", "itl_p99", "itl_p1", "itl_p999", "itl_mean", -}) +_SECONDS_TO_MS_COLUMNS = frozenset( + { + "ttft_median", + "ttft_p95", + "ttft_p99", + "ttft_p1", + "ttft_p999", + "ttft_mean", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p1", + "tpot_p999", + "itl_median", + "itl_p95", + "itl_p99", + "itl_p1", + "itl_p999", + "itl_mean", + } +) # KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { @@ -143,9 +159,7 @@ def export_kpis_to_csv( version = labels.get("version", "") run_name = ( - f"{acc}-{cluster_tag}-{model_id}-{tp}" - if cluster_tag - else f"{acc}-{model_id}-{tp}" + f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" ) row: dict[str, Any] = dict.fromkeys(FIELDNAMES, "") diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 1b524f7fa..6187a52f1 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -14,7 +14,9 @@ RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" -def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None, channel_id: str | None = None) -> bool: +def _send_via_topsail_bot( + message: str, *, notification_vault: str | None = None, channel_id: str | None = None +) -> bool: """Send a Slack message using the topsail bot token from the forge notifications vault.""" vault_name = notification_vault or "psap-forge-notifications" try: @@ -49,7 +51,11 @@ def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, - "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, + "request_latency_median": { + "label": "Median E2E Latency", + "higher_is_better": False, + "threshold": 5, + }, } DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" @@ -161,10 +167,16 @@ def send_regression_notification( detail_lines = [] all_results = analysis_result.get("all_results", []) - profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + profiles = sorted( + {r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")} + ) for profile in profiles: - profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + profile_results = [ + r + for r in all_results + if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement")) + ] detail_lines.append(f"\n*{profile}:*") for r in profile_results: if r.get("is_regression"): @@ -226,7 +238,9 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) def send_failure_notification( @@ -275,7 +289,9 @@ def send_failure_notification( parallelism_parts.append(f"TP={tp}") if dp: parallelism_parts.append(f"DP={dp}") - parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + parallelism_line = ( + f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + ) profiles_line = "" if workload_keys: @@ -303,4 +319,6 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index c648ce9e7..060288284 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -72,6 +72,7 @@ def sync_csv_to_s3( return {"status": "failed", "error": str(e)} finally: import os + if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) @@ -98,9 +99,9 @@ def upload_profiler_traces_to_s3( return {"status": "skipped", "reason": f"traces dir not found: {traces_dir}"} trace_files = [ - f for f in traces_dir.iterdir() - if f.name.startswith("trace") and "rank0" in f.name - and f.suffix in (".json", ".gz") + f + for f in traces_dir.iterdir() + if f.name.startswith("trace") and "rank0" in f.name and f.suffix in (".json", ".gz") ] if not trace_files: return {"status": "skipped", "reason": "no trace files found"} diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py index b1783b09c..c370953ed 100644 --- a/projects/rhaiis/toolbox/enable_profiler_gate/main.py +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -44,8 +44,7 @@ def find_predictor_pod(args, context): def toggle_profiler_gate(args, context): if args.disable: shell.run( - f"oc exec {context.pod_name} -n {args.namespace} " - "-- sh -c 'rm -f /tmp/profiler_gate'", + f"oc exec {context.pod_name} -n {args.namespace} -- sh -c 'rm -f /tmp/profiler_gate'", check=False, ) return f"Profiler gate disabled on {context.pod_name}" diff --git a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py index dc2e95e50..2fd23542d 100644 --- a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py +++ b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py @@ -24,8 +24,7 @@ def run(*, namespace: str): @task def check_webhook_pod(args, context): result = shell.run( - "oc get pods -n vllm-profiler -l app=env-injector " - "-o jsonpath='{.items[0].status.phase}'", + "oc get pods -n vllm-profiler -l app=env-injector -o jsonpath='{.items[0].status.phase}'", check=False, ) phase = result.stdout.strip().strip("'") @@ -99,8 +98,7 @@ def check_tls_secret(args, context): @task def check_configmap(args, context): result = shell.run( - f"oc get configmap env-injector-files -n {args.namespace} " - "-o jsonpath='{.data}'", + f"oc get configmap env-injector-files -n {args.namespace} -o jsonpath='{{.data}}'", check=False, ) if result.returncode != 0 or "sitecustomize.py" not in result.stdout: @@ -117,9 +115,7 @@ def check_configmap(args, context): def raise_on_errors(args, context): errors = getattr(context, "errors", []) if errors: - raise ValueError( - "Profiler prerequisites not met:\n - " + "\n - ".join(errors) - ) + raise ValueError("Profiler prerequisites not met:\n - " + "\n - ".join(errors)) return "All profiler prerequisites verified" From 61fdcd1808e94480e789ae23a5ab92776f852499 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 11:59:30 -0400 Subject: [PATCH 203/344] refactor: move CSV regression analysis from caliper engine to rhaiis plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move compare_runs, run_regression_analysis, RegressionResult, and geometric_mean out of the generic caliper/engine/kpi/analyze.py into rhaiis/postprocess/regression.py. The caliper engine should contain no project-specific code (no references to model, accelerator, TP, tokens, etc.). This is a temporary transition — these CSV-based comparison functions will be replaced by generic Caliper KPI-based analysis once the framework lands. Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 257 +------------------- projects/rhaiis/orchestration/analysis.py | 3 +- projects/rhaiis/postprocess/regression.py | 273 +++++++++++++++++++++- 3 files changed, 274 insertions(+), 259 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 59e624f57..47ae439ff 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -1,271 +1,16 @@ -"""Regression analysis vs baseline KPI set.""" +"""Generic KPI analysis against historical baselines.""" from __future__ import annotations import json import logging import time -from dataclasses import dataclass from pathlib import Path from typing import Any -import numpy as np - logger = logging.getLogger(__name__) -@dataclass -class RegressionResult: - profile: str - metric_column: str - metric_label: str - current_value: float - baseline_value: float - pct_diff: float - is_regression: bool - is_improvement: bool = False - - -def geometric_mean(values) -> float | None: - positive = ( - values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] - ) - if len(positive) == 0: - return None - return float(np.exp(np.mean(np.log(positive)))) - - -def compare_runs( - current_csv_path: str, - consolidated_csv_path: str, - compare_version: str, - current_version: str, - profile_map: dict[tuple[int, int], str], - metrics: dict[str, dict[str, Any]], - restrict_profiles: list[str] | None = None, -) -> tuple[list[RegressionResult], str]: - """Compare current run metrics against a baseline version. - - Args: - profile_map: Mapping of (input_tokens, output_tokens) -> profile name. - metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. - - Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. - """ - import pandas as pd - - current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") - baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") - - for col in current_df.columns: - if current_df[col].dtype == object: - current_df[col] = current_df[col].str.strip() - for col in baseline_df.columns: - if baseline_df[col].dtype == object: - baseline_df[col] = baseline_df[col].str.strip() - - if current_df.empty: - return [], "Current run produced no benchmark data" - - model = current_df["model"].iloc[0] - accelerator = current_df["accelerator"].iloc[0] - tp = current_df["TP"].iloc[0] - - logger.info( - "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", - current_version, - compare_version, - model, - accelerator, - tp, - ) - - def _assign_profile(row): - try: - isl = int(float(row["prompt toks"])) - osl = int(float(row["output toks"])) - except (ValueError, TypeError): - return None - return profile_map.get((isl, osl)) - - current_df["_profile"] = current_df.apply(_assign_profile, axis=1) - baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) - - baseline_filtered = baseline_df[ - (baseline_df["version"] == compare_version) - & (baseline_df["model"] == model) - & (baseline_df["accelerator"] == accelerator) - & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) - ] - - if baseline_filtered.empty: - return ( - [], - f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", - ) - - current_profiles = set(current_df["_profile"].dropna().unique()) - baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) - common_profiles = sorted(current_profiles & baseline_profiles) - - if restrict_profiles: - common_profiles = [p for p in common_profiles if p in restrict_profiles] - - if not common_profiles: - return [], "No common ISL/OSL profiles between current and baseline" - - results: list[RegressionResult] = [] - - for profile in common_profiles: - cur = current_df[current_df["_profile"] == profile].copy() - base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() - - cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") - base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") - - common_conc = set(cur["intended concurrency"].dropna().unique()) & set( - base["intended concurrency"].dropna().unique() - ) - common_conc.discard(1) - - if not common_conc: - continue - - cur_common = cur[cur["intended concurrency"].isin(common_conc)] - base_common = base[base["intended concurrency"].isin(common_conc)] - - for col, meta in metrics.items(): - if col not in cur_common.columns or col not in base_common.columns: - continue - - cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() - base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() - - cur_gm = geometric_mean(cur_vals) - base_gm = geometric_mean(base_vals) - - if cur_gm is None or base_gm is None: - continue - - pct_diff = ((cur_gm - base_gm) / base_gm) * 100 - threshold = meta["threshold"] - - if meta["higher_is_better"]: - is_regression = pct_diff < -threshold - is_improvement = pct_diff > threshold - else: - is_regression = pct_diff > threshold - is_improvement = pct_diff < -threshold - - results.append( - RegressionResult( - profile=profile, - metric_column=col, - metric_label=meta["label"], - current_value=cur_gm, - baseline_value=base_gm, - pct_diff=pct_diff, - is_regression=is_regression, - is_improvement=is_improvement, - ) - ) - - regressions = [r for r in results if r.is_regression] - logger.info( - "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) - ) - return results, "" - - -def run_regression_analysis( - current_csv_path: Path, - consolidated_csv_path: Path, - compare_version: str, - current_version: str, - output_file: Path, - profile_map: dict[tuple[int, int], str], - metrics: dict[str, dict[str, Any]], - restrict_profiles: list[str] | None = None, -) -> dict[str, Any]: - """Run regression analysis comparing current run against baseline. - - Returns dict with status, regressions, improvements, and skip_reason. - """ - try: - results, skip_reason = compare_runs( - str(current_csv_path), - str(consolidated_csv_path), - compare_version, - current_version, - profile_map=profile_map, - metrics=metrics, - restrict_profiles=restrict_profiles, - ) - - if skip_reason: - analysis = { - "status": "skipped", - "reason": skip_reason, - "current_version": current_version, - "compare_version": compare_version, - } - else: - regressions = [r for r in results if r.is_regression] - improvements = [r for r in results if r.is_improvement] - - analysis = { - "status": "completed", - "current_version": current_version, - "compare_version": compare_version, - "total_comparisons": len(results), - "regression_count": len(regressions), - "improvement_count": len(improvements), - "regressions": [ - { - "profile": r.profile, - "metric": r.metric_label, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - } - for r in regressions - ], - "improvements": [ - { - "profile": r.profile, - "metric": r.metric_label, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - } - for r in improvements - ], - "all_results": [ - { - "profile": r.profile, - "metric": r.metric_label, - "metric_column": r.metric_column, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - "is_regression": r.is_regression, - "is_improvement": r.is_improvement, - } - for r in results - ], - } - - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis, f, indent=2) - - return analysis - - except Exception as e: - logger.exception("Regression analysis failed") - return {"status": "failed", "error": str(e)} - - def run_kpi_analysis( current_kpi_file: Path, historical_data_dir: Path, diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 236e420a1..f41390aab 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -122,9 +122,8 @@ def run_regression_check( vllm_args: dict | None = None, ) -> None: from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.caliper.engine.kpi.analyze import run_regression_analysis from projects.core.library import config - from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP, run_regression_analysis s3_cfg = config.project.get_config("rhaiis.s3", {}) csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 6187a52f1..b71bec30d 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -1,9 +1,19 @@ -"""RHAIIS-specific profile/metric definitions and Slack notification for regression analysis.""" +"""RHAIIS-specific regression analysis and Slack notifications. + +Temporary home for CSV-based regression comparison ported from model-furnace. +Will be replaced by generic Caliper KPI-based analysis once the framework lands. +""" from __future__ import annotations +import json import logging import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np import projects.core.notifications.slack.api as slack_api from projects.core.library import vault @@ -39,6 +49,267 @@ def _send_via_topsail_bot( return ok +# --------------------------------------------------------------------------- +# CSV-based regression comparison (ported from model-furnace, temporary) +# --------------------------------------------------------------------------- + + +@dataclass +class RegressionResult: + profile: str + metric_column: str + metric_label: str + current_value: float + baseline_value: float + pct_diff: float + is_regression: bool + is_improvement: bool = False + + +def geometric_mean(values) -> float | None: + positive = ( + values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + ) + if len(positive) == 0: + return None + return float(np.exp(np.mean(np.log(positive)))) + + +def compare_runs( + current_csv_path: str, + consolidated_csv_path: str, + compare_version: str, + current_version: str, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> tuple[list[RegressionResult], str]: + """Compare current run metrics against a baseline version. + + Args: + profile_map: Mapping of (input_tokens, output_tokens) -> profile name. + metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. + + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. + """ + import pandas as pd + + current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") + baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") + + for col in current_df.columns: + if current_df[col].dtype == object: + current_df[col] = current_df[col].str.strip() + for col in baseline_df.columns: + if baseline_df[col].dtype == object: + baseline_df[col] = baseline_df[col].str.strip() + + if current_df.empty: + return [], "Current run produced no benchmark data" + + model = current_df["model"].iloc[0] + accelerator = current_df["accelerator"].iloc[0] + tp = current_df["TP"].iloc[0] + + logger.info( + "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, + compare_version, + model, + accelerator, + tp, + ) + + def _assign_profile(row): + try: + isl = int(float(row["prompt toks"])) + osl = int(float(row["output toks"])) + except (ValueError, TypeError): + return None + return profile_map.get((isl, osl)) + + current_df["_profile"] = current_df.apply(_assign_profile, axis=1) + baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) + + baseline_filtered = baseline_df[ + (baseline_df["version"] == compare_version) + & (baseline_df["model"] == model) + & (baseline_df["accelerator"] == accelerator) + & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) + ] + + if baseline_filtered.empty: + return ( + [], + f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", + ) + + current_profiles = set(current_df["_profile"].dropna().unique()) + baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) + common_profiles = sorted(current_profiles & baseline_profiles) + + if restrict_profiles: + common_profiles = [p for p in common_profiles if p in restrict_profiles] + + if not common_profiles: + return [], "No common ISL/OSL profiles between current and baseline" + + results: list[RegressionResult] = [] + + for profile in common_profiles: + cur = current_df[current_df["_profile"] == profile].copy() + base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() + + cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") + base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") + + common_conc = set(cur["intended concurrency"].dropna().unique()) & set( + base["intended concurrency"].dropna().unique() + ) + common_conc.discard(1) + + if not common_conc: + continue + + cur_common = cur[cur["intended concurrency"].isin(common_conc)] + base_common = base[base["intended concurrency"].isin(common_conc)] + + for col, meta in metrics.items(): + if col not in cur_common.columns or col not in base_common.columns: + continue + + cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() + base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() + + cur_gm = geometric_mean(cur_vals) + base_gm = geometric_mean(base_vals) + + if cur_gm is None or base_gm is None: + continue + + pct_diff = ((cur_gm - base_gm) / base_gm) * 100 + threshold = meta["threshold"] + + if meta["higher_is_better"]: + is_regression = pct_diff < -threshold + is_improvement = pct_diff > threshold + else: + is_regression = pct_diff > threshold + is_improvement = pct_diff < -threshold + + results.append( + RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + ) + ) + + regressions = [r for r in results if r.is_regression] + logger.info( + "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) + ) + return results, "" + + +def run_regression_analysis( + current_csv_path: Path, + consolidated_csv_path: Path, + compare_version: str, + current_version: str, + output_file: Path, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> dict[str, Any]: + """Run regression analysis comparing current run against baseline. + + Returns dict with status, regressions, improvements, and skip_reason. + """ + try: + results, skip_reason = compare_runs( + str(current_csv_path), + str(consolidated_csv_path), + compare_version, + current_version, + profile_map=profile_map, + metrics=metrics, + restrict_profiles=restrict_profiles, + ) + + if skip_reason: + analysis = { + "status": "skipped", + "reason": skip_reason, + "current_version": current_version, + "compare_version": compare_version, + } + else: + regressions = [r for r in results if r.is_regression] + improvements = [r for r in results if r.is_improvement] + + analysis = { + "status": "completed", + "current_version": current_version, + "compare_version": compare_version, + "total_comparisons": len(results), + "regression_count": len(regressions), + "improvement_count": len(improvements), + "regressions": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in regressions + ], + "improvements": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in improvements + ], + "all_results": [ + { + "profile": r.profile, + "metric": r.metric_label, + "metric_column": r.metric_column, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + "is_regression": r.is_regression, + "is_improvement": r.is_improvement, + } + for r in results + ], + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis, f, indent=2) + + return analysis + + except Exception as e: + logger.exception("Regression analysis failed") + return {"status": "failed", "error": str(e)} + + +# --------------------------------------------------------------------------- +# RHAIIS constants +# --------------------------------------------------------------------------- + PROFILE_MAP = { (1000, 1000): "profile1", (512, 2048): "profile2", From 5668caf0cccafe700e3c56bde240c3c07453415f Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 13:58:12 -0400 Subject: [PATCH 204/344] fix: separate test nodes per workload for correct dashboard CSV per-profile data Previously, all workload benchmarks ran under a single NextArtifactDir ("testing") with one set of test labels (workload_key=profile1). The Caliper parser merged all benchmarks.json files into one record, causing: - Dashboard showing only "profile1" regardless of which profiles ran - Multiple data points on the same chart from different profiles Now each workload's benchmark phase runs in its own NextArtifactDir (e.g., "benchmark_profile1", "benchmark_profile2") with per-workload test labels. The parser creates separate records per profile, producing correct per-profile CSV rows and dashboard entries. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 95 +++++++++++---------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ceb138947..f94812476 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -109,19 +109,6 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) - _create_test_labels( - model_key, - workload_keys[0], - accelerator, - vllm_args, - hf_model_id=model_cfg["hf_model_id"], - version=version, - vllm_image=vllm_image, - cluster_tag=cluster_tag, - accelerator_chip=gpu_type.upper(), - run_uuid=run_uuid, - ) - import subprocess from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -279,6 +266,8 @@ def _run_test( endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, run_uuid=run_uuid, + version=version, + cluster_tag=cluster_tag, ) try: @@ -342,8 +331,15 @@ def _run_workload_benchmark( endpoint_url: str, benchmark_timeout: int, run_uuid: str, + version: str, + cluster_tag: str, ) -> None: - """Run benchmark and post-processing for a single workload.""" + """Run benchmark and post-processing for a single workload. + + Each workload gets its own NextArtifactDir with per-workload test labels so + the Caliper parser treats them as separate test nodes and produces one + dashboard CSV row-set per profile. + """ logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) @@ -357,41 +353,52 @@ def _run_workload_benchmark( run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - if not run_benchmark: - logger.info("run_benchmark=false, skipping main benchmark") - try: - from projects.rhaiis.orchestration.analysis import run_standalone_analysis + with env.NextArtifactDir(f"benchmark_{workload_key}"): + _create_test_labels( + model_key, + workload_key, + accelerator, + vllm_args, + hf_model_id=model_cfg["hf_model_id"], + version=version, + vllm_image=vllm_image, + cluster_tag=cluster_tag, + accelerator_chip=gpu_type.upper(), + run_uuid=run_uuid, + ) - run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Standalone analysis failed", exc_info=True) - _warnings.append(f"Standalone analysis failed for {workload_key}") - else: - logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + try: + from projects.rhaiis.orchestration.analysis import run_standalone_analysis - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append(f"Standalone analysis failed for {workload_key}") + else: + logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), - ) + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) - # Dashboard CSV generation and S3 sync is handled by the Caliper - # postprocessing pipeline (kpis_to_csv step) + _sync_postprocessed_dashboard_csv + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + ) def _create_test_labels( From d57c523f16c9ef9d30000065f26d1256007d4e24 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:34:02 +0200 Subject: [PATCH 205/344] [core] dsl: task: log the end time --- projects/core/dsl/task.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/projects/core/dsl/task.py b/projects/core/dsl/task.py index 40e31017c..bd593a512 100644 --- a/projects/core/dsl/task.py +++ b/projects/core/dsl/task.py @@ -59,6 +59,7 @@ def _log_retry_attempt( result=None, exc=None, artifact_dirname_suffix=None, + retry_backoff=1.0, ): """ Log a retry attempt with consistent formatting. @@ -115,7 +116,21 @@ def _log_retry_attempt( retry_reason = f"returned: {result}" logger.warning(f"~~ RETRY ATTEMPT #{attempt + 1}/{retry_attempts} ({retry_reason})") - logger.info(f"~~ RETRY in {current_delay:.0f}s") + + # Calculate total remaining time for all future retries with backoff + remaining_attempts = retry_attempts - 1 - attempt + total_remaining_time = 0 + delay_for_calculation = current_delay + + for _i in range(remaining_attempts): + total_remaining_time += delay_for_calculation + delay_for_calculation *= retry_backoff + + # Calculate when retries will end + end_time = time.time() + total_remaining_time + end_timestamp = datetime.fromtimestamp(end_time).strftime("%H:%M") + + logger.info(f"~~ RETRY in {current_delay:.0f}s until {end_timestamp}") logger.info("~" * LINE_WIDTH) logger.info("") @@ -187,6 +202,7 @@ def _execute_with_retry( retry_reason=retry_reason, result=result, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff else: @@ -239,6 +255,7 @@ def _execute_with_retry( current_delay, exc=exc, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff From 407ca5012b95ecaab153878a9c24afb0ff53534d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:03:59 +0200 Subject: [PATCH 206/344] [core] dsl: runtime: reduce the log level --- projects/core/dsl/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index 78ff145d6..c0c4b8b88 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -674,7 +674,7 @@ def _generate_failure_file_for_agent(exception: Exception) -> None: with open(failure_file_path, "w", encoding="utf-8") as f: f.write(failure_content) - logger.info(f"Generated FAILURE file for agent analysis: {failure_file_path}") + logger.debug(f"Generated FAILURE file for agent analysis: {failure_file_path}") except Exception as failure_error: logger.warning(f"Failed to generate FAILURE file for agent: {failure_error}") From b3f7964c5059d142c4ab6633f58a0ce26a69c86b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:09:30 +0200 Subject: [PATCH 207/344] [core] library: export: improve the test status --- projects/core/library/export.py | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index c1eff68f1..a115a84c3 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -729,6 +729,12 @@ def _build_enhanced_notification( notification_parts.append("**Test Logs**") notification_parts.extend(step_log_links) + # Add distinct test and post-processing status right under Test Logs + test_status_section = _build_test_status_section(status) + if test_status_section: + notification_parts.append("") + notification_parts.extend(test_status_section) + if postprocess_status_links: notification_parts.append("") notification_parts.extend(postprocess_status_links) @@ -740,6 +746,42 @@ def _build_enhanced_notification( return "\n".join(notification_parts) +def _build_test_status_section(status: dict[str, Any]) -> list[str]: + """Build distinct test and post-processing status section.""" + try: + test_phase = status.get("test_phase", {}) + if not test_phase: + return [] + + test_status = test_phase.get("phase", "UNKNOWN") + test_message = test_phase.get("message", "") + + # Determine post-processing status based on final status and test outcome + final_status = status.get("final_status", "unknown") + if test_status == "FAILED": + post_processing_status = "skipped" # Don't run post-processing if test failed + elif final_status == "success": + post_processing_status = "success" + elif "failed" in final_status.lower(): + post_processing_status = "failed" + else: + post_processing_status = "unknown" + + status_lines = [f"**test:** {test_status}"] + + if test_message: + # Format message with blockquote-style prefix + status_lines.append(f"> {test_message}") + + status_lines.append(f"**post-processing:** {post_processing_status}") + + return status_lines + + except Exception as e: + logger.warning(f"Failed to build test status section: {e}") + return [] + + def _extract_project_from_status(status: dict[str, Any]) -> str: """Extract project name from status object or environment.""" # Try to get project from environment variables From 7635cda01d981fdebbb4eb16a456119040ce7c92 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:45:24 +0200 Subject: [PATCH 208/344] [core] library: config: write_variables_override: accept comma separated presets --- projects/core/library/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index 9dd5d51f2..0d1fa1303 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -558,6 +558,9 @@ def write_variables_override(presets=None, variables_dict=None): # Set presets in project.args if presets: + if len(presets) == 1 and "," in presets[0]: + presets = list(presets[0].split(",")) + overrides["project.args"] = presets logger.info(f"write_variables_override: project.args --> {presets}") From 0c9ce6066d2b8d438639dab4185a4ee30b442aca Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:38:37 +0200 Subject: [PATCH 209/344] [core] library: env: add a MuteStdOut context --- projects/core/library/env.py | 78 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 4def14e1e..e2ef4bbb2 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -3,6 +3,8 @@ import pathlib import threading import time +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO # ARTIFACT_DIR is accessed via __getattr__ for thread-local support BASE_ARTIFACT_DIR = None # Immutable copy of the initial ARTIFACT_DIR @@ -33,7 +35,7 @@ def __getattr__(name): ) return globals().get("_GLOBAL_ARTIFACT_DIR") - return globals()[name] + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") def get_tls_artifact_dir(): @@ -215,3 +217,77 @@ def next_artifact_index(): return 0 return len(list(current_artifact_dir.glob("*__*"))) + + +class MuteStdOut: + """Context manager to mute stdout, stderr, and logging with a startup message.""" + + def __init__(self, reason: str): + self.reason = reason + self.captured_stdout = StringIO() + self.captured_stderr = StringIO() + self.original_logging_level = None + self.original_handlers_streams = [] + + def __enter__(self): + # Print the startup message before muting + logger = logging.getLogger(__name__) + logger.info(self.reason) + + # Redirect logging handlers that have direct stream references + self._redirect_logging_handlers() + + # Suppress logging by setting root logger level to CRITICAL + root_logger = logging.getLogger() + self.original_logging_level = root_logger.level + root_logger.setLevel(logging.CRITICAL) + + # Start capturing stdout and stderr + self.stdout_redirect = redirect_stdout(self.captured_stdout) + self.stderr_redirect = redirect_stderr(self.captured_stderr) + + self.stdout_redirect.__enter__() + self.stderr_redirect.__enter__() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Restore stdout and stderr + self.stdout_redirect.__exit__(exc_type, exc_val, exc_tb) + self.stderr_redirect.__exit__(exc_type, exc_val, exc_tb) + + # Restore original logging level + if self.original_logging_level is not None: + root_logger = logging.getLogger() + root_logger.setLevel(self.original_logging_level) + + # Restore original logging handler streams + self._restore_logging_handlers() + + return False + + def _redirect_logging_handlers(self): + """Redirect logging handlers with direct stream references to captured streams.""" + import sys + + # Save and redirect handlers from root logger and all existing loggers + all_loggers = [logging.getLogger()] + [ + logging.getLogger(name) for name in logging.Logger.manager.loggerDict + ] + + for logger in all_loggers: + for handler in getattr(logger, "handlers", []): + if hasattr(handler, "stream"): + original_stream = handler.stream + self.original_handlers_streams.append((handler, original_stream)) + + if original_stream is sys.stdout: + handler.stream = self.captured_stdout + elif original_stream is sys.stderr: + handler.stream = self.captured_stderr + + def _restore_logging_handlers(self): + """Restore original streams to logging handlers.""" + for handler, original_stream in self.original_handlers_streams: + handler.stream = original_stream + self.original_handlers_streams.clear() From 8aa9b770d364686e1798548c54549e630c2fe4e0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 28 Jul 2026 14:18:53 -0400 Subject: [PATCH 210/344] fix: address PR review comments (coderabbit + colleague) - Use pd.to_numeric for TP filtering in analysis.py to handle non-numeric values - Guard against empty ARTIFACT_BASE_DIR in ci.py pipeline failure detection - Anchor resource cleanup suffix match with endswith in prepare_rhaiis.py - Clear _warnings at start of _run_test to prevent cross-invocation leaks - Add pipefail and post-extraction verification to profiler trace copy - Pass restrict_profiles=[workload_key] in per-workload standalone analysis - Deduplicate notification functions into shared _send_alert helper - Stop swallowing exceptions: surface CSV sync, profiler upload failures - Use descriptive artifact dir name with model_key and workload_keys - Factor shared warmup/profiler step args into step_kwargs dict Co-authored-by: Cursor --- projects/rhaiis/orchestration/analysis.py | 2 +- projects/rhaiis/orchestration/ci.py | 5 +- .../rhaiis/orchestration/notifications.py | 91 ++++++++----------- .../rhaiis/orchestration/prepare_rhaiis.py | 4 +- projects/rhaiis/orchestration/test_phase.py | 53 +++++------ .../toolbox/copy_profiler_traces/main.py | 10 +- 6 files changed, 79 insertions(+), 86 deletions(-) diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index f41390aab..df6ee38b2 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -73,7 +73,7 @@ def run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) + & (pd.to_numeric(df["TP"], errors="coerce").fillna(-1).astype(int).astype(str) == tp) ] if current_rows.empty: diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 9ba9c2e26..c0d70d18b 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -25,7 +25,10 @@ def _check_pipeline_failure_and_notify() -> None: produced FAILURE artifacts or the test step was skipped entirely. """ try: - base_dir = Path(os.environ.get("ARTIFACT_BASE_DIR", "")) + base_dir_env = os.environ.get("ARTIFACT_BASE_DIR", "") + if not base_dir_env: + return + base_dir = Path(base_dir_env) if not base_dir.is_dir(): return diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index 98b35a931..b500188b3 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -11,6 +11,41 @@ logger = logging.getLogger(__name__) +def _send_alert( + error_message: str, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Shared implementation for pipeline failure/warning Slack alerts.""" + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=error_message, + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + + def send_pipeline_failure_alert( exc: Exception, *, @@ -19,32 +54,8 @@ def send_pipeline_failure_alert( ) -> None: """Best-effort Slack alert when the pipeline fails.""" try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - - send_failure_notification( - error=traceback.format_exception_only(type(exc), exc)[-1].strip(), - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) + error_message = traceback.format_exception_only(type(exc), exc)[-1].strip() + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) except Exception: logger.warning("Failed to send pipeline failure alert", exc_info=True) @@ -57,32 +68,8 @@ def send_pipeline_warning( ) -> None: """Best-effort Slack alert for non-critical pipeline warnings.""" try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - warning_text = "\n".join(f"• {w}" for w in warnings) - send_failure_notification( - error=f"Pipeline completed with warnings:\n{warning_text}", - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) + error_message = f"Pipeline completed with warnings:\n{warning_text}" + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) except Exception: logger.warning("Failed to send pipeline warning alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index 29d111ff7..ca182f4f3 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -46,13 +46,13 @@ def prepare(): def _delete_resources_by_suffix(resource_type: str, ns: str, suffix: str) -> None: - """Delete resources whose names contain the FJOB suffix.""" + """Delete resources whose names end with the FJOB suffix.""" result = oc("get", resource_type, "-n", ns, "-o", "name", check=False, log_stdout=False) if result.returncode != 0 or not result.stdout: return for line in result.stdout.strip().splitlines(): name = line.strip() - if suffix in name: + if name.endswith(f"-{suffix}"): oc("delete", name, "-n", ns, "--ignore-not-found", check=False) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f94812476..bef05895d 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -40,7 +40,8 @@ def run( try: _sync_postprocessed_dashboard_csv(model_key, workload_keys) except Exception: - logger.warning("Dashboard CSV S3 sync after postprocessing failed", exc_info=True) + logger.exception("Dashboard CSV S3 sync after postprocessing failed") + ret = 1 return ret @@ -53,7 +54,8 @@ def do_test( deployment_name: str | None = None, ) -> int: try: - with env.NextArtifactDir("testing"): + dir_name = f"test_{model_key}_{'_'.join(workload_keys)}" + with env.NextArtifactDir(dir_name): return _run_test( model_key=model_key, workload_keys=workload_keys, @@ -74,6 +76,7 @@ def _run_test( namespace: str, deployment_name: str | None = None, ) -> int: + _warnings.clear() model_cfg = runtime_config.get_model(model_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator @@ -140,7 +143,7 @@ def _run_test( ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: - logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + logger.warning("Failed to annotate FournosJob with UUID", exc_info=True) from projects.core.library import config @@ -217,37 +220,29 @@ def _run_test( # Phase 1: warmup or profiler for ALL workloads first for wl_key in workload_keys: - workload = runtime_config.get_workload(wl_key) + step_kwargs = dict( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=runtime_config.get_workload(wl_key), + workload_key=wl_key, + benchmark_timeout=benchmark_timeout, + ) if profiler_enabled: logger.info("Running profiler for workload=%s", wl_key) - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=wl_key, - benchmark_timeout=benchmark_timeout, - ) + _run_profiler_step(**step_kwargs) elif warmup_enabled: logger.info("Running warmup for workload=%s", wl_key) - _run_warmup_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=wl_key, - benchmark_timeout=benchmark_timeout, - ) + _run_warmup_step(**step_kwargs) if profiler_enabled: try: _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: - logger.warning("Profiler trace upload failed; continuing", exc_info=True) + logger.exception("Profiler trace upload failed") + _warnings.append("Profiler trace upload failed") # Phase 2: benchmark + post-processing for ALL workloads for wl_key in workload_keys: @@ -372,7 +367,13 @@ def _run_workload_benchmark( try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis - run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + run_standalone_analysis( + model_cfg, + accelerator_key, + vllm_args, + run_uuid=run_uuid, + restrict_profiles=[workload_key], + ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) _warnings.append(f"Standalone analysis failed for {workload_key}") diff --git a/projects/rhaiis/toolbox/copy_profiler_traces/main.py b/projects/rhaiis/toolbox/copy_profiler_traces/main.py index 0d9331aad..147fe8dfe 100644 --- a/projects/rhaiis/toolbox/copy_profiler_traces/main.py +++ b/projects/rhaiis/toolbox/copy_profiler_traces/main.py @@ -56,11 +56,13 @@ def list_trace_files(args, context): def copy_traces(args, context): traces_dir = args.artifact_dir / "artifacts/traces" shell.run( - f"oc exec {context.pod_name} -n {args.namespace} " - "-- sh -c 'cd /tmp && tar cf - trace_*.json*' " - f"| tar --no-same-owner -xf - -C {traces_dir}", + 'bash -o pipefail -c "' + f"oc exec {context.pod_name} -n {args.namespace}" + " -- sh -c 'cd /tmp && tar cf - trace_*.json*'" + f' | tar --no-same-owner -xf - -C {traces_dir}"', ) - return f"Copied {context.trace_count} trace files to {traces_dir}" + copied = list(traces_dir.glob("trace_*.json*")) + return f"Copied {len(copied)} trace files to {traces_dir}" if __name__ == "__main__": From 44902bfc63cd0e21fd7dd4ecde617e2fc5b0e4e0 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:12:17 +0300 Subject: [PATCH 211/344] Address PR review: stricter error handling in mlflow verifier, fix duplicate directive - mlflow_verifier: raise FileNotFoundError/ValueError instead of returning empty strings on missing vault secrets, missing experiments, no runs, and version extraction failures - mlflow_verifier: suppress InsecureRequestWarning from urllib3 - mlflow_verifier: restore MLFLOW_WORKSPACE env var in finally block - base_verifier: update docstring to reflect exception-based contract - pr_args: remove duplicate /pipeline directive from parsed_directives to avoid double entry in GitHub notifications Co-authored-by: Cursor --- .../caliper/orchestration/mlflow_verifier.py | 67 ++++++++++++------- projects/core/nightly/base_verifier.py | 3 +- projects/mcp_gateway/orchestration/pr_args.py | 3 +- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 8d7d00adb..1941c5490 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -18,6 +18,7 @@ import os import re import time +import warnings from projects.caliper.engine.file_export.mlflow_secrets import ( load_mlflow_secrets_yaml, @@ -62,8 +63,6 @@ def __init__(self, _cfg): def get_last_tested_version(self) -> str: connection = self._load_connection() - if not connection: - return "" last_error = None for attempt in range(1, MAX_ATTEMPTS + 1): @@ -87,47 +86,65 @@ def _load_connection(self) -> dict: secret_path = vault.get_vault_content_path(self.secret_name, self.secret_file) if secret_path is None or not secret_path.exists(): - logger.warning("MLflow secret not found in vault, assuming no previous run") - return {} + raise FileNotFoundError( + f"MLflow secret '{self.secret_name}/{self.secret_file}' not found in vault. " + "Ensure the vault configuration is correct." + ) data = load_mlflow_secrets_yaml(secret_path) validate_mlflow_secrets(data) return data def _query_latest_run(self) -> str: + import urllib3 + import mlflow - if self.workspace: - os.environ["MLFLOW_WORKSPACE"] = self.workspace + warnings.filterwarnings("ignore", category=urllib3.exceptions.InsecureRequestWarning) + + previous_workspace = os.environ.get("MLFLOW_WORKSPACE") + try: + if self.workspace: + os.environ["MLFLOW_WORKSPACE"] = self.workspace - client = mlflow.tracking.MlflowClient() + client = mlflow.tracking.MlflowClient() - experiment = client.get_experiment_by_name(self.experiment) - if experiment is None: - logger.warning("Experiment '%s' not found in MLflow", self.experiment) - return "" + experiment = client.get_experiment_by_name(self.experiment) + if experiment is None: + raise ValueError( + f"MLflow experiment '{self.experiment}' not found. " + "Create the experiment first or check the configuration." + ) - runs = client.search_runs( - experiment_ids=[experiment.experiment_id], - order_by=["start_time DESC"], - max_results=1, - ) + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + order_by=["start_time DESC"], + max_results=1, + ) - if not runs: - logger.info("No runs found in experiment '%s'", self.experiment) - return "" + if not runs: + raise ValueError( + f"No runs found in MLflow experiment '{self.experiment}'." + ) - latest_run = runs[0] - run_name = latest_run.info.run_name or "" - logger.info("Latest MLflow run: %s", run_name) + latest_run = runs[0] + run_name = latest_run.info.run_name or "" + logger.info("Latest MLflow run: %s", run_name) - return self._extract_version(run_name) + return self._extract_version(run_name) + finally: + if previous_workspace is None: + os.environ.pop("MLFLOW_WORKSPACE", None) + else: + os.environ["MLFLOW_WORKSPACE"] = previous_workspace def _extract_version(self, run_name: str) -> str: match = self._version_re.search(run_name) if not match: - logger.warning("Could not extract version from run name: %s", run_name) - return "" + raise ValueError( + f"Could not extract version from run name '{run_name}' " + f"using pattern '{self._version_re.pattern}'." + ) version = match.group(1) diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py index a0c644d6d..de42c2037 100644 --- a/projects/core/nightly/base_verifier.py +++ b/projects/core/nightly/base_verifier.py @@ -16,6 +16,7 @@ class NightlyVerifier(ABC): def get_last_tested_version(self) -> str: """Return the version string of the last successful run. - Returns empty string if no previous run exists. + Raises an exception if the version cannot be determined + (missing credentials, experiment not found, etc.). """ ... diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 1b11aade6..1a6f7d879 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -80,13 +80,14 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st logger.info("Parsed mcp_gateway version: %s", parts[0]) # Parse /pipeline nightly — extract source from pipeline directive + # The directive line itself is already captured by the FOURNOS /pipeline handler, + # so only extract the config overrides here to avoid duplicating it in notifications. if line.startswith("/pipeline nightly "): parts = line.split() if len(parts) == 3: source = parts[2] config_overrides["nightly.source"] = source config_overrides["fournos.job.pipeline_name"] = "nightly" - parsed_directives.append(line) logger.info("Parsed nightly source from pipeline directive: %s", source) return config_overrides, parsed_directives From c4bc7af57f04c58e704f19636debfdfcbaa881d1 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:14:30 +0300 Subject: [PATCH 212/344] fix: sort imports in mlflow_verifier to satisfy ruff I001 Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 1941c5490..2ada0b728 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -96,9 +96,8 @@ def _load_connection(self) -> dict: return data def _query_latest_run(self) -> str: - import urllib3 - import mlflow + import urllib3 warnings.filterwarnings("ignore", category=urllib3.exceptions.InsecureRequestWarning) From 024bd90e9c417b73b366d061646f3cda68e85594 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:16:31 +0300 Subject: [PATCH 213/344] style: apply ruff formatting to mlflow_verifier Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 2ada0b728..7433c9e0b 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -122,9 +122,7 @@ def _query_latest_run(self) -> str: ) if not runs: - raise ValueError( - f"No runs found in MLflow experiment '{self.experiment}'." - ) + raise ValueError(f"No runs found in MLflow experiment '{self.experiment}'.") latest_run = runs[0] run_name = latest_run.info.run_name or "" From 08bf719fb8ddce096193108ba319140ac585b095 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:05:04 +0200 Subject: [PATCH 214/344] [caliper] orchestration: postprocess: re-enable the hierarchical KPI format --- projects/caliper/orchestration/postprocess.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index c0a594075..898a9929c 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -527,6 +527,45 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): + # Transform JSONL (schema v1) to hierarchical JSON (schema v2) + try: + logger.info(f"Transforming KPI output to hierarchical format: {output_file}") + + # Read the generated JSONL file + kpis = [] + with open(output_file) as f: + for line in f: + line = line.strip() + if line: + import json + + kpis.append(json.loads(line)) + + if kpis: + # Transform to hierarchical format + hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) + + # Write back as JSON (schema v2) + import json + + with open(output_file, "w") as f: + json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) + + logger.info( + f"Successfully transformed {len(kpis)} KPI records to hierarchical format" + ) + else: + logger.warning("No KPI records found in output file") + + except Exception as transform_error: + logger.error(f"Failed to transform KPIs to hierarchical format: {transform_error}") + return { + "status": "failed", + "error": f"KPI transformation failed: {transform_error}", + "completed_at": time.time(), + "log_file": log_file, + } + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) logger.info( f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" From b61e356becaaa72727dda419f768ee17e1f0e0be Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 13:39:08 -0400 Subject: [PATCH 215/344] feat: add SGLang and TRT-LLM engine support Replace all vllm-specific references with generic engine abstraction. Configuration, runtime helpers, CLI, templates, and callers now support vllm, sglang, and trtllm engines with automatic argument translation. Fix parity gaps with model-furnace: add securityContext, XDG env vars, NCCL_DEBUG for multi-GPU, 64Gi shared memory, TRT-LLM termination grace period and resource limits, and disable prometheus for SGLang. Co-authored-by: Cursor --- projects/rhaiis/README.md | 251 ++++++++++++------ projects/rhaiis/orchestration/analysis.py | 24 +- projects/rhaiis/orchestration/ci.py | 4 +- projects/rhaiis/orchestration/cli.py | 45 +++- .../rhaiis/orchestration/config.d/models.yaml | 56 ++-- .../rhaiis/orchestration/config.d/rhaiis.yaml | 34 ++- .../rhaiis/orchestration/notifications.py | 17 +- .../orchestration/presets.d/presets.yaml | 10 + .../rhaiis/orchestration/runtime_config.py | 83 +++++- projects/rhaiis/orchestration/test_phase.py | 102 ++++--- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 13 +- .../templates/inferenceservice.yaml.j2 | 29 +- .../templates/servingruntime.yaml.j2 | 129 ++++++++- 13 files changed, 595 insertions(+), 202 deletions(-) diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index f0e425e8f..1ccc40af6 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -4,30 +4,46 @@ using KServe InferenceService. The current implementation focuses on single-replica vLLM deployments benchmarked -with GuideLLM. The workflow deploys an InferenceService, waits for readiness, -runs a GuideLLM benchmark job, extracts results, and cleans up. +with GuideLLM. The workflow deploys an InferenceService, runs warmup and/or +profiler passes, executes multi-profile benchmarks, generates a dashboard CSV via +the Caliper postprocessing pipeline, syncs results to S3, and cleans up. ## Workflow sequence ``` -deploy_kserve_isvc Deploy ServingRuntime + InferenceService +deploy_kserve_isvc Deploy ServingRuntime + InferenceService | -wait_isvc_ready Poll InferenceService status + health check +wait_isvc_ready Poll InferenceService status + health check | -run_guidellm_benchmark Create PVC, run GuideLLM job, copy results, cleanup + ┌─────┴─────┐ + │ Phase 1 │ For each workload profile: + │ (warmup │ - warmup run (optional) OR profiler-gated run + │ /profiler)│ - profiler trace upload to S3 + └─────┬─────┘ + ┌─────┴─────┐ + │ Phase 2 │ For each workload profile (in its own artifact dir): + │ (benchmark)│ - write per-workload test labels + │ │ - run GuideLLM benchmark at configured rates + └─────┬─────┘ | -capture_isvc_state Capture ISVC YAML, pod logs, events +capture_isvc_state Capture ISVC YAML, pod logs, events | -cleanup_isvc Delete InferenceService + ServingRuntime +cleanup_isvc Delete InferenceService + ServingRuntime + | +Caliper postprocessing Parse → KPI → CSV export (per-profile rows) + | +S3 dashboard sync Append CSV to consolidated dashboard on S3 + | +Regression analysis Compare against baseline version (optional) ``` On failure at any step, `capture_isvc_state` and `cleanup_isvc` still run -(try/finally in the orchestration layer). +(try/finally in the orchestration layer). Pipeline failures and non-critical +warnings are reported via Slack. -After the benchmark, `run_and_postprocess()` automatically runs: -- KPI computation (22 metrics via rhaiis postprocess plugin) -- PSAP payload generation (model-furnace compatible JSON) -- MLflow metadata tagging +Each workload profile runs in its own `NextArtifactDir` with per-workload test +labels, so the Caliper parser creates separate records per profile. This ensures +each profile gets its own dashboard CSV rows. The benchmark step uses the canonical `projects.guidellm.toolbox.run_guidellm_benchmark` (shared with llm_d). rhaiis builds `guidellm_args` from its config and passes them @@ -37,8 +53,8 @@ to the canonical runner. Config is split into `config.yaml` (base) and `config.d/` (per-domain): -- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper -- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types +- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper postprocessing +- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types, S3 settings, profiler - [`orchestration/config.d/models.yaml`](./orchestration/config.d/models.yaml) — 52 model definitions - [`orchestration/config.d/workloads.yaml`](./orchestration/config.d/workloads.yaml) — workload profiles @@ -46,11 +62,14 @@ Key sections: | Section | Purpose | |---------|---------| -| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args | +| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args, S3 config | +| `rhaiis.s3` | S3 bucket, vault, and credentials for dashboard CSV and profiler trace uploads | +| `rhaiis.profiler` | PyTorch profiler settings (enable, S3 prefix, rates, labels) | | `models` | Model definitions (hf_model_id, per-model vLLM overrides) | | `workloads` | Benchmark profiles (data shape, rates, max_seconds) | -| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size | -| `tests` | CI test mapping (model_key, workload_key) | +| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret | +| `tests` | CI test mapping (model_key, workload_keys, version) | +| `caliper.postprocess` | Caliper postprocessing pipeline (parse, KPI, CSV export) | ## Fournos integration @@ -58,38 +77,60 @@ rhaiis supports Fournos-driven execution via `ci.py`: ``` bin/run_ci rhaiis ci resolve-fournos-config # Populate spec.secretRefs + hardware -bin/run_ci rhaiis ci pre-cleanup # Delete leftover jobs/pods +bin/run_ci rhaiis ci pre-cleanup # No-op (avoid cleaning running resources) bin/run_ci rhaiis ci prepare # Verify cluster, ensure namespace/SA +bin/run_ci rhaiis ci preflight # Preflight checks bin/run_ci rhaiis ci test # Deploy, benchmark, capture, cleanup +bin/run_ci rhaiis ci post-cleanup # Detect pipeline failures, cleanup resources bin/run_ci rhaiis ci export-artifacts # Caliper export to MLflow ``` -FournosJob example: +### FournosJob YAML + ```yaml apiVersion: fournos.dev/v1 kind: FournosJob metadata: generateName: rhaiis-benchmark- spec: - owner: mehul - displayName: rhaiis-qwen3-benchmark - pipeline: forge-prepare-test - cluster: fournos-fg - priority: manual - hardware: - gpuType: l40s - gpuCount: 1 + owner: haumesh + displayName: rhaiis-benchmark + pipeline: forge-full + cluster: zeus executionEngine: forge: project: rhaiis - args: [nvidia, profile1] + args: [nvidia] configOverrides: - tests.rhaiis.model_key: qwen3-0_6b - rhaiis.namespace: kserve-e2e-perf + tests.rhaiis.model_key: nemotron3super-120b-fp8 + rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 + tests.rhaiis.version: "vLLM-0.24.0" + tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] + rhaiis.vllm_args.tensor-parallel-size: 2 + rhaiis.cluster_tag: "zeus2" rhaiis.deploy.image_pull_secret: npalaska-image-pull - rhaiis.deploy.storage_pvc: model-pvc + caliper.postprocess.csv_dashboard.enabled: true + env: + PULL_PULL_SHA: "" ``` +### GitHub PR workflow + +Jobs can also be triggered via PR comments on `openshift-psap/forge`: + +``` +/test fournos rhaiis nvidia +/pipeline forge-full +/cluster zeus +/var tests.rhaiis.model_key: nemotron3super-120b-fp8 +/var tests.rhaiis.version: vLLM-0.24.0 +/var tests.rhaiis.workload_keys: ["profile1"] +/var rhaiis.vllm_args.tensor-parallel-size: 2 +``` + +Note: `/var` directives use `key: value` format (colon required). +If `PULL_PULL_SHA` is not set, the PR HEAD commit is used automatically. + Presets from `args` are applied via `project.args` → `presets.d/presets.yaml`. Config overrides use dot-notation to set any nested config value. @@ -98,13 +139,23 @@ Available configOverrides: | Key | Description | |-----|-------------| | `tests.rhaiis.model_key` | Model key from config.d/models.yaml | +| `tests.rhaiis.version` | Version label for dashboard and regression | +| `tests.rhaiis.workload_keys` | List of workload profiles to run | +| `tests.rhaiis.warmup` | Enable warmup pass before benchmarks | +| `tests.rhaiis.slack_user` | Slack user ID for failure notifications | +| `tests.rhaiis.compare_version` | Baseline version for regression comparison | | `rhaiis.namespace` | Kubernetes namespace | +| `rhaiis.cluster_tag` | Cluster identifier for dashboard grouping | | `rhaiis.deploy.image_pull_secret` | Image pull secret name | | `rhaiis.deploy.storage_pvc` | PVC name for model storage | | `rhaiis.deploy.replicas` | Number of predictor replicas | | `rhaiis.images.nvidia` | vLLM container image override | | `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | +| `rhaiis.profiler.enabled` | Enable PyTorch profiler | +| `rhaiis.agent_analysis.enabled` | Enable AI agent regression analysis | +| `caliper.postprocess.csv_dashboard.enabled` | Enable dashboard CSV S3 sync | | `benchmarks.guidellm.timeout` | Benchmark timeout in seconds | +| `benchmarks.guidellm.hf_token_secret` | K8s secret name for HF_TOKEN (omit to skip) | Monitoring Fournos jobs: ```bash @@ -122,16 +173,22 @@ oc patch fournosjob -n psap-automation \ - CLI: [`orchestration/cli.py`](./orchestration/cli.py) - CI: [`orchestration/ci.py`](./orchestration/ci.py) (Fournos pipeline) - CI test: [`orchestration/test_rhaiis.py`](./orchestration/test_rhaiis.py) +- Test phase: [`orchestration/test_phase.py`](./orchestration/test_phase.py) +- Analysis: [`orchestration/analysis.py`](./orchestration/analysis.py) +- Notifications: [`orchestration/notifications.py`](./orchestration/notifications.py) ## Toolbox commands | Command | Source | Purpose | |---------|--------|---------| -| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime | +| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime with configurable labels | | `wait_isvc_ready` | [rhaiis](./toolbox/wait_isvc_ready/) | Poll InferenceService readiness with health check | | `run_guidellm_benchmark` | [canonical](../guidellm/toolbox/run_guidellm_benchmark/) | Run GuideLLM benchmark (shared with llm_d) | | `capture_isvc_state` | [rhaiis](./toolbox/capture_isvc_state/) | Capture InferenceService YAML, pod logs, events, describe output | | `cleanup_isvc` | [rhaiis](./toolbox/cleanup_isvc/) | Delete InferenceService, ServingRuntime, wait for deletion | +| `enable_profiler_gate` | [rhaiis](./toolbox/enable_profiler_gate/) | Enable/disable the PyTorch profiler gate file on the vLLM pod | +| `verify_profiler_prereqs` | [rhaiis](./toolbox/verify_profiler_prereqs/) | Verify profiler prerequisites (gate file, sitecustomize.py) | +| `copy_profiler_traces` | [rhaiis](./toolbox/copy_profiler_traces/) | Copy Chrome trace JSON files from the vLLM pod | ## Usage @@ -208,68 +265,66 @@ Available overrides: ## Result extraction -GuideLLM results are extracted using the copy-pod pattern (same as llm_d): +GuideLLM results are extracted using `oc cp` with a gzip fallback for large files: 1. GuideLLM job writes `benchmarks.json` to a PVC mounted at `/results` 2. A copy pod is created on the same node (required for ReadWriteOnce PVC) -3. Results are extracted via `oc exec cat /results/benchmarks.json` -4. Written to local `artifacts/results/benchmarks.json` +3. Results are copied via `oc cp` from the copy pod +4. If `oc cp` fails for large files (e.g. `unexpected EOF`), the file is gzipped + inside the pod, copied compressed, and decompressed locally 5. PVC, job, and copy pod are deleted -Artifacts are stored under `/tmp/forge_/002__run_guidellm_benchmark/artifacts/`. +## Postprocessing and dashboard CSV -## Available models +After benchmarks complete, `run_and_postprocess()` runs the Caliper pipeline: -52 models from model_furnace are defined in `config.yaml`. Key families: +1. **Parse**: `RhaiisParser` extends `GuideLLMParser` with extra metrics (p1/p999 + percentiles, mean latencies, token counts, throughput) from raw `benchmarks.json` +2. **KPI generate**: `RhaiisKpiHandler` emits 33 KPI types per rate point per profile +3. **CSV export**: `RhaiisPlugin.export_kpis_to_csv` maps KPIs to the dashboard CSV + schema with all metadata columns (model, version, TP, accelerator, UUID, etc.) +4. **S3 sync**: The CSV is appended to the consolidated dashboard CSV on S3 +5. **Regression check**: Optionally compares current vs. baseline version -| Family | Key examples | TP size | -|--------|-------------|---------| -| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | -| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | -| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | -| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | -| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | -| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | -| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | -| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | -| Validation | `qwen3-0_6b` | 1 | +Dashboard CSV columns include throughput (output tok/s, total tok/s), latency +percentiles (TTFT/TPOT/ITL at median, p1, p95, p99, p999, mean), request latency, +token counts, concurrency, and run metadata. -Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` +## Notifications -## Workload profiles +Pipeline events are reported to Slack via the topsail bot: -From model_furnace `guidellm_profiles.iterations`: +- **Failure alerts**: Sent when the pipeline crashes (exceptions, image pull failures, + infrastructure errors). Includes error details, model, accelerator, TP, version, + and job ID. +- **Warning alerts**: Sent when the pipeline completes with non-critical warnings + (e.g. profiler trace upload failed, predictor log upload failed). +- **Regression alerts**: Sent when regression analysis detects performance changes, + with a link to the dashboard filtered by the relevant configuration. -| Key | Prompt tokens | Output tokens | Rates | Max seconds | -|-----|--------------|---------------|-------|-------------| -| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | -| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | -| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | -| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | +Notifications are sent to the `psap-rhaiis-alerts` Slack channel. The post-cleanup +step also detects infrastructure failures that occur before the test step runs. -## Postprocessing and KPIs +## PyTorch profiling -After each benchmark run, the rhaiis postprocess plugin (`projects/rhaiis/postprocess/`) -automatically computes 22 KPIs with model-furnace parity: +When `rhaiis.profiler.enabled: true`, the pipeline runs profiler-gated benchmarks +before the main benchmarks: -| Category | KPIs | -|----------|------| -| Throughput | request_rate, tokens/s, input_tokens/s, output_tokens/s | -| Latency | ttft (median, p95, p99), itl (median, p95, p99), tpot (median, p95, p99) | -| Request latency | request_latency (median, p95) | -| Token efficiency | input_tokens/request, output_tokens/request | -| Request stats | concurrency, completed_requests, failed_requests | -| Benchmark | duration, prompt_token_count_mean | +1. Verify profiler prerequisites on the vLLM pod (gate file, sitecustomize.py) +2. Enable the profiler gate with a workload-specific label (e.g. `isl1000_osl1000`) +3. Run a GuideLLM benchmark at the configured profiler rates +4. Disable the profiler gate +5. Copy Chrome trace JSON files from the vLLM pod +6. Upload traces to S3 organized by accelerator/model/TP/version/profile -KPIs are written to `kpis.json` in the postprocess output directory. +Traces are viewable in `chrome://tracing` or Perfetto UI. -## PSAP payload +## Parallel job isolation -A model-furnace compatible `PSAP_perf____.json` -is generated after each benchmark. It wraps the raw `benchmarks.json` with experiment -metadata (model, vLLM image/args, accelerator, timestamps) for downstream compatibility. +Multiple FournosJobs can run concurrently on the same cluster. Each job gets a +unique deployment name by appending the last 5 characters of the FJOB name +(e.g. `nvidia-nemotron-3-super-120b-a1-54r5f`). Resource cleanup is scoped to +the current job's suffix to avoid interfering with other running jobs. ## MLflow export @@ -281,18 +336,48 @@ MLflow tags set on each run: | Tag | Example | |-----|---------| | `project` | rhaiis | -| `model_key` | qwen3-0_6b | -| `hf_model_id` | Qwen/Qwen3-0.6B | +| `model_key` | nemotron3super-120b-fp8 | +| `hf_model_id` | nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 | | `accelerator` | nvidia | -| `tensor_parallel_size` | 1 | -| `vllm_image` | quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2 | -| `vllm_version` | 3.4.0-ea.2-1773886296 | -| `workload_key` | profile1 | -| `rates` | 1,5,10 | +| `tensor_parallel_size` | 2 | +| `vllm_image` | vllm/vllm-openai:v0.24.0 | +| `vllm_version` | v0.24.0 | +| `workload_key` | profile1,profile2,profile4 | +| `rates` | 1,50,100,200,300 | | `guidellm_backend_type` | openai_http | MLflow experiment: `forge-rhaiis` +## Available models + +52 models are defined in `config.d/models.yaml`. Key families: + +| Family | Key examples | TP size | +|--------|-------------|---------| +| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | +| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | +| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | +| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | +| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | +| Nemotron 120B | `nemotron3super-120b-fp8` | 2 | +| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | +| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | +| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | +| Validation | `qwen3-0_6b` | 1 | + +Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` + +## Workload profiles + +| Key | Prompt tokens | Output tokens | Rates | Max seconds | +|-----|--------------|---------------|-------|-------------| +| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | +| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | +| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | +| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | + ## Presets Presets in `presets.d/presets.yaml` provide shortcuts for common configurations: diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index df6ee38b2..724b99b5f 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -15,7 +15,7 @@ def run_standalone_analysis( model_cfg: dict, accelerator_key: str, - vllm_args: dict, + engine_args: dict, *, run_uuid: str = "", restrict_profiles: list[str] | None = None, @@ -67,7 +67,12 @@ def run_standalone_analysis( df[col] = df[col].str.strip() model_id = model_cfg.get("hf_model_id", "") - tp = str(vllm_args.get("tensor-parallel-size", 1)) + tp = str( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) current_rows = df[ (df["version"] == version) @@ -99,7 +104,7 @@ def run_standalone_analysis( accelerator, run_uuid=run_uuid, restrict_profiles=restrict_profiles, - vllm_args=vllm_args, + engine_args=engine_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -119,7 +124,7 @@ def run_regression_check( *, run_uuid: str = "", restrict_profiles: list[str] | None = None, - vllm_args: dict | None = None, + engine_args: dict | None = None, ) -> None: from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials from projects.core.library import config @@ -176,7 +181,9 @@ def run_regression_check( from projects.rhaiis.postprocess.regression import send_regression_notification - _vllm = vllm_args or {} + _ea = engine_args or {} + tp = _ea.get("tensor-parallel-size") or _ea.get("tp-size") or _ea.get("tp_size") or "" + dp = _ea.get("data-parallel-size") or _ea.get("dp-size") or "" send_regression_notification( analysis, model=model_cfg.get("hf_model_id", ""), @@ -185,8 +192,8 @@ def run_regression_check( slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), notification_vault="psap-forge-notifications", report_url=report_url, - tp=str(_vllm.get("tensor-parallel-size", "")), - dp=str(_vllm.get("data-parallel-size", "")), + tp=str(tp), + dp=str(dp), ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) @@ -233,7 +240,8 @@ def run_agent_analysis( logger.warning("Agent not reachable, skipping analysis: %s", detail) return "" - tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + ea = model_cfg.get("engine_args", {}) + tp = str(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) model = model_cfg.get("hf_model_id", "") improvements = analysis.get("improvements", []) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index c0d70d18b..89a72f62e 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -97,8 +97,8 @@ def resolve_hardware_request(hardware_spec: dict) -> dict: model_key = runtime_config.get_test_model_key() model = runtime_config.get_model(model_key) - vllm_args = model.get("vllm_args", {}) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) + ea = model.get("engine_args", {}) + tp_size = int(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) diff --git a/projects/rhaiis/orchestration/cli.py b/projects/rhaiis/orchestration/cli.py index b65a07c54..cea6215e6 100644 --- a/projects/rhaiis/orchestration/cli.py +++ b/projects/rhaiis/orchestration/cli.py @@ -51,7 +51,13 @@ def cli(ctx): @click.option("--namespace", "-n", default=None, help="Kubernetes namespace") @click.option("--deployment-name", default=None, help="Deployment name (defaults to model name)") @click.option("--accelerator", type=click.Choice(["nvidia", "amd"]), default=None) -@click.option("--vllm-image", help="vLLM container image override") +@click.option( + "--engine", + type=click.Choice(["vllm", "sglang", "trtllm"]), + default=None, + help="Inference engine", +) +@click.option("--serving-image", help="Serving engine container image override") @click.option("--tensor-parallel", "-tp", type=int, help="Tensor parallel size override") @click.option("--replicas", "-r", type=int, default=None) @click.option("--storage-source", type=click.Choice(["hf", "pvc"]), default=None) @@ -70,7 +76,8 @@ def test( namespace: str | None, deployment_name: str | None, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -93,7 +100,8 @@ def test( _apply_cli_overrides( workload_key=workload_key, accelerator=accelerator, - vllm_image=vllm_image, + engine=engine, + serving_image=serving_image, tensor_parallel=tensor_parallel, replicas=replicas, storage_source=storage_source, @@ -128,7 +136,8 @@ def _apply_cli_overrides( *, workload_key: str, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -140,11 +149,21 @@ def _apply_cli_overrides( ) -> None: if accelerator: config.project.set_config("rhaiis.accelerator", accelerator) - if vllm_image: + if engine: + config.project.set_config("rhaiis.engine", engine) + resolved_engine = runtime_config.get_engine() + if serving_image: resolved_accel = runtime_config.get_accelerator() - config.project.set_config(f"rhaiis.images.{resolved_accel}", vllm_image) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.images.{resolved_accel}", serving_image + ) if tensor_parallel is not None: - config.project.set_config("rhaiis.vllm_args.tensor-parallel-size", tensor_parallel) + tp_key = {"sglang": "tp-size", "trtllm": "tp_size"}.get( + resolved_engine, "tensor-parallel-size" + ) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.args.{tp_key}", tensor_parallel + ) if replicas is not None: config.project.set_config("rhaiis.deploy.replicas", replicas) if storage_source: @@ -169,9 +188,10 @@ def _print_dry_run( workload_cfg = runtime_config.get_workload(workload_key) deploy_cfg = runtime_config.get_deploy_config() accelerator = runtime_config.get_accelerator() - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload_cfg) + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_args = runtime_config.merge_engine_args(engine_defaults, model_cfg, workload_cfg, engine) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) if not deployment_name: @@ -182,9 +202,10 @@ def _print_dry_run( click.echo(f" Workload: {workload_key}") click.echo(f" Namespace: {namespace}") click.echo(f" Deployment: {deployment_name}") + click.echo(f" Engine: {engine}") click.echo(f" Accelerator: {accelerator}") - click.echo(f" Image: {vllm_image}") - click.echo(f" vLLM args: {vllm_args}") + click.echo(f" Image: {serving_image}") + click.echo(f" Engine args: {engine_args}") click.echo(f" Env vars: {env_vars}") click.echo(f" Replicas: {deploy_cfg.get('replicas', 1)}") click.echo( diff --git a/projects/rhaiis/orchestration/config.d/models.yaml b/projects/rhaiis/orchestration/config.d/models.yaml index 806ce0442..1a839f919 100644 --- a/projects/rhaiis/orchestration/config.d/models.yaml +++ b/projects/rhaiis/orchestration/config.d/models.yaml @@ -2,46 +2,46 @@ qwen3-0_6b: name: Qwen3-0.6B hf_model_id: Qwen/Qwen3-0.6B - vllm_args: {} + engine_args: {} # Llama-4 Scout (MoE) llama-4-scout: name: Llama-4-Scout-17B-16E-Instruct hf_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-4-scout-fp8: name: Llama-4-Scout-17B-16E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 2 kv-cache-dtype: fp8 llama-4-scout-int4: name: Llama-4-Scout-17B-16E-Instruct-INT4 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 2 # Llama-4 Maverick (MoE) llama-4-maverick: name: Llama-4-Maverick-17B-128E-Instruct hf_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 8 llama-4-maverick-fp8: name: Llama-4-Maverick-17B-128E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 8 kv-cache-dtype: fp8 llama-4-maverick-w4a16: name: Llama-4-Maverick-17B-128E-Instruct-W4A16 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 enable-expert-parallel: true @@ -66,25 +66,25 @@ mistral-2503-fp8: llama-3-3-70b: name: Llama-3.3-70B-Instruct hf_model_id: meta-llama/Llama-3.3-70B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-fp8: name: Llama-3.3-70B-Instruct-FP8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w8a8: name: Llama-3.3-70B-Instruct-W8A8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w8a8 - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w4a16: name: Llama-3.3-70B-Instruct-W4A16 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 4 # Granite 3.1 8B @@ -125,7 +125,7 @@ qwen25-7b-w4a16: qwen3-n-80b-a3b-instruct: name: Qwen3-Next-80B-A3B-Instruct hf_model_id: Qwen/Qwen3-Next-80B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-30b-a3b-instruct: @@ -135,13 +135,13 @@ qwen3-30b-a3b-instruct: qwen3-30b-a3b: name: Qwen3-30B-A3B hf_model_id: Qwen/Qwen3-30B-A3B - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-235b-instruct: name: Qwen3-235B-A22B-Instruct hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507 - vllm_args: + engine_args: tensor-parallel-size: 4 env_vars: VLLM_ROCM_USE_AITER: "0" @@ -149,7 +149,7 @@ qwen3-235b-instruct: qwen3-235b-instruct-fp8: name: Qwen3-235B-A22B-Instruct-FP8 hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true kv-cache-dtype: fp8 @@ -157,20 +157,20 @@ qwen3-235b-instruct-fp8: qwen3-235b-fp8-dynamic: name: Qwen3-235B-A22B-FP8-dynamic hf_model_id: RedHatAI/Qwen3-235B-A22B-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true qwen3-vl-235b-fp8: name: Qwen3-VL-235B-A22B-FP8 hf_model_id: Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-vl-30b-a3b: name: Qwen3-VL-30B-A3B-Instruct hf_model_id: Qwen/Qwen3-VL-30B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 # GPT-OSS @@ -200,7 +200,7 @@ mixtral-8x7b: nemotron-3-1-70b: name: Nemotron-70B-Instruct hf_model_id: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF - vllm_args: + engine_args: tensor-parallel-size: 2 nemotron-3-1-70b-fp8: @@ -224,25 +224,25 @@ gemma-9b-w8a16: deepseek-v4-pro: name: DeepSeek-V4-Pro hf_model_id: deepseek-ai/DeepSeek-V4-Pro - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528: name: DeepSeek-R1-0528 hf_model_id: deepseek-ai/DeepSeek-R1-0528 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528-w4a16: name: DeepSeek-R1-0528-W4A16 hf_model_id: RedHatAI/DeepSeek-R1-0528-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-v3-2: name: DeepSeek-V3.2 hf_model_id: deepseek-ai/DeepSeek-V3.2 - vllm_args: + engine_args: tensor-parallel-size: 8 # Mistral 7B @@ -258,30 +258,30 @@ nemotron3nano-30b: nemotron3nano-30b-fp8: name: Nemotron-3-Nano-30B-A3B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 nemotron3super-120b-fp8: name: Nemotron-3-Super-120B-A12B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 # Ministral ministral-3-14b-reasoning: name: Ministral-3-14B-Reasoning hf_model_id: mistralai/Ministral-3-14B-Reasoning-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp16: name: Ministral-3-14B-Instruct hf_model_id: mistralai/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp8: name: Ministral-3-14B-FP8 hf_model_id: RedHatAI/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 90faeab5b..ae95755d3 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -11,6 +11,33 @@ images: nvidia: quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2-1773886296 amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5-1766067105 +engine: vllm + +engines: + vllm: + images: + nvidia: vllm/vllm-openai:v0.24.0 + amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5 + port: 8080 + args: + gpu-memory-utilization: 0.92 + trust-remote-code: true + no-enable-log-requests: true + no-enable-prefix-caching: true + uvicorn-log-level: debug + sglang: + images: + nvidia: lmsysorg/sglang:v0.5.11 + amd: lmsysorg/sglang:v0.5.4.post2-rocm700-mi30x + port: 8080 + args: {} + trtllm: + images: + nvidia: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc13 + port: 8888 + args: {} + trtllm_config: {} + deploy: replicas: 1 cpu_request: "4" @@ -22,13 +49,6 @@ deploy: ready_timeout: 3600 health_check_timeout: 120 -vllm_args: - gpu-memory-utilization: 0.92 - trust-remote-code: true - no-enable-log-requests: true - no-enable-prefix-caching: true - uvicorn-log-level: debug - env_vars: {} accelerator_env_vars: diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index b500188b3..fcaddc5f6 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -25,11 +25,20 @@ def _send_alert( accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), + engine = runtime_config.get_engine() + engine_args = runtime_config.merge_engine_args( + runtime_config.get_engine_args(engine), model_cfg, runtime_config.get_workload(workload_keys[0]), + engine, ) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or "" + ) + dp = engine_args.get("data-parallel-size") or engine_args.get("dp-size") or "" send_failure_notification( error=error_message, @@ -38,8 +47,8 @@ def _send_alert( job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), + tp=str(tp), + dp=str(dp), version=_cfg.project.get_config("tests.rhaiis.version", ""), workload_keys=workload_keys, cluster=cluster_tag, diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index 86c620f6b..a2ed40b80 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -75,3 +75,13 @@ nvidia: amd: rhaiis.accelerator: amd + +# Engine +vllm: + rhaiis.engine: vllm + +sglang: + rhaiis.engine: sglang + +trtllm: + rhaiis.engine: trtllm diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 1abbc2492..3f898fa81 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -39,12 +39,30 @@ def get_benchmark_config() -> dict: return dict(config.project.get_config("benchmarks.guidellm")) -def get_vllm_image(accelerator: str) -> str: - return config.project.get_config(f"rhaiis.images.{accelerator}") +def get_engine() -> str: + return config.project.get_config("rhaiis.engine", "vllm") -def get_vllm_defaults() -> dict: - return dict(config.project.get_config("rhaiis.vllm_args")) +def get_serving_image(accelerator: str, engine: str | None = None) -> str: + engine = engine or get_engine() + try: + return config.project.get_config(f"rhaiis.engines.{engine}.images.{accelerator}") + except Exception: + return config.project.get_config(f"rhaiis.images.{accelerator}") + + +def get_engine_args(engine: str | None = None) -> dict: + engine = engine or get_engine() + return dict(config.project.get_config(f"rhaiis.engines.{engine}.args", {})) + + +def get_engine_port(engine: str | None = None) -> int: + engine = engine or get_engine() + return int(config.project.get_config(f"rhaiis.engines.{engine}.port", 8080)) + + +def get_trtllm_config() -> dict: + return dict(config.project.get_config("rhaiis.engines.trtllm.trtllm_config", {})) def get_model(model_key: str) -> dict: @@ -71,15 +89,62 @@ def get_profiler_config() -> dict: return dict(config.project.get_config("rhaiis.profiler", {}) or {}) -def merge_vllm_args( +_COMMON_ARG_TRANSLATIONS: dict[str, dict[str, str]] = { + "sglang": { + "tensor-parallel-size": "tp-size", + "data-parallel-size": "dp-size", + }, + "trtllm": { + "tensor-parallel-size": "tp_size", + "data-parallel-size": "dp_size", + }, +} + + +def _translate_args(args: dict, engine: str) -> dict: + """Translate vLLM-style engine_args to another engine's arg naming. + + Only shared args (TP, DP) are translated; vLLM-specific args are dropped. + """ + mapping = _COMMON_ARG_TRANSLATIONS.get(engine, {}) + translated = {} + for key, val in args.items(): + if key in mapping: + translated[mapping[key]] = val + # Drop vLLM-only args that have no equivalent + return translated + + +def merge_engine_args( overrides: dict, model: dict, workload: dict, + engine: str | None = None, ) -> dict: - merged = dict(model.get("vllm_args", {})) - merged.update(workload.get("vllm_args", {})) - merged.update(overrides) - return merged + engine = engine or get_engine() + engine_key = f"{engine}_args" + + # Use engine-specific block if present, otherwise fall back to engine_args + # and auto-translate common args when the engine isn't vLLM + model_args = model.get(engine_key) if engine != "vllm" else None + if model_args is None: + base = dict(model.get("engine_args", {})) + if engine != "vllm": + base = _translate_args(base, engine) + else: + base = dict(model_args) + + wl_args = workload.get(engine_key) if engine != "vllm" else None + if wl_args is None: + wl = dict(workload.get("engine_args", {})) + if engine != "vllm": + wl = _translate_args(wl, engine) + else: + wl = dict(wl_args) + + base.update(wl) + base.update(overrides) + return base def merge_env_vars(accelerator: str, model: dict) -> dict: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index bef05895d..951793a28 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -100,10 +100,14 @@ def _run_test( else: deployment_name = base_name - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_port = runtime_config.get_engine_port(engine) first_workload = runtime_config.get_workload(workload_keys[0]) - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + engine_args = runtime_config.merge_engine_args( + engine_defaults, model_cfg, first_workload, engine + ) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) from projects.core.library import config as _cfg @@ -160,7 +164,7 @@ def _run_test( run_standalone_analysis( model_cfg, accelerator_key, - vllm_args, + engine_args, run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) @@ -174,17 +178,22 @@ def _run_test( try: isvc_labels = {"opendatahub.io/dashboard": "true"} - if profiler_enabled: + if profiler_enabled and engine == "vllm": isvc_labels["vllm-profiler/enabled"] = "true" + elif profiler_enabled and engine != "vllm": + logger.warning("Profiler is only supported with vLLM engine, skipping profiler") + profiler_enabled = False logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) - deploy_kserve_isvc( + deploy_kwargs = dict( deployment_name=deployment_name, namespace=namespace, model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, + serving_image=serving_image, accelerator=accelerator, - vllm_args=vllm_args, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, env_vars=env_vars, replicas=deploy_cfg.get("replicas", 1), cpu_request=deploy_cfg.get("cpu_request", "4"), @@ -195,6 +204,9 @@ def _run_test( service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) + if engine == "trtllm": + deploy_kwargs["trtllm_config"] = runtime_config.get_trtllm_config() or None + deploy_kserve_isvc(**deploy_kwargs) logger.info("Waiting for InferenceService to be ready") wait_isvc_ready( @@ -204,7 +216,9 @@ def _run_test( health_check_timeout=deploy_cfg.get("health_check_timeout", 120), ) - endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" + endpoint_url = ( + f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:{engine_port}" + ) warmup_enabled = ( config.project.get_config("tests.rhaiis.warmup", True) @@ -239,7 +253,7 @@ def _run_test( if profiler_enabled: try: - _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) + _upload_profiler_traces(model_cfg, gpu_type, engine_args, profiler_cfg) except Exception: logger.exception("Profiler trace upload failed") _warnings.append("Profiler trace upload failed") @@ -253,8 +267,8 @@ def _run_test( accelerator=accelerator, accelerator_key=accelerator_key, gpu_type=gpu_type, - vllm_image=vllm_image, - vllm_args=vllm_args, + serving_image=serving_image, + engine_args=engine_args, benchmark_cfg=benchmark_cfg, deployment_name=deployment_name, namespace=namespace, @@ -274,8 +288,8 @@ def _run_test( ",".join(workload_keys), model_cfg, accelerator, - vllm_image, - vllm_args, + serving_image, + engine_args, benchmark_cfg, first_rates, first_max_seconds, @@ -318,8 +332,8 @@ def _run_workload_benchmark( accelerator: str, accelerator_key: str, gpu_type: str, - vllm_image: str, - vllm_args: dict, + serving_image: str, + engine_args: dict, benchmark_cfg: dict, deployment_name: str, namespace: str, @@ -353,10 +367,10 @@ def _run_workload_benchmark( model_key, workload_key, accelerator, - vllm_args, + engine_args, hf_model_id=model_cfg["hf_model_id"], version=version, - vllm_image=vllm_image, + serving_image=serving_image, cluster_tag=cluster_tag, accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, @@ -370,7 +384,7 @@ def _run_workload_benchmark( run_standalone_analysis( model_cfg, accelerator_key, - vllm_args, + engine_args, run_uuid=run_uuid, restrict_profiles=[workload_key], ) @@ -406,22 +420,28 @@ def _create_test_labels( model_key: str, workload_key: str, accelerator: str, - vllm_args: dict, + engine_args: dict, *, hf_model_id: str = "", version: str = "", - vllm_image: str = "", + serving_image: str = "", cluster_tag: str = "", accelerator_chip: str = "", run_uuid: str = "", ) -> None: - _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") - runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) + _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") + runtime_args = "; ".join(f"{k}: {v}" for k, v in engine_args.items()) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) labels = { "model_key": model_key, "workload_key": workload_key, "accelerator": accelerator_chip or accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), + "tensor_parallel_size": str(tp), "hf_model_id": hf_model_id, "version": version, "image_tag": image_tag, @@ -438,8 +458,8 @@ def _set_mlflow_metadata( workload_key: str, model_cfg: dict, accelerator: str, - vllm_image: str, - vllm_args: dict, + serving_image: str, + engine_args: dict, benchmark_cfg: dict, rates: list[int], max_seconds: int, @@ -448,18 +468,24 @@ def _set_mlflow_metadata( ) -> None: from projects.core.library import config - image_name, image_tag = runtime_config.split_image_tag(vllm_image) + image_name, image_tag = runtime_config.split_image_tag(serving_image) guidellm_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") benchmark_args = benchmark_cfg.get("args", {}) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) tags = { "project": "rhaiis", "model_key": model_key, "hf_model_id": model_cfg["hf_model_id"], "accelerator": accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), - "vllm_image": vllm_image, - "vllm_version": image_tag, + "tensor_parallel_size": str(tp), + "serving_image": serving_image, + "serving_version": image_tag, "workload_key": workload_key, "rates": ",".join(str(r) for r in rates), "max_seconds": str(max_seconds), @@ -521,9 +547,10 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) model_cfg = runtime_config.get_model(model_key) accelerator = runtime_config.get_accelerator() - vllm_defaults = runtime_config.get_vllm_defaults() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) first_workload = runtime_config.get_workload(workload_keys[0]) - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) from projects.rhaiis.orchestration.analysis import run_regression_check @@ -534,7 +561,7 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) model_cfg, accelerator, run_uuid="", - vllm_args=vllm_args, + engine_args=ea, ) @@ -717,7 +744,7 @@ def _infer_profile_labels_from_traces(trace_files: list) -> list[str]: def _upload_profiler_traces( model_cfg: dict, accelerator: str, - vllm_args: dict, + engine_args: dict, profiler_cfg: dict, ) -> None: from pathlib import Path @@ -757,7 +784,12 @@ def _upload_profiler_traces( traces_dir, model_name=model_cfg.get("hf_model_id", ""), accelerator=accelerator, - tp_size=int(vllm_args.get("tensor-parallel-size", 1)), + tp_size=int( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ), version=version, profile_labels=profile_labels, s3_bucket=s3_cfg.get("bucket", ""), diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index e8908595a..c5d4e0b43 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -15,9 +15,12 @@ def run( deployment_name: str, namespace: str, model_id: str, - vllm_image: str, + serving_image: str, accelerator: str = "nvidia", - vllm_args: dict | None = None, + engine: str = "vllm", + engine_args: dict | None = None, + engine_port: int = 8080, + trtllm_config: dict | None = None, env_vars: dict | None = None, replicas: int = 1, cpu_request: str = "4", @@ -37,8 +40,8 @@ def prepare_args(args, context): for key, value in (args.env_vars or {}).items(): context.env_vars_list.append({"name": key, "value": str(value)}) - vllm_args = args.vllm_args or {} - tp_size = vllm_args.get("tensor-parallel-size", 1) + ea = args.engine_args or {} + tp_size = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 context.gpu_count = int(tp_size) if args.storage_pvc: @@ -48,7 +51,7 @@ def prepare_args(args, context): context.use_pvc = False context.pvc_name = "" - return f"GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" + return f"engine={args.engine}, GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" @task diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index 5ca599383..5d728620f 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -3,41 +3,66 @@ kind: InferenceService metadata: annotations: serving.kserve.io/deploymentMode: RawDeployment - serving.kserve.io/enable-prometheus-scraping: "true" storage.kserve.io/readonly: "false" +{%- if args.engine != "sglang" %} + serving.kserve.io/enable-prometheus-scraping: "true" prometheus.io/scrape: "true" prometheus.io/path: /metrics - prometheus.io/port: "8080" + prometheus.io/port: "{{ args.engine_port }}" +{%- endif %} {%- if args.labels %} labels: {%- for key, val in args.labels.items() %} {{ key }}: "{{ val }}" {%- endfor %} +{%- if args.engine == "sglang" or args.engine == "trtllm" %} + monitoring.opendatahub.io/scrape: "false" +{%- endif %} {%- endif %} name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: predictor: +{%- if args.engine == "trtllm" %} + terminationGracePeriodSeconds: 120 +{%- endif %} minReplicas: {{ args.replicas }} maxReplicas: {{ args.replicas }} model: modelFormat: +{%- if args.engine == "sglang" %} + name: sglang +{%- elif args.engine == "trtllm" %} + name: trtllm +{%- else %} name: vLLM +{%- endif %} runtime: {{ args.deployment_name }} resources: {%- if args.accelerator == "nvidia" %} limits: nvidia.com/gpu: "{{ ctx.gpu_count }}" +{%- if args.engine == "trtllm" %} + memory: "{{ args.memory_request }}" + ephemeral-storage: "50Gi" +{%- endif %} requests: nvidia.com/gpu: "{{ ctx.gpu_count }}" +{%- if args.engine == "trtllm" %} + cpu: "{{ args.cpu_request }}" + memory: "{{ args.memory_request }}" + ephemeral-storage: "50Gi" +{%- endif %} {%- elif args.accelerator == "amd" %} limits: amd.com/gpu: "{{ ctx.gpu_count }}" requests: amd.com/gpu: "{{ ctx.gpu_count }}" {%- endif %} +{%- if not (args.accelerator == "nvidia" and args.engine == "trtllm") %} cpu: "{{ args.cpu_request }}" memory: "{{ args.memory_request }}" +{%- endif %} {%- if args.storage_source == "hf" and ctx.use_pvc %} storageUri: pvc://{{ ctx.pvc_name }}/ {%- elif args.storage_source != "hf" %} diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index 42d53cd73..2b077c658 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: ServingRuntime metadata: annotations: - opendatahub.io/template-display-name: ServingRuntime for vLLM | RHAIIS + opendatahub.io/template-display-name: ServingRuntime for {{ args.engine }} | RHAIIS labels: opendatahub.io/dashboard: "true" name: {{ args.deployment_name }} @@ -11,17 +11,117 @@ spec: {%- if args.image_pull_secret %} imagePullSecrets: - name: {{ args.image_pull_secret }} +{%- if args.engine == "trtllm" %} + - name: ngc-secret +{%- endif %} +{%- elif args.engine == "trtllm" %} + imagePullSecrets: + - name: ngc-secret {%- endif %} builtInAdapter: modelLoadingTimeoutMillis: 300000 multiModel: false supportedModelFormats: - autoSelect: true +{%- if args.engine == "sglang" %} + name: sglang +{%- elif args.engine == "trtllm" %} + name: trtllm +{%- else %} name: vLLM +{%- endif %} containers: +{%- if args.engine == "trtllm" %} + - name: kserve-container + image: "{{ args.serving_image }}" + command: + - /bin/bash + - -c + - | + set -ex + if [ ! -d "/mnt/models/hub/models--{{ args.model_id | replace('/', '--') }}/snapshots" ]; then + hf download {{ args.model_id }} + fi + python3 -c " + from flashinfer.norm import get_norm_module + get_norm_module() + print('FlashInfer norm kernel pre-compiled successfully') + " +{%- if args.trtllm_config %} + cat > /tmp/trtllm-config.yml << EOF + {{ args.trtllm_config | indent(10) }} + EOF +{%- endif %} + trtllm-serve {{ args.model_id }} \ + --host 0.0.0.0 \ + --port={{ args.engine_port }} \ +{%- for key, val in (args.engine_args or {}).items() %} + {%- if key != "trtllm-config" %} + {%- if val is boolean %} + {%- if val %} + --{{ key }} \ + {%- endif %} + {%- else %} + --{{ key }}={{ val }} \ + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if args.trtllm_config %} + --extra_llm_api_options=/tmp/trtllm-config.yml +{%- endif %} + + env: + - name: HF_HUB_OFFLINE + value: "0" + - name: HF_HOME + value: /mnt/models + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: storage-config + key: HF_TOKEN + - name: TRTLLM_ENABLE_PDL + value: "1" + - name: HOME + value: /tmp + - name: XDG_CACHE_HOME + value: /tmp/.cache + - name: FLASHINFER_WORKSPACE_BASE + value: /mnt/models +{%- for env_item in ctx.env_vars_list %} + - name: {{ env_item.name }} + value: "{{ env_item.value }}" +{%- endfor %} + securityContext: + runAsUser: 0 + ports: + - containerPort: {{ args.engine_port }} + protocol: TCP + volumeMounts: + - name: shared-memory + mountPath: /dev/shm + volumes: + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: 128Gi +{%- else %} - name: kserve-container - image: "{{ args.vllm_image }}" + image: "{{ args.serving_image }}" command: +{%- if args.engine == "sglang" %} + - python3 + - -m + - sglang.launch_server + args: +{%- if args.storage_source == "hf" %} + - --model-path={{ args.model_id }} +{%- else %} + - --model-path=/mnt/models +{%- endif %} + - --port=8080 + - --host=0.0.0.0 +{%- else %} - python3 - -m - vllm.entrypoints.openai.api_server @@ -33,7 +133,8 @@ spec: - --served-model-name={{ args.model_id }} {%- endif %} - --port=8080 -{%- for key, val in (args.vllm_args or {}).items() %} +{%- endif %} +{%- for key, val in (args.engine_args or {}).items() %} {%- if val is boolean %} {%- if val %} - --{{ key }} @@ -43,8 +144,14 @@ spec: {%- endif %} {%- endfor %} env: - - name: USER - value: "vllm" + - name: XDG_CACHE_HOME + value: /tmp/.cache + - name: XDG_CONFIG_HOME + value: /tmp/.config +{%- if ctx.gpu_count > 1 %} + - name: NCCL_DEBUG + value: "WARN" +{%- endif %} {%- if args.storage_source == "hf" %} - name: HF_HUB_OFFLINE value: "0" @@ -52,8 +159,10 @@ spec: value: /mnt/models - name: HF_HOME value: /mnt/models +{%- if args.engine != "sglang" %} - name: VLLM_CACHE_DIR value: /mnt/models/.cache/vllm +{%- endif %} - name: HF_DATASETS_CACHE value: /mnt/models/.cache/huggingface/datasets - name: HF_TOKEN @@ -61,13 +170,18 @@ spec: secretKeyRef: name: storage-config key: HF_TOKEN +{%- else %} + - name: HOME + value: /tmp {%- endif %} {%- for env_item in ctx.env_vars_list %} - name: {{ env_item.name }} value: "{{ env_item.value }}" {%- endfor %} + securityContext: + runAsUser: 0 ports: - - containerPort: 8080 + - containerPort: {{ args.engine_port }} protocol: TCP {%- if ctx.gpu_count > 1 %} volumeMounts: @@ -77,5 +191,6 @@ spec: - name: shared-memory emptyDir: medium: Memory - sizeLimit: 8Gi + sizeLimit: 64Gi +{%- endif %} {%- endif %} From cf28237922e1c4dae832838eb1467a527927b27b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 13:56:54 -0400 Subject: [PATCH 216/344] fix: remove runAsUser:0 from vLLM/SGLang ServingRuntime OpenShift SCC rejects runAsUser:0 when the service account lacks the anyuid SCC. vLLM and SGLang don't require root, so drop it. TRT-LLM retains runAsUser:0 as it needs root for flashinfer compilation. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index 2b077c658..fbf63f759 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -178,8 +178,6 @@ spec: - name: {{ env_item.name }} value: "{{ env_item.value }}" {%- endfor %} - securityContext: - runAsUser: 0 ports: - containerPort: {{ args.engine_port }} protocol: TCP From ea8f55569e479adbd4d15f6b4cf64f90d613465a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 14:47:33 -0400 Subject: [PATCH 217/344] fix: add USER env var back for vLLM/SGLang ServingRuntime PyTorch's torch._inductor calls getpass.getuser() which falls back to pwd.getpwuid(). OpenShift assigns arbitrary UIDs not in /etc/passwd, causing a KeyError. Setting USER env var avoids the passwd lookup. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index fbf63f759..c4bec6e4c 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -144,6 +144,8 @@ spec: {%- endif %} {%- endfor %} env: + - name: USER + value: "{{ args.engine }}" - name: XDG_CACHE_HOME value: /tmp/.cache - name: XDG_CONFIG_HOME From fa0dab0db93b823f610eeb752c4a4a4055d59421 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 15:09:53 -0400 Subject: [PATCH 218/344] refactor: replace image_pull_secret string with image_pull_secrets list Remove hardcoded ngc-secret from ServingRuntime template. Users now configure all pull secrets through rhaiis.deploy.image_pull_secrets as a list, including any NGC secrets needed for TRT-LLM. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- projects/rhaiis/orchestration/test_phase.py | 2 +- projects/rhaiis/toolbox/deploy_kserve_isvc/main.py | 2 +- .../templates/servingruntime.yaml.j2 | 12 ++++-------- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index ae95755d3..2b0d89124 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -44,7 +44,7 @@ deploy: memory_request: "16Gi" storage_source: hf storage_pvc: model-pvc - image_pull_secret: "" + image_pull_secrets: [] service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 951793a28..32747fa84 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -200,7 +200,7 @@ def _run_test( memory_request=deploy_cfg.get("memory_request", "16Gi"), storage_source=deploy_cfg.get("storage_source", "hf"), storage_pvc=deploy_cfg.get("storage_pvc", ""), - image_pull_secret=deploy_cfg.get("image_pull_secret", ""), + image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index c5d4e0b43..1a8b0e4a8 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -27,7 +27,7 @@ def run( memory_request: str = "16Gi", storage_source: str = "hf", storage_pvc: str = "", - image_pull_secret: str = "", + image_pull_secrets: list | None = None, service_account_name: str = "", labels: dict | None = None, ): diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index c4bec6e4c..e5f8d80f6 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -8,15 +8,11 @@ metadata: name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: -{%- if args.image_pull_secret %} +{%- if args.image_pull_secrets %} imagePullSecrets: - - name: {{ args.image_pull_secret }} -{%- if args.engine == "trtllm" %} - - name: ngc-secret -{%- endif %} -{%- elif args.engine == "trtllm" %} - imagePullSecrets: - - name: ngc-secret +{%- for secret in args.image_pull_secrets %} + - name: {{ secret }} +{%- endfor %} {%- endif %} builtInAdapter: modelLoadingTimeoutMillis: 300000 From 6c1c3970700bd24a9f8011c8d1733c9d1763ab3c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 15:17:26 -0400 Subject: [PATCH 219/344] refactor: move manifest building to orchestration layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Jinja2 templates with Python dict construction in a new manifests.py module. The deploy_kserve_isvc script is now generic — it accepts pre-built manifest dicts and applies them, with no engine awareness. This follows the pattern established by llm_d. Co-authored-by: Cursor --- projects/rhaiis/orchestration/manifests.py | 376 ++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 34 +- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 73 ++-- .../templates/inferenceservice.yaml.j2 | 73 ---- .../templates/servingruntime.yaml.j2 | 192 --------- 5 files changed, 433 insertions(+), 315 deletions(-) create mode 100644 projects/rhaiis/orchestration/manifests.py delete mode 100644 projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 delete mode 100644 projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py new file mode 100644 index 000000000..c181288a4 --- /dev/null +++ b/projects/rhaiis/orchestration/manifests.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def build_servingruntime( + *, + deployment_name: str, + namespace: str, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + image_pull_secrets: list[str] | None = None, + env_vars: dict | None = None, + trtllm_config: dict | None = None, +) -> dict[str, Any]: + """Build a KServe ServingRuntime manifest dict.""" + env_vars_list = [{"name": k, "value": str(v)} for k, v in (env_vars or {}).items()] + + manifest: dict[str, Any] = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "annotations": { + "opendatahub.io/template-display-name": f"ServingRuntime for {engine} | RHAIIS", + }, + "labels": {"opendatahub.io/dashboard": "true"}, + "name": deployment_name, + "namespace": namespace, + }, + "spec": { + "builtInAdapter": {"modelLoadingTimeoutMillis": 300000}, + "multiModel": False, + "supportedModelFormats": [ + {"autoSelect": True, "name": _model_format_name(engine)}, + ], + }, + } + + if image_pull_secrets: + manifest["spec"]["imagePullSecrets"] = [{"name": s} for s in image_pull_secrets] + + if engine == "trtllm": + container = _build_trtllm_container( + model_id=model_id, + serving_image=serving_image, + engine_args=engine_args, + engine_port=engine_port, + env_vars_list=env_vars_list, + trtllm_config=trtllm_config, + ) + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "128Gi"}}, + ] + else: + container = _build_vllm_sglang_container( + model_id=model_id, + serving_image=serving_image, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, + storage_source=storage_source, + gpu_count=gpu_count, + env_vars_list=env_vars_list, + ) + if gpu_count > 1: + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "64Gi"}}, + ] + + manifest["spec"]["containers"] = [container] + return manifest + + +def build_inferenceservice( + *, + deployment_name: str, + namespace: str, + engine: str, + engine_port: int, + accelerator: str, + gpu_count: int, + replicas: int, + cpu_request: str, + memory_request: str, + storage_source: str, + storage_pvc: str, + model_id: str, + service_account_name: str = "", + labels: dict | None = None, +) -> dict[str, Any]: + """Build a KServe InferenceService manifest dict.""" + annotations: dict[str, str] = { + "serving.kserve.io/deploymentMode": "RawDeployment", + "storage.kserve.io/readonly": "false", + } + if engine != "sglang": + annotations.update( + { + "serving.kserve.io/enable-prometheus-scraping": "true", + "prometheus.io/scrape": "true", + "prometheus.io/path": "/metrics", + "prometheus.io/port": str(engine_port), + } + ) + + metadata: dict[str, Any] = { + "annotations": annotations, + "name": deployment_name, + "namespace": namespace, + } + + if labels: + meta_labels = dict(labels) + if engine in ("sglang", "trtllm"): + meta_labels["monitoring.opendatahub.io/scrape"] = "false" + metadata["labels"] = meta_labels + + resources = _build_resources( + accelerator=accelerator, + gpu_count=gpu_count, + cpu_request=cpu_request, + memory_request=memory_request, + engine=engine, + ) + + model_spec: dict[str, Any] = { + "modelFormat": {"name": _model_format_name(engine)}, + "runtime": deployment_name, + "resources": resources, + } + + if storage_source == "hf" and storage_pvc: + model_spec["storageUri"] = f"pvc://{storage_pvc}/" + elif storage_source != "hf": + model_spec["storageUri"] = f"{storage_source}://{model_id}" + + predictor: dict[str, Any] = { + "minReplicas": replicas, + "maxReplicas": replicas, + "model": model_spec, + } + + if engine == "trtllm": + predictor["terminationGracePeriodSeconds"] = 120 + + if service_account_name: + predictor["serviceAccountName"] = service_account_name + + return { + "apiVersion": "serving.kserve.io/v1beta1", + "kind": "InferenceService", + "metadata": metadata, + "spec": {"predictor": predictor}, + } + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _model_format_name(engine: str) -> str: + if engine == "sglang": + return "sglang" + if engine == "trtllm": + return "trtllm" + return "vLLM" + + +def _build_resources( + *, + accelerator: str, + gpu_count: int, + cpu_request: str, + memory_request: str, + engine: str, +) -> dict[str, Any]: + resources: dict[str, Any] = {} + gpu_key = "nvidia.com/gpu" if accelerator == "nvidia" else "amd.com/gpu" + + if accelerator in ("nvidia", "amd"): + limits: dict[str, str] = {gpu_key: str(gpu_count)} + requests: dict[str, str] = {gpu_key: str(gpu_count)} + + if accelerator == "nvidia" and engine == "trtllm": + limits["memory"] = memory_request + limits["ephemeral-storage"] = "50Gi" + requests["cpu"] = cpu_request + requests["memory"] = memory_request + requests["ephemeral-storage"] = "50Gi" + + resources["limits"] = limits + resources["requests"] = requests + + if not (accelerator == "nvidia" and engine == "trtllm"): + resources.setdefault("requests", {}) + resources["requests"]["cpu"] = cpu_request + resources["requests"]["memory"] = memory_request + + return resources + + +def _build_vllm_sglang_container( + *, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + env_vars_list: list[dict], +) -> dict[str, Any]: + if engine == "sglang": + command = ["python3", "-m", "sglang.launch_server"] + if storage_source == "hf": + args = [f"--model-path={model_id}", "--port=8080", "--host=0.0.0.0"] + else: + args = ["--model-path=/mnt/models", "--port=8080", "--host=0.0.0.0"] + else: + command = ["python3", "-m", "vllm.entrypoints.openai.api_server"] + if storage_source == "hf": + args = [f"--model={model_id}", "--port=8080"] + else: + args = ["--model=/mnt/models", f"--served-model-name={model_id}", "--port=8080"] + + for key, val in (engine_args or {}).items(): + if isinstance(val, bool): + if val: + args.append(f"--{key}") + else: + args.append(f"--{key}={val}") + + env: list[dict[str, Any]] = [ + {"name": "USER", "value": engine}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "XDG_CONFIG_HOME", "value": "/tmp/.config"}, + ] + + if gpu_count > 1: + env.append({"name": "NCCL_DEBUG", "value": "WARN"}) + + if storage_source == "hf": + env.extend( + [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HOME", "value": "/mnt/models"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + ] + ) + if engine != "sglang": + env.append({"name": "VLLM_CACHE_DIR", "value": "/mnt/models/.cache/vllm"}) + env.extend( + [ + {"name": "HF_DATASETS_CACHE", "value": "/mnt/models/.cache/huggingface/datasets"}, + { + "name": "HF_TOKEN", + "valueFrom": { + "secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}, + }, + }, + ] + ) + else: + env.append({"name": "HOME", "value": "/tmp"}) + + env.extend(env_vars_list) + + container: dict[str, Any] = { + "name": "kserve-container", + "image": serving_image, + "command": command, + "args": args, + "env": env, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + } + + if gpu_count > 1: + container["volumeMounts"] = [{"name": "shared-memory", "mountPath": "/dev/shm"}] + + return container + + +def _build_trtllm_launch_script( + model_id: str, + engine_port: int, + engine_args: dict, + trtllm_config: dict | None, +) -> str: + safe_model = model_id.replace("/", "--") + parts = [ + "set -ex", + f'if [ ! -d "/mnt/models/hub/models--{safe_model}/snapshots" ]; then', + f" hf download {model_id}", + "fi", + 'python3 -c "', + "from flashinfer.norm import get_norm_module", + "get_norm_module()", + "print('FlashInfer norm kernel pre-compiled successfully')", + '"', + ] + + if trtllm_config: + import yaml + + config_yaml = yaml.dump(trtllm_config, default_flow_style=False).rstrip() + parts.extend(["cat > /tmp/trtllm-config.yml << EOF", config_yaml, "EOF"]) + + cmd = [f"trtllm-serve {model_id} \\", " --host 0.0.0.0 \\", f" --port={engine_port}"] + + for key, val in (engine_args or {}).items(): + if key == "trtllm-config": + continue + if isinstance(val, bool): + if val: + cmd.append(f" --{key}") + else: + cmd.append(f" --{key}={val}") + + if trtllm_config: + cmd.append(" --extra_llm_api_options=/tmp/trtllm-config.yml") + + # Add line continuations between cmd parts (except last) + cmd_lines = [] + for i, line in enumerate(cmd): + if i < len(cmd) - 1 and not line.endswith("\\"): + cmd_lines.append(f"{line} \\") + else: + cmd_lines.append(line) + + parts.extend(cmd_lines) + return "\n".join(parts) + "\n" + + +def _build_trtllm_container( + *, + model_id: str, + serving_image: str, + engine_args: dict, + engine_port: int, + env_vars_list: list[dict], + trtllm_config: dict | None, +) -> dict[str, Any]: + script = _build_trtllm_launch_script(model_id, engine_port, engine_args, trtllm_config) + + env: list[dict[str, Any]] = [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + { + "name": "HF_TOKEN", + "valueFrom": {"secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}}, + }, + {"name": "TRTLLM_ENABLE_PDL", "value": "1"}, + {"name": "HOME", "value": "/tmp"}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "FLASHINFER_WORKSPACE_BASE", "value": "/mnt/models"}, + ] + env.extend(env_vars_list) + + return { + "name": "kserve-container", + "image": serving_image, + "command": ["/bin/bash", "-c", script], + "env": env, + "securityContext": {"runAsUser": 0}, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + "volumeMounts": [{"name": "shared-memory", "mountPath": "/dev/shm"}], + } diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 32747fa84..3ffbf7529 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -121,6 +121,10 @@ def _run_test( from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) + from projects.rhaiis.orchestration.manifests import ( + build_inferenceservice, + build_servingruntime, + ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready @@ -185,28 +189,46 @@ def _run_test( profiler_enabled = False logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) - deploy_kwargs = dict( + ea = engine_args or {} + gpu_count = int( + ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 + ) + + sr_manifest = build_servingruntime( deployment_name=deployment_name, namespace=namespace, model_id=model_cfg["hf_model_id"], serving_image=serving_image, - accelerator=accelerator, engine=engine, engine_args=engine_args, engine_port=engine_port, + storage_source=deploy_cfg.get("storage_source", "hf"), + gpu_count=gpu_count, + image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], env_vars=env_vars, + trtllm_config=runtime_config.get_trtllm_config() if engine == "trtllm" else None, + ) + isvc_manifest = build_inferenceservice( + deployment_name=deployment_name, + namespace=namespace, + engine=engine, + engine_port=engine_port, + accelerator=accelerator, + gpu_count=gpu_count, replicas=deploy_cfg.get("replicas", 1), cpu_request=deploy_cfg.get("cpu_request", "4"), memory_request=deploy_cfg.get("memory_request", "16Gi"), storage_source=deploy_cfg.get("storage_source", "hf"), storage_pvc=deploy_cfg.get("storage_pvc", ""), - image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], + model_id=model_cfg["hf_model_id"], service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) - if engine == "trtllm": - deploy_kwargs["trtllm_config"] = runtime_config.get_trtllm_config() or None - deploy_kserve_isvc(**deploy_kwargs) + deploy_kserve_isvc( + namespace=namespace, + servingruntime_manifest=sr_manifest, + inferenceservice_manifest=isvc_manifest, + ) logger.info("Waiting for InferenceService to be ready") wait_isvc_ready( diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index 1a8b0e4a8..4a0f7887e 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 +from __future__ import annotations + +from pathlib import Path + +import yaml + from projects.core.dsl import ( entrypoint, execute_tasks, task, - template, ) from projects.core.dsl.utils.k8s import oc, oc_resource_exists @@ -12,46 +17,18 @@ @entrypoint def run( *, - deployment_name: str, namespace: str, - model_id: str, - serving_image: str, - accelerator: str = "nvidia", - engine: str = "vllm", - engine_args: dict | None = None, - engine_port: int = 8080, - trtllm_config: dict | None = None, - env_vars: dict | None = None, - replicas: int = 1, - cpu_request: str = "4", - memory_request: str = "16Gi", - storage_source: str = "hf", - storage_pvc: str = "", - image_pull_secrets: list | None = None, - service_account_name: str = "", - labels: dict | None = None, + servingruntime_manifest: dict, + inferenceservice_manifest: dict, ): - return execute_tasks(locals()) - - -@task -def prepare_args(args, context): - context.env_vars_list = [] - for key, value in (args.env_vars or {}).items(): - context.env_vars_list.append({"name": key, "value": str(value)}) - - ea = args.engine_args or {} - tp_size = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 - context.gpu_count = int(tp_size) - - if args.storage_pvc: - context.use_pvc = True - context.pvc_name = args.storage_pvc - else: - context.use_pvc = False - context.pvc_name = "" + """Deploy a KServe InferenceService from pre-built manifests. - return f"engine={args.engine}, GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" + Args: + namespace: Target namespace for the deployment. + servingruntime_manifest: Pre-built ServingRuntime manifest dict. + inferenceservice_manifest: Pre-built InferenceService manifest dict. + """ + return execute_tasks(locals()) @task @@ -63,19 +40,27 @@ def ensure_namespace(args, context): @task -def render_and_apply_servingruntime(args, context): +def apply_servingruntime(args, context): output_path = args.artifact_dir / "servingruntime.yaml" - template.render_template_to_file("servingruntime.yaml.j2", output_path) + _write_manifest(args.servingruntime_manifest, output_path) oc("apply", "-f", str(output_path)) - return f"Applied ServingRuntime {args.deployment_name}" + name = args.servingruntime_manifest.get("metadata", {}).get("name", "") + return f"Applied ServingRuntime {name}" @task -def render_and_apply_inferenceservice(args, context): +def apply_inferenceservice(args, context): output_path = args.artifact_dir / "inferenceservice.yaml" - template.render_template_to_file("inferenceservice.yaml.j2", output_path) + _write_manifest(args.inferenceservice_manifest, output_path) oc("apply", "-f", str(output_path)) - return f"Applied InferenceService {args.deployment_name}" + name = args.inferenceservice_manifest.get("metadata", {}).get("name", "") + return f"Applied InferenceService {name}" + + +def _write_manifest(manifest: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) if __name__ == "__main__": diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 deleted file mode 100644 index 5d728620f..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - storage.kserve.io/readonly: "false" -{%- if args.engine != "sglang" %} - serving.kserve.io/enable-prometheus-scraping: "true" - prometheus.io/scrape: "true" - prometheus.io/path: /metrics - prometheus.io/port: "{{ args.engine_port }}" -{%- endif %} -{%- if args.labels %} - labels: -{%- for key, val in args.labels.items() %} - {{ key }}: "{{ val }}" -{%- endfor %} -{%- if args.engine == "sglang" or args.engine == "trtllm" %} - monitoring.opendatahub.io/scrape: "false" -{%- endif %} -{%- endif %} - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: - predictor: -{%- if args.engine == "trtllm" %} - terminationGracePeriodSeconds: 120 -{%- endif %} - minReplicas: {{ args.replicas }} - maxReplicas: {{ args.replicas }} - model: - modelFormat: -{%- if args.engine == "sglang" %} - name: sglang -{%- elif args.engine == "trtllm" %} - name: trtllm -{%- else %} - name: vLLM -{%- endif %} - runtime: {{ args.deployment_name }} - resources: -{%- if args.accelerator == "nvidia" %} - limits: - nvidia.com/gpu: "{{ ctx.gpu_count }}" -{%- if args.engine == "trtllm" %} - memory: "{{ args.memory_request }}" - ephemeral-storage: "50Gi" -{%- endif %} - requests: - nvidia.com/gpu: "{{ ctx.gpu_count }}" -{%- if args.engine == "trtllm" %} - cpu: "{{ args.cpu_request }}" - memory: "{{ args.memory_request }}" - ephemeral-storage: "50Gi" -{%- endif %} -{%- elif args.accelerator == "amd" %} - limits: - amd.com/gpu: "{{ ctx.gpu_count }}" - requests: - amd.com/gpu: "{{ ctx.gpu_count }}" -{%- endif %} -{%- if not (args.accelerator == "nvidia" and args.engine == "trtllm") %} - cpu: "{{ args.cpu_request }}" - memory: "{{ args.memory_request }}" -{%- endif %} -{%- if args.storage_source == "hf" and ctx.use_pvc %} - storageUri: pvc://{{ ctx.pvc_name }}/ -{%- elif args.storage_source != "hf" %} - storageUri: {{ args.storage_source }}://{{ args.model_id }} -{%- endif %} -{%- if args.service_account_name %} - serviceAccountName: {{ args.service_account_name }} -{%- endif %} diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 deleted file mode 100644 index e5f8d80f6..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ /dev/null @@ -1,192 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - annotations: - opendatahub.io/template-display-name: ServingRuntime for {{ args.engine }} | RHAIIS - labels: - opendatahub.io/dashboard: "true" - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: -{%- if args.image_pull_secrets %} - imagePullSecrets: -{%- for secret in args.image_pull_secrets %} - - name: {{ secret }} -{%- endfor %} -{%- endif %} - builtInAdapter: - modelLoadingTimeoutMillis: 300000 - multiModel: false - supportedModelFormats: - - autoSelect: true -{%- if args.engine == "sglang" %} - name: sglang -{%- elif args.engine == "trtllm" %} - name: trtllm -{%- else %} - name: vLLM -{%- endif %} - containers: -{%- if args.engine == "trtllm" %} - - name: kserve-container - image: "{{ args.serving_image }}" - command: - - /bin/bash - - -c - - | - set -ex - if [ ! -d "/mnt/models/hub/models--{{ args.model_id | replace('/', '--') }}/snapshots" ]; then - hf download {{ args.model_id }} - fi - python3 -c " - from flashinfer.norm import get_norm_module - get_norm_module() - print('FlashInfer norm kernel pre-compiled successfully') - " -{%- if args.trtllm_config %} - cat > /tmp/trtllm-config.yml << EOF - {{ args.trtllm_config | indent(10) }} - EOF -{%- endif %} - trtllm-serve {{ args.model_id }} \ - --host 0.0.0.0 \ - --port={{ args.engine_port }} \ -{%- for key, val in (args.engine_args or {}).items() %} - {%- if key != "trtllm-config" %} - {%- if val is boolean %} - {%- if val %} - --{{ key }} \ - {%- endif %} - {%- else %} - --{{ key }}={{ val }} \ - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if args.trtllm_config %} - --extra_llm_api_options=/tmp/trtllm-config.yml -{%- endif %} - - env: - - name: HF_HUB_OFFLINE - value: "0" - - name: HF_HOME - value: /mnt/models - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: storage-config - key: HF_TOKEN - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: HOME - value: /tmp - - name: XDG_CACHE_HOME - value: /tmp/.cache - - name: FLASHINFER_WORKSPACE_BASE - value: /mnt/models -{%- for env_item in ctx.env_vars_list %} - - name: {{ env_item.name }} - value: "{{ env_item.value }}" -{%- endfor %} - securityContext: - runAsUser: 0 - ports: - - containerPort: {{ args.engine_port }} - protocol: TCP - volumeMounts: - - name: shared-memory - mountPath: /dev/shm - volumes: - - name: shared-memory - emptyDir: - medium: Memory - sizeLimit: 128Gi -{%- else %} - - name: kserve-container - image: "{{ args.serving_image }}" - command: -{%- if args.engine == "sglang" %} - - python3 - - -m - - sglang.launch_server - args: -{%- if args.storage_source == "hf" %} - - --model-path={{ args.model_id }} -{%- else %} - - --model-path=/mnt/models -{%- endif %} - - --port=8080 - - --host=0.0.0.0 -{%- else %} - - python3 - - -m - - vllm.entrypoints.openai.api_server - args: -{%- if args.storage_source == "hf" %} - - --model={{ args.model_id }} -{%- else %} - - --model=/mnt/models - - --served-model-name={{ args.model_id }} -{%- endif %} - - --port=8080 -{%- endif %} -{%- for key, val in (args.engine_args or {}).items() %} - {%- if val is boolean %} - {%- if val %} - - --{{ key }} - {%- endif %} - {%- else %} - - --{{ key }}={{ val }} - {%- endif %} -{%- endfor %} - env: - - name: USER - value: "{{ args.engine }}" - - name: XDG_CACHE_HOME - value: /tmp/.cache - - name: XDG_CONFIG_HOME - value: /tmp/.config -{%- if ctx.gpu_count > 1 %} - - name: NCCL_DEBUG - value: "WARN" -{%- endif %} -{%- if args.storage_source == "hf" %} - - name: HF_HUB_OFFLINE - value: "0" - - name: HOME - value: /mnt/models - - name: HF_HOME - value: /mnt/models -{%- if args.engine != "sglang" %} - - name: VLLM_CACHE_DIR - value: /mnt/models/.cache/vllm -{%- endif %} - - name: HF_DATASETS_CACHE - value: /mnt/models/.cache/huggingface/datasets - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: storage-config - key: HF_TOKEN -{%- else %} - - name: HOME - value: /tmp -{%- endif %} -{%- for env_item in ctx.env_vars_list %} - - name: {{ env_item.name }} - value: "{{ env_item.value }}" -{%- endfor %} - ports: - - containerPort: {{ args.engine_port }} - protocol: TCP -{%- if ctx.gpu_count > 1 %} - volumeMounts: - - name: shared-memory - mountPath: /dev/shm - volumes: - - name: shared-memory - emptyDir: - medium: Memory - sizeLimit: 64Gi -{%- endif %} -{%- endif %} From 70a55ead57b0d4a27cc0f623184a1bbb702111c9 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 16:13:41 -0400 Subject: [PATCH 220/344] fix: write hierarchical KPIs to separate file to preserve JSONL The hierarchical format transformation was overwriting kpis.json (JSONL) with a single indented JSON object, breaking the downstream csv-export step which expects JSON Lines input. Co-authored-by: Cursor --- projects/caliper/orchestration/postprocess.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 898a9929c..e94868754 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -542,17 +542,18 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Transform to hierarchical format + # Transform to hierarchical format and write to a separate file + # so the original JSONL is preserved for csv-export hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) - # Write back as JSON (schema v2) import json - with open(output_file, "w") as f: + hierarchical_file = output_file.with_suffix(".hierarchical.json") + with open(hierarchical_file, "w") as f: json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {hierarchical_file}" ) else: logger.warning("No KPI records found in output file") From 0f061ab4f73b65a259fcb3afe1d535ad56fca543 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 23:35:22 -0400 Subject: [PATCH 221/344] add default trtllm_config structure to base config The config override system cannot create multiple levels of nested keys at once. Pre-populate trtllm_config with sensible defaults so FournosJob overrides for kv_cache_config, cuda_graph_config, and moe_config can resolve correctly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 2b0d89124..c88c057f3 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,7 +36,19 @@ engines: nvidia: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc13 port: 8888 args: {} - trtllm_config: {} + trtllm_config: + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.90 + cuda_graph_config: + enable_padding: false + max_batch_size: 256 + enable_attention_dp: false + moe_config: + backend: CUTLASS + stream_interval: 0 + print_iter_log: false deploy: replicas: 1 From b69e3eb6de48c2f148f62496ce4f5b964332ca11 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 11:35:36 +0300 Subject: [PATCH 222/344] Add artifact export and MLflow config to nightly pipeline The nightly pipeline lacked an export-artifacts step, so logs never reached MLflow and GitHub notifications relied solely on submit.py. - Add export-artifacts finally step to pipeline-nightly.yaml - Write __test_labels__.yaml with outcome (launched/noop) in handler - Set nightly-specific MLflow experiment and run_naming overrides via FournosJob configOverrides in pr_args.py Co-authored-by: Cursor --- .../base/workflows/pipeline-nightly.yaml | 20 +++++++++++++++++++ projects/core/nightly/handler.py | 19 ++++++++++++++++++ .../mcp_gateway/orchestration/config.yaml | 5 +++++ projects/mcp_gateway/orchestration/pr_args.py | 9 +++++++++ 4 files changed, 53 insertions(+) diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml index a3e7a38ab..f39b7de29 100644 --- a/fournos/gitops/base/workflows/pipeline-nightly.yaml +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -41,3 +41,23 @@ spec: value: nightly - name: job-step-name value: 00__nightly + + finally: + - name: 01-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 01__export-artifacts diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 02a536efe..1d7112c07 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -13,6 +13,8 @@ import logging import os +import yaml + from projects.core.library import ci as ci_lib from projects.core.library import config, env from projects.core.library.config import requires @@ -116,6 +118,21 @@ def _create_fournos_job(project: str, version: str, source_cfg: dict, _cfg) -> N ) +def _write_test_labels() -> None: + """Write ``__test_labels__.yaml`` so the caliper export ``run_naming`` + templates can resolve the ``{outcome}`` placeholder.""" + result_file = env.ARTIFACT_DIR / "result.txt" + if result_file.exists(): + outcome = "noop" if result_file.read_text().strip() == "NO-OP" else "launched" + else: + outcome = "error" + + labels = {"outcome": outcome} + labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + labels_path.write_text(yaml.dump({"labels": labels})) + logger.info("Wrote test labels: %s", labels) + + def run(): """Entrypoint for the nightly phase.""" logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -137,3 +154,5 @@ def run(): logger.error("confirm-nightly failed: %s", e) ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") raise + finally: + _write_test_labels() diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 7b33d74c9..3357725c3 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,4 +1,9 @@ nightly: + caliper_export: + experiment: nightly-pipeline-experiment + run_naming: + prefix: "mcp-gw-nightly" + single_run: "{prefix}-{timestamp}-{outcome}" sources: ghcr: receiver: "projects.mcp_gateway.nightly_resolvers.ghcr_receiver.GHCRReceiver" diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 1a6f7d879..1523160bd 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -88,6 +88,15 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st source = parts[2] config_overrides["nightly.source"] = source config_overrides["fournos.job.pipeline_name"] = "nightly" + config_overrides["caliper.export.backend.mlflow.config.experiment"] = ( + "nightly-pipeline-experiment" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.prefix"] = ( + "mcp-gw-nightly" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.single_run"] = ( + "{prefix}-{timestamp}-{outcome}" + ) logger.info("Parsed nightly source from pipeline directive: %s", source) return config_overrides, parsed_directives From 1f64f22dcff33c9e27b42970bb6ecbac9e18ba5c Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 12:00:44 +0300 Subject: [PATCH 223/344] Decouple verifier experiment from export experiment The MLflow verifier now reads its experiment from nightly.mlflow_verifier.experiment (mcp-gw-cpt-ghcr) instead of caliper.export.backend.mlflow.config.experiment, so the export override to nightly-pipeline-experiment no longer affects version checks. Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 4 ++-- projects/mcp_gateway/orchestration/config.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 7433c9e0b..bfcdb56e0 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -39,7 +39,7 @@ class CaliperMLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. Config requirements (via @requires on __init__): - caliper.export.backend.mlflow.config.experiment + nightly.mlflow_verifier.experiment (experiment to query for previous runs) caliper.export.backend.mlflow.config.workspace caliper.export.backend.mlflow.secrets.vault (dict with 'name' and 'mlflow_secret') nightly.mlflow_verifier.version_pattern (regex with one capture group) @@ -47,7 +47,7 @@ class CaliperMLflowVerifier(NightlyVerifier): """ @requires( - experiment="caliper.export.backend.mlflow.config.experiment", + experiment="nightly.mlflow_verifier.experiment", workspace="caliper.export.backend.mlflow.config.workspace", vault_cfg="caliper.export.backend.mlflow.secrets.vault", version_pattern="nightly.mlflow_verifier.version_pattern", diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 3357725c3..2c8b22a34 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -12,6 +12,7 @@ nightly: mlflow_experiment: "mcp-gw-cpt-ghcr" verifier: "projects.caliper.orchestration.mlflow_verifier.CaliperMLflowVerifier" mlflow_verifier: + experiment: "mcp-gw-cpt-ghcr" version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" strip_prefix: "sha-" cluster: "forge-smoke-testing" From f088034abada14ad9e6c40f6eefef964c1c65f2e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 12:06:05 +0300 Subject: [PATCH 224/344] Handle verifier bootstrap: treat failures as no previous version When the MLflow verifier cannot extract a version from existing run names (e.g. different naming convention during bootstrap), catch the exception and proceed as if no previous version exists, launching a new test run instead of failing the pipeline. Co-authored-by: Cursor --- projects/core/nightly/handler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 1d7112c07..e8e614659 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -57,7 +57,15 @@ def _confirm_nightly(project: str, image_version: str, source_cfg: dict) -> None verifier = verifier_cls() logger.info("Using verifier: %s", type(verifier).__name__) - last_version = verifier.get_last_tested_version() + try: + last_version = verifier.get_last_tested_version() + except Exception as e: + logger.warning( + "Verifier failed (bootstrap scenario?): %s. " + "Treating as no previous version — will launch a new run.", + e, + ) + last_version = "" logger.info("Last tested version: '%s'", last_version or "") if image_version == last_version: From 42ad31acec2639e8081bdf93fde3ba2431540a27 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:19:06 +0200 Subject: [PATCH 225/344] [kserve] toolbox: deploy_llmisvc: correctly resolve the kserve address --- projects/kserve/toolbox/deploy_llmisvc/main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index cf064757c..4e3fc062d 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -35,7 +35,7 @@ def run( *, namespace: str, inference_service_manifest_path: str, - gateway_status_address_name: str = "gateway-external", + gateway_status_address_name: str | None = "gateway-external", dry_run: bool = False, ) -> str: """ @@ -387,16 +387,25 @@ def resolve_endpoint_task(args, ctx): ctx.endpoint_url = endpoint_url write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") return f"Endpoint resolved: {endpoint_url}" - return False # Retry + return False, "No endpoint URL available" def try_resolve_endpoint_url( - *, namespace: str, inference_service_name: str, gateway_status_address_name: str + *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None ) -> str | None: payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) for address in payload.get("status", {}).get("addresses", []): - if address.get("name") == gateway_status_address_name and address.get("url"): + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 + if gateway_status_address_name is None: + if address.get("url"): + url = address["url"] + # Append port 8000 when not using gateway if no port is already specified + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + return url + # Otherwise, match by name + elif address.get("name") == gateway_status_address_name and address.get("url"): return address["url"] return None From 74f94fd5f9fd45116d420aceeeffc864d7c167dc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:39:53 +0200 Subject: [PATCH 226/344] [kserve] toolbox: deploy_llmisvc: improve wait_old_pods_gone --- projects/kserve/toolbox/deploy_llmisvc/main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 4e3fc062d..266323704 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -119,10 +119,20 @@ def delete_existing_service(args, ctx): def wait_old_pods_gone(args, ctx): """Wait for old llm-d pods to disappear""" - pods = oc_get_json( - "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + result = oc( + "get", + "pods", + "-n", + args.namespace, + "-l", + ctx.selector, + "--ignore-not-found=true", + "--no-headers", + check=False, ) - if not pods or not pods.get("items"): + + # Check if output is empty (no pods found) + if not result.stdout.strip(): return f"Old pods gone for {ctx.inference_service_name}" return False # Retry From ce0d075eb3981fd63e9c95962e89fae1ae1335d2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:00:30 +0200 Subject: [PATCH 227/344] [kserve] toolbox: prepare_hf_model_cache: wait for the pod to start running in a dedicated task --- .../toolbox/prepare_hf_model_cache/main.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index a25544e87..d6d502d8e 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -286,6 +286,44 @@ def create_download_job(args, ctx): return f"Download job {cache_spec['download_job_name']} created" +@retry(attempts=24, delay=5, backoff=1.0) # 24 attempts * 5 seconds = 120 seconds (2 minutes) +@task +def wait_for_pods_running(args, ctx): + """Wait for the download job pods to start running within 2 minutes""" + + if getattr(ctx, "cache_ready", False): + return "Skipping pod wait - cache already ready" + + cache_spec = ctx.cache_spec + job_name = cache_spec["download_job_name"] + namespace = cache_spec["namespace"] + + # Get pod status using plain text output + result = oc( + "get", + "pod", + "--no-headers", + "-l", + f"job-name={job_name}", + "-n", + namespace, + "-o", + "custom-columns=NAME:.metadata.name,STATUS:.status.phase", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False # No pods yet, retry + + # Keep waiting only if "Pending" appears in the output + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + # All pods are past Pending state + return "All pods past Pending state" + + @retry(attempts=30, delay=30, backoff=1.0) @task def wait_for_download(args, ctx): From ca2bceca5ac69cfdb31e2f63054d3c6d588f13c6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:12 +0200 Subject: [PATCH 228/344] [kserve] toolbox: capture_llmisvc_state: capture more information --- .../toolbox/capture_llmisvc_state/main.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index 12efa1935..060963447 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -26,6 +26,7 @@ def setup_directories(args, context): """Create the artifacts directory""" shell.mkdir("artifacts") + shell.mkdir("artifacts/logs") return "Artifacts directory created" @@ -121,7 +122,7 @@ def capture_namespace_pods(args, context): """Capture all pods in the namespace with wide output""" shell.run( f"oc get pods -owide -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status.txt", check=False, ) return "Namespace pods status captured" @@ -132,7 +133,7 @@ def capture_namespace_services(args, context): """Capture all services in the namespace""" shell.run( f"oc get svc -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.services.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.services.status.txt", check=False, ) return "Namespace services captured" @@ -173,20 +174,19 @@ def capture_pod_logs(args, context): if not pod_names or not result.stdout.strip(): return "No pods found to capture logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod logs captured for {len(pod_names)} pods" + return f"Pod logs captured for {captured_count} pods in dedicated files" @task @@ -195,26 +195,26 @@ def capture_pod_previous_logs(args, context): result = shell.run( f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].metadata.name}}"', check=False, + log_stdout=False, ) pod_names = result.stdout.strip().split() if not pod_names or not result.stdout.strip(): return "No pods found to capture previous logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.previous.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Previous logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.previous.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod previous logs captured for {len(pod_names)} pods" + return f"Pod previous logs captured for {captured_count} pods in dedicated files" @task From 049bdcc16f358d141d04daa8971cc34a8cf37f15 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:26 +0200 Subject: [PATCH 229/344] [kserve] toolbox: deploy_llmisvc: capture more information --- .../kserve/toolbox/deploy_llmisvc/main.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 266323704..d2768aa9e 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -267,6 +267,197 @@ def capture_replicaset_description(args, ctx): return f"Failed to capture ReplicaSet description: {e}" +@always +@task +def capture_final_llmisvc_yaml(args, ctx): + """Capture the final YAML state of the LLMInferenceService""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" + + # Capture final YAML state + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + llmisvc_yaml_path = artifacts_dir / "llmisvc_final.yaml" + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured final LLMInferenceService YAML to {llmisvc_yaml_path}" + + except Exception as e: + return f"Failed to capture final LLMInferenceService YAML: {e}" + + +@always +@task +def capture_pod_status(args, ctx): + """Capture pod status for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture pod status with wide output + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + log_stdout=False, + check=False, + ) + + pod_status_path = artifacts_dir / "pod_status.txt" + with open(pod_status_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod status to {pod_status_path}" + + except Exception as e: + return f"Failed to capture pod status: {e}" + + +@always +@task +def capture_pod_descriptions(args, ctx): + """Capture pod descriptions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Get pod names + pod_result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "jsonpath={.items[*].metadata.name}", + log_stdout=False, + check=False, + ) + + pod_names = pod_result.stdout.strip().split() + if not pod_names or not pod_result.stdout.strip(): + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("No pods found for the service") + return f"No pods found, wrote empty file to {pod_desc_path}" + + # Describe each pod + pod_descriptions = [] + for pod_name in pod_names: + describe_result = oc( + "describe", + "pod", + pod_name, + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + pod_descriptions.append( + f"=== Description for pod: {pod_name} ===\n{describe_result.stdout}" + ) + + # Save all pod descriptions + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("\n\n".join(pod_descriptions)) + + return f"Captured descriptions for {len(pod_names)} pods to {pod_desc_path}" + + except Exception as e: + return f"Failed to capture pod descriptions: {e}" + + +@always +@task +def capture_pod_yaml(args, ctx): + """Capture pod YAML definitions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture all pod YAMLs + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + pod_yaml_path = artifacts_dir / "pod_definitions.yaml" + with open(pod_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod YAML definitions to {pod_yaml_path}" + + except Exception as e: + return f"Failed to capture pod YAML: {e}" + + @task def query_service_status(args, ctx): """Query the status of the LLMInferenceService""" From 35efb5176209ee3db411c55e1e4b14184de7dc86 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:37:10 +0200 Subject: [PATCH 230/344] [kserve] toolbox: prepare_hf_model_cache: wait 1h for the download completion --- projects/kserve/toolbox/prepare_hf_model_cache/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index d6d502d8e..1e01c1737 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -324,7 +324,7 @@ def wait_for_pods_running(args, ctx): return "All pods past Pending state" -@retry(attempts=30, delay=30, backoff=1.0) +@retry(attempts=120, delay=30, backoff=1.0) @task def wait_for_download(args, ctx): """Wait for the download job to complete""" From d8f7538b888f0d1f7f296c119009ff0833ab3985 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 14:48:32 +0200 Subject: [PATCH 231/344] [kserve] toolbox: deploy_llmisvc: add flag to wait_pods_scheduled --- .../kserve/toolbox/deploy_llmisvc/main.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index d2768aa9e..8d75691e2 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -37,6 +37,7 @@ def run( inference_service_manifest_path: str, gateway_status_address_name: str | None = "gateway-external", dry_run: bool = False, + wait_pods_scheduled: bool = False, ) -> str: """ Deploy an LLMInferenceService and wait for its endpoint. @@ -46,6 +47,7 @@ def run( inference_service_manifest_path: Path to the InferenceService YAML manifest file gateway_status_address_name: Gateway status address name for endpoint resolution dry_run: If True, only prepare the manifest without deploying + wait_pods_scheduled: If True, wait for all pods to be scheduled before checking service readiness """ ctx = execute_tasks(locals()) @@ -521,6 +523,40 @@ def query_service_message(args, ctx): return "No Ready condition found in status" +@retry(attempts=999999, delay=30, backoff=1.0) +@task +def wait_pods_scheduled(args, ctx): + """Wait for all pods to be scheduled (optional task)""" + + # Check if this task is enabled + if not args.wait_pods_scheduled: + return "Pod scheduling wait disabled by parameter" + + service_name = ctx.inference_service_name + + # Get pod status using plain text output + result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "--no-headers", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False, "No pods found for the service yet" + + # Keep waiting if any pod is Pending + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + return f"All pods for {service_name} are scheduled successfully" + + @retry(attempts=90, delay=10, backoff=1.0) @task def wait_service_ready(args, ctx): From b95774139a18ceab77f18f4f79513ce915239a81 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:03:26 +0200 Subject: [PATCH 232/344] [kserve] toolbox: deploy_llmisvc: capture_final_llmisvc_yaml: better error handling --- .../kserve/toolbox/deploy_llmisvc/main.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 8d75691e2..3f5433410 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path import yaml @@ -24,6 +25,8 @@ from .on_failure_helpers import on_wait_pods_appear_failure +logger = logging.getLogger(__name__) + def load_yaml(path: Path): with path.open(encoding="utf-8") as handle: @@ -301,6 +304,30 @@ def capture_final_llmisvc_yaml(args, ctx): ) llmisvc_yaml_path = artifacts_dir / "llmisvc_final.yaml" + + if result.returncode != 0: + # Handle the case where the LLMInferenceService is not found + if "not found" in result.stderr.lower(): + logger.warning( + f"LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write( + f"# LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'\n" + ) + f.write(f"# Error: {result.stderr.strip()}\n") + return f"LLMInferenceService not found, logged error to {llmisvc_yaml_path}" + else: + # Handle other error cases + logger.error( + f"Failed to get LLMInferenceService '{service_name}': {result.stderr.strip()}" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(f"# Error getting LLMInferenceService '{service_name}'\n") + f.write(f"# Error: {result.stderr.strip()}\n") + return f"Error getting LLMInferenceService, logged error to {llmisvc_yaml_path}" + + # Success case - write the YAML content with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: f.write(result.stdout) From e9c25ddf320cd65a0578c08d6b137a38a440f995 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 12:04:22 +0200 Subject: [PATCH 233/344] [kserve] toolbox: deploy_llmisvc: abort on Pod restart --- .../kserve/toolbox/deploy_llmisvc/main.py | 400 ++++++++++-------- 1 file changed, 213 insertions(+), 187 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 3f5433410..5dd994582 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -169,6 +169,219 @@ def wait_pods_appear(args, ctx): return False # Retry +@task +def query_service_status(args, ctx): + """Query the status of the LLMInferenceService""" + + service_name = ctx.inference_service_name + + # Query only the Ready condition status + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')].status}", + log_stdout=False, + ) + + ready_status = result.stdout.strip() + ctx.is_ready = ready_status == "True" + + if ctx.is_ready: + return f"LLMInferenceService {service_name} status: Ready" + else: + return f"LLMInferenceService {service_name} status: Not Ready" + + +@task +def query_service_message(args, ctx): + """Query detailed message from LLMInferenceService""" + + service_name = ctx.inference_service_name + + # Query the Ready condition details + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')]}", + log_stdout=False, + ) + + if result.stdout.strip(): + try: + import json + + condition = json.loads(result.stdout) + reason = condition.get("reason", "Unknown") + message = condition.get("message", "No message") + + if not ctx.is_ready: + return f"Not ready - Reason: {reason}, Message: {message}" + else: + return "Ready - Service is operational" + except (json.JSONDecodeError, KeyError) as e: + return f"Failed to parse Ready condition: {e}" + else: + return "No Ready condition found in status" + + +@retry(attempts=999999, delay=30, backoff=1.0) +@task +def wait_pods_scheduled(args, ctx): + """Wait for all pods to be scheduled (optional task)""" + + # Check if this task is enabled + if not args.wait_pods_scheduled: + return "Pod scheduling wait disabled by parameter" + + service_name = ctx.inference_service_name + + # Get pod status using plain text output + result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "--no-headers", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False, "No pods found for the service yet" + + # Keep waiting if any pod is Pending + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + return f"All pods for {service_name} are scheduled successfully" + + +@retry(attempts=90, delay=10, backoff=1.0) +@task +def wait_service_ready(args, ctx): + """Wait for LLMInferenceService to be ready""" + + service_name = ctx.inference_service_name + + # Query the current status and show diagnostic info + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')]}", + ) + + # Also show pod status for debugging + oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + ) + + # Check for pod restarts and abort if any pods have restarted + restart_result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "-o", + "jsonpath={range .items[*]}{.metadata.name}:{.status.containerStatuses[*].restartCount}{'\\n'}{end}", + ) + + if restart_result.stdout.strip(): + for line in restart_result.stdout.strip().split("\n"): + if ":" in line: + pod_name, restart_counts = line.split(":", 1) + # Check if any container has restarted + counts = restart_counts.split() + for count_str in counts: + try: + if int(count_str) > 0: + raise RuntimeError( + f"Pod {pod_name} has restarted (restart count: {count_str}). Aborting wait due to pod restart." + ) + except ValueError: + # Skip non-numeric restart counts + pass + + if result.stdout.strip(): + try: + import json + + condition = json.loads(result.stdout) + status = condition.get("status", "Unknown") + reason = condition.get("reason", "Unknown") + message = condition.get("message", "No message") + + if status == "True": + return f"LLMInferenceService {service_name} is ready" + else: + return ( + False, + f"Service not ready - Status: {status}, Reason: {reason}, Message: {message}", + ) + + except (json.JSONDecodeError, KeyError) as e: + return (False, f"Failed to parse Ready condition: {e}") + else: + return (False, f"No Ready condition found in status for {service_name}") + + +def try_resolve_endpoint_url( + *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None +) -> str | None: + payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) + + for address in payload.get("status", {}).get("addresses", []): + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 + if gateway_status_address_name is None: + if address.get("url"): + url = address["url"] + # Append port 8000 when not using gateway if no port is already specified + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + return url + # Otherwise, match by name + elif address.get("name") == gateway_status_address_name and address.get("url"): + return address["url"] + return None + + +@retry(attempts=90, delay=10, backoff=1.0) +@task +def resolve_endpoint_task(args, ctx): + """Resolve the gateway endpoint URL""" + + endpoint_url = try_resolve_endpoint_url( + namespace=args.namespace, + inference_service_name=ctx.inference_service_name, + gateway_status_address_name=args.gateway_status_address_name, + ) + if endpoint_url: + ctx.endpoint_url = endpoint_url + write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") + return f"Endpoint resolved: {endpoint_url}" + return False, "No endpoint URL available" + + @always @task def capture_llmisv_description(args, ctx): @@ -487,192 +700,5 @@ def capture_pod_yaml(args, ctx): return f"Failed to capture pod YAML: {e}" -@task -def query_service_status(args, ctx): - """Query the status of the LLMInferenceService""" - - service_name = ctx.inference_service_name - - # Query only the Ready condition status - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')].status}", - log_stdout=False, - ) - - ready_status = result.stdout.strip() - ctx.is_ready = ready_status == "True" - - if ctx.is_ready: - return f"LLMInferenceService {service_name} status: Ready" - else: - return f"LLMInferenceService {service_name} status: Not Ready" - - -@task -def query_service_message(args, ctx): - """Query detailed message from LLMInferenceService""" - - service_name = ctx.inference_service_name - - # Query the Ready condition details - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')]}", - log_stdout=False, - ) - - if result.stdout.strip(): - try: - import json - - condition = json.loads(result.stdout) - reason = condition.get("reason", "Unknown") - message = condition.get("message", "No message") - - if not ctx.is_ready: - return f"Not ready - Reason: {reason}, Message: {message}" - else: - return "Ready - Service is operational" - except (json.JSONDecodeError, KeyError) as e: - return f"Failed to parse Ready condition: {e}" - else: - return "No Ready condition found in status" - - -@retry(attempts=999999, delay=30, backoff=1.0) -@task -def wait_pods_scheduled(args, ctx): - """Wait for all pods to be scheduled (optional task)""" - - # Check if this task is enabled - if not args.wait_pods_scheduled: - return "Pod scheduling wait disabled by parameter" - - service_name = ctx.inference_service_name - - # Get pod status using plain text output - result = oc( - "get", - "pods", - "-l", - ctx.selector, - "-n", - args.namespace, - "--no-headers", - check=False, - log_stdout=False, - ) - - if not result.stdout.strip(): - return False, "No pods found for the service yet" - - # Keep waiting if any pod is Pending - if "Pending" in result.stdout: - return False, "Waiting for pods to exit Pending state" - - return f"All pods for {service_name} are scheduled successfully" - - -@retry(attempts=90, delay=10, backoff=1.0) -@task -def wait_service_ready(args, ctx): - """Wait for LLMInferenceService to be ready""" - - service_name = ctx.inference_service_name - - # Query the current status and show diagnostic info - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')]}", - log_stdout=True, - ) - - # Also show pod status for debugging - oc( - "get", - "pods", - "-l", - ctx.selector, - "-n", - args.namespace, - log_stdout=True, # Show pod status in logs - ) - - if result.stdout.strip(): - try: - import json - - condition = json.loads(result.stdout) - status = condition.get("status", "Unknown") - reason = condition.get("reason", "Unknown") - message = condition.get("message", "No message") - - if status == "True": - return f"LLMInferenceService {service_name} is ready" - else: - return ( - False, - f"Service not ready - Status: {status}, Reason: {reason}, Message: {message}", - ) - - except (json.JSONDecodeError, KeyError) as e: - return (False, f"Failed to parse Ready condition: {e}") - else: - return (False, f"No Ready condition found in status for {service_name}") - - -@retry(attempts=90, delay=10, backoff=1.0) -@task -def resolve_endpoint_task(args, ctx): - """Resolve the gateway endpoint URL""" - - endpoint_url = try_resolve_endpoint_url( - namespace=args.namespace, - inference_service_name=ctx.inference_service_name, - gateway_status_address_name=args.gateway_status_address_name, - ) - if endpoint_url: - ctx.endpoint_url = endpoint_url - write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") - return f"Endpoint resolved: {endpoint_url}" - return False, "No endpoint URL available" - - -def try_resolve_endpoint_url( - *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None -) -> str | None: - payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) - - for address in payload.get("status", {}).get("addresses", []): - # When gateway_status_address_name is None, return the first address with a URL and append port 8000 - if gateway_status_address_name is None: - if address.get("url"): - url = address["url"] - # Append port 8000 when not using gateway if no port is already specified - if ":" not in url.split("/")[-1]: # Check if no port in the hostname part - url = f"{url}:8000" - return url - # Otherwise, match by name - elif address.get("name") == gateway_status_address_name and address.get("url"): - return address["url"] - return None - - if __name__ == "__main__": run.main() From 3d8055036af27cad11c0eb775d4373d972a0fb68 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:48:28 +0200 Subject: [PATCH 234/344] [kserve] toolbox: deploy_llmisvc: improve the URL resolution --- .../kserve/toolbox/deploy_llmisvc/main.py | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 5dd994582..553582617 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -348,24 +348,112 @@ def wait_service_ready(args, ctx): def try_resolve_endpoint_url( *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None ) -> str | None: + logger.info( + f"=== Resolving endpoint URL for {inference_service_name} in namespace {namespace} ===" + ) + logger.info(f"Target gateway_status_address_name: {gateway_status_address_name}") + payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) - for address in payload.get("status", {}).get("addresses", []): + # Log the entire status section for debugging + status = payload.get("status", {}) + logger.info(f"Status section keys found: {list(status.keys())}") + + # Check status.address first + status_address = status.get("address") + logger.info(f"status.address content: {status_address}") + + if status_address: + logger.info("✓ status.address exists") + if isinstance(status_address, dict): + logger.info("✓ status.address is a dict") + if status_address.get("url"): + url = status_address["url"] + logger.info(f"✓ Found URL in status.address: '{url}'") + + # When gateway_status_address_name is None, append port 8000 if needed + if gateway_status_address_name is None: + logger.info("Mode: No gateway (will append port 8000 if needed)") + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + logger.info(f"✓ Appended port 8000: '{url}'") + else: + logger.info("✓ URL already has port, using as-is") + else: + logger.info( + f"Mode: Gateway '{gateway_status_address_name}' (no port modification)" + ) + + logger.info(f"🎯 RESOLVED from status.address: '{url}'") + return url + else: + logger.info("✗ status.address has no 'url' field") + logger.info(f" Available fields: {list(status_address.keys())}") + else: + logger.info(f"✗ status.address is not a dict, type: {type(status_address)}") + else: + logger.info("✗ No status.address found") + + # Fallback to existing status.addresses logic + logger.info("--- Falling back to status.addresses lookup ---") + status_addresses = status.get("addresses", []) + logger.info(f"status.addresses content: {status_addresses}") + + if not status_addresses: + logger.info("✗ No status.addresses found - RESOLUTION FAILED") + return None + + logger.info(f"✓ Found {len(status_addresses)} address(es) in status.addresses") + + for i, address in enumerate(status_addresses): + logger.info(f"Checking address[{i}]: {address}") + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 if gateway_status_address_name is None: + logger.info(" Mode: No gateway filter (looking for any URL)") if address.get("url"): url = address["url"] + logger.info(f" ✓ Found URL in address[{i}]: '{url}'") # Append port 8000 when not using gateway if no port is already specified if ":" not in url.split("/")[-1]: # Check if no port in the hostname part url = f"{url}:8000" + logger.info(f" ✓ Appended port 8000: '{url}'") + else: + logger.info(" ✓ URL already has port, using as-is") + + logger.info(f"🎯 RESOLVED from status.addresses[{i}]: '{url}'") return url + else: + logger.info(f" ✗ Address[{i}] has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") # Otherwise, match by name - elif address.get("name") == gateway_status_address_name and address.get("url"): - return address["url"] + else: + address_name = address.get("name") + logger.info( + f" Mode: Gateway filter (looking for name='{gateway_status_address_name}')" + ) + logger.info(f" Address[{i}] name: '{address_name}'") + + if address_name == gateway_status_address_name: + logger.info(f" ✓ Name matches '{gateway_status_address_name}'") + if address.get("url"): + url = address["url"] + logger.info(f" ✓ Found URL: '{url}'") + logger.info(f"🎯 RESOLVED from status.addresses[{i}] by name: '{url}'") + return url + else: + logger.info(" ✗ Matching address has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") + else: + logger.info( + f" ✗ Name mismatch: '{address_name}' != '{gateway_status_address_name}'" + ) + + logger.info("❌ RESOLUTION FAILED - No suitable URL found in any address") return None -@retry(attempts=90, delay=10, backoff=1.0) +@retry(attempts=30, delay=10, backoff=1.0) @task def resolve_endpoint_task(args, ctx): """Resolve the gateway endpoint URL""" From 02247d69a6486928ddcbb4ec9be22c7ad9e8a72a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:20:56 +0200 Subject: [PATCH 235/344] [kserve] toolbox: deploy_llmisvc: also wait on SchedulingGated --- projects/kserve/toolbox/deploy_llmisvc/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 553582617..59ba34f83 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -259,10 +259,13 @@ def wait_pods_scheduled(args, ctx): if not result.stdout.strip(): return False, "No pods found for the service yet" - # Keep waiting if any pod is Pending + # Keep waiting if any pod is Pending or SchedulingGated if "Pending" in result.stdout: return False, "Waiting for pods to exit Pending state" + if "SchedulingGated" in result.stdout: + return False, "Waiting for pods to exit SchedulingGated state" + return f"All pods for {service_name} are scheduled successfully" @@ -304,6 +307,7 @@ def wait_service_ready(args, ctx): args.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}:{.status.containerStatuses[*].restartCount}{'\\n'}{end}", + log_stdout=False, ) if restart_result.stdout.strip(): From ef3abf5ec53094582fe3a00a06421860e314e517 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:58:14 +0200 Subject: [PATCH 236/344] [kserve] toolbox: deploy_llmisvc: expose the try_resolve_endpoint_url method --- projects/kserve/toolbox/deploy_llmisvc/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/__init__.py b/projects/kserve/toolbox/deploy_llmisvc/__init__.py index e69de29bb..68f67b1fb 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/__init__.py +++ b/projects/kserve/toolbox/deploy_llmisvc/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from projects.kserve.toolbox.deploy_llmisvc.main import try_resolve_endpoint_url + +__all__ = ["try_resolve_endpoint_url"] From 8ef80c1cca50415add3ae364b681419202351d56 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 21:04:34 +0200 Subject: [PATCH 237/344] [kserve] toolbox: deploy_llmisvc: save the deploy/rs/pod description --- .../kserve/toolbox/deploy_llmisvc/main.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 59ba34f83..c3efade89 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -642,6 +642,42 @@ def capture_final_llmisvc_yaml(args, ctx): return f"Failed to capture final LLMInferenceService YAML: {e}" +@always +@task +def capture_workload_overview(args, ctx): + """Capture deployment, replicaset, and pod overview for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + workload_overview_path = artifacts_dir / "workload_overview.txt" + + # Capture deployment, replicaset, and pod overview + oc( + "get", + "deploy,rs,pod", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + check=False, + stdout_dest=workload_overview_path, + ) + + return f"Captured workload overview to {workload_overview_path}" + + @always @task def capture_pod_status(args, ctx): From 96b00a0a20a109893cc133a57101d731c40568ff Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 17:28:11 +0200 Subject: [PATCH 238/344] [guidellm] toolbox: run_smoke_request: save the pod info in a file --- projects/guidellm/toolbox/run_smoke_request/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index 5905beff2..e131e995c 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -247,6 +247,7 @@ def capture_smoke_pod_debug_info(args, ctx): "-o", "yaml", check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.yaml", ) # Capture pod description From 68b26a81a640bb6159d41fe4f007b935bdcd608c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:08:43 +0200 Subject: [PATCH 239/344] [guidellm] toolbox: run_smoke_request: don't wait for the pod deletion --- projects/guidellm/toolbox/run_smoke_request/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index e131e995c..eba195b84 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -284,6 +284,7 @@ def cleanup_smoke_pod(args, ctx): "-n", args.namespace, "--ignore-not-found=true", + "--wait=false", check=False, ) return f"Cleaned up smoke pod {ctx.pod_name}" From bcde3d06b9af5efc72a6980208c6f5ede37155e4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:18:36 +0200 Subject: [PATCH 240/344] [guidellm] toolbox: run_guidellm_benchmark: give more time --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index fa82ff1ec..12b6f3446 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -147,7 +147,7 @@ def create_guidellm_resources_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} created" -@retry(attempts=180, delay=10, backoff=1.0) +@retry(attempts=1080, delay=10, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" From b4e8c2cf5666b55ad9902c508425ef4dec4c0abe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:04:58 +0200 Subject: [PATCH 241/344] [guidellm] toolbox: run_guidellm_benchmark: set the owner of the PVC --- .../toolbox/run_guidellm_benchmark/main.py | 38 ++++++++++++++----- .../toolbox/run_guidellm_benchmark/utils.py | 13 ++++++- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 12b6f3446..c7fe0b2ae 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -120,19 +120,12 @@ def _best_effort_delete(description: str, *oc_args: str) -> None: @task def create_guidellm_resources_task(args, ctx): - """Create the GuideLLM benchmark PVC and job""" + """Create the GuideLLM benchmark job and PVC with job as owner""" # Ensure src directory exists (args.artifact_dir / "src").mkdir(parents=True, exist_ok=True) - oc_apply( - args.artifact_dir / "src" / "guidellm-pvc.yaml", - render_guidellm_pvc_from_parts( - namespace=ctx.target_namespace, - name=ctx.benchmark_name, - pvc_size=args.pvc_size, - ), - ) + # Create the job first oc_apply( args.artifact_dir / "src" / "guidellm-job.yaml", render_guidellm_job_from_parts( @@ -144,7 +137,32 @@ def create_guidellm_resources_task(args, ctx): hf_token_secret=args.hf_token_secret, ), ) - return f"GuideLLM benchmark {ctx.benchmark_name} created" + + # Get the job metadata for owner reference + job_data = oc_get_json("job", name=ctx.benchmark_name, namespace=ctx.target_namespace) + + # Create owner reference from job metadata + owner_reference = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": job_data["metadata"]["name"], + "uid": job_data["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } + + # Create the PVC with job as owner + oc_apply( + args.artifact_dir / "src" / "guidellm-pvc.yaml", + render_guidellm_pvc_from_parts( + namespace=ctx.target_namespace, + name=ctx.benchmark_name, + pvc_size=args.pvc_size, + owner_reference=owner_reference, + ), + ) + + return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" @retry(attempts=1080, delay=10, backoff=1.0) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 7cab6a2c2..6366acce8 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -146,13 +146,16 @@ def _build_multi_run_script(*, endpoint_url: str, runs: list[GuideLLMRun]) -> st return "\n".join(lines) -def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) -> dict[str, Any]: +def render_guidellm_pvc_from_parts( + *, namespace: str, name: str, pvc_size: str, owner_reference: dict[str, Any] | None = None +) -> dict[str, Any]: """Render a GuideLL-M PVC manifest from individual components. Args: namespace: Target namespace name: Name of the benchmark job and PVC pvc_size: Size of the PVC + owner_reference: Optional owner reference to set (e.g., for job ownership) Returns: PVC manifest as dict @@ -165,7 +168,13 @@ def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) "pvc_size": pvc_size, }, ) - return yaml.safe_load(rendered_yaml) + manifest = yaml.safe_load(rendered_yaml) + + # Add owner reference if provided + if owner_reference: + manifest["metadata"]["ownerReferences"] = [owner_reference] + + return manifest def render_guidellm_job_from_parts( From c5547eb47f07ad895b6e467d84f91e98ef10e8ae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 09:44:13 +0200 Subject: [PATCH 242/344] [caliper] engine: parameter_matrix: don't truncate long parameters that merges unrelated legend names ... --- projects/caliper/engine/parameter_matrix.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/caliper/engine/parameter_matrix.py b/projects/caliper/engine/parameter_matrix.py index 69c95d3ee..03e720e10 100644 --- a/projects/caliper/engine/parameter_matrix.py +++ b/projects/caliper/engine/parameter_matrix.py @@ -237,8 +237,4 @@ def create_legend_name( legend_name = ", ".join(param_pairs) - # Truncate if too long - if len(legend_name) > max_length: - legend_name = legend_name[: max_length - 3] + "..." - return legend_name From b7f46d8e8492e851e31e899ce954bfed6e1dc636 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 10:17:24 +0200 Subject: [PATCH 243/344] [llm_d] tests: test_deployment_profiles: add a 60s timeout --- .../llm_d/tests/test_deployment_profiles.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 631ab48a0..0663760a3 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -67,13 +67,25 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): ci_script = PROJECT_ROOT / "projects" / "llm_d" / "orchestration" / "ci.py" cmd = [str(ci_script), "--preset", preset, "test"] - result = subprocess.run( - cmd, - capture_output=True, - text=True, - cwd=PROJECT_ROOT, - env=env_vars, - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + env=env_vars, + timeout=60, # 60-second timeout + ) + except subprocess.TimeoutExpired as e: + # Show stdout when timeout occurs + stdout_output = e.stdout.decode() if e.stdout else "No stdout available" + stderr_output = e.stderr.decode() if e.stderr else "No stderr available" + pytest.fail( + f"CI command timed out after 60 seconds for preset {preset}:\n" + f"Command: {' '.join(cmd)}\n\n" + f"STDOUT (before timeout):\n{stdout_output}\n\n" + f"STDERR (before timeout):\n{stderr_output}\n\n" + ) # Check that the command succeeded if result.returncode != 0: From b4577af93e4e4b43da8dab7fcfefb4d7fe564a26 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 09:46:07 +0200 Subject: [PATCH 244/344] [caliper] postprocess: helpers: visualization_utils: update the size/scale of the plots --- .../helpers/visualization_utils.py | 163 +++++------------- 1 file changed, 45 insertions(+), 118 deletions(-) diff --git a/projects/caliper/postprocess/helpers/visualization_utils.py b/projects/caliper/postprocess/helpers/visualization_utils.py index 68d45c361..a6e6f4c2b 100644 --- a/projects/caliper/postprocess/helpers/visualization_utils.py +++ b/projects/caliper/postprocess/helpers/visualization_utils.py @@ -3,8 +3,11 @@ from __future__ import annotations import base64 +import logging from pathlib import Path +logger = logging.getLogger(__name__) + def save_figure( fig, @@ -12,8 +15,8 @@ def save_figure( filename: str, as_image: bool = True, report_number: int | None = None, - width: int = 800, - height: int = 500, + width: int = 1200, + height: int = 650, ) -> str | None: """ Save a plotly figure as either an image or HTML file with optional report numbering. @@ -38,135 +41,34 @@ def save_figure( final_filename = filename if as_image: - print(f"💾 Saving {final_filename} as PNG image...") + logger.info(f"Saving {final_filename} as PNG image...") output_file = output_dir / f"{final_filename}.png" - fig.write_image(output_file, width=width, height=height, scale=2) + fig.write_image(output_file, width=width, height=height) else: - print(f"💾 Saving {final_filename} as full-page interactive HTML...") + logger.info(f"Saving {final_filename} as full-page interactive HTML...") output_file = output_dir / f"{final_filename}.html" - # Configure the figure for full-page display + # Configure figure for full-screen responsive behavior fig.update_layout( autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - ) - - # Save with full-page configuration - fig.write_html( - output_file, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, + width=None, # Remove any fixed width + height=None, # Remove any fixed height ) + fig.write_html(output_file) - # Make it full page - _make_html_full_page(output_file) - - print(f"✅ {final_filename} saved successfully") + logger.info(f"{final_filename} saved successfully") return str(output_file) except Exception as e: final_filename = ( f"report_{report_number:02d}_{filename}" if report_number is not None else filename ) - print(f"❌ Failed to save figure {final_filename}: {e}") + logger.error(f"Failed to save figure {final_filename}: {e}") return None -def _make_html_full_page(html_file_path: str) -> None: - """ - Modify an HTML file to make the plot full page by adding custom CSS. - - Args: - html_file_path: Path to the HTML file to modify - """ - try: - with open(html_file_path, encoding="utf-8") as f: - content = f.read() - - # Insert full-page CSS styles - full_page_css = """ - -""" - - # Insert the CSS before the closing tag - content = content.replace("", f"{full_page_css}") - - with open(html_file_path, "w", encoding="utf-8") as f: - f.write(content) - - except Exception as e: - print(f"⚠️ Warning: Failed to make HTML full page: {e}") - - -def write_full_page_html(fig, output_file_path: str, title: str = "Plot") -> bool: - """ - Save a Plotly figure as a full-page HTML file. - - Args: - fig: Plotly figure object - output_file_path: Path where to save the HTML file - title: Title for the HTML page - - Returns: - True if successful, False otherwise - """ - try: - # Configure the figure for full-page display - fig.update_layout( - autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - title_text=title if title != "Plot" else None, - ) - - # Save with full-page configuration - fig.write_html( - output_file_path, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, - ) - - # Make it full page - _make_html_full_page(output_file_path) - return True - - except Exception as e: - print(f"❌ Failed to save full-page HTML: {e}") - return False - - def figure_to_base64( - fig, width: int = 800, height: int = 500, plot_name: str = "plot" + fig, width: int = 1200, height: int = 650, plot_name: str = "plot" ) -> str | None: """ Convert a plotly figure to base64-encoded PNG for embedding in HTML. @@ -181,20 +83,20 @@ def figure_to_base64( Base64-encoded image string with data URI prefix, or None if failed """ try: - print(f"🖼️ Converting {plot_name} to high-quality PNG ({width}x{height})...") + logger.info(f"Converting {plot_name} to high-quality PNG ({width}x{height})...") # Convert figure to PNG bytes - img_bytes = fig.to_image(format="png", width=width, height=height, scale=2) + img_bytes = fig.to_image(format="png", width=width, height=height) - print(f"📦 Encoding {plot_name} as base64 for HTML embedding...") + logger.info(f"Encoding {plot_name} as base64 for HTML embedding...") # Encode as base64 img_base64 = base64.b64encode(img_bytes).decode() - print(f"✅ {plot_name} image ready ({len(img_base64) // 1024}KB)") + logger.info(f"{plot_name} image ready ({len(img_base64) // 1024}KB)") return f"data:image/png;base64,{img_base64}" except Exception as e: - print(f"❌ Failed to convert {plot_name} to base64: {e}") + logger.error(f"Failed to convert {plot_name} to base64: {e}") return None @@ -265,3 +167,28 @@ def create_report_title_display(base_title: str, report_number: int | None = Non return f"Report {report_number:02d}: {base_title}" else: return base_title + + +def write_full_page_html(fig, output_path: str, title: str) -> bool: + """ + Write a plotly figure as a full-page HTML file. + + Args: + fig: Plotly figure object + output_path: Full path to the output HTML file + title: Title for the HTML document (currently unused but kept for compatibility) + + Returns: + True if successful, False otherwise + """ + try: + output_file = Path(output_path) + output_dir = output_file.parent + filename = output_file.stem # Get filename without extension + + # Use save_figure to write HTML + result = save_figure(fig, output_dir, filename, as_image=False) + return result is not None + except Exception as e: + logger.error(f"Failed to write full-page HTML to {output_path}: {e}") + return False From 5ebeae81640a96e36726009e4b744c77e760864a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:42:54 +0200 Subject: [PATCH 245/344] [caliper] cli: commands: don't swallow the stack trace --- projects/caliper/cli/commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 3bcafe341..181d8c03b 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -173,13 +173,19 @@ def parse_cmd( click.echo("⚠️ No test directories found") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() status.update( { "success": False, "error": str(e), + "traceback": full_traceback, } ) click.echo(f"parse failed: {e}", err=True) + click.echo("Full traceback:", err=True) + click.echo(full_traceback, err=True) finally: # Write status file if requested if status_file: From b80af82167e94a6bf91ec3fb6a2dd2e1c2487bdf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:43:13 +0200 Subject: [PATCH 246/344] [caliper] engine: parse: show the parsing duration --- projects/caliper/engine/parse.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index 132e4428e..3a81f8fd0 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import time from pathlib import Path from projects.caliper.engine.cache import ( @@ -48,9 +49,15 @@ def run_parse( cache_refs = [] all_warnings = [] + # Track timing statistics + parse_start_time = time.time() + cached_count = 0 + parsed_count = 0 + parse_fn = plugin.parse for node in nodes: + start_time = time.time() test_base_dir = node.directory cache_file = cache_path_for_test_base(test_base_dir, plugin_module) fp = fingerprint_test_base(test_base_dir, plugin_module) @@ -71,6 +78,10 @@ def run_parse( # Use cached records all_records.extend(cached_records) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + cached_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed from cache in {elapsed_time:.3f}s: {relative_path}") else: # Parse this test base result = parse_fn([node]) # Parse just this node @@ -88,6 +99,16 @@ def run_parse( fingerprint=fp, ) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + parsed_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed fresh in {elapsed_time:.3f}s: {relative_path}") + + # Log timing summary + total_parse_time = time.time() - parse_start_time + logger.info( + f"⏱️ Parse timing summary: {total_parse_time:.3f}s total ({cached_count} cached, {parsed_count} parsed)" + ) # Create unified model with all records cache_ref_summary = f"per-test-base: {len(cache_refs)} cache files" From ae6b755bf2edae0f79a04cf4c53e2b64b2e45840 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:43:31 +0200 Subject: [PATCH 247/344] [caliper] orchestration: postprocess: show the stack trace --- projects/caliper/orchestration/postprocess.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 898a9929c..1cdee02f0 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1131,7 +1131,10 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") elif status == "failed": error_msg = result.get("error", "unknown error") + traceback_msg = result.get("traceback") logger.error(f"Step '{step_name}' failed: {error_msg}") + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) return True return False @@ -1233,7 +1236,10 @@ def _run_parse_step(self) -> None: error_msg = (status_data or {}).get( "error", f"Command failed with exit code {result.returncode}" ) + traceback_msg = (status_data or {}).get("traceback") logger.error("Caliper parse failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "parse", { @@ -1347,7 +1353,10 @@ def _run_visualize_step(self) -> None: error_msg = (status_data or {}).get( "error", f"Command failed with exit code {result.returncode}" ) + traceback_msg = (status_data or {}).get("traceback") logger.error("Caliper visualize failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "visualize", { From ec6b56d2c30d15f5efac4facd4201f3a30ebe7a2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 16:25:41 +0200 Subject: [PATCH 248/344] [caliper] improve the --include-filter behavior --- projects/caliper/cli/commands.py | 19 +++ projects/caliper/engine/parse.py | 4 +- projects/caliper/engine/traverse.py | 192 ++++++++++++++++++++++++++- projects/caliper/engine/visualize.py | 21 ++- 4 files changed, 225 insertions(+), 11 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 181d8c03b..c18a8b639 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -108,6 +108,11 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) +@click.option( + "--include-label", + multiple=True, + help="Include only test directories with matching labels (format: key=value).", +) @click.option( "--status-file", type=click.Path(path_type=Path), @@ -120,6 +125,7 @@ def parse_cmd( no_cache: bool, cache_dir: Path | None, show_matrix: bool, + include_label: tuple[str, ...], status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, @@ -135,6 +141,18 @@ def parse_cmd( mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + # Parse include-label options into filter dict + label_filter = None + if include_label: + label_filter = {} + for label_spec in include_label: + if "=" not in label_spec: + raise click.BadParameter( + f"Invalid label format '{label_spec}'. Use key=value format." + ) + key, value = label_spec.split("=", 1) + label_filter[key.strip()] = value.strip() + status = {"success": False} try: @@ -144,6 +162,7 @@ def parse_cmd( plugin=plugin, use_cache=not no_cache, show_parameter_matrix=show_matrix, + label_filter=label_filter, ) # Extract test directories from test nodes diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index 3a81f8fd0..368f712cb 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -5,6 +5,7 @@ import logging import time from pathlib import Path +from typing import Any from projects.caliper.engine.cache import ( cache_path_for_test_base, @@ -33,6 +34,7 @@ def run_parse( use_cache: bool, force_report_partial: bool = True, show_parameter_matrix: bool = True, + label_filter: dict[str, Any] | None = None, ) -> UnifiedRunModel: """ Run full parse or load valid cache. @@ -42,7 +44,7 @@ def run_parse( base_dir = base_dir.resolve() # Discover test bases - nodes = discover_test_bases(base_dir) + nodes = discover_test_bases(base_dir, label_filter=label_filter) # Always use per-test-base caching all_records = [] diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index 461e4f7d6..fa0214c41 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -14,8 +14,15 @@ MATRIXBENCHMARKING_MARKER = "settings.yaml" -def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: - """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode.""" +def discover_test_bases( + base_dir: Path, *, label_filter: dict[str, Any] | None = None +) -> list[TestBaseNode]: + """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode. + + Args: + base_dir: Base directory to search + label_filter: Optional dict of label key-value pairs that must match for inclusion + """ base_dir = base_dir.resolve() if not base_dir.is_dir(): raise FileNotFoundError(f"Base directory does not exist: {base_dir}") @@ -33,9 +40,10 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: path = Path(dirpath) marker_path = path / marker_found - labels = _load_labels( - marker_path, is_matrixbenchmarking=(marker_found == MATRIXBENCHMARKING_MARKER) - ) + # Apply label filtering if specified + if label_filter is not None and not _matches_label_filter(labels, label_filter): + continue + nodes.append( TestBaseNode( directory=path, @@ -47,6 +55,180 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: return sorted(nodes, key=lambda n: str(n.directory)) +def _load_hierarchical_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge __test_labels__.yaml files hierarchically from base_dir down to test_dir. + + Merges in order: + 1. base_dir/__test_labels__.*.yaml (all variants) + 2. parent_dir/__test_labels__.*.yaml (all variants) + 3. test_dir/__test_labels__.*.yaml (all variants) + 4. test_dir/__test_labels__.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main __test_labels__.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge __test_labels__.*.yaml files (excluding plain __test_labels__.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all __test_labels__.*.yaml files (but not __test_labels__.yaml itself) + pattern = str(dir_path / "__test_labels__.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=False) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main __test_labels__.yaml from the test directory (final priority) + main_labels_path = test_dir / MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=False) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _load_hierarchical_labels_matrixbenchmarking(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge settings.yaml files hierarchically from base_dir down to test_dir for MatrixBenchmarking. + + Merges in order: + 1. base_dir/settings.*.yaml (all variants) + 2. parent_dir/settings.*.yaml (all variants) + 3. test_dir/settings.*.yaml (all variants) + 4. test_dir/settings.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main settings.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge settings.*.yaml files (excluding plain settings.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all settings.*.yaml files (but not settings.yaml itself) + pattern = str(dir_path / "settings.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=True) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main settings.yaml from the test directory (final priority) + main_labels_path = test_dir / MATRIXBENCHMARKING_MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=True) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _matches_label_filter(labels: dict[str, Any], label_filter: dict[str, Any]) -> bool: + """Check if labels match the specified filter criteria. + + Args: + labels: The hierarchically merged labels from the test directory + label_filter: Dict of key-value pairs that must match + + Returns: + True if all filter criteria match, False otherwise + + Examples: + _matches_label_filter({"labels": {"platform": "CKS", "version": "3.5"}}, + {"platform": "CKS"}) -> True + _matches_label_filter({"labels": {"platform": "OCP"}}, + {"platform": "CKS"}) -> False + """ + # Extract the labels section if it exists (for __test_labels__.yaml structure) + actual_labels = labels.get("labels", {}) if isinstance(labels.get("labels"), dict) else labels + + for filter_key, filter_value in label_filter.items(): + # Support nested key access with dot notation (e.g., "labels.platform") + if "." in filter_key: + keys = filter_key.split(".") + current = labels + for key in keys: + if not isinstance(current, dict) or key not in current: + return False + current = current[key] + if current != filter_value: + return False + else: + # Direct key access in labels section or root level + if filter_key not in actual_labels and filter_key not in labels: + return False + value = actual_labels.get(filter_key, labels.get(filter_key)) + if value != filter_value: + return False + + return True + + +def _deep_merge_dict(target: dict[str, Any], source: dict[str, Any]) -> None: + """Deep merge source dict into target dict, with source values taking precedence.""" + for key, value in source.items(): + if key in target and isinstance(target[key], dict) and isinstance(value, dict): + _deep_merge_dict(target[key], value) + else: + target[key] = value + + def _load_labels(path: Path, is_matrixbenchmarking: bool = False) -> dict[str, Any]: """Load labels from either __test_labels__.yaml or MatrixBenchmarking settings.yaml.""" raw = path.read_text(encoding="utf-8") diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index e2947c151..529a3d03d 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -70,19 +70,30 @@ def run_visualize( use_cache: bool, cache_path: Path | None, ) -> list[str]: + # Convert include pairs to directory-level filter for discovery + directory_filter = None + if include_pairs: + inc = parse_filter_kv(include_pairs) + directory_filter = inc # Use include filters for directory discovery + + # Apply directory-level filtering during parsing model = run_parse( base_dir=base_dir, plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, + label_filter=directory_filter, ) + + # Still apply record-level filtering for exclude filters and any remaining include filters inc = parse_filter_kv(include_pairs) exc = parse_filter_kv(exclude_pairs) - model.unified_result_records = filter_records( - model.unified_result_records, - include=inc, - exclude=exc, - ) + if exc: # Only filter records if there are exclude filters + model.unified_result_records = filter_records( + model.unified_result_records, + include={}, # Include filtering already done at directory level + exclude=exc, + ) cfg = resolve_visualize_config(base_dir, visualize_config_path) ids = resolve_report_ids( reports_csv=reports_csv, From 9283f387b41bd8d9667e961b8fac8dedf3ecb57d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 11:54:53 +0200 Subject: [PATCH 249/344] Revert "[caliper] remove unnecessary reference to the hierachical format" This reverts commit 5b392f0d2f9597830f38ff17ff2b77b912e5b6a4. --- projects/caliper/cli/commands.py | 6 +- projects/caliper/engine/kpi/analyze.py | 349 +----------------- .../engine/kpi/analyze_hierarchical.py | 214 +++++++++++ projects/caliper/orchestration/postprocess.py | 59 +-- .../caliper/orchestration/step_logging.py | 348 +++++++++++++++++ 5 files changed, 622 insertions(+), 354 deletions(-) create mode 100644 projects/caliper/engine/kpi/analyze_hierarchical.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index c18a8b639..a6937755c 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -639,9 +639,9 @@ def kpi_analyze( output: Path, plugin_module: str, ) -> None: - """Analyze KPIs for regression.""" + """Analyze KPIs for regressions using hierarchical format.""" try: - from projects.caliper.engine.kpi.analyze import find_baseline_kpis + from projects.caliper.engine.kpi.analyze_hierarchical import find_most_recent_baseline from projects.caliper.engine.load_plugin import load_plugin # Load plugin for KPI definitions and analysis rules @@ -652,7 +652,7 @@ def kpi_analyze( click.echo(f"🔌 Using plugin: {plugin_module}") - baseline_kpis = find_baseline_kpis(baseline_dir) + baseline_kpis = find_most_recent_baseline(baseline_dir) if not baseline_kpis: click.echo( f"❌ No kpis.json files found in baseline directory: {baseline_dir}", err=True diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 47ae439ff..246c317e5 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,343 +2,32 @@ from __future__ import annotations -import json -import logging -import time -from pathlib import Path from typing import Any -logger = logging.getLogger(__name__) - - -def run_kpi_analysis( - current_kpi_file: Path, - historical_data_dir: Path, - output_file: Path, - plugin_module: str, -) -> int: - """Run KPI analysis and generate output file. - - Args: - current_kpi_file: Path to current KPI JSON file - historical_data_dir: Directory containing historical KPI files - output_file: Path where analysis results will be written - plugin_module: Plugin module name for analysis - - Returns: - Exit code (0 for success, non-zero for failure) - """ - try: - logger.info(f"Running KPI analysis for: {current_kpi_file}") - logger.info(f"Historical data directory: {historical_data_dir}") - logger.info(f"Output file: {output_file}") - logger.info(f"Plugin module: {plugin_module}") - - # Check if current KPI file exists - if not current_kpi_file.exists(): - logger.error(f"Current KPI file not found: {current_kpi_file}") - return 1 - - # Check if historical data directory exists - if not historical_data_dir.exists(): - logger.error(f"Historical data directory not found: {historical_data_dir}") - return 1 - - # Find all historical KPI files - historical_kpi_files = [] - for kpi_file in historical_data_dir.rglob("kpis.json"): - historical_kpi_files.append(str(kpi_file)) - logger.info(f"Found historical KPI file: {kpi_file}") - - if not historical_kpi_files: - logger.warning("No historical KPI files found for analysis") - # Return special exit code 2 to indicate warning (no historical data) - analysis_result = { - "status": "warning", - "message": "no historical KPI found for regression testing", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": 0, - "plugin_module": plugin_module, - "completed_at": time.time(), - } - # Write warning result to output file - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - return 2 # Special exit code for warning - - logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") - - # Create analysis result - analysis_result = { - "status": "success (stub)", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": len(historical_kpi_files), - "plugin_module": plugin_module, - "completed_at": time.time(), - } - - # Create output directory if needed - output_file.parent.mkdir(parents=True, exist_ok=True) - - # Write analysis results to output file - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - f.write("\n") # Add EOL at EOF - - logger.info(f"Analysis completed successfully, results written to: {output_file}") - return 0 - - except Exception as e: - logger.exception("KPI analysis failed") - - # Create failure result - failure_result = { - "status": "failed", - "error": str(e), - "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, - "completed_at": time.time(), - } - - try: - # Attempt to write failure result - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(failure_result, f, indent=2) - f.write("\n") # Add EOL at EOF - except Exception: - logger.exception("Failed to write error result to output file") - - return 1 - - -def analyze_kpis( - postprocess_config, # CaliperOrchestrationPostprocessConfig - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Run KPI analysis step and return result status. - - This is the orchestration interface for KPI analysis. - """ - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled"} - - # Determine paths for analysis - analyze_config = postprocess_config.analyze - - # Historical KPIs directory path - historical_kpis_dir = Path(analyze_config.historical_kpis) - if not historical_kpis_dir.is_absolute(): - historical_kpis_dir = output_dir / historical_kpis_dir - - # Output path for analysis results - output_path = output_dir / analyze_config.output - - try: - # Check if required files exist - if not current_kpis_file.exists(): - return { - "status": "failed", - "error": f"Current KPI file not found: {current_kpis_file}", - "completed_at": time.time(), - } - - if not historical_kpis_dir.exists(): - return { - "status": "failed", - "error": f"Historical KPIs directory not found: {historical_kpis_dir}", - "completed_at": time.time(), - } - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=current_kpis_file, - historical_data_dir=historical_kpis_dir, - output_file=output_path, - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read the results from the output file - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return success with output file path for notifications - return { - "status": "success", - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": "Analysis completed but no output file was generated", - "completed_at": time.time(), - } - elif exit_code == 2: - # Warning: no historical data found - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return warning status with message - return { - "status": "warning", - "message": result_data.get( - "message", "no historical KPI found for regression testing" - ), - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "baseline_files_count": 0, - "completed_at": time.time(), - } - else: - # Read error from output file if it exists - error_msg = f"Analysis failed with exit code {exit_code}" - if output_path.exists(): - try: - with open(output_path) as f: - result_data = json.load(f) - error_msg = result_data.get("error", error_msg) - except Exception: - pass - - return { - "status": "failed", - "error": error_msg, - "completed_at": time.time(), - } - - except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Analysis step failed with {error_msg}") - - # Don't suppress the exception - provide detailed error info and re-raise - import traceback - - full_traceback = traceback.format_exc() - logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") - - # Re-raise the original exception so the caller can see the full details - raise RuntimeError(f"KPI analysis failed: {error_msg}") from e - - -def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: - """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. - - Args: - historical_dir: Directory to search for historical kpis.json files - - Returns: - Dictionary mapping file paths to loaded KPI JSON objects - """ - baseline_kpis = {} - kpi_files = list(historical_dir.rglob("kpis.json")) - - if not kpi_files: - logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") - return baseline_kpis - - logger.info(f"Found {len(kpi_files)} historical KPI files to load") - - for kpi_file in kpi_files: - try: - with open(kpi_file) as f: - kpi_data = json.load(f) - - # Validate that it's a hierarchical format (schema_version 2) - schema_version = kpi_data.get("schema_version", "unknown") - if schema_version != "2": - logger.warning( - f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" - ) - continue - - baseline_kpis[kpi_file] = kpi_data - logger.debug(f"Loaded KPI file: {kpi_file}") - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON in {kpi_file}: {e}") - continue - except FileNotFoundError as e: - logger.error(f"KPI file not found: {kpi_file}: {e}") - continue - except Exception as e: - logger.error(f"Failed to load KPI file {kpi_file}: {e}") - continue - - logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") - return baseline_kpis +from projects.caliper.engine.model import RegressionFinding def run_analyze( *, current_path: Any, - baseline_kpis: dict[Path, dict[str, Any]], + baseline_path: Any, output_path: Any, plugin: Any = None, -) -> dict[str, Any]: - """Run KPI analysis against ALL baseline files (CLI interface).""" - # For CLI use, we need to extract the baseline directory from the dict - # This is a bit of a hack, but the CLI interface expects a directory - if baseline_kpis: - # Get the parent directory of the first baseline file - first_baseline_path = next(iter(baseline_kpis.keys())) - historical_dir = first_baseline_path.parent - - # Extract plugin module name if available - plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=Path(current_path), - historical_data_dir=historical_dir, - output_file=Path(output_path), - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read results and convert to expected format for CLI - try: - with open(Path(output_path)) as f: - result_data = json.load(f) - return { - "status": "success", - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": result_data.get("completed_at", time.time()), - } - except Exception as e: - return { - "status": "failed", - "error": f"Failed to read analysis results: {e}", - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } +) -> list[RegressionFinding]: + """Run KPI analysis - hierarchical format only.""" + from pathlib import Path + + from .analyze_hierarchical import analyze_hierarchical_kpis + + result = analyze_hierarchical_kpis( + current_kpis_path=Path(current_path), + baseline_kpis_path=Path(baseline_path), + output_path=Path(output_path), + plugin=plugin, + ) + + # Convert to expected format for CLI + if result["status"] == "success": + return [] # Return empty list for now, actual findings are in the output file else: - return { - "status": "failed", - "error": "No baseline KPI files provided", - "completed_at": time.time(), - } - - -# EOF + raise RuntimeError(f"Analysis failed: {result.get('error', 'Unknown error')}") diff --git a/projects/caliper/engine/kpi/analyze_hierarchical.py b/projects/caliper/engine/kpi/analyze_hierarchical.py new file mode 100644 index 000000000..c76a9514d --- /dev/null +++ b/projects/caliper/engine/kpi/analyze_hierarchical.py @@ -0,0 +1,214 @@ +"""KPI analysis for hierarchical format.""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def analyze_hierarchical_kpis( + current_kpis_path: Path, + baseline_kpis_path: Path, + output_path: Path, + plugin: Any = None, +) -> dict[str, Any]: + """ + Analyze hierarchical KPI format for regressions. + + Args: + current_kpis_path: Path to current KPIs JSON file (hierarchical format) + baseline_kpis_path: Path to baseline KPIs JSON file (hierarchical format) + output_path: Path where analysis results will be written + plugin: Caliper plugin instance for KPI definitions and analysis rules + + Returns: + Analysis result dictionary with status, findings, etc. + """ + try: + # Load KPI files + with open(current_kpis_path) as f: + current_data = json.load(f) + + with open(baseline_kpis_path) as f: + baseline_data = json.load(f) + + # Validate format + if not isinstance(current_data, dict) or not isinstance(baseline_data, dict): + return { + "status": "failed", + "error": "KPI files must be in hierarchical format (dict), not list format", + "completed_at": time.time(), + } + + # Extract schema version and validate + current_schema = current_data.get("schema_version", "unknown") + baseline_schema = baseline_data.get("schema_version", "unknown") + + if current_schema != "2" or baseline_schema != "2": + return { + "status": "failed", + "error": f"Only schema_version 2 supported (current: {current_schema}, baseline: {baseline_schema})", + "completed_at": time.time(), + } + + # Analyze metrics using plugin-aware logic + current_metrics = current_data.get("metrics", {}) + baseline_metrics = baseline_data.get("metrics", {}) + + findings = [] + regressions = 0 + improvements = 0 + + # Get plugin-specific KPI definitions if available + if plugin and hasattr(plugin, "compute_kpis"): + try: + # Try to get KPI metadata from plugin + # This is a stub - plugins may expose KPI definitions differently + logger.debug(f"Plugin available for analysis: {plugin.__class__.__name__}") + except Exception as e: + logger.warning(f"Could not get KPI definitions from plugin: {e}") + + # Compare common metrics + for metric_name in current_metrics: + if metric_name in baseline_metrics: + current_metric = current_metrics[metric_name] + baseline_metric = baseline_metrics[metric_name] + + current_value = current_metric.get("value") + baseline_value = baseline_metric.get("value") + + if current_value is not None and baseline_value is not None: + try: + curr_val = float(current_value) + base_val = float(baseline_value) + + # Get direction preference from metric metadata or use default + higher_is_better = current_metric.get("higher_is_better", False) + if "labels" in current_metric: + higher_is_better = current_metric["labels"].get( + "higher_is_better", False + ) + + # Calculate change + change_percent = ( + ((curr_val - base_val) / base_val) * 100 + if base_val != 0 + else float("inf") + ) + + # Default threshold: 5% change (can be made configurable via plugin) + threshold_percent = 5.0 + + # Determine if this is a regression based on direction + is_regression = False + is_improvement = False + + if abs(change_percent) > threshold_percent: + if higher_is_better: + # For metrics where higher is better + if curr_val < base_val: # Decreased + is_regression = True + elif curr_val > base_val: # Increased + is_improvement = True + else: + # For metrics where lower is better + if curr_val > base_val: # Increased + is_regression = True + elif curr_val < base_val: # Decreased + is_improvement = True + + if is_regression: + regressions += 1 + findings.append( + { + "metric": metric_name, + "type": "regression", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + elif is_improvement: + improvements += 1 + findings.append( + { + "metric": metric_name, + "type": "improvement", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + + except (TypeError, ValueError): + logger.warning( + f"Could not compare metric {metric_name}: non-numeric values" + ) + + # Create analysis results + analysis_result = { + "analysis_timestamp": time.time(), + "current_file": str(current_kpis_path), + "baseline_file": str(baseline_kpis_path), + "schema_version": "2", + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "findings": findings, + "summary": f"Found {regressions} regressions and {improvements} improvements across {len([m for m in current_metrics if m in baseline_metrics])} metrics", + } + + # Write results + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(analysis_result, f, indent=2) + + logger.info(f"Analysis completed: {regressions} regressions, {improvements} improvements") + + return { + "status": "success", + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "output_file": str(output_path), + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "completed_at": time.time(), + } + + except FileNotFoundError as e: + return { + "status": "failed", + "error": f"File not found: {e}", + "completed_at": time.time(), + } + except json.JSONDecodeError as e: + return { + "status": "failed", + "error": f"Invalid JSON format: {e}", + "completed_at": time.time(), + } + except Exception as e: + logger.exception("Analysis failed") + return { + "status": "failed", + "error": f"Analysis failed: {e}", + "completed_at": time.time(), + } + + +def find_most_recent_baseline(historical_dir: Path) -> Path | None: + """Find the most recently modified kpis.json file in historical directory.""" + kpi_files = list(historical_dir.rglob("kpis.json")) + if not kpi_files: + return None + return max(kpi_files, key=lambda p: p.stat().st_mtime) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 1cdee02f0..eef73c57d 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -21,7 +21,6 @@ from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, - build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -896,23 +895,35 @@ def _run_analyse_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "analyse_kpis_status.yaml" - # Build CLI command - command = build_analyse_kpis_command( - config=postprocess_config, - tree_root=base_dir, - manifest_path=manifest_path, - status_file=status_file, - output_file=output_file, - current_kpis_file=current_kpis_file, - historical_kpis_dir=historical_kpis_dir, + # Note: CLI command building removed in favor of direct function call + + # Load plugin for KPI definitions + from projects.caliper.engine.load_plugin import load_plugin + + plugin = load_plugin(plugin_module) + + # Find the most recent baseline file + from projects.caliper.engine.kpi.analyze_hierarchical import ( + analyze_hierarchical_kpis, + find_most_recent_baseline, ) - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper kpi analyse-kpis", - status_file=status_file, - step_logs_dir=step_logs_dir, + baseline_file = find_most_recent_baseline(historical_kpis_dir) + if not baseline_file: + return { + "status": "failed", + "error": f"No baseline KPI files found in {historical_kpis_dir}", + "completed_at": time.time(), + "log_file": None, + } + + # Run hierarchical KPI analysis using core engine + logger.info(f"Running KPI analysis: {current_kpis_file} vs {baseline_file}") + result = analyze_hierarchical_kpis( + current_kpis_path=current_kpis_file, + baseline_kpis_path=baseline_file, + output_path=output_file, + plugin=plugin, ) # Clean up temporary status file @@ -921,20 +932,26 @@ def _run_analyse_kpis( except FileNotFoundError: pass - # Convert to expected format - if status_data.get("success"): + # Handle the analysis result + if result["status"] == "success": + logger.info( + f"Analysis completed: {result['regressions_count']} regressions, {result['improvements_count']} improvements" + ) return { "status": "success", "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "findings_count": result["findings_count"], + "regressions_count": result["regressions_count"], + "improvements_count": result["improvements_count"], "completed_at": time.time(), - "log_file": log_file, + "log_file": None, } else: return { "status": "failed", - "error": status_data.get("error", "Unknown error"), + "error": result.get("error", "Analysis failed"), "completed_at": time.time(), - "log_file": log_file, + "log_file": None, } except Exception as e: diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 66cd83137..0febd3ecb 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -11,6 +11,7 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path +from typing import Any # Thread-local storage for step-specific handlers _step_local_handlers = threading.local() @@ -234,3 +235,350 @@ def cleanup_step_logging(): if hasattr(_step_local_handlers, "file_handler"): _step_local_handlers.file_handler.close() del _step_local_handlers.file_handler + + +def _log_command_banner( + step_name: str, + command_line: str, + description: str = "", + step_args: dict[str, Any] | None = None, +) -> None: + """ + Log a command reproduction banner at the start of a step. + + Args: + step_name: Name of the step (e.g., "caliper_parse") + command_line: The CLI command to reproduce this step + description: Brief description of what this step does + step_args: Dictionary of step arguments for reference + """ + logger = logging.getLogger(__name__) + + LINE_WIDTH = 80 + + logger.info("") + logger.info("=" * LINE_WIDTH) + logger.info(f"CALIPER STEP: {step_name.upper()}") + if description: + logger.info(f"DESCRIPTION: {description}") + logger.info("=" * LINE_WIDTH) + logger.info(f"COMMAND TO REPRODUCE:\n{command_line}") + + if step_args: + logger.info("") + logger.info("STEP PARAMETERS:") + for key, value in step_args.items(): + if value is not None: + # Truncate very long values + value_str = str(value) + if len(value_str) > 100: + value_str = value_str[:97] + "..." + logger.info(f" {key}: {value_str}") + + logger.info("=" * LINE_WIDTH) + logger.info("") + + +def log_parse_command( + base_dir: Path, + plugin_module: str, + use_cache: bool, + manifest_path: Path | None = None, +) -> None: + """Log the CLI command to reproduce the parse step.""" + cache_flag = "" if use_cache else " --no-cache" + command = f'caliper parse --base-dir "{base_dir}" --plugin {plugin_module}{cache_flag}' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "use_cache": use_cache, + } + + # Only include manifest_path if it's provided and safe to log + if manifest_path: + step_args["manifest_path"] = str(manifest_path) + + _log_command_banner( + "caliper_parse", + command, + "Parse test results from artifact tree into unified model", + step_args, + ) + + +def log_visualize_command( + base_dir: Path, + plugin_module: str, + output_dir: Path, + reports_csv: str | None = None, + report_group: str | None = None, + visualize_config_path: Path | None = None, + include_pairs: tuple[str, ...] = (), + exclude_pairs: tuple[str, ...] = (), + use_cache: bool = True, +) -> None: + """Log the CLI command to reproduce the visualize step.""" + command_parts = [ + "caliper visualize", + f'--base-dir "{base_dir}"', + f'--output-dir "{output_dir}"', + f"--plugin {plugin_module}", + ] + + if not use_cache: + command_parts.append("--no-cache") + if reports_csv: + command_parts.append(f'--reports "{reports_csv}"') + if report_group: + command_parts.append(f'--report-group "{report_group}"') + if visualize_config_path: + command_parts.append(f'--visualize-config "{visualize_config_path}"') + + for include_pair in include_pairs: + command_parts.append(f'--include "{include_pair}"') + for exclude_pair in exclude_pairs: + command_parts.append(f'--exclude "{exclude_pair}"') + + command = " \\\n ".join(command_parts) + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_dir": str(output_dir), + "use_cache": use_cache, + } + + # Only include optional parameters if they're provided + if reports_csv: + step_args["reports_csv"] = reports_csv + if report_group: + step_args["report_group"] = report_group + if visualize_config_path: + step_args["visualize_config_path"] = str(visualize_config_path) + if include_pairs: + step_args["include_pairs"] = list(include_pairs) + if exclude_pairs: + step_args["exclude_pairs"] = list(exclude_pairs) + + _log_command_banner( + "caliper_visualize", + command, + "Generate visualization reports and plots from parsed data", + step_args, + ) + + +def log_artifacts_to_kpis_command( + base_dir: Path, + plugin_module: str, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the KPI generation step.""" + command = f'caliper kpi generate --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_kpi_generate", command, "Generate KPI metrics from parsed test results", step_args + ) + + +def log_kpi_export_command( + input_path: Path, + target_system: str = "opensearch", +) -> None: + """Log the CLI command to reproduce the KPI export step.""" + command = f'caliper kpi export --input "{input_path}" --target {target_system}' + + step_args = { + "input_path": str(input_path), + "target_system": target_system, + } + + _log_command_banner( + "caliper_kpi_export", command, "Export KPI metrics to external monitoring system", step_args + ) + + +def log_ai_data_command( + base_dir: Path, + plugin_module: str, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the AI evaluation export step.""" + command = f'caliper ai-eval export --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_ai_data_export", + command, + "Export AI evaluation payload with structured test directories and artifacts", + step_args, + ) + + +def log_s3_import_command( + bucket: str, + prefix: str, + output_dir: Path, +) -> None: + """Log the CLI command to reproduce the S3 import step.""" + command = ( + f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' + ) + + step_args = { + "bucket": bucket, + "prefix": prefix, + "output_dir": str(output_dir), + } + + _log_command_banner( + "caliper_s3_import", + command, + "Import historical data from S3 for analysis", + step_args, + ) + + +def log_s3_export_command( + bucket: str, + export_path: str, + from_dir: Path, + include_csv: bool, + include_kpis_json: bool, + include_ai_data: bool, +) -> None: + """Log the CLI command to reproduce the S3 export step.""" + command_parts = [f'caliper s3-export --from-dir "{from_dir}" --bucket "{bucket}"'] + + if export_path: + # Parse the export path to get prefix, instance, directory + parts = export_path.split("/") + if len(parts) >= 3: + instance = parts[0] + directory = parts[1] + prefix = "/".join(parts[2:]) if len(parts) > 2 else "" + command_parts.append(f'--instance "{instance}"') + command_parts.append(f'--directory "{directory}"') + if prefix: + command_parts.append(f'--prefix "{prefix}"') + + # Note: Include flags use default values (--include-csv, --include-kpis-json, --include-ai-data all default to True) + + command = " ".join(command_parts) + + step_args = { + "bucket": bucket, + "export_path": export_path, + "from_dir": str(from_dir), + "include_csv": include_csv, + "include_kpis_json": include_kpis_json, + "include_ai_data": include_ai_data, + } + + _log_command_banner( + "caliper_s3_export", + command, + "Export postprocess artifacts to S3", + step_args, + ) + + +def log_analyse_kpis_command( + current_kpi_file: Path, + historical_dir: Path, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the KPI analysis step.""" + command = f'caliper analyse-kpis --current "{current_kpi_file}" --historical-dir "{historical_dir}" --output "{output_file}"' + + step_args = { + "current_kpi_file": str(current_kpi_file), + "historical_dir": str(historical_dir), + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_analyse_kpis", + command, + "Analyze current KPIs against historical data", + step_args, + ) + + +def log_analyze_command( + base_dir: Path, + plugin_module: str, + current_kpis_path: Path | None = None, + baseline_file: Path | None = None, + historical_kpis_dir: Path | None = None, + output_path: Path | None = None, +) -> None: + """Log the CLI command to reproduce the analyze step.""" + # Use the correct command structure: caliper kpi analyze + command = "caliper kpi analyze" + + # Add current KPI file path if provided + if current_kpis_path: + command += f' --current "{current_kpis_path}"' + + # Add the historical directory, not the specific file + if historical_kpis_dir: + command += f' --baseline-dir "{historical_kpis_dir}"' + + # Add output path if provided + if output_path: + command += f' --output "{output_path}"' + + # Add plugin module + command += f" --plugin {plugin_module}" + + # Add helpful context + command += "\n# Analyzes hierarchical KPI format for regressions" + command += "\n# The tool will automatically find the most recent kpis.json file in the baseline directory" + command += "\n# To re-run this analysis step independently, use the command above" + command += f"\n# Plugin: {plugin_module}" + command += f"\n# Base dir: {base_dir}" + if baseline_file: + command += f"\n# Most recent baseline file found: {baseline_file}" + + step_args = { + "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, + "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, + "baseline_file_selected": str(baseline_file) if baseline_file else None, + "output_path": str(output_path) if output_path else None, + "plugin_module": plugin_module, + "base_dir": str(base_dir), + } + + _log_command_banner( + "caliper_analyze", command, "Run regression analysis on test results", step_args + ) + + +def log_kpis_to_csv_command( + input_path: Path, + output_path: Path, +) -> None: + """Log the CLI command to reproduce the KPI CSV export step.""" + command = f'caliper kpi csv-export --input "{input_path}" --output "{output_path}"' + + step_args = { + "input_path": str(input_path), + "output_path": str(output_path), + } + + _log_command_banner( + "caliper_kpi_csv_export", command, "Export KPI metrics to CSV format", step_args + ) From 9901b0baf5ef3d30359946a5105e3b6a1fb20c6c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 16:26:49 +0200 Subject: [PATCH 250/344] [caliper] engine: traverse: include hierarchical label parsing --- projects/caliper/engine/traverse.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index fa0214c41..add739209 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -39,7 +39,18 @@ def discover_test_bases( continue path = Path(dirpath) - marker_path = path / marker_found + + # Use hierarchical label loading for both marker types + if marker_found == MARKER: + labels = _load_hierarchical_labels(path, base_dir) + else: + # Use hierarchical loading for MatrixBenchmarking settings.yaml too + labels = _load_hierarchical_labels_matrixbenchmarking(path, base_dir) + + # Skip directory if skip: true is set in labels + if labels.get("skip") is True: + continue + # Apply label filtering if specified if label_filter is not None and not _matches_label_filter(labels, label_filter): continue From 802a178ddaa3b711500c5a9c2f7deb8eb055ae69 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 11:27:36 -0400 Subject: [PATCH 251/344] fix: add init container to fix PVC permissions for GuideLLM The /results PVC is mounted with root ownership, but OpenShift runs the GuideLLM container as a non-root UID. Add an init container that chmod's the mount so GuideLLM can write benchmark results. Co-authored-by: Cursor --- .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8f1ef15b6..8ca1f3253 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -17,6 +17,8 @@ spec: spec: serviceAccountName: default restartPolicy: Never + securityContext: + fsGroup: 0 containers: - name: guidellm image: {{ image }} From 27be386a96d80498e760e0495a068aab14825fc6 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 13:15:14 -0400 Subject: [PATCH 252/344] fix: make fsGroup opt-in for GuideLLM job pods fsGroup: 0 is incompatible with restricted/nonroot SCCs on shared clusters. Move it behind an optional benchmarks.guidellm.fs_group config key (disabled by default). Clusters where the CSI driver provisions root-owned volumes can opt in via configOverrides. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 3 +++ .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 4 +++- projects/guidellm/toolbox/run_guidellm_benchmark/utils.py | 5 +++++ projects/rhaiis/orchestration/test_phase.py | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index fa82ff1ec..17b51493c 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -34,6 +34,7 @@ def run( pvc_size: str = "1Gi", guidellm_args: list[str] | None = None, hf_token_secret: str = "", + fs_group: int | None = None, ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -47,6 +48,7 @@ def run( pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds securityContext.fsGroup to the GuideLLM job pod. """ execute_tasks(locals()) @@ -142,6 +144,7 @@ def create_guidellm_resources_task(args, ctx): endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, hf_token_secret=args.hf_token_secret, + fs_group=args.fs_group, ), ) return f"GuideLLM benchmark {ctx.benchmark_name} created" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8ca1f3253..10b40fe0a 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -17,8 +17,10 @@ spec: spec: serviceAccountName: default restartPolicy: Never +{% if fs_group is not none %} securityContext: - fsGroup: 0 + fsGroup: {{ fs_group }} +{% endif %} containers: - name: guidellm image: {{ image }} diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 7cab6a2c2..e3ef5a0a9 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -176,6 +176,7 @@ def render_guidellm_job_from_parts( endpoint_url: str, guidellm_args: list[str], hf_token_secret: str = "", + fs_group: int | None = None, ) -> dict[str, Any]: """Render a GuideLL-M job manifest from individual components. @@ -186,6 +187,9 @@ def render_guidellm_job_from_parts( endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds a pod-level securityContext.fsGroup to ensure + the PVC is writable by the container. Needed on clusters where the + CSI driver provisions volumes with root-only permissions. Returns: Job manifest as dict @@ -198,6 +202,7 @@ def render_guidellm_job_from_parts( "name": name, "image": image, "hf_token_secret": hf_token_secret, + "fs_group": fs_group, }, ) manifest = yaml.safe_load(rendered_yaml) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 3ffbf7529..d630519f9 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -435,6 +435,7 @@ def _run_workload_benchmark( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) @@ -655,6 +656,7 @@ def _run_warmup_step( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) logger.info("Warmup completed") except Exception: @@ -720,6 +722,7 @@ def _run_profiler_step( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) finally: enable_profiler_gate( From a5de6caa3cf75f160096bc30bd70dff3be74fc19 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 14:14:58 -0400 Subject: [PATCH 253/344] chore: add missing trtllm_config defaults for Nemotron-H Add mamba_ssm_cache_dtype, mamba_ssm_stochastic_rounding, mamba_ssm_philox_rounds, enable_chunked_prefill, and num_postprocess_workers to the base trtllm_config so they can be overridden via FournosJob configOverrides. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index c88c057f3..89b904e48 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -41,10 +41,15 @@ engines: dtype: auto enable_block_reuse: false free_gpu_memory_fraction: 0.90 + mamba_ssm_cache_dtype: "" + mamba_ssm_stochastic_rounding: false + mamba_ssm_philox_rounds: 0 cuda_graph_config: enable_padding: false max_batch_size: 256 enable_attention_dp: false + enable_chunked_prefill: false + num_postprocess_workers: 0 moe_config: backend: CUTLASS stream_interval: 0 From f221ef179526acea205cce8455af8a11e31bc2c1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 15:35:58 -0400 Subject: [PATCH 254/344] fix: include trtllm_config in dashboard runtime_args The runtime_args field on the dashboard was only populated from engine_args, missing the trtllm_config entries (kv_cache, CUDA graphs, MoE, etc.). Thread trtllm_config through to _create_test_labels so it appears in the dashboard alongside engine args. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index d630519f9..b0e6c8bb6 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging import os import uuid as _uuid_mod @@ -281,6 +282,7 @@ def _run_test( _warnings.append("Profiler trace upload failed") # Phase 2: benchmark + post-processing for ALL workloads + trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None for wl_key in workload_keys: _run_workload_benchmark( model_key=model_key, @@ -299,6 +301,7 @@ def _run_test( run_uuid=run_uuid, version=version, cluster_tag=cluster_tag, + trtllm_config=trtllm_cfg, ) try: @@ -364,6 +367,7 @@ def _run_workload_benchmark( run_uuid: str, version: str, cluster_tag: str, + trtllm_config: dict | None = None, ) -> None: """Run benchmark and post-processing for a single workload. @@ -396,6 +400,7 @@ def _run_workload_benchmark( cluster_tag=cluster_tag, accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, + trtllm_config=trtllm_config, ) if not run_benchmark: @@ -451,9 +456,16 @@ def _create_test_labels( cluster_tag: str = "", accelerator_chip: str = "", run_uuid: str = "", + trtllm_config: dict | None = None, ) -> None: _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") - runtime_args = "; ".join(f"{k}: {v}" for k, v in engine_args.items()) + parts = [f"{k}: {v}" for k, v in engine_args.items()] + for key, value in (trtllm_config or {}).items(): + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, (dict, list)) else value + ) + parts.append(f"trtllm.{key}: {formatted_value}") + runtime_args = "; ".join(parts) tp = ( engine_args.get("tensor-parallel-size") or engine_args.get("tp-size") From c4d582e7a97fcc743eddb7299d545bb7cbdafe53 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 19:05:31 -0400 Subject: [PATCH 255/344] chore: use recommended sglang serve entrypoint Replace deprecated 'python -m sglang.launch_server' with the recommended 'sglang serve' command. Co-authored-by: Cursor --- projects/rhaiis/orchestration/manifests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py index c181288a4..caa96c821 100644 --- a/projects/rhaiis/orchestration/manifests.py +++ b/projects/rhaiis/orchestration/manifests.py @@ -220,7 +220,7 @@ def _build_vllm_sglang_container( env_vars_list: list[dict], ) -> dict[str, Any]: if engine == "sglang": - command = ["python3", "-m", "sglang.launch_server"] + command = ["sglang", "serve"] if storage_source == "hf": args = [f"--model-path={model_id}", "--port=8080", "--host=0.0.0.0"] else: From f88e6a7b57dc2a340056753f31fb4f1bba016db2 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 21:57:28 -0400 Subject: [PATCH 256/344] docs: update README with multi-engine support Add SGLang and TRT-LLM documentation: engine abstraction overview, per-engine FournosJob YAML examples, updated configOverrides table with new engine/deploy keys, and manifests.py in entrypoints list. Co-authored-by: Cursor --- projects/rhaiis/README.md | 171 +++++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 19 deletions(-) diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index 1ccc40af6..e59c8e29e 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -3,10 +3,11 @@ `rhaiis` is the Forge project for benchmarking AI inference engines on OpenShift using KServe InferenceService. -The current implementation focuses on single-replica vLLM deployments benchmarked -with GuideLLM. The workflow deploys an InferenceService, runs warmup and/or -profiler passes, executes multi-profile benchmarks, generates a dashboard CSV via -the Caliper postprocessing pipeline, syncs results to S3, and cleans up. +Supports **vLLM**, **SGLang**, and **TRT-LLM** engines via a generic abstraction. +The workflow deploys an InferenceService with the selected engine, runs warmup +and/or profiler passes, executes multi-profile GuideLLM benchmarks, generates a +dashboard CSV via the Caliper postprocessing pipeline, syncs results to S3, and +cleans up. ## Workflow sequence @@ -54,7 +55,7 @@ to the canonical runner. Config is split into `config.yaml` (base) and `config.d/` (per-domain): - [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper postprocessing -- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types, S3 settings, profiler +- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, engine config (vLLM/SGLang/TRT-LLM), images, gpu_types, S3 settings, profiler - [`orchestration/config.d/models.yaml`](./orchestration/config.d/models.yaml) — 52 model definitions - [`orchestration/config.d/workloads.yaml`](./orchestration/config.d/workloads.yaml) — workload profiles @@ -62,15 +63,44 @@ Key sections: | Section | Purpose | |---------|---------| -| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args, S3 config | +| `rhaiis` | Namespace, accelerator, deploy settings, S3 config | +| `rhaiis.engine` | Active engine: `vllm` (default), `sglang`, or `trtllm` | +| `rhaiis.engines.{vllm,sglang,trtllm}` | Per-engine images, port, and default args | +| `rhaiis.engines.trtllm.trtllm_config` | TRT-LLM server config (KV cache, CUDA graphs, MoE) | +| `rhaiis.deploy` | Deploy settings (replicas, CPU/memory, image_pull_secrets list, storage) | | `rhaiis.s3` | S3 bucket, vault, and credentials for dashboard CSV and profiler trace uploads | | `rhaiis.profiler` | PyTorch profiler settings (enable, S3 prefix, rates, labels) | -| `models` | Model definitions (hf_model_id, per-model vLLM overrides) | +| `models` | Model definitions (hf_model_id, per-model `engine_args` overrides) | | `workloads` | Benchmark profiles (data shape, rates, max_seconds) | -| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret | +| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret, fs_group | | `tests` | CI test mapping (model_key, workload_keys, version) | | `caliper.postprocess` | Caliper postprocessing pipeline (parse, KPI, CSV export) | +## Engine support + +rhaiis supports three inference engines via a generic abstraction: + +| Engine | Config key | Image source | TP arg | Notes | +|--------|-----------|-------------|--------|-------| +| vLLM | `rhaiis.engine: vllm` (default) | `rhaiis.engines.vllm.images` | `tensor-parallel-size` | No root required | +| SGLang | `rhaiis.engine: sglang` | `rhaiis.engines.sglang.images` | `tp-size` | No root required, uses `sglang serve` entrypoint | +| TRT-LLM | `rhaiis.engine: trtllm` | `rhaiis.engines.trtllm.images` | `tp_size` | Requires root (`anyuid` SCC), NGC pull secret | + +Models define `engine_args` with vLLM-style keys (e.g. `tensor-parallel-size`). +When running with SGLang or TRT-LLM, arguments are automatically translated +(e.g. `tensor-parallel-size` → `tp-size` for SGLang, `tp_size` for TRT-LLM). + +Engine-specific args can also be set directly via `rhaiis.engines..args.*` +in FournosJob `configOverrides`, which take precedence over model-level `engine_args`. + +TRT-LLM additionally supports a `trtllm_config` block for server-side configuration +(KV cache, CUDA graphs, MoE backend, etc.) that is serialized as a JSON config file +inside the container. + +KServe manifests (ServingRuntime + InferenceService) are built as Python dicts in +`orchestration/manifests.py` with engine-specific container construction, then +passed to the generic `deploy_kserve_isvc` tool for application. + ## Fournos integration rhaiis supports Fournos-driven execution via `ci.py`: @@ -85,7 +115,7 @@ bin/run_ci rhaiis ci post-cleanup # Detect pipeline failures, cleanu bin/run_ci rhaiis ci export-artifacts # Caliper export to MLflow ``` -### FournosJob YAML +### FournosJob YAML (vLLM) ```yaml apiVersion: fournos.dev/v1 @@ -97,6 +127,12 @@ spec: displayName: rhaiis-benchmark pipeline: forge-full cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications executionEngine: forge: project: rhaiis @@ -106,10 +142,100 @@ spec: rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 tests.rhaiis.version: "vLLM-0.24.0" tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] - rhaiis.vllm_args.tensor-parallel-size: 2 + rhaiis.engines.vllm.args.tensor-parallel-size: 2 rhaiis.cluster_tag: "zeus2" - rhaiis.deploy.image_pull_secret: npalaska-image-pull + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 # opt-in for IBM Cloud clusters + env: + PULL_PULL_SHA: "" +``` + +### FournosJob YAML (SGLang) + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: sglang-benchmark- +spec: + owner: haumesh + displayName: sglang-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "SGLang-0.5.11" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: sglang + rhaiis.engines.sglang.args.tp-size: 2 + rhaiis.engines.sglang.args.disable-radix-cache: true + rhaiis.engines.sglang.args.mem-fraction-static: 0.90 + rhaiis.engines.sglang.args.context-length: 8192 + rhaiis.engines.sglang.args.trust-remote-code: true + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 + env: + PULL_PULL_SHA: "" +``` + +### FournosJob YAML (TRT-LLM) + +TRT-LLM requires the `anyuid` SCC on the target cluster (`oc adm policy add-scc-to-user anyuid -z default -n `). + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: trtllm-benchmark- +spec: + owner: haumesh + displayName: trtllm-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "TRT-LLM-1.3.0rc13" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: trtllm + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull", "ngc-secret"] + rhaiis.deploy.memory_request: "256Gi" + # Engine args + rhaiis.engines.trtllm.args.tp_size: 2 + rhaiis.engines.trtllm.args.ep_size: 2 + rhaiis.engines.trtllm.args.max_batch_size: 256 + rhaiis.engines.trtllm.args.max_num_tokens: 8192 + rhaiis.engines.trtllm.args.trust_remote_code: true + # TRT-LLM server config + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.dtype: fp8 + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.free_gpu_memory_fraction: 0.8 + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.enable_padding: true + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.max_batch_size: 256 + rhaiis.engines.trtllm.trtllm_config.enable_attention_dp: true + rhaiis.engines.trtllm.trtllm_config.moe_config.backend: TRTLLM caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 env: PULL_PULL_SHA: "" ``` @@ -125,7 +251,7 @@ Jobs can also be triggered via PR comments on `openshift-psap/forge`: /var tests.rhaiis.model_key: nemotron3super-120b-fp8 /var tests.rhaiis.version: vLLM-0.24.0 /var tests.rhaiis.workload_keys: ["profile1"] -/var rhaiis.vllm_args.tensor-parallel-size: 2 +/var rhaiis.engines.vllm.args.tensor-parallel-size: 2 ``` Note: `/var` directives use `key: value` format (colon required). @@ -144,18 +270,23 @@ Available configOverrides: | `tests.rhaiis.warmup` | Enable warmup pass before benchmarks | | `tests.rhaiis.slack_user` | Slack user ID for failure notifications | | `tests.rhaiis.compare_version` | Baseline version for regression comparison | +| `rhaiis.engine` | Inference engine: `vllm` (default), `sglang`, `trtllm` | | `rhaiis.namespace` | Kubernetes namespace | | `rhaiis.cluster_tag` | Cluster identifier for dashboard grouping | -| `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `rhaiis.deploy.image_pull_secrets` | List of image pull secret names | | `rhaiis.deploy.storage_pvc` | PVC name for model storage | | `rhaiis.deploy.replicas` | Number of predictor replicas | -| `rhaiis.images.nvidia` | vLLM container image override | -| `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | +| `rhaiis.deploy.memory_request` | Memory request for predictor (e.g. `256Gi` for TRT-LLM) | +| `rhaiis.engines.vllm.args.*` | vLLM CLI args (e.g. `tensor-parallel-size`, `gpu-memory-utilization`) | +| `rhaiis.engines.sglang.args.*` | SGLang CLI args (e.g. `tp-size`, `mem-fraction-static`, `context-length`) | +| `rhaiis.engines.trtllm.args.*` | TRT-LLM CLI args (e.g. `tp_size`, `ep_size`, `max_batch_size`) | +| `rhaiis.engines.trtllm.trtllm_config.*` | TRT-LLM server config (kv_cache, cuda_graph, moe) | | `rhaiis.profiler.enabled` | Enable PyTorch profiler | | `rhaiis.agent_analysis.enabled` | Enable AI agent regression analysis | | `caliper.postprocess.csv_dashboard.enabled` | Enable dashboard CSV S3 sync | | `benchmarks.guidellm.timeout` | Benchmark timeout in seconds | | `benchmarks.guidellm.hf_token_secret` | K8s secret name for HF_TOKEN (omit to skip) | +| `benchmarks.guidellm.fs_group` | Pod-level fsGroup for PVC permissions (disabled by default) | Monitoring Fournos jobs: ```bash @@ -174,6 +305,8 @@ oc patch fournosjob -n psap-automation \ - CI: [`orchestration/ci.py`](./orchestration/ci.py) (Fournos pipeline) - CI test: [`orchestration/test_rhaiis.py`](./orchestration/test_rhaiis.py) - Test phase: [`orchestration/test_phase.py`](./orchestration/test_phase.py) +- Manifests: [`orchestration/manifests.py`](./orchestration/manifests.py) (KServe ServingRuntime + InferenceService builders) +- Runtime config: [`orchestration/runtime_config.py`](./orchestration/runtime_config.py) (engine-aware config helpers) - Analysis: [`orchestration/analysis.py`](./orchestration/analysis.py) - Notifications: [`orchestration/notifications.py`](./orchestration/notifications.py) @@ -181,7 +314,7 @@ oc patch fournosjob -n psap-automation \ | Command | Source | Purpose | |---------|--------|---------| -| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime with configurable labels | +| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Apply pre-built KServe InferenceService + ServingRuntime manifests | | `wait_isvc_ready` | [rhaiis](./toolbox/wait_isvc_ready/) | Poll InferenceService readiness with health check | | `run_guidellm_benchmark` | [canonical](../guidellm/toolbox/run_guidellm_benchmark/) | Run GuideLLM benchmark (shared with llm_d) | | `capture_isvc_state` | [rhaiis](./toolbox/capture_isvc_state/) | Capture InferenceService YAML, pod logs, events, describe output | @@ -253,13 +386,13 @@ Available overrides: |------|---------------|-------------| | `--rates` | `workloads..rates` | Comma-separated concurrency levels (e.g. `1,5,50`) | | `--max-seconds` | `workloads..max_seconds` | Max benchmark duration per rate | -| `--tensor-parallel` | `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | -| `--vllm-image` | `rhaiis.images.` | vLLM container image | +| `--tensor-parallel` | engine args | Tensor parallel size (translated per engine) | +| `--vllm-image` | `rhaiis.engines..images.` | Serving container image | | `--accelerator` | `rhaiis.accelerator` | `nvidia` or `amd` | | `--replicas` | `rhaiis.deploy.replicas` | Number of predictor replicas | | `--storage-source` | `rhaiis.deploy.storage_source` | `hf` (HuggingFace download) or `pvc` | | `--storage-pvc` | `rhaiis.deploy.storage_pvc` | PVC name for model storage | -| `--image-pull-secret` | `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `--image-pull-secret` | `rhaiis.deploy.image_pull_secrets` | Image pull secret name | | `--service-account-name` | `rhaiis.deploy.service_account_name` | Service account for predictor | | `--deployment-name` | derived from model HF ID | InferenceService name | From 618599e1dcd70df80ec5dee289f5a4f0a940fdfe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:02:56 +0200 Subject: [PATCH 257/344] [cluster] toolbox: create_sa_kubeconfig: new toolbox script --- .../toolbox/create_sa_kubeconfig/main.py | 349 ++++++++++++++++++ 1 file changed, 349 insertions(+) create mode 100644 projects/cluster/toolbox/create_sa_kubeconfig/main.py diff --git a/projects/cluster/toolbox/create_sa_kubeconfig/main.py b/projects/cluster/toolbox/create_sa_kubeconfig/main.py new file mode 100644 index 000000000..16f1a277e --- /dev/null +++ b/projects/cluster/toolbox/create_sa_kubeconfig/main.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import base64 +import json +import logging +import os +from pathlib import Path + +import yaml + +from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists + +logger = logging.getLogger("DSL") + + +@entrypoint +def run( + sa_name: str, + *, + namespace: str = "", + kubeconfig_file: str, + create_token_secret: bool = True, + token_duration: str = "8760h", +) -> int: + """ + Generate kubeconfig file from a Service Account using non-persisting credential delivery. + + This tool generates a working kubeconfig file that can be used to authenticate as a specific + Service Account. It supports both long-lived token secrets and temporary tokens. + + Args: + sa_name: Name of the Service Account + namespace: Namespace where the Service Account exists/will be created (default: current project) + kubeconfig_file: Output file path for working kubeconfig + create_token_secret: Create a long-lived token secret instead of using kubectl create token + token_duration: Duration for temporary tokens when create_token_secret=False (default: 8760h) + """ + + execute_tasks(locals()) + return 0 + + +@task +def setup_configuration(args, ctx): + """Setup configuration and validate parameters""" + + # Set default namespace from current project if not provided + if args.namespace: + ctx.namespace = args.namespace + else: + result = oc("project", "-q") + ctx.namespace = result.stdout.strip() + if not ctx.namespace: + raise ValueError("Couldn't find the current namespace") + + return f"Prepared to create kubeconfig for SA {args.sa_name} in namespace {ctx.namespace}" + + +@task +def validate_parameters(args, ctx): + """Validate command parameters""" + + if not args.sa_name: + raise ValueError("sa_name cannot be empty") + + # Validate token duration format for temporary tokens + if not args.create_token_secret: + if not args.token_duration.endswith(("s", "m", "h")): + raise ValueError( + "token_duration must end with 's', 'm', or 'h' (e.g., '1h', '30m', '3600s')" + ) + + return f"Validated parameters for Service Account {args.sa_name}" + + +@task +def ensure_service_account(args, ctx): + """Create Service Account if it doesn't exist""" + + if oc_resource_exists("serviceaccount", args.sa_name, namespace=ctx.namespace): + return f"Service Account {args.sa_name} already exists in namespace {ctx.namespace}" + + oc( + "create", + "serviceaccount", + args.sa_name, + "-n", + ctx.namespace, + ) + + return f"Created Service Account {args.sa_name} in namespace {ctx.namespace}" + + +@task +def create_token_secret(args, ctx): + """Create a token secret for the Service Account if requested""" + + if not args.create_token_secret: + return "Using temporary token approach, skipping token secret creation" + + ctx.secret_name = f"{args.sa_name}-token" + + # Check if secret already exists + if oc_resource_exists("secret", ctx.secret_name, namespace=ctx.namespace): + return f"Token secret {ctx.secret_name} already exists in namespace {ctx.namespace}" + + # Create token secret manifest + secret_manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": ctx.secret_name, + "namespace": ctx.namespace, + "annotations": {"kubernetes.io/service-account.name": args.sa_name}, + }, + "type": "kubernetes.io/service-account-token", + } + + # Serialize manifest to YAML and apply through stdin + secret_yaml = yaml.dump(secret_manifest, default_flow_style=False) + oc("apply", "-f", "-", input_text=secret_yaml, handled_secretly=True) + + return f"Created token secret {ctx.secret_name}" + + +@retry(attempts=30, delay=2, backoff=1.0) +@task +def wait_for_token_secret_ready(args, ctx): + """Wait for token secret to be populated""" + + if not args.create_token_secret: + return "Using temporary token approach, no secret to wait for" + + try: + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + return False, f"Secret {ctx.secret_name} not found" + + data = secret_data.get("data", {}) + if not data.get("token") or not data.get("ca.crt"): + return False, f"Secret {ctx.secret_name} not fully populated yet" + + return f"Token secret {ctx.secret_name} is ready" + + except Exception as e: + return False, f"Error checking secret {ctx.secret_name}: {e}" + + +@task +def extract_cluster_info(args, ctx): + """Extract cluster API server URL""" + + # Get API server URL + result = oc("config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}") + ctx.api_server = result.stdout.strip() + + if not ctx.api_server: + raise RuntimeError("Could not extract API server URL from current kubeconfig") + + return f"Extracted cluster info: API server {ctx.api_server}" + + +def _build_cluster_config(ctx): + """Build cluster configuration section for kubeconfig""" + # Use certificate verification by default for security + config = {"server": ctx.api_server} + + # Check if current KUBECONFIG already uses insecure settings (development context) + is_development_context = _is_development_cluster() + + if is_development_context: + config["insecure-skip-tls-verify"] = True + logger.warning("TLS verification disabled - detected development/testing environment") + + return config + + +def _is_development_cluster() -> bool: + """Determine if we're connecting to a development cluster based on current KUBECONFIG.""" + try: + # Get current cluster configuration + result = oc("config", "view", "--minify", "-o", "json", log_stdout=False) + if result.returncode != 0: + return False + + current_config = json.loads(result.stdout) + clusters = current_config.get("clusters", []) + + if not clusters: + return False + + cluster_config = clusters[0].get("cluster", {}) + + if cluster_config.get("insecure-skip-tls-verify"): + logger.debug( + "Development context detected: current cluster uses insecure-skip-tls-verify" + ) + return True + + return False + + except Exception as e: + logger.debug(f"Could not determine cluster context: {e}") + return False + + +@task +def generate_kubeconfig(args, ctx): + """Generate working kubeconfig file using non-persisting credential delivery""" + + # Extract token locally (not stored in context) + if args.create_token_secret: + # Extract token from secret (with secret handling flag to prevent logging) + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + raise RuntimeError(f"Token secret {ctx.secret_name} not found") + + token_b64 = secret_data.get("data", {}).get("token") + if not token_b64: + raise RuntimeError(f"Token not found in secret {ctx.secret_name}") + + token = base64.b64decode(token_b64).decode("utf-8") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + else: + # Create temporary token + result = oc( + "create", + "token", + args.sa_name, + "-n", + ctx.namespace, + f"--duration={args.token_duration}", + handled_secretly=True, + ) + + token = result.stdout.strip() + if not token: + raise RuntimeError("Failed to create temporary token") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + # Create working kubeconfig with token + working_kubeconfig = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "cluster": _build_cluster_config(ctx), + "name": "cluster", + } + ], + "contexts": [ + { + "context": { + "cluster": "cluster", + "namespace": ctx.namespace, + "user": args.sa_name, + }, + "name": args.sa_name, + } + ], + "current-context": args.sa_name, + "users": [{"name": args.sa_name, "user": {"token": token}}], # Working token included + } + + # Write working kubeconfig to specified file path with secure permissions + kubeconfig_path = Path(args.kubeconfig_file) + + with open(kubeconfig_path, "w") as f: + yaml.dump(working_kubeconfig, f, default_flow_style=False) + + # Ensure permissions are correctly set after writing + os.chmod(kubeconfig_path, 0o600) + + # Store path for reference (no tokens or structure stored in context) + ctx.kubeconfig_path = str(kubeconfig_path) + + return f"Generated working kubeconfig: {ctx.kubeconfig_path}" + + +@task +def validate_kubeconfig(args, ctx): + """Validate that the generated kubeconfig works""" + + # Test the kubeconfig by running a simple command + + result = oc( + "--kubeconfig", + ctx.kubeconfig_path, + "whoami", + check=True, + ) + + if result.returncode == 0 and result.stdout.strip(): + username = result.stdout.strip() + return f"Kubeconfig validation successful - authenticated as: {username}" + else: + raise RuntimeError("Authentication test returned no username") + + +@task +def display_usage_instructions(args, ctx): + """Display usage instructions""" + + instructions = f""" +# Working kubeconfig created successfully! + +## Usage: +# Use the kubeconfig file directly +export KUBECONFIG={ctx.kubeconfig_path} +oc get pods + +# Or specify the kubeconfig file explicitly +oc --kubeconfig={ctx.kubeconfig_path} get pods + +## Files created: +- {ctx.kubeconfig_path} (working kubeconfig file with authentication) + +## Security Note: +- Kubeconfig file contains authentication token for the Service Account +- File has restrictive permissions (0600) - only readable by owner +- Uses non-persisting credential delivery (token not stored in execution context) +""" + + if args.create_token_secret: + instructions += f"\n## Token source: Long-lived secret {ctx.secret_name}" + else: + instructions += f"\n## Token source: Temporary token (expires in {args.token_duration})" + + print(instructions) + + return f"Working kubeconfig ready: {ctx.kubeconfig_path}" + + +if __name__ == "__main__": + run.main() From 39fe524325a20b9e09e6f56dfc3ce0358237e85d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 19:15:25 +0200 Subject: [PATCH 258/344] [core] dsl: utils: k8s: add the handled secretly arg to flag the calls handling secret data --- projects/core/dsl/utils/k8s.py | 59 ++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 381dd1bed..88381a91a 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -24,31 +24,52 @@ def oc( log_stdout: bool = True, log_stderr: bool = True, stdout_dest: Path | None = None, + handled_secretly: bool = False, ) -> shell.CommandResult: """Run an oc command. Args: *args: Arguments to pass to oc check: Raise subprocess.CalledProcessError if command fails - input_text: Input to send to command (not supported) - timeout_seconds: Command timeout (not supported) + input_text: Input to send to command + timeout_seconds: Command timeout log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) stdout_dest: Optional file path to write stdout to + handled_secretly: If True, disable all logging to prevent secret leakage Returns: CommandResult with execution details """ cmd = ["oc", *args] - return shell.run( - cmd, - check=check, - shell=False, - log_stdout=log_stdout, - log_stderr=log_stderr, - stdout_dest=stdout_dest, - ) + # When handled secretly, disable all logging to prevent credential leakage + if handled_secretly: + log_stdout = False + log_stderr = False + + try: + return shell.run( + cmd, + check=check, + shell=False, + log_stdout=log_stdout, + log_stderr=log_stderr, + stdout_dest=stdout_dest, + input_text=input_text, + timeout_seconds=timeout_seconds, + ) + except subprocess.CalledProcessError as e: + if not handled_secretly: + raise + + # When handled secretly, sanitize error to prevent credential leakage + raise subprocess.CalledProcessError( + e.returncode, + "oc command failed", + "", + "Error output suppressed for secret handling", + ) from e def oc_get_json( @@ -58,6 +79,7 @@ def oc_get_json( namespace: str | None = None, selector: str | None = None, ignore_not_found: bool = False, + handled_secretly: bool = False, ) -> dict[str, Any] | None: """Get a Kubernetes resource as JSON using oc. @@ -67,6 +89,7 @@ def oc_get_json( namespace: Namespace (optional) selector: Label selector (optional) ignore_not_found: Return None instead of raising error if not found + handled_secretly: If True, disable all logging to prevent secret leakage Returns: Resource data as dict, or None if not found and ignore_not_found=True @@ -83,20 +106,30 @@ def oc_get_json( args.extend(["-l", selector]) args.extend(["-o", "json"]) + # When handled secretly, disable all logging to prevent credential leakage + log_stderr_enabled = not ignore_not_found and not handled_secretly + result = oc( *args, check=not ignore_not_found, - log_stdout=False, - log_stderr=not ignore_not_found, + log_stdout=False, # Always disabled to prevent data exposure + log_stderr=log_stderr_enabled, # Disabled when handled secretly + handled_secretly=handled_secretly, ) if result.returncode != 0: if ignore_not_found and _is_oc_not_found_error(result.stderr): return None + # Re-raise error (oc() already sanitized if handled_secretly=True) raise subprocess.CalledProcessError( result.returncode, f"oc {' '.join(args)}", result.stdout, result.stderr ) if not result.stdout: - raise ValueError(f"oc {' '.join(args)} returned no output") + error_msg = ( + f"oc get {kind} returned no output" + if handled_secretly + else f"oc {' '.join(args)} returned no output" + ) + raise ValueError(error_msg) return json.loads(result.stdout) From c5ce243b395197c3c0163d0fcbd699ec20b17eae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 09:47:23 +0200 Subject: [PATCH 259/344] AGENTS.md: adjust the secret-handling guidelines --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index b2043d04f..0d2f0c8c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Key paths: If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys), it MUST NOT be written to any path under `env.ARTIFACT_DIR`. -2. **Use `oc_apply_secret()` or `oc("apply", "-f", "-", input_text=...)` for secrets.** +2. **Use `oc("apply", "-f", "-", input_text=..., handled_secretly=True)` for secrets.** Apply secrets via stdin or a dedicated helper that does not persist the payload. Never use the standard `oc_apply()` for Secret-kind manifests. From cec558c97944f243b6ffff202e48d1d8a5b71a19 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 09:53:23 +0200 Subject: [PATCH 260/344] [core] dsl: shell: add support for input and timeout --- projects/core/dsl/shell.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/core/dsl/shell.py b/projects/core/dsl/shell.py index fbd833587..8a493e5bd 100644 --- a/projects/core/dsl/shell.py +++ b/projects/core/dsl/shell.py @@ -32,6 +32,8 @@ def run( stdout_dest: str | Path | None = None, log_stdout: bool = True, log_stderr: bool = True, + input_text: str | None = None, + timeout_seconds: float | None = None, ) -> CommandResult: """ Execute a shell command @@ -43,6 +45,8 @@ def run( stdout_dest: Optional file path to write stdout to log_stdout: Optional. If False, don't log the content of stdout. log_stderr: Optional. If False, don't log the content of stderr. + input_text: Optional text to send to command's stdin + timeout_seconds: Optional timeout in seconds Returns: CommandResult with execution details """ @@ -65,6 +69,8 @@ def run( check=False, # We handle check ourselves capture_output=True, text=True, + input=input_text, + timeout=timeout_seconds, ) cmd_result = CommandResult( From 89f906a692da8a7d1f1218398c446bff4e9ef977 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 10:34:35 -0400 Subject: [PATCH 261/344] fix: address PR review comments - Add UTF-8 encoding to hierarchical JSON read/write - Validate output paths reject absolute/parent-traversal values - Honor configured output name for hierarchical JSON, save JSONL separately - Fix vLLM config override key in README examples - Mention release tags in FournosJob YAML examples - Accept YAML file paths instead of dicts in deploy_kserve_isvc toolbox - Write manifest files to src/ subdirectory in artifact dir Co-authored-by: Cursor --- projects/caliper/orchestration/postprocess.py | 38 +++++++++++++------ projects/rhaiis/README.md | 8 ++-- projects/rhaiis/orchestration/test_phase.py | 18 ++++++++- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 32 ++++++---------- 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e94868754..ed60c6af2 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -495,8 +495,13 @@ def _run_artifacts_to_kpis( } try: - # Prepare paths - output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + # Prepare paths — reject absolute or parent-traversal output names + configured_output = postprocess_config.kpi.artifacts_to_kpis.output + if Path(configured_output).is_absolute() or ".." in Path(configured_output).parts: + raise ValueError( + f"kpi.artifacts_to_kpis.output must be a relative path without '..': {configured_output}" + ) + output_file = output_dir / configured_output output_file.parent.mkdir(parents=True, exist_ok=True) # Create temporary status file for subprocess communication @@ -533,7 +538,7 @@ def _run_artifacts_to_kpis( # Read the generated JSONL file kpis = [] - with open(output_file) as f: + with open(output_file, encoding="utf-8") as f: for line in f: line = line.strip() if line: @@ -542,18 +547,20 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Transform to hierarchical format and write to a separate file - # so the original JSONL is preserved for csv-export + # Save the JSONL to a separate file for csv-export, + # then write the hierarchical JSON to the configured output name + jsonl_file = output_file.with_suffix(".jsonl") + output_file.rename(jsonl_file) + hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) import json - hierarchical_file = output_file.with_suffix(".hierarchical.json") - with open(hierarchical_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {hierarchical_file}" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {output_file}" ) else: logger.warning("No KPI records found in output file") @@ -807,8 +814,12 @@ def _run_kpis_to_csv( } try: - # Create output file path - output_file = output_dir / postprocess_config.kpi.kpis_to_csv.output + csv_output = postprocess_config.kpi.kpis_to_csv.output + if Path(csv_output).is_absolute() or ".." in Path(csv_output).parts: + raise ValueError( + f"kpi.kpis_to_csv.output must be a relative path without '..': {csv_output}" + ) + output_file = output_dir / csv_output output_file.parent.mkdir(parents=True, exist_ok=True) # Create temporary status file for subprocess communication @@ -1526,8 +1537,11 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - # Path to the JSON file for reference in command logging - kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + # csv-export reads the JSONL file (not the hierarchical JSON) + configured_output = Path(self.config.kpi.artifacts_to_kpis.output) + kpi_json_path = output_dir / configured_output.with_suffix(".jsonl") + if not kpi_json_path.exists(): + kpi_json_path = output_dir / configured_output result = _run_kpis_to_csv( self.config, plugin, diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index e59c8e29e..5991f39ff 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -139,7 +139,7 @@ spec: args: [nvidia] configOverrides: tests.rhaiis.model_key: nemotron3super-120b-fp8 - rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 + rhaiis.engines.vllm.images.nvidia: vllm/vllm-openai:v0.24.0 tests.rhaiis.version: "vLLM-0.24.0" tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] rhaiis.engines.vllm.args.tensor-parallel-size: 2 @@ -148,7 +148,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 # opt-in for IBM Cloud clusters env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### FournosJob YAML (SGLang) @@ -187,7 +187,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### FournosJob YAML (TRT-LLM) @@ -237,7 +237,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### GitHub PR workflow diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b0e6c8bb6..246aa4195 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -4,6 +4,9 @@ import logging import os import uuid as _uuid_mod +from pathlib import Path + +import yaml from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels @@ -15,6 +18,12 @@ _warnings: list[str] = [] +def _write_manifest(manifest: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -225,10 +234,15 @@ def _run_test( service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) + sr_file = env.ARTIFACT_DIR / "src" / "servingruntime.yaml" + isvc_file = env.ARTIFACT_DIR / "src" / "inferenceservice.yaml" + _write_manifest(sr_manifest, sr_file) + _write_manifest(isvc_manifest, isvc_file) + deploy_kserve_isvc( namespace=namespace, - servingruntime_manifest=sr_manifest, - inferenceservice_manifest=isvc_manifest, + servingruntime_file=str(sr_file), + inferenceservice_file=str(isvc_file), ) logger.info("Waiting for InferenceService to be ready") diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index 4a0f7887e..add66bb25 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - import yaml from projects.core.dsl import ( @@ -18,15 +16,15 @@ def run( *, namespace: str, - servingruntime_manifest: dict, - inferenceservice_manifest: dict, + servingruntime_file: str, + inferenceservice_file: str, ): - """Deploy a KServe InferenceService from pre-built manifests. + """Deploy a KServe InferenceService from pre-rendered YAML files. Args: namespace: Target namespace for the deployment. - servingruntime_manifest: Pre-built ServingRuntime manifest dict. - inferenceservice_manifest: Pre-built InferenceService manifest dict. + servingruntime_file: Path to a ServingRuntime YAML file. + inferenceservice_file: Path to an InferenceService YAML file. """ return execute_tasks(locals()) @@ -41,27 +39,19 @@ def ensure_namespace(args, context): @task def apply_servingruntime(args, context): - output_path = args.artifact_dir / "servingruntime.yaml" - _write_manifest(args.servingruntime_manifest, output_path) - oc("apply", "-f", str(output_path)) - name = args.servingruntime_manifest.get("metadata", {}).get("name", "") + oc("apply", "-f", args.servingruntime_file) + with open(args.servingruntime_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") return f"Applied ServingRuntime {name}" @task def apply_inferenceservice(args, context): - output_path = args.artifact_dir / "inferenceservice.yaml" - _write_manifest(args.inferenceservice_manifest, output_path) - oc("apply", "-f", str(output_path)) - name = args.inferenceservice_manifest.get("metadata", {}).get("name", "") + oc("apply", "-f", args.inferenceservice_file) + with open(args.inferenceservice_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") return f"Applied InferenceService {name}" -def _write_manifest(manifest: dict, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: - yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) - - if __name__ == "__main__": run.main() From f5274f7dbdfb857efd1f616dfe348d3bf935e030 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 13:14:47 -0400 Subject: [PATCH 262/344] fix: use format.py from PR #136 for csv-export v2 support csv-export now handles both hierarchical JSON (v2) and JSONL formats via read_kpis_from_file, eliminating the need to rename files or maintain separate JSONL copies. The configured output name is honored. Co-authored-by: Cursor --- projects/caliper/cli/commands.py | 14 +- projects/caliper/engine/kpi/format.py | 319 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 13 +- 3 files changed, 325 insertions(+), 21 deletions(-) create mode 100644 projects/caliper/engine/kpi/format.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index a6937755c..4ae1710be 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -544,16 +544,10 @@ def kpi_csv_export( status_data = {"success": False} try: - # Read KPI JSON Lines file (one JSON object per line) - import json - - kpi_records = [] - with open(input_file, encoding="utf-8") as f: - for line in f: - line = line.strip() - if line: # Skip empty lines - kpi_record = json.loads(line) - kpi_records.append(kpi_record) + # Read KPI file (supports both hierarchical JSON and JSONL formats) + from projects.caliper.engine.kpi.format import read_kpis_from_file + + kpi_records = read_kpis_from_file(input_file) # Export to CSV from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py new file mode 100644 index 000000000..3e5eb057d --- /dev/null +++ b/projects/caliper/engine/kpi/format.py @@ -0,0 +1,319 @@ +"""KPI output format transformations and utilities.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: + """ + Transform flat KPI list into hierarchical JSON structure. + + Groups KPIs by test (run_id), extracting common labels and organizing + KPI metadata (name, help, unit, etc.) for improved readability. + + Args: + kpis: List of flat KPI records from compute_kpis + model: Unified model for accessing plugin metadata + + Returns: + Hierarchical JSON structure organized by test + """ + + if not kpis: + return {"schema_version": "2", "tests": []} + + # Group KPIs by test (run_id) + tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) + + # Get KPI function metadata from the plugin module + from projects.caliper.engine.kpi.decorators import get_kpi_functions + + try: + plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) + kpi_functions = get_kpi_functions(plugin_module_obj) + except (ImportError, AttributeError): + kpi_functions = {} + + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + test_data = tests_data[run_id] + + # Extract common labels (excluding KPI-specific ones) + kpi_labels = kpi.get("labels", {}) + test_labels = { + k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] + } # Exclude KPI-specific labels + + # Validate and merge test labels from all KPIs in the test + for key, value in test_labels.items(): + if key in test_data["labels"] and test_data["labels"][key] != value: + raise ValueError( + f"Label mismatch for run_id '{run_id}': key '{key}' has conflicting values " + f"'{test_data['labels'][key]}' vs '{value}'" + ) + test_data["labels"].update(test_labels) + + # Store test metadata from first KPI + if not test_data["metadata"]: + test_data["metadata"] = { + "timestamp": kpi.get("timestamp"), + "source": kpi.get("source", {}), + "run_id": run_id, + } + + # Create KPI record with metadata + kpi_id = kpi.get("kpi_id") + raw_value = kpi.get("value") + + # Resolve _kpi_is_2d first as source of truth when metadata exists + is_2d = False + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + is_2d = getattr(func, "_kpi_is_2d", False) + else: + # Consistent fallback heuristic when metadata is absent + is_2d = bool( + isinstance(raw_value, list) + and raw_value + and isinstance(raw_value[0], (list, tuple)) + and len(raw_value[0]) == 2 + ) + + # Apply tuple-pair structural transform only for confirmed 2D KPIs + if is_2d and isinstance(raw_value, list) and raw_value and len(raw_value) > 0: + first_item = raw_value[0] + if isinstance(first_item, (list, tuple)) and len(first_item) == 2: + try: + # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format + structured_value = { + "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], + "count": len(raw_value), + } + final_value = structured_value + except (ValueError, TypeError, IndexError): + # If conversion fails, preserve original list-of-tuples representation + final_value = raw_value + else: + # Preserve original list-of-tuples representation + final_value = raw_value + else: + # Not 2D or no valid tuple-pair structure, keep original value + final_value = raw_value + + kpi_record = { + "id": kpi_id, + "value": final_value, + "unit": kpi.get("unit"), + "higher_is_better": kpi_labels.get("higher_is_better", True), + "is_2d": is_2d, # Set is_2d consistently based on resolved value + } + + # Add KPI metadata from function decorator if available + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + kpi_record.update( + { + "name": ( + func.__doc__.replace(" KPI.", "") + if func.__doc__ + else kpi_id.replace("_", " ").title() + ), + "help": getattr(func, "_kpi_help", ""), + } + ) + + # Add 2D-specific metadata if this is a 2D KPI + if is_2d: + kpi_record.update( + { + "x_unit": getattr(func, "_kpi_x_unit", ""), + "x_help": getattr(func, "_kpi_x_help", ""), + "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_help": getattr(func, "_kpi_y_help", None) + or getattr(func, "_kpi_help", ""), + } + ) + + # Add formatting info if available + if hasattr(func, "_kpi_format"): + kpi_record["format"] = func._kpi_format + else: + # Fallback if no function metadata available + kpi_record.update( + { + "name": kpi_id.replace("_", " ").title(), + "help": f"KPI: {kpi_id}", + } + ) + + test_data["kpis"].append(kpi_record) + + # Convert to final structure + tests_list = [] + for run_id, test_data in tests_data.items(): + tests_list.append( + { + "run_id": run_id, + "labels": test_data["labels"], + "metadata": test_data["metadata"], + "kpis": test_data["kpis"], + } + ) + + return {"schema_version": "2", "tests": tests_list} + + +def write_kpis_in_format( + kpis: list[dict], output_file: Path, format_type: str = "hierarchical", model: Any = None +) -> None: + """ + Write KPIs to file in the specified format. + + Args: + kpis: List of KPI records + output_file: Path to output file + format_type: Format type - "hierarchical" (default) or "jsonl" + model: Unified model (required for hierarchical format) + """ + output_file.parent.mkdir(parents=True, exist_ok=True) + + if format_type == "hierarchical": + if model is None: + raise ValueError("Model is required for hierarchical format") + + # Transform to hierarchical format (schema v2) + hierarchical_data = transform_kpis_to_hierarchical_format(kpis, model) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) + # Add EOL at EOF if we have data + if kpis: + f.write("\n") + + elif format_type == "jsonl": + # Write as JSONL (schema v1) - matches original behavior + text = "\n".join(json.dumps(kpi, ensure_ascii=False) for kpi in kpis) + ( + "\n" if kpis else "" + ) + with open(output_file, "w", encoding="utf-8") as f: + f.write(text) + + else: + raise ValueError(f"Unknown format type: {format_type}. Use 'hierarchical' or 'jsonl'") + + +def _convert_to_schema_v1_value(raw_value: Any) -> Any: + """ + Convert structured KPI value back to schema-v1 list-of-pairs representation. + + Args: + raw_value: The KPI value, either scalar or structured with data_points/count + + Returns: + Converted value - list of pairs for 2D data, scalar values unchanged + """ + if isinstance(raw_value, dict): + # Check if this is a structured value with data_points + if "data_points" in raw_value and isinstance(raw_value["data_points"], list): + # Convert data_points list back to list-of-pairs format + data_points = raw_value["data_points"] + return [ + [point.get("x"), point.get("y")] + for point in data_points + if isinstance(point, dict) and "x" in point and "y" in point + ] + # For other dict structures, return as-is (preserve existing format) + return raw_value + else: + # Preserve scalar values unchanged + return raw_value + + +def read_kpis_from_file(file_path: Path) -> list[dict]: + """ + Read KPIs from a file, handling both JSONL and hierarchical JSON formats. + + Args: + file_path: Path to the KPI file + + Returns: + List of KPI records in flat format + """ + kpis = [] + + with open(file_path, encoding="utf-8") as f: + content = f.read().strip() + + if not content: + return kpis + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Hierarchical format - flatten it + # Accumulate records in temporary list to ensure atomicity + temp_kpis = [] + + for test in data.get("tests", []): + test_metadata = test.get("metadata", {}) + test_labels = test.get("labels", {}) + + for kpi_record in test.get("kpis", []): + # Convert structured value to schema-v1 format if needed + raw_value = kpi_record.get("value") + converted_value = _convert_to_schema_v1_value(raw_value) + + flat_kpi = { + "schema_version": "1", + "kpi_id": kpi_record.get("id"), + "value": converted_value, + "unit": kpi_record.get("unit"), + "run_id": test_metadata.get("run_id"), + "timestamp": test_metadata.get("timestamp"), + "labels": { + **test_labels, + "higher_is_better": kpi_record.get("higher_is_better", True), + }, + "source": test_metadata.get("source", {}), + } + + # Preserve schema-v2 metadata fields if present + metadata_fields = [ + "is_2d", + "name", + "help", + "x_unit", + "y_unit", + "x_help", + "y_help", + "format", + ] + for field in metadata_fields: + if field in kpi_record: + flat_kpi[field] = kpi_record[field] + temp_kpis.append(flat_kpi) + + # Only merge into main kpis list after entire branch succeeds + kpis.extend(temp_kpis) + else: + # Unknown JSON format + raise ValueError("Unknown JSON format") + + except json.JSONDecodeError: + # Try to parse as JSONL + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + return kpis diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 65c2bae51..71ea9e4ba 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -546,11 +546,6 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Save the JSONL to a separate file for csv-export, - # then write the hierarchical JSON to the configured output name - jsonl_file = output_file.with_suffix(".jsonl") - output_file.rename(jsonl_file) - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) import json @@ -559,7 +554,7 @@ def _run_artifacts_to_kpis( json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {output_file}" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format" ) else: logger.warning("No KPI records found in output file") @@ -1563,11 +1558,7 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - # csv-export reads the JSONL file (not the hierarchical JSON) - configured_output = Path(self.config.kpi.artifacts_to_kpis.output) - kpi_json_path = output_dir / configured_output.with_suffix(".jsonl") - if not kpi_json_path.exists(): - kpi_json_path = output_dir / configured_output + kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv( self.config, plugin, From e10f7213b69e2a2fe950cb66c25a99f3fed86957 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 13:47:07 -0400 Subject: [PATCH 263/344] fix: preserve per-KPI labels (rate_index) in hierarchical JSON Labels that vary across KPIs within the same run_id (e.g. rate_index, intended_concurrency) are now stored on individual KPI records instead of being merged into test-level labels. read_kpis_from_file restores them when flattening, so csv-export produces one row per rate point. Co-authored-by: Cursor --- projects/caliper/engine/kpi/format.py | 54 ++++++++++++------- projects/caliper/orchestration/postprocess.py | 31 +++++++++-- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index 3e5eb057d..7273bb326 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -38,24 +38,33 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: except (ImportError, AttributeError): kpi_functions = {} + # First pass: determine which labels vary across KPIs in the same run + run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + for k, v in kpi.get("labels", {}).items(): + if k == "higher_is_better": + continue + run_label_values[run_id][k].add(str(v)) + + per_kpi_label_keys: dict[str, set[str]] = {} + for run_id, label_vals in run_label_values.items(): + per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} + for kpi in kpis: run_id = kpi.get("run_id", "unknown") test_data = tests_data[run_id] + varying_keys = per_kpi_label_keys.get(run_id, set()) - # Extract common labels (excluding KPI-specific ones) kpi_labels = kpi.get("labels", {}) test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels - - # Validate and merge test labels from all KPIs in the test - for key, value in test_labels.items(): - if key in test_data["labels"] and test_data["labels"][key] != value: - raise ValueError( - f"Label mismatch for run_id '{run_id}': key '{key}' has conflicting values " - f"'{test_data['labels'][key]}' vs '{value}'" - ) - test_data["labels"].update(test_labels) + k: v + for k, v in kpi_labels.items() + if k not in ("higher_is_better",) and k not in varying_keys + } + + if not test_data["labels"]: + test_data["labels"] = test_labels # Store test metadata from first KPI if not test_data["metadata"]: @@ -65,6 +74,9 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: "run_id": run_id, } + # Per-KPI labels that vary across rate points + kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} + # Create KPI record with metadata kpi_id = kpi.get("kpi_id") raw_value = kpi.get("value") @@ -104,13 +116,15 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: # Not 2D or no valid tuple-pair structure, keep original value final_value = raw_value - kpi_record = { + kpi_record: dict[str, Any] = { "id": kpi_id, "value": final_value, "unit": kpi.get("unit"), "higher_is_better": kpi_labels.get("higher_is_better", True), - "is_2d": is_2d, # Set is_2d consistently based on resolved value + "is_2d": is_2d, } + if kpi_specific_labels: + kpi_record["labels"] = kpi_specific_labels # Add KPI metadata from function decorator if available if kpi_id in kpi_functions: @@ -269,6 +283,13 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: raw_value = kpi_record.get("value") converted_value = _convert_to_schema_v1_value(raw_value) + # Merge test-level labels with any per-KPI labels + merged_labels = { + **test_labels, + **kpi_record.get("labels", {}), + "higher_is_better": kpi_record.get("higher_is_better", True), + } + flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), @@ -276,10 +297,7 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: "unit": kpi_record.get("unit"), "run_id": test_metadata.get("run_id"), "timestamp": test_metadata.get("timestamp"), - "labels": { - **test_labels, - "higher_is_better": kpi_record.get("higher_is_better", True), - }, + "labels": merged_labels, "source": test_metadata.get("source", {}), } diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 71ea9e4ba..a32ef2736 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -354,17 +354,33 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: except (ImportError, AttributeError): kpi_functions = {} + # First pass: determine which labels are common (same value for all KPIs in a run) + # vs per-KPI (vary across KPIs, e.g. rate_index, intended_concurrency) + run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + for k, v in kpi.get("labels", {}).items(): + if k == "higher_is_better": + continue + run_label_values[run_id][k].add(str(v)) + + # Labels with more than one distinct value per run are per-KPI + per_kpi_label_keys: dict[str, set[str]] = {} + for run_id, label_vals in run_label_values.items(): + per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} + for kpi in kpis: run_id = kpi.get("run_id", "unknown") test_data = tests_data[run_id] + varying_keys = per_kpi_label_keys.get(run_id, set()) - # Extract common labels (excluding KPI-specific ones) kpi_labels = kpi.get("labels", {}) test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels + k: v + for k, v in kpi_labels.items() + if k not in ("higher_is_better",) and k not in varying_keys + } - # Merge test labels (they should be the same for all KPIs in a test) if not test_data["labels"]: test_data["labels"] = test_labels @@ -376,6 +392,9 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: "run_id": run_id, } + # Per-KPI labels that vary across rate points + kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} + # Create KPI record with metadata kpi_id = kpi.get("kpi_id") raw_value = kpi.get("value") @@ -400,12 +419,14 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: else: final_value = raw_value - kpi_record = { + kpi_record: dict[str, Any] = { "id": kpi_id, "value": final_value, "unit": kpi.get("unit"), "higher_is_better": kpi_labels.get("higher_is_better", True), } + if kpi_specific_labels: + kpi_record["labels"] = kpi_specific_labels # Add KPI metadata from function decorator if available if kpi_id in kpi_functions: From b95b5c266f3796d083a512921d3079ea0aaad231 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 15:01:20 -0400 Subject: [PATCH 264/344] fix: restore run_path when flattening hierarchical KPIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_path was lost during hierarchical→flat conversion, causing all profiles to collapse into the same CSV group key. Now set from run_id. Co-authored-by: Cursor --- projects/caliper/engine/kpi/format.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index 7273bb326..ee7ea1409 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -290,12 +290,14 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: "higher_is_better": kpi_record.get("higher_is_better", True), } + run_id = test_metadata.get("run_id") flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), "value": converted_value, "unit": kpi_record.get("unit"), - "run_id": test_metadata.get("run_id"), + "run_id": run_id, + "run_path": run_id, "timestamp": test_metadata.get("timestamp"), "labels": merged_labels, "source": test_metadata.get("source", {}), From 445a5602809d365211e64a602bccfc3198b4df17 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:12 +0200 Subject: [PATCH 265/344] [kserve] toolbox: capture_llmisvc_state: capture more information --- .../toolbox/capture_llmisvc_state/main.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index 060963447..c921d45be 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -228,6 +228,62 @@ def capture_llminferenceservice_describe(args, context): return "LLMInferenceService describe captured" +@task +def capture_workload_overview(args, context): + """Capture deployment, replicaset, and pod overview for debugging""" + + workload_overview_path = args.artifact_dir / "artifacts/workload_overview.txt" + + # Capture deployment, replicaset, and pod overview with wide output + shell.run( + f'oc get deploy,rs,pod -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o wide', + stdout_dest=workload_overview_path, + check=False, + ) + + return f"Captured workload overview to {workload_overview_path}" + + +@task +def capture_workload_descriptions(args, context): + """Capture workload descriptions for deployments, replicasets, and pods in a single file""" + + descriptions_file = args.artifact_dir / "artifacts/workload_descriptions.txt" + + with open(descriptions_file, "w") as handle: + # Capture deployments descriptions + handle.write("=== DEPLOYMENTS ===\n") + deploy_result = shell.run( + f'oc describe deployments -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(deploy_result.stdout) + handle.write("\n\n") + + # Capture replicasets descriptions + handle.write("=== REPLICASETS ===\n") + rs_result = shell.run( + f'oc describe replicasets -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(rs_result.stdout) + handle.write("\n\n") + + # Capture pods descriptions + handle.write("=== PODS ===\n") + pods_result = shell.run( + f'oc describe pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(pods_result.stdout) + handle.write("\n") + + return f"Captured workload descriptions to {descriptions_file}" + + @task def capture_pods_describe(args, context): """Capture describe output for related pods""" From 3938c159d7d34a45a8285831986af5638bce4826 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:22:47 +0200 Subject: [PATCH 266/344] [kserve] toolbox: deploy_llmisvc: raise an exception if name isn't k8s compliant --- projects/kserve/toolbox/deploy_llmisvc/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index c3efade89..f49f33cf2 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -16,7 +16,7 @@ retry, task, ) -from projects.core.dsl.utils import write_text +from projects.core.dsl.utils import slugify_identifier, write_text from projects.core.dsl.utils.k8s import ( oc, oc_apply, @@ -77,6 +77,17 @@ def copy_manifest_to_src(args, ctx): # Load manifest to extract the service name manifest = load_yaml(original_path) ctx.inference_service_name = manifest["metadata"]["name"] + + # Validate that the service name is Kubernetes compliant + normalized_name = slugify_identifier(ctx.inference_service_name) + if normalized_name != ctx.inference_service_name: + raise ValueError( + f"LLMInferenceService name '{ctx.inference_service_name}' is not Kubernetes compliant. " + f"Expected: '{normalized_name}'. " + f"Names must be lowercase, contain only letters, numbers, and hyphens, " + f"and be 63 characters or less." + ) + ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}" # Ensure the src directory exists From 88ff61b726ecc0c396545b746947695a2c138e78 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 24 Jul 2026 11:11:42 +0200 Subject: [PATCH 267/344] [kserve] toolbox: deploy_llmisvc: wait_pods_scheduled: ensure the LLMISVC exists --- .../kserve/toolbox/deploy_llmisvc/main.py | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index f49f33cf2..566ac3660 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -172,10 +172,17 @@ def apply_inference_service(args, ctx): def wait_pods_appear(args, ctx): """Wait for llm-d pods to appear""" - pods = oc_get_json( - "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + # Use plain text output for better readability in logs + result = oc( + "get", "pods", "-n", args.namespace, "-l", ctx.selector, check=False, log_stdout=True ) - if pods and pods.get("items"): + + # Check if pods appeared (successful command with actual pod output) + if ( + result.returncode == 0 + and result.stdout.strip() + and "No resources found" not in result.stdout + ): return f"Pods appeared for {ctx.inference_service_name}" return False # Retry @@ -254,6 +261,19 @@ def wait_pods_scheduled(args, ctx): service_name = ctx.inference_service_name + # First, check if the LLMInferenceService exists + llmisvc_result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + check=False, + ) + + if llmisvc_result.returncode != 0: + return False, f"LLMInferenceService {service_name} does not exist yet" + # Get pod status using plain text output result = oc( "get", @@ -264,9 +284,16 @@ def wait_pods_scheduled(args, ctx): args.namespace, "--no-headers", check=False, - log_stdout=False, ) + # Check for nonzero return code from pod query + if result.returncode != 0: + return False, f"Failed to query pods for service {service_name}" + + # Check if pod query result indicates no resources found + if "No resources found" in result.stdout: + return False, "No pods found for the service yet" + if not result.stdout.strip(): return False, "No pods found for the service yet" @@ -280,7 +307,7 @@ def wait_pods_scheduled(args, ctx): return f"All pods for {service_name} are scheduled successfully" -@retry(attempts=90, delay=10, backoff=1.0) +@retry(attempts=180, delay=10, backoff=1.0) @task def wait_service_ready(args, ctx): """Wait for LLMInferenceService to be ready""" From 0711987278472618da53b9e5c41df7c75bf851cd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:18:36 +0200 Subject: [PATCH 268/344] [guidellm] toolbox: run_guidellm_benchmark: give more time --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 3cb443f9d..dc0db6ea8 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -168,7 +168,7 @@ def create_guidellm_resources_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" -@retry(attempts=1080, delay=10, backoff=1.0) +@retry(attempts=360, delay=30, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" From 2de3103bf077f813ea289d7c78d838e397c3ae6b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:25:16 +0200 Subject: [PATCH 269/344] [guidellm] toolbox: run_guidellm_benchmark: compress and trim the benchmarks.json file --- .../toolbox/run_guidellm_benchmark/main.py | 181 +++++++++++++----- 1 file changed, 133 insertions(+), 48 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index dc0db6ea8..65cafe544 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -2,6 +2,8 @@ from __future__ import annotations +import gzip +import json import logging import subprocess from pathlib import Path @@ -23,6 +25,23 @@ logger = logging.getLogger(__name__) +def trim_benchmark_json(obj): + """Remove 'requests' field recursively from JSON data""" + if isinstance(obj, dict): + return { + k: trim_benchmark_json(v) + for k, v in obj.items() + if k + not in [ + "requests", + ] + } + elif isinstance(obj, list): + return [trim_benchmark_json(item) for item in obj] + else: + return obj + + @entrypoint def run( *, @@ -35,6 +54,7 @@ def run( guidellm_args: list[str] | None = None, hf_token_secret: str = "", fs_group: int | None = None, + keep_full_benchmark_file: bool = False, ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -49,6 +69,7 @@ def run( guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. fs_group: If set, adds securityContext.fsGroup to the GuideLLM job pod. + keep_full_benchmark_file: Whether to keep the full untrimmed benchmark JSON files alongside trimmed ones (default: False) """ execute_tasks(locals()) @@ -313,7 +334,6 @@ def extract_results(args, ctx): results_dir = args.artifact_dir / "artifacts" / "results" results_dir.mkdir(parents=True, exist_ok=True) - copy_pod = f"{ctx.benchmark_name}-copy" extracted_files: list[dict[str, str | None]] = [] for run in ctx.guidellm_runs: if run.rate is None: @@ -323,10 +343,57 @@ def extract_results(args, ctx): remote_path = f"/results/benchmarks-{run.label}.json" local_path = results_dir / f"benchmarks-{run.label}.json" - local_path = _copy_result_file(ctx.target_namespace, copy_pod, remote_path, local_path) - if local_path is None: + logger.info(f"Retrieving the compressed benchmark file for {run.label}...") + + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") + + result = oc( + "exec", + "-n", + ctx.target_namespace, + f"{ctx.benchmark_name}-copy", + "--", + "gzip", + "-c", + remote_path, + check=False, + log_stdout=False, + stdout_dest=temp_gz_path, + text=False, + ) + if result.returncode != 0: raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") + # Extract and parse JSON from local compressed file + logger.info(f"Extracting compressed benchmark file for {run.label}...") + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + raise RuntimeError( + f"Failed to extract compressed results for run {run.label}: {e}" + ) from e + + # Always create trimmed version + logger.info(f"Trimming the benchmark.json file for {run.label}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON for {run.label}, writing raw data") + write_text(local_path, json_content) + + # Manage full benchmark file based on flag + if not args.keep_full_benchmark_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full benchmark file {temp_gz_path.name}") + else: + logger.info(f"Keeping full benchmark file {temp_gz_path.name}") extracted_files.append( { "label": run.label, @@ -347,69 +414,87 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" -def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> Path | None: - """Copy a result file from a pod, compressing first to handle large files. +def _copy_result_file( + namespace: str, + pod: str, + remote_path: str, + local_path: Path, + *, + trim_json: bool = True, + keep_full_file: bool = False, +) -> Path | None: + """Copy a result file from a pod, with optional compression and trimming. - Falls back to direct ``oc cp`` for small files. + Args: + namespace: Kubernetes namespace + pod: Pod name + remote_path: Path to file inside the pod + local_path: Local destination path + trim_json: Whether to trim large fields from JSON files (default: True) + keep_full_file: Whether to keep the full untrimmed file alongside trimmed one (default: False) + + Returns: + Path to the copied file, or None if copy failed """ import gzip - import shutil - # Try direct oc cp first - result = oc( - "cp", - f"{namespace}/{pod}:{remote_path}", - str(local_path), - "-c", - "copy-helper", - check=False, - log_stdout=False, - ) - if result.returncode == 0 and local_path.exists() and local_path.stat().st_size > 0: - return local_path + # Use compression approach for better handling of large files + logger.info(f"Retrieving compressed file from {remote_path}...") - logging.getLogger(__name__).info( - "Direct oc cp failed (rc=%d), retrying with gzip compression", - result.returncode, - ) + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") - # Compress inside the pod, copy the gz, decompress locally - gz_remote = f"{remote_path}.gz" - compress = oc( + # Compress and stream directly from pod + result = oc( "exec", - pod, "-n", namespace, - "-c", - "copy-helper", + pod, "--", "gzip", - "-kf", + "-c", remote_path, check=False, + log_stdout=False, + stdout_dest=temp_gz_path, + text=False, ) - if compress.returncode != 0: - logging.getLogger(__name__).warning("gzip inside pod failed (rc=%d)", compress.returncode) + if result.returncode != 0: + logging.getLogger(__name__).warning("gzip extraction failed (rc=%d)", result.returncode) return None - local_gz = local_path.parent / f"{local_path.name}.gz" - cp_result = oc( - "cp", - f"{namespace}/{pod}:{gz_remote}", - str(local_gz), - "-c", - "copy-helper", - check=False, - log_stdout=False, - ) - if cp_result.returncode != 0 or not local_gz.exists() or local_gz.stat().st_size == 0: - logging.getLogger(__name__).warning("oc cp of gzipped file failed") - local_gz.unlink(missing_ok=True) + # Extract and parse JSON from local compressed file + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + logging.getLogger(__name__).warning("Failed to extract compressed results: %s", e) + temp_gz_path.unlink(missing_ok=True) return None - with gzip.open(local_gz, "rb") as f_in, open(local_path, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) - local_gz.unlink(missing_ok=True) + # Create trimmed version if requested and file is JSON + if trim_json and local_path.suffix.lower() == ".json": + logger.info(f"Trimming JSON file {local_path.name}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON, writing raw data to {local_path}") + write_text(local_path, json_content) + else: + # Write raw content for non-JSON files or when trimming is disabled + write_text(local_path, json_content) + + # Manage full file based on keep_full_file flag + if not keep_full_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full file {temp_gz_path.name}") + else: + logger.info(f"Keeping full file {temp_gz_path.name}") if local_path.exists() and local_path.stat().st_size > 0: return local_path From 244d836e320aeff9415ef27b9f24095472e66935 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:12:15 +0200 Subject: [PATCH 270/344] [guidellm] postprocess: guidellm: improve the plotting --- .../guidellm/plotting/performance_analysis.py | 389 +++++++++++++++--- 1 file changed, 331 insertions(+), 58 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index d1794e2ef..6e8ef1c78 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -637,7 +637,7 @@ def generate_comprehensive_performance_report( report_title: str = "GuideLLM Performance Analysis", ) -> str | None: """ - Generate comprehensive performance analysis report with separate plot files. + Generate comprehensive performance analysis report with separate plots for each loadshape. Args: records: List of unified result records @@ -653,13 +653,21 @@ def generate_comprehensive_performance_report( logger.info(f"\n🚀 Starting {display_title} generation...") logger.info("=" * 70) - df = create_dataframe_from_records(records) - if df.empty: + # Group records by guidellm_loadshape + loadshape_groups = {} + for record in records: + loadshape = record.distinguishing_labels.get("guidellm_loadshape", "default") + if loadshape not in loadshape_groups: + loadshape_groups[loadshape] = [] + loadshape_groups[loadshape].append(record) + + if not loadshape_groups: logger.info("❌ No data available for analysis") return None - logger.info("\n📊 Generating performance analysis plots...") - logger.info(" This may take a moment as we create high-quality visualizations...") + logger.info( + f"\n🔍 Found {len(loadshape_groups)} loadshape(s): {', '.join(loadshape_groups.keys())}" + ) # Generate all the plots as figures plot_functions = [ @@ -680,74 +688,339 @@ def generate_comprehensive_performance_report( report_dir.mkdir(exist_ok=True) logger.info(f"\n📁 Created report directory: {report_dir_name}") - plots_data = [] - for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - logger.info(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") - try: - fig = plot_func(df, title_context) - if fig: - # Save as both PNG and HTML in the report directory - filename = plot_name.lower().replace(" ", "_") - logger.info(f"💾 Saving {plot_name} as PNG and HTML...") - - # Save PNG image - width = 900 if "Percentiles" in plot_name else 800 - height = 600 if "Percentiles" in plot_name else 500 - png_path = save_figure( - fig, report_dir, filename, as_image=True, width=width, height=height - ) - - # Save HTML version - html_path = save_figure(fig, report_dir, filename, as_image=False) + all_plots_data = [] + + # Process each loadshape separately + for loadshape, loadshape_records in loadshape_groups.items(): + logger.info(f"\n📊 Processing loadshape: {loadshape}") + logger.info(f" Records: {len(loadshape_records)}") + + # Create DataFrame for this loadshape + df = create_dataframe_from_records(loadshape_records) + if df.empty: + logger.info(f" ⚠️ No data available for loadshape: {loadshape}") + continue + + logger.info(f" 📈 Generating {len(plot_functions)} plots for {loadshape}...") + + # Create subdirectory for this loadshape + loadshape_dir = report_dir / loadshape + loadshape_dir.mkdir(exist_ok=True) + + # Extract model information for this loadshape + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + loadshape_plots = [] + for i, (plot_name, plot_func) in enumerate(plot_functions, 1): + logger.info( + f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for {loadshape}..." + ) + try: + # Create subtitle with model and loadshape info + subtitle = f"Model: {model_info} • Load Shape: {loadshape}" + + # Add original title context if provided + if title_context: + loadshape_title_context = f"{title_context}
{subtitle}" + else: + loadshape_title_context = f"
{subtitle}" + + fig = plot_func(df, loadshape_title_context) + if fig: + # Save as both PNG and HTML in the loadshape directory + filename = f"{loadshape}_{plot_name.lower().replace(' ', '_')}" + + # Save PNG image + width = 900 if "Percentiles" in plot_name else 800 + height = 600 if "Percentiles" in plot_name else 500 + png_path = save_figure( + fig, loadshape_dir, filename, as_image=True, width=width, height=height + ) - if png_path and html_path: - # Store relative paths for linking - plots_data.append( - ( - plot_name, - f"{report_dir_name}/{Path(png_path).name}", # PNG path - f"{report_dir_name}/{Path(html_path).name}", # HTML path + # Save HTML version + html_path = save_figure(fig, loadshape_dir, filename, as_image=False) + + if png_path and html_path: + # Store relative paths for linking + loadshape_plots.append( + ( + plot_name, + f"{report_dir_name}/{loadshape}/{Path(png_path).name}", # PNG path + f"{report_dir_name}/{loadshape}/{Path(html_path).name}", # HTML path + ) ) + logger.info(f" ✅ {plot_name} saved for {loadshape}") + else: + logger.info( + f" ⚠️ {plot_name} could not be created for {loadshape} (no figure returned)" ) - logger.info(f"✅ {plot_name} saved as PNG and HTML") - else: - logger.info(f"⚠️ {plot_name} could not be created (no figure returned)") - except Exception as e: - logger.info(f"❌ Failed to generate {plot_name}: {e}") + except Exception as e: + logger.info(f" ❌ Failed to generate {plot_name} for {loadshape}: {e}") - logger.info(f"\n✅ Successfully generated {len(plots_data)} visualizations!") + # Store loadshape plots data + if loadshape_plots: + all_plots_data.append((loadshape, loadshape_plots)) - # Generate summary statistics - logger.info("\n📊 Computing performance statistics and insights...") - summary_stats = _generate_performance_summary(df) + if not all_plots_data: + logger.info("❌ No plots were successfully generated") + return None - if summary_stats.get("best_tokens"): - best = summary_stats["best_tokens"] - logger.info(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") + logger.info(f"\n✅ Successfully generated plots for {len(all_plots_data)} loadshape(s)!") - # Create comprehensive HTML report + # Create comprehensive HTML report with sections for each loadshape logger.info("\n📝 Assembling comprehensive HTML report...") - logger.info(" 🔗 Creating report with images linking to interactive HTML versions...") + logger.info(" 🔗 Creating report with loadshape sections and interactive HTML links...") - html_content = _create_comprehensive_html_report_with_images( - plots_data, summary_stats, title_context, display_title - ) + html_content = f""" + + + + + + {display_title} + + + + + +

Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.

+ +
+

{display_title}

+

Generated performance analysis with separate visualizations for each loadshape

+
+ +
+

Quick Navigation:

+
    """ + + # Add navigation links for each loadshape + for loadshape, _ in all_plots_data: + html_content += f'\n
  • {loadshape}
    • ' + + html_content += """ +
+
+""" + + # Add sections for each loadshape with tabbed interface + container_id = 0 + for loadshape, plots in all_plots_data: + # Extract model information for this loadshape + loadshape_records = loadshape_groups[loadshape] + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + html_content += f""" +
+

Loadshape: {loadshape}

+

+ Model: {model_info} • Load Shape: {loadshape} +

+ +
+
""" + + # Define tab mapping with icons + tab_mapping = { + "Token Throughput vs Concurrency": "🚀 Throughput", + "TTFT Analysis": "⏱️ TTFT", + "Token Throughput Percentiles": "📊 Throughput Percentiles", + "Throughput Scaling": "📈 Scaling", + "Latency vs Throughput": "⚖️ Latency Trade-off", + } + + # Create tab headers + for tab_idx, (plot_name, _png_path, _html_path) in enumerate(plots): + tab_title = tab_mapping.get(plot_name, plot_name) + active_class = " active" if tab_idx == 0 else "" + html_content += f""" + """ + + html_content += """ +
""" + + # Create tab contents + for tab_idx, (plot_name, png_path, html_path) in enumerate(plots): + active_class = " active" if tab_idx == 0 else "" + + # Create descriptions for each plot type + descriptions = { + "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", + "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", + "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", + "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", + "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", + } + + description = descriptions.get(plot_name, f"{plot_name} performance analysis.") + + html_content += f""" +
+
+

{plot_name}

+

{description}

+

{plot_name}

+
+
""" + + html_content += """ +
+
""" + container_id += 1 + + html_content += """ + +""" + + # Save the main HTML report + main_html_filename = create_report_filename( + report_title.lower().replace(" ", "_"), report_number, report_title, "html" ) - report_file = output_dir / filename - logger.info(f"\n💾 Writing {display_title} to: {report_file.name}") - report_file.write_text(html_content, encoding="utf-8") + main_html_path = output_dir / main_html_filename + + with open(main_html_path, "w", encoding="utf-8") as f: + f.write(html_content) + + logger.info(f"✅ Comprehensive report saved as: {main_html_filename}") + logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") - file_size_mb = report_file.stat().st_size / (1024 * 1024) - logger.info(f"📄 Report complete! File size: {file_size_mb:.1f}MB") logger.info("=" * 70) - logger.info(f"🎉 {display_title} ready: {report_file.name}") + logger.info(f"🎉 {display_title} ready: {main_html_path.name}") - return str(report_file) + return str(main_html_path) except Exception as e: logger.info(f"❌ Failed to generate comprehensive performance report: {e}") From cd1f5e712707f970ff659ebadf71161134e5b2d1 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:02:11 +0200 Subject: [PATCH 271/344] [rhoai] library: deploy: improve the error message --- projects/rhoai/library/deploy.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index f62bd84ba..cefc7b28f 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -79,9 +79,10 @@ def custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: ) if secret_path is None: raise RuntimeError( - "RHOAI pull secret content " - f"'{catalog.pull_secret.vault.content}' was not found in vault " - f"'{catalog.pull_secret.vault.name}'" + f"RHOAI custom catalog pull secret is required but not available. " + f"Vault '{catalog.pull_secret.vault.name}' with content " + f"'{catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." ) return secret_path @@ -286,9 +287,10 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) if pull_secret_path is None: raise RuntimeError( - "RHOAI pull secret content " - f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " - f"'{custom_catalog.pull_secret.vault.name}'" + f"RHOAI custom catalog is enabled, but the required pull secret is not available. " + f"Vault '{custom_catalog.pull_secret.vault.name}' with content " + f"'{custom_catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." ) current_secret = _current_pull_secret_json() @@ -325,9 +327,11 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) if staging_pull_secret_path is None: raise RuntimeError( - "RHOAI staging pull secret content " - f"'{custom_catalog.staging_pull_secret.vault.content}' was not found in vault " - f"'{custom_catalog.staging_pull_secret.vault.name}'" + f"RHOAI custom catalog is enabled with staging pull secret configured, but the " + f"required staging pull secret is not available. " + f"Vault '{custom_catalog.staging_pull_secret.vault.name}' with content " + f"'{custom_catalog.staging_pull_secret.vault.content}' is MANDATORY when staging pull secret is configured. " + f"Please ensure the vault is properly configured and accessible." ) current_secret = _apply_vault_pull_secret( From 5edff8c625264ca4c7423beffe89c96e34c30bda Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:15:10 +0200 Subject: [PATCH 272/344] [rhoai] toolbox: wait_datasciencecluster_ready: better detect deployment errors --- .../wait_datasciencecluster_ready/main.py | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py index cd1a7f79f..9053068b0 100644 --- a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py +++ b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py @@ -61,16 +61,14 @@ def capture_initial_dsc(args, ctx): def wait_for_datasciencecluster_ready(args, ctx): """Wait for the DataScienceCluster phase to become Ready""" - # Query only the status.phase field and show output + # Use plain text output to show both READY and REASON columns result = oc( "get", "datasciencecluster", args.datasciencecluster_name, "-n", args.namespace, - "-o", - "jsonpath={.status.phase}", - log_stdout=True, # Show the output + log_stdout=True, # Show the table output check=False, ) @@ -81,18 +79,46 @@ def wait_for_datasciencecluster_ready(args, ctx): error_msg += f": {result.stderr.strip()}" raise RuntimeError(error_msg) - phase = result.stdout.strip() if result.stdout else None + # Parse the table output into a dict + lines = result.stdout.strip().split("\n") + if len(lines) < 2: + parsed_data = {} + else: + header = lines[0].split() + data = lines[1].split() + # Handle case where REASON column might be empty + while len(data) < len(header): + data.append("") + parsed_data = dict(zip(header, data, strict=True)) + + # Check if we have the required columns + if not parsed_data: + return (False, "Unable to parse DataScienceCluster output, waiting...") + + ready_value = parsed_data.get("READY", "").strip() + reason_value = parsed_data.get("REASON", "").strip() + + # Check if READY column shows True + if ready_value.lower() == "true": + return f"DataScienceCluster ready (READY={ready_value})" + + # Fail fast if READY shows False and REASON shows Error + if ready_value.lower() == "false" and "error" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster is in Error state (READY={ready_value}, REASON={reason_value}). Check the DSC status and logs for details." + ) - if phase == "Ready": - return "DataScienceCluster ready" - if phase in {"Failed", "Error"}: - raise RuntimeError(f"DataScienceCluster entered terminal phase {phase}") + # Fail fast if REASON shows Failed + if "failed" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster failed (REASON={reason_value}). Check the DSC status and logs for details." + ) - # Provide specific reason for retry - if not phase: - return (False, "DataScienceCluster status.phase is empty, retrying...") - else: - return (False, f"DataScienceCluster is in {phase} phase, waiting for Ready...") + # Still waiting - provide context for retry + return ( + False, + f"DataScienceCluster is not ready yet (READY={ready_value}, REASON={reason_value or 'N/A'}), waiting...", + ) @always From 875babcad4a0d5b9ca82b410c35746ba05f9c125 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 17:59:45 +0200 Subject: [PATCH 273/344] [cluster] toolbox: deploy_custom_catalog: wait for the Pod to start running --- .../toolbox/deploy_custom_catalog/main.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 2b70724f0..113913a54 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -77,6 +77,50 @@ def apply_catalog_source(args, ctx): return f"Applied CatalogSource {args.catalog_source_name}" +@retry(attempts=24, delay=5) # 2 minutes total (24 * 5s) +@task +def wait_for_catalog_source_pod_running(args, ctx): + """Wait for the CatalogSource pod to start running""" + + # Search for pod with the catalog source label + result = shell.run( + f"oc get pods -n {args.catalog_namespace} -l olm.catalogSource={args.catalog_source_name} --no-headers", + check=False, + ) + if not result.success: + return False + + # Check if we have actual pods (not "No resources found") + if "No resources found" in result.stdout: + logger.info(f"No pods found yet for catalogSource {args.catalog_source_name}") + return False + + # Count pods and check for multiple matches + pod_count = len(result.stdout.strip().splitlines()) + if pod_count != 1: + logger.warning( + f"Multiple pods ({pod_count}) found for catalogSource {args.catalog_source_name}" + ) + + # Check for image issues + if "ErrImagePull" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull error: {result.stdout.strip()}") + + if "ImagePullBackOff" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull back off: {result.stdout.strip()}") + + # Check for failed pods first - any failure should cause immediate error + if "Failed" in result.stdout: + raise RuntimeError(f"CatalogSource pod failed to start: {result.stdout.strip()}") + + # Look for Running status in the output + if "Running" in result.stdout: + return "CatalogSource pod is running" + + # Pods exist but not running yet + return False + + @retry(attempts=60, delay=15) @task def wait_for_catalog_source_ready(args, ctx): From f5457e70c571464253e21315457ca925657c14f7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 18:31:18 +0200 Subject: [PATCH 274/344] [llm_d] orchestration: render_inference_service: make the deployment k8s compliant --- .../orchestration/render_inference_service.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 29c254333..682021fec 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -73,6 +73,11 @@ def render_inference_service_from_parts( is_pd_deployment = "pd_config" in deployment_profile name = inference_service["name"] + if deployment_profile_name: + name = f"{name}-{deployment_profile_name}" + # Normalize name to be Kubernetes compliant and limit to 25 characters + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name manifest["metadata"]["namespace"] = namespace manifest["metadata"].setdefault("labels", {}) @@ -188,7 +193,10 @@ def _render_standard_deployment( scheduler = deployment_profile.get("scheduler") has_scheduler_manifest = "scheduler_manifest" in deployment_profile - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name manifest["spec"]["replicas"] = deployment_profile["replicas"] @@ -235,7 +243,10 @@ def _render_pd_deployment( from .runtime_config import get_decode_pod_count, get_prefill_pod_count # Set manifest name with deployment profile - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name # Configure prefill section manifest["spec"]["prefill"] = { From 37a512577104833dd8c78f0b93cb95a96457b52b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 18:40:25 +0200 Subject: [PATCH 275/344] [llm_d] tests: reference_deployments: normalize the guidellm names --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 2 +- .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 6265d7c82..5be1b330d 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -1,7 +1,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-intelligentrouting-tp4-x4 + name: llm-d-intelligentr-4e3b09 namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index bc538b5ce..ad5475ddb 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -1,7 +1,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + name: llm-d-pd-d-x2-p-tp-bfb506 namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' From d75dbfe81b06277854c1737cbf2610c88f8f64ef Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 13:49:48 +0200 Subject: [PATCH 276/344] [core] dsl: utils: k8s: add the text flag for the oc command --- projects/core/dsl/utils/k8s.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 88381a91a..a843da00c 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -24,6 +24,7 @@ def oc( log_stdout: bool = True, log_stderr: bool = True, stdout_dest: Path | None = None, + text: bool = True, handled_secretly: bool = False, ) -> shell.CommandResult: """Run an oc command. @@ -36,6 +37,7 @@ def oc( log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) stdout_dest: Optional file path to write stdout to + text: Optional. If False, handle binary output (default True) handled_secretly: If True, disable all logging to prevent secret leakage Returns: @@ -58,6 +60,7 @@ def oc( stdout_dest=stdout_dest, input_text=input_text, timeout_seconds=timeout_seconds, + text=text, ) except subprocess.CalledProcessError as e: if not handled_secretly: From f9a05ed1309dadafafffdebfc409f95821cdf4c7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 13:49:29 +0200 Subject: [PATCH 277/344] [core] dsl: shell: add text field for the run command --- projects/core/dsl/shell.py | 93 ++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/projects/core/dsl/shell.py b/projects/core/dsl/shell.py index 8a493e5bd..019a0feec 100644 --- a/projects/core/dsl/shell.py +++ b/projects/core/dsl/shell.py @@ -34,6 +34,7 @@ def run( log_stderr: bool = True, input_text: str | None = None, timeout_seconds: float | None = None, + text: bool = True, ) -> CommandResult: """ Execute a shell command @@ -47,6 +48,7 @@ def run( log_stderr: Optional. If False, don't log the content of stderr. input_text: Optional text to send to command's stdin timeout_seconds: Optional timeout in seconds + text: Optional. If False, handle binary output (default True) Returns: CommandResult with execution details """ @@ -68,14 +70,23 @@ def run( shell=shell, check=False, # We handle check ourselves capture_output=True, - text=True, input=input_text, timeout=timeout_seconds, + text=text, ) + # Handle text vs binary output + if text: + stdout_str = result.stdout or "" + stderr_str = result.stderr or "" + else: + # For binary mode, convert to string representation for CommandResult + stdout_str = f"" if result.stdout else "" + stderr_str = result.stderr.decode("utf-8", errors="replace") if result.stderr else "" + cmd_result = CommandResult( - stdout=result.stdout or "", - stderr=result.stderr or "", + stdout=stdout_str, + stderr=stderr_str, returncode=result.returncode, command=command, ) @@ -84,34 +95,60 @@ def run( if stdout_dest: stdout_path = Path(stdout_dest) stdout_path.parent.mkdir(parents=True, exist_ok=True) - with open(stdout_path, "w") as f: - f.write(cmd_result.stdout) + with open(stdout_path, "w" if text else "wb") as f: + f.write(stdout_str if text else result.stdout) # Print output in verbose format - if result.stdout and result.stdout.strip(): - if stdout_dest: - logger.info(f"| ") - elif log_stdout: - stdout = result.stdout.strip().splitlines() - if len(stdout) == 1: - logger.info(f"| {stdout[0]}") + if text: + if result.stdout: + if stdout_dest: + logger.info(f"| ") + elif log_stdout: + stdout_lines = stdout_str.strip().splitlines() + if len(stdout_lines) == 1: + logger.info(f"| {stdout_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stdout_lines) + "\n| ") + else: + logger.info("| ") + + if result.stderr: + if log_stderr: + stderr_lines = stderr_str.strip().splitlines() + if len(stderr_lines) == 1: + logger.info(f"| {stderr_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr_lines) + "\n| ") else: - logger.info("| \n" + "\n| ".join(stdout) + "\n| ") - else: - logger.info("| ") - - if result.stderr and result.stderr.strip(): - if log_stderr: - stderr = result.stderr.strip().splitlines() - if len(stderr) == 1: - logger.info(f"| {stderr[0]}") + logger.info("| ") + + if not (result.stdout or result.stderr): + logger.info("| ") + else: + # Binary mode - always show lengths + if result.stdout: + if stdout_dest: + logger.info( + f"| ({len(result.stdout)} bytes)" + ) else: - logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + logger.info(f"| {len(result.stdout)} bytes") - if not (result.stdout.strip() or result.stderr.strip()): - logger.info("| ") + if result.stderr: + # Try to decode stderr as text for logging (it's usually text even in binary mode) + try: + stderr_text = result.stderr.decode("utf-8", errors="replace").strip() + if stderr_text: + stderr = stderr_text.splitlines() + if len(stderr) == 1: + logger.info(f"| {stderr[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + except Exception: + logger.info(f"| {len(result.stderr)} bytes") + + if not (result.stdout or result.stderr): + logger.info("| ") if result.returncode != 0: logger.info(f"| {result.returncode}") @@ -122,9 +159,9 @@ def run( # Create a more informative error message error_msg = f"Command failed with exit code {result.returncode}: {command}" if result.stderr: - error_msg += f"\nSTDERR: {result.stderr.strip()}" + error_msg += f"\nSTDERR: {stderr_str.strip()}" if result.stdout: - error_msg += f"\nSTDOUT: {result.stdout.strip()}" + error_msg += f"\nSTDOUT: {stdout_str.strip()}" # Create exception with enhanced message error = subprocess.CalledProcessError( From fdb42e08d383f92c567cd6c1007f73bf253bb83d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 19:34:37 +0200 Subject: [PATCH 278/344] [llm_d] orchestration: config: disable kpis_to_csv and analyze --- projects/llm_d/orchestration/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index de1e08125..76bc188dc 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -59,7 +59,7 @@ caliper: enabled: true output: kpis/kpis.json kpis_to_csv: - enabled: true + enabled: false output: kpis/kpis.csv include_header_comments: true artifacts_to_ai_data: @@ -67,7 +67,7 @@ caliper: output_dir: ai_data analyze: - enabled: true + enabled: false historical_kpis: historical_data output: regression_analyze/kpi_analyze.json From 492ac3364121e9e6d7905c64c7c56a646258e5f6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:10:24 +0200 Subject: [PATCH 279/344] [fournos_launcher] toolbox: submit_and_wait: Avi's early return --- .../toolbox/submit_and_wait/main.py | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index d0a26d2b1..b9bc9e778 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -216,7 +216,86 @@ def submit_fournos_job(args, ctx): return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" -@retry(attempts=3000, delay=10, backoff=1.0) +@retry(attempts=30, delay=10, backoff=1.0) +@task +def wait_for_job_to_resolve(args, ctx): + """Wait for FOURNOS job to successfully resolve (reach Pending state)""" + + # Check job status + status_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.phase}}"', + check=False, + ) + + if not status_result.success: + # Check if it's a "not found" error (permanent failure) vs temporary error + if "not found" in status_result.stderr.lower(): + raise FournosJobFailureError( + ctx.final_job_name, + "Job not found during resolve phase - may have been deleted or never created", + args.namespace, + "not_found", + ) + + # Other errors might be temporary, retry + logger.info( + f"Failed to get job status during resolve phase, retrying... (stderr: {status_result.stderr.strip()})" + ) + return False # Retry + + status = status_result.stdout.strip() + + if not status: + logger.info(f"Job {ctx.final_job_name} not processed.") + return False, "Not processed" # Retry + + if status == "Resolving": + logger.info(f"Job {ctx.final_job_name} status: {status}. Still resolving, keep waiting.") + return False, "Resolving" # Retry + + if status == "Stopping": + logger.warning(f"Job {ctx.final_job_name} status: {status}.") + raise FournosJobFailureError( + ctx.final_job_name, "Job stopped in its early stages", args.namespace + ) + + if status == "Failed": + # Get failure details + failure_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.message}}"', + check=False, + ) + failure_msg = ( + failure_result.stdout.strip() + if failure_result.success + else "Unknown failure during resolve" + ) + raise FournosJobFailureError( + ctx.final_job_name, f"Job failed in its early stages: {failure_msg}", args.namespace + ) + + if status in ["Admitted", "Running", "Succeeded"]: + # Job moved past Pending - this means resolve succeeded + return f"Job {ctx.final_job_name} resolved successfully (current status: {status})" + + if status == "Pending": + return f"Job {ctx.final_job_name} successfully resolved and reached Pending state" + + logger.info(f"Unknown job {ctx.final_job_name} status: {status}") + return False, "Unknown status" # Unknown status, retry + + +@task +def check_early_return(args, ctx): + """Check for early return""" + + if not args.wait: + return EarlyReturn(f"Fournos job successfully launched: {ctx.final_job_name} (wait=False)") + + return f"Fournos job successfully launched: {ctx.final_job_name}" + + +@retry(attempts=3000, delay=30, backoff=1.0) @task def wait_for_job_completion(args, ctx): """Wait for FOURNOS job to complete""" From 7b7f5509bc70ea09df2476108f1205374e79767e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:02:54 +0200 Subject: [PATCH 280/344] docs: caliper: test-labels-format: new doc --- docs/caliper/test-labels-format.md | 130 +++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 docs/caliper/test-labels-format.md diff --git a/docs/caliper/test-labels-format.md b/docs/caliper/test-labels-format.md new file mode 100644 index 000000000..bc2fbec3c --- /dev/null +++ b/docs/caliper/test-labels-format.md @@ -0,0 +1,130 @@ +# Test Labels Format (`__test_labels__.yaml`) + +## Purpose + +The `__test_labels__.yaml` file marks directories as Caliper test bases and provides metadata for test identification, grouping, and status tracking. + +## File Format + +```yaml +version: "1" +labels: + # Test distinguishing characteristics + key1: value1 + key2: value2 +kpi_labels: + # System/environment context for KPI labeling + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + # Test execution status (added at completion) + success: true|false + message: "Status description" +``` + +### Required Fields +- **`version`**: Schema version (currently `"1"`) +- **`labels`**: Key-value pairs describing test characteristics + +### Optional Fields +- **`kpi_labels`**: System/environment context labels for KPI records + - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`) + - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`) + - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`) +- **`completion`**: Test execution status + - **`success`**: `true` if succeeded, `false` if failed + - **`message`**: Human-readable status description + +## Labels vs KPI Labels + +### Labels (Test Configuration) +- **Purpose**: Distinguish different test configurations and variations +- **Usage**: Used for test filtering, grouping, and analysis +- **Examples**: `model_name`, `deployment_profile`, `guidellm_loadshape` +- **Scope**: Specific to the test configuration + +### KPI Labels (System Context) +- **Purpose**: Provide system/environment context for KPI records +- **Usage**: Applied to all KPI metrics from the test, enables cross-environment analysis +- **Examples**: `platform`, `gpu_type`, `test_harness` +- **Scope**: Describes the execution environment + +## Common Labels + +- **`model_name`**: AI/ML model name (e.g., `"llama-3.1-8b"`) +- **`deployment_profile`**: Deployment variant (e.g., `"simple-tp2-x4"`) +- **`guidellm_loadshape`**: Benchmark load pattern (e.g., `"heavy-heterogeneous"`) +- **`benchmark_type`**: Benchmark type (e.g., `"throughput"`, `"latency"`) + +## Common KPI Labels + +- **`platform`**: Target platform (e.g., `"CKS"`, `"RHOAI"`) +- **`gpu_type`**: GPU hardware type (e.g., `"H100"`, `"A100"`) +- **`test_harness`**: Testing framework (e.g., `"guidellm"`, `"vllm"`) + +## Examples + +### Basic Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple" + guidellm_loadshape: "default" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +``` + +### Completed Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple-tp2-x4" + guidellm_loadshape: "heavy-heterogeneous" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + success: true + message: "Test completed successfully" +``` + +### Failed Test +```yaml +version: "1" +labels: + model_name: "gpt-4o" + deployment_profile: "distributed" +kpi_labels: + platform: "RHOAI" + gpu_type: "A100" + test_harness: "vllm" +completion: + success: false + message: "Connection timeout to inference service" +``` + +## Usage + +- **Test Discovery**: Caliper scans for `__test_labels__.yaml` files to find test results +- **Test Analysis**: Labels enable filtering and comparative analysis of test configurations +- **KPI Labeling**: KPI labels are automatically applied to all generated KPI metrics +- **Cross-Environment Analysis**: KPI labels enable performance comparisons across different platforms/hardware +- **Report Generation**: Both labels and KPI labels provide context for charts and report organization +- **Status Tracking**: Completion field tracks test execution outcome + +## Best Practices + +- **Labels**: Use for test-specific configuration and distinguishing characteristics +- **KPI Labels**: Use for system/environment context that applies to all KPIs +- Generate both label sets early, add completion status at test end +- Use consistent, descriptive keys across related tests +- Use lowercase with underscores (e.g., `model_name`, `gpu_type`) +- Keep label schemas stable within projects +- Populate KPI labels from configuration to ensure consistency (e.g., from `cpt.kpi.labels.*`) +- Missing KPI labels should be handled gracefully with fallback values From 6ceec5035e42f1c03f55772267edb2300535c8f5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:56:52 +0200 Subject: [PATCH 281/344] [llm_d] work on P/D --- projects/llm_d/orchestration/runtime_config.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 19d3ba066..03cd2e7c6 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -161,6 +161,11 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: match = re.search(r"p\.tp(\d+)", profile_name) if match: return int(match.group(1)) + elif field_name == "decode_tensor_parallelism": + # Look for d.tp4 pattern (decode tensor_parallelism) + match = re.search(r"d\.tp(\d+)", profile_name) + if match: + return int(match.group(1)) else: # Standard patterns: simple-tp4-x4, intelligentrouting-tp4-x4 if field_name == "tensor_parallelism": @@ -200,7 +205,17 @@ def resolve_value(obj: Any, path: str = "") -> Any: return [resolve_value(item, path) for item in obj] elif obj == "FROM_NAME": # Determine which field to extract based on the path - field_name = path.split(".")[-1] # Get the last part of the path + path_parts = path.split(".") + field_name = path_parts[-1] # Get the last part of the path + + # Map field names for P/D deployments + if len(path_parts) >= 2 and path_parts[-2] in ("prefill", "decode"): + container_type = path_parts[-2] # prefill or decode + if field_name == "replicas": + field_name = f"{container_type}_pods" + elif field_name == "tensor_parallelism": + field_name = f"{container_type}_tensor_parallelism" + try: return _extract_value_from_profile_name(profile_name, field_name) except ValueError as e: From 2f2aa7ed561a8bf529934b35a22aea4894da5cbd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:05:34 +0200 Subject: [PATCH 282/344] [llm_d] orchestration: add the ability to skip the test finalizers --- projects/llm_d/orchestration/config.d/runtime.yaml | 2 ++ projects/llm_d/orchestration/test_phase.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 81abc4371..05e8739b0 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -15,3 +15,5 @@ router: file: manifests/scheduler_config/__router_template__.yaml placeholder: __ENDPOINT_PICKER_CONFIG__ epp: manifests/scheduler_config + +run_test_finalizers: true diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 8402815d8..93d1d2d7a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -144,7 +144,7 @@ def do_test() -> int: except SignalInterrupt: primary_exc = sys.exc_info() finally: - do_finalizers = True + do_finalizers = config.project.get_config("runtime.run_test_finalizers") if primary_exc and isinstance(primary_exc[1], SignalInterrupt): logging.warning("Caught a SignalInterrupt, skipping the finalizers") do_finalizers = False From 4ac8f62d6173545082cef90b5a088ecb46835041 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:07:15 +0200 Subject: [PATCH 283/344] [llm_d] orchestration: presets: cks: new preset file --- .../llm_d/orchestration/presets.d/cks.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/cks.yaml diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml new file mode 100644 index 000000000..e96a453c3 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -0,0 +1,58 @@ +__multiple: true + +cks: + runtime.namespace: kpouget-dev + platform.cluster.namespace.labels: {} + + runtime.kserve.wait_readiness: false + runtime.kueue.enabled: true + runtime.kueue.queue_name: llm-d-cluster-queue + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + platform.gateway.status_address_name: gateway-internal + platform.preflight.check_crds: false + runtime.kserve.wait_long_scheduling: true + platform.inference_service.scheduler.node_selector: + gpu.nvidia.com/class: H200 + +cks-intelligent: + extends: + - cks + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-pd: + extends: + - cks + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-smoke: + extends: + - cks + #- llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - intelligentrouting-tp1-x2 + - simple-tp1-x1 + - pd-d.x1-p.tp1-d.tp1-p.x1 + + runtime.benchmark_key: + - short From 0af9e35932b17d9ac74ba84100b0281f3f9b043a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 17:22:54 +0200 Subject: [PATCH 284/344] [llm_d] orchestration: allow skipping the kserver readiness wait --- projects/llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 05e8739b0..509634702 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -7,6 +7,7 @@ job_name: null namespace: forge-llm-d kserve: dry_run: false + wait_readiness: true kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 93d1d2d7a..2326bc15a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -180,8 +180,8 @@ def deploy_inference_service() -> str: platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # Check if dry-run mode is enabled early - dry_run = config.project.get_config("runtime.kserve.dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run") + wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: @@ -190,7 +190,7 @@ def deploy_inference_service() -> str: logger.info("Skipping model cache preparation - dry-run mode enabled") # Step 2: Wait for the serving control plane to settle before creating the service. - if not dry_run: + if not dry_run and wait_readiness: rhoai_namespace = platform["rhoai"]["namespace"] wait_kserve_ready.run(namespace=rhoai_namespace) From 5f774852111561de514e39a8c8dc76602c3d074e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 17:39:07 +0200 Subject: [PATCH 285/344] [llm_d] improve the kueue handling --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../manifests/kueue-localqueue.yaml | 8 ++++ .../orchestration/render_inference_service.py | 19 +++++++++ projects/llm_d/orchestration/test_phase.py | 42 +++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 projects/llm_d/orchestration/manifests/kueue-localqueue.yaml diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 509634702..d1ad2e850 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -11,6 +11,7 @@ kserve: kueue: enabled: false queue_name: default-queue + local_queue_manifest: manifests/kueue-localqueue.yaml router: template: file: manifests/scheduler_config/__router_template__.yaml diff --git a/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml new file mode 100644 index 000000000..d2232815c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml @@ -0,0 +1,8 @@ +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + name: "QUEUE_NAME" + namespace: "NAMESPACE" +spec: + clusterQueue: "QUEUE_NAME" + stopPolicy: None diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 682021fec..95c64f6c3 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -407,12 +407,19 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: scheduler_template["metadata"] = {} if "annotations" not in scheduler_template["metadata"]: scheduler_template["metadata"]["annotations"] = {} + if "labels" not in scheduler_template["metadata"]: + scheduler_template["metadata"]["labels"] = {} # Apply the same Kueue annotations to the scheduler pod template for annotation_key, annotation_value in kueue_annotations.items(): full_annotation_key = f"{kueue_prefix}{annotation_key}" scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + # Apply the same Kueue labels to the scheduler pod template + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + scheduler_template["metadata"]["labels"][full_label_key] = label_value + # Update the scheduler template back to the data structure manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template @@ -437,3 +444,15 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: manifest["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( pod_group_total_count ) + + # Add required pod-group-name label using the ISVC name + pod_group_name = manifest["metadata"]["name"] + manifest["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name + + # Also add pod-group-name label to scheduler template if it exists + if has_scheduler: + scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + if "metadata" in scheduler_template and "labels" in scheduler_template["metadata"]: + scheduler_template["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = ( + pod_group_name + ) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2326bc15a..5c40fc1d7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -5,8 +5,11 @@ from pathlib import Path from typing import Any +import yaml + from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier +from projects.core.dsl.utils.k8s import oc from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -28,6 +31,42 @@ logger = logging.getLogger(__name__) +def ensure_kueue_local_queue() -> None: + """Create LocalQueue when kueue is enabled.""" + enable_kueue = config.project.get_config("runtime.kueue.enabled") + if not enable_kueue: + return + + queue_name = config.project.get_config("runtime.kueue.queue_name") + manifest_path = config.project.get_config("runtime.kueue.local_queue_manifest") + namespace = runtime_config.get_namespace() + + logger.info("Creating LocalQueue: %s", queue_name) + + # Read and parse the YAML template + config_dir = runtime_config.get_config_dir() + template_file = config_dir / manifest_path + + with template_file.open(encoding="utf-8") as f: + local_queue_manifest = yaml.safe_load(f) + + # Update the fields + local_queue_manifest["metadata"]["name"] = queue_name + local_queue_manifest["metadata"]["namespace"] = namespace + local_queue_manifest["spec"]["clusterQueue"] = queue_name + + # Write manifest to manifests directory and apply + manifests_dir = env.ARTIFACT_DIR / "manifests" + manifests_dir.mkdir(parents=True, exist_ok=True) + manifest_file = manifests_dir / f"{queue_name}-localqueue.yaml" + + write_yaml(manifest_file, local_queue_manifest) + + # Apply the manifest + oc("apply", "-f", str(manifest_file)) + logger.info("LocalQueue %s created successfully", queue_name) + + def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" @@ -119,6 +158,9 @@ def do_test() -> int: namespace, labels=config.project.get_config("platform.cluster.namespace.labels") ) + # Ensure LocalQueue exists when kueue is enabled + ensure_kueue_local_queue() + endpoint_url: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None From 4c6237b4eb03b46f13c55cae6117a36b4c81718f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 23:21:50 +0200 Subject: [PATCH 286/344] [llm_d] orchestration: correctly cleanup the llmisvc --- projects/llm_d/orchestration/test_phase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 5c40fc1d7..df25e3c61 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -391,9 +391,14 @@ def cleanup_test_resources() -> None: inference_service = platform["inference_service"] smoke = platform["smoke"] + # Build the actual inference service name (includes deployment profile) + base_name = inference_service["name"] + deployment_profile_name = runtime_config.get_deployment_profile_name() + actual_service_name = f"{base_name}-{deployment_profile_name}" + cleanup_test_resources_command.run( namespace=namespace, - inference_service_name=inference_service["name"], + inference_service_name=actual_service_name, smoke_pod_name=smoke["pod_name"], benchmark_job_name=runtime_config.get_benchmark_job_name(), ) From 5a631ea79bbbd33d162c6af6347b2a032011bd33 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 23:22:17 +0200 Subject: [PATCH 287/344] [llm_d] toolbox: cleanup_test_resources: delete_inference_service: more accurate message --- projects/llm_d/toolbox/cleanup_test_resources/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 475bf49ec..464c9f420 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -193,7 +193,13 @@ def delete_inference_service(args, ctx): f"in {args.namespace} (rc={result.returncode})" ) - return f"Deleted llminferenceservice {args.inference_service_name}" + # Check if something was actually deleted by looking at the output + if result.stdout.strip(): + return f"Deleted llminferenceservice {args.inference_service_name}" + else: + return ( + f"LLMInferenceService {args.inference_service_name} was not found (already cleaned up)" + ) @retry(attempts=90, delay=10, backoff=1.0) From 03e3f43d1ece410bcba6e84155d42ae0eb5d540b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:24:23 +0200 Subject: [PATCH 288/344] [llm_d] orchestration: preflight_phase: make more flexible and easier to extend --- .../orchestration/config.d/platform.yaml | 3 + .../llm_d/orchestration/preflight_phase.py | 62 +++++++++++++++---- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 4e666be30..a90b9fd0f 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -95,3 +95,6 @@ smoke: client_image: quay.io/curl/curl:8.21.0 endpoint_path: /v1/completions request_timeout_seconds: 60 + +preflight: + check_crds: true diff --git a/projects/llm_d/orchestration/preflight_phase.py b/projects/llm_d/orchestration/preflight_phase.py index e6c0b17c0..86ab3eddd 100644 --- a/projects/llm_d/orchestration/preflight_phase.py +++ b/projects/llm_d/orchestration/preflight_phase.py @@ -3,22 +3,19 @@ import logging from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.library import ci as ci_lib from projects.llm_d.orchestration import runtime_config logger = logging.getLogger(__name__) -def run() -> int: - """Run preflight checks before testing phase. - - Validates that required CRDs exist in the cluster: - - leaderworkersets.leaderworkerset.x-k8s.io - - llminferenceservices.serving.kserve.io +def check_crds() -> bool: + """Check that required CRDs exist in the cluster. Returns: - 0 on success, non-zero on failure + True if all CRDs are present, False otherwise """ - logger.info("Starting preflight checks") + logger.info("Checking required CRDs") # Get required CRDs from platform configuration platform = runtime_config.get_platform_config() @@ -36,10 +33,49 @@ def run() -> int: logger.info(f"CRD found: {crd_name}") if missing_crds: - logger.error( - f"Preflight check failed - missing {len(missing_crds)} required CRDs: {', '.join(missing_crds)}" + error_message = f"Missing {len(missing_crds)} required CRDs:\n" + "\n".join( + f"- {crd}" for crd in missing_crds ) - return 1 + logger.error(f"CRD check failed - {error_message}") + ci_lib.add_notification_file("MISSING_CRDS", error_message) + return False + + logger.info("CRD check passed - all required CRDs are available") + return True + + +def run() -> int: + """Run preflight checks before testing phase. + + Validates that required components exist in the cluster based on + platform configuration settings. - logger.info("Preflight checks completed successfully - all required CRDs are available") - return 0 + Returns: + 0 on success, non-zero on failure + """ + logger.info("Starting preflight checks") + + platform = runtime_config.get_platform_config() + preflight_config = platform.get("preflight", {}) + + # Track overall success across all checks + all_checks_passed = True + + # Run CRD checks if enabled + if preflight_config.get("check_crds", True): + if not check_crds(): + all_checks_passed = False + else: + logger.info("CRD checks disabled by platform configuration") + + # Future checks can be added here: + # if preflight_config.get("check_operators", True): + # if not check_operators(): + # all_checks_passed = False + + if all_checks_passed: + logger.info("Preflight checks completed successfully") + return 0 + else: + logger.error("One or more preflight checks failed") + return 1 From 6f8ec99f95496c5eafe887567c381be027c40cc3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:17:41 +0200 Subject: [PATCH 289/344] [llm_d] orchestration: prepare_phase: make the storage_class_name field mandatory --- projects/llm_d/orchestration/prepare_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 533596f69..309b2a714 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -287,7 +287,7 @@ def prepare_model_cache() -> None: "model_uri": model_uri, "pvc_size": model_cache["pvc"]["size"], "access_mode": model_cache["pvc"]["access_mode"], - "storage_class_name": model_cache["pvc"].get("storage_class_name"), + "storage_class_name": model_cache["pvc"]["storage_class_name"], "pvc_name_prefix": pvc_name_prefix, "model_directory_name": model_cache["pvc"]["model_directory_name"], } From 679f735f48fd9f5411cd51a1b24c59de5d43097a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:17:58 +0200 Subject: [PATCH 290/344] [llm_d] orchestration: presets: janus: new preset file --- .../llm_d/orchestration/presets.d/janus.yaml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/janus.yaml diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml new file mode 100644 index 000000000..4026bc346 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -0,0 +1,50 @@ +__multiple: true + +janus: + runtime.namespace: kpouget-dev + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + model_cache.pvc.storage_class_name: nfs + deployments.pd.resources: composite.dra.io/gpu-nic-pair + platform.gateway.status_address_name: null + platform.gateway.status_address_name: gateway-internal + +janus-intelligent: + extends: + - janus + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-pd: + extends: + - janus + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-smoke: + extends: + - janus + - llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + + runtime.benchmark_key: + - short From f134f63cace93ff9693c2835e1749f650d165757 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:39:51 +0200 Subject: [PATCH 291/344] [llm_d] orchestration: test_phase: use MuteStdOut to mute the finalizers --- projects/llm_d/orchestration/test_phase.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index df25e3c61..d091df649 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -109,6 +109,7 @@ def _run_finalizer( **kwargs, ): try: + # with MuteStdOut(reason=f"Finalizer: {description}"): callback(**kwargs) except Exception: if primary_exc is None: @@ -122,24 +123,24 @@ def _run_finalizer( capture_namespace_events = platform["artifacts"]["capture_namespace_events"] finalizer_exc = _run_finalizer( - "capture inference-service state", + "capturing inference-service state", capture_inference_service_state, ) finalizer_exc = _run_finalizer( - "write endpoint URL", + "writing endpoint URL", write_endpoint_url, artifact_dir=env.ARTIFACT_DIR, endpoint_url=endpoint_url, ) finalizer_exc = _run_finalizer( - "capture namespace events", + "capturing namespace events", capture_namespace_events_after_test, artifact_dir=env.ARTIFACT_DIR, namespace=namespace, capture_namespace_events=capture_namespace_events, ) finalizer_exc = _run_finalizer( - "cleanup runtime resources", + "cleaning up runtime resources", cleanup_test_resources, ) From 0aea34d1fc8a35857805d0066b8f814f41a79f05 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 14:50:00 +0200 Subject: [PATCH 292/344] [llm_d] orchestration: test_phase: add flag to wait for pod scheduling --- projects/llm_d/orchestration/test_phase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index d091df649..8271a81f7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -243,11 +243,15 @@ def deploy_inference_service() -> str: # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) + # Get scheduling wait configuration + wait_pods_scheduled = config.project.get_config("runtime.kserve.wait_long_scheduling") + endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), gateway_status_address_name=gateway["status_address_name"], dry_run=dry_run, + wait_pods_scheduled=wait_pods_scheduled, ) if dry_run: From 6cc61128c8f8ee5618b6d6043312b9641e905e55 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 16:29:35 +0200 Subject: [PATCH 293/344] [llm_d] orchestration: config: runtime: allow waiting for a long time on kserve readiness --- projects/llm_d/orchestration/config.d/runtime.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index d1ad2e850..90c370119 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -8,6 +8,7 @@ namespace: forge-llm-d kserve: dry_run: false wait_readiness: true + wait_long_scheduling: false kueue: enabled: false queue_name: default-queue From 61bbe24ab19b7c514c3ea6d05335473bee84a5cf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:23:06 +0200 Subject: [PATCH 294/344] [llm_d] orchestration: render_inference_service: slugginfy the llmisvc name --- projects/llm_d/orchestration/render_inference_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 95c64f6c3..c893c75b9 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -197,7 +197,6 @@ def _render_standard_deployment( name = slugify_identifier(name) name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name - manifest["spec"]["replicas"] = deployment_profile["replicas"] serving_container = manifest["spec"]["template"]["containers"][0] From 0cd797de64ba9670abfa038ade19fb5f2fa74194 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:52:55 +0200 Subject: [PATCH 295/344] [llm_d] orchestration: make the scheduler node selector configurable --- .../orchestration/config.d/platform.yaml | 3 +++ .../manifests/llminferenceservice.yaml | 2 -- .../llm_d/orchestration/runtime_config.py | 20 ++++++++++--------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index a90b9fd0f..3f911b06d 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -86,6 +86,9 @@ inference_service: name: llm-d template: manifests/llminferenceservice.yaml workload_deployment_name_suffix: -kserve + scheduler: + node_selector: + nvidia.com/gpu.present: "true" artifacts: capture_namespace_events: true diff --git a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml index c3043bf31..822fcdcbc 100644 --- a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml +++ b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml @@ -17,8 +17,6 @@ spec: template: containers: - name: main - nodeSelector: - nvidia.com/gpu.present: "true" route: {} gateway: {} template: diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 03cd2e7c6..93c071269 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -490,17 +490,19 @@ def get_deployment_profile() -> dict[str, Any]: resolved_profile = deep_merge(resolved_profile, scheduler_content) - # force the scheduler to run on a GPU node. - # 'gpu.present: true' doesn't work on AKS/CKS - scheduler_node_selector = { - "scheduler": { - "template": { - "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + # Configure scheduler node selector from platform config + platform_node_selector = config.project.get_config( + "platform.inference_service.scheduler.node_selector" + ) + if platform_node_selector: + scheduler_node_selector = { + "scheduler": { + "template": { + "nodeSelector": platform_node_selector, + } } } - } - - resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) + resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) return resolved_profile From ec20b6a013fce8c0621e6167c114ee0246c61ed9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 09:35:28 +0200 Subject: [PATCH 296/344] [llm_d] orchestration: cli: new 'deploy_and_test' entrypoint --- projects/llm_d/orchestration/ci.py | 38 +++++++++--- projects/llm_d/orchestration/cli.py | 71 ++++++++++++++++------ projects/llm_d/orchestration/test_phase.py | 34 +++++++++++ 3 files changed, 118 insertions(+), 25 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index 152603660..de85604bc 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -10,6 +10,7 @@ import click +from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome from projects.core.agentic.config_review import trigger_config_review_for_ci from projects.core.agentic.on_failure import agent_review_on_failure from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint @@ -21,11 +22,11 @@ caliper_export_list_optional_vaults, caliper_export_list_vaults, ) +from projects.core.library.postprocess import run_postprocess_after_test from projects.core.library.replot import caliper_replot_entrypoint from projects.llm_d.orchestration.cleanup_phase import run as cleanup_toolbox_run from projects.llm_d.orchestration.preflight_phase import run as preflight_toolbox_run from projects.llm_d.orchestration.prepare_sequence import run_prepare_sequence -from projects.llm_d.orchestration.test_phase import run as test_toolbox_run logger = logging.getLogger(__name__) RHOAI_CUSTOM_CATALOG_VAULTS = [ @@ -121,17 +122,38 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" - from projects.llm_d.orchestration import runtime_config # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) - exit_code = 0 - for run_spec in runtime_config.get_run_specs(): - with runtime_config.activate_run_spec(run_spec): - with env.NextArtifactDir(run_spec.artifact_dirname): - exit_code = max(exit_code, test_toolbox_run()) - return exit_code + # Run all tests first + from projects.llm_d.orchestration.test_phase import run_all_tests + + max_exit_code = run_all_tests(stop_on_error=False) + + test_failed = max_exit_code != 0 + failure_message = f"Tests completed with exit code {max_exit_code}" if test_failed else None + + # Run post-processing once after all tests + try: + if test_failed and failure_message: + outcome = TestPhaseOutcome("FAILED", failure_message) + elif max_exit_code == 0: + outcome = TestPhaseOutcome("SUCCESS") + else: + outcome = TestPhaseOutcome("FAILED", f"exit_code={max_exit_code}") + + status = run_postprocess_after_test(env.BASE_ARTIFACT_DIR, test_outcome=outcome) + + if status is None or not status.get("success", False): + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + except Exception: + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + return max_exit_code @main.command() diff --git a/projects/llm_d/orchestration/cli.py b/projects/llm_d/orchestration/cli.py index 189b81ecc..362f73791 100755 --- a/projects/llm_d/orchestration/cli.py +++ b/projects/llm_d/orchestration/cli.py @@ -4,7 +4,6 @@ """ import logging -import sys import types from pathlib import Path @@ -16,13 +15,6 @@ logger = logging.getLogger(__name__) -def init(): - """Initialize LLM-D orchestration environment""" - env.init() - run.init() - config.init(Path(__file__).parent) - - @click.group() @click.option( "--preset", @@ -33,18 +25,63 @@ def init(): def main(ctx, preset): """CLI Operations.""" ctx.ensure_object(types.SimpleNamespace) - init() - if not preset: - return + env.init() + run.init() + + if preset: + config.write_variables_override(presets=preset) + + config.init(Path(__file__).parent) + +@main.command() +@click.option( + "--deployment-profile", + multiple=True, + help="The deployment profile(s) to deploy", +) +@click.option( + "--benchmark-key", + multiple=True, + help="The benchmark key(s) to launch.", +) +@click.option( + "--stop-on-error/--continue-on-error", + default=True, + help="Stop on the first test error (default: true)", +) +@click.pass_context +def deploy_and_test(ctx, deployment_profile, benchmark_key, stop_on_error) -> int: + """Test LLM-D with specified deployment profiles and benchmark keys.""" try: - for preset_name in preset: - logger.info(f"Applying preset: {preset_name}") - config.project.apply_preset(preset_name) - except ValueError as e: - logger.error(f"Failed to apply preset '{preset_name}': {e}") - sys.exit(1) + # Configure runtime settings based on provided options + if deployment_profile: + config.project.set_config("runtime.deployment_profile", list(deployment_profile)) + + if benchmark_key: + config.project.set_config("runtime.benchmark_key", list(benchmark_key)) + + # Initialize vaults for deployment + from projects.llm_d.orchestration.ci import init_vaults_for_phase + from projects.llm_d.orchestration.test_phase import run_all_tests + + # Test execution + logger.info("Starting test phase...") + init_vaults_for_phase("test") + + max_exit_code = run_all_tests(stop_on_error=stop_on_error) + + if max_exit_code != 0: + logger.error(f"Test phase completed with exit code {max_exit_code}") + return max_exit_code + + logger.info("All tests completed successfully!") + return 0 + + except Exception: + logger.exception("Deploy and test failed") + return 1 main.add_command(postprocess_command) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 8271a81f7..c5d87a337 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -86,6 +86,40 @@ def create_test_labels() -> None: logger.info("Created test labels: %s", labels) +def run_all_tests(stop_on_error: bool = False) -> int: + """Run tests for all run specifications without post-processing. + + Args: + stop_on_error: If True, stop on the first test failure + + Returns: + Maximum exit code from all tests + """ + from projects.llm_d.orchestration import runtime_config + + max_exit_code = 0 + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + with env.NextArtifactDir(run_spec.artifact_dirname): + try: + exit_code = do_test() + max_exit_code = max(max_exit_code, exit_code) + + if exit_code != 0 and stop_on_error: + logger.error( + f"Test failed with exit code {exit_code}, stopping due to stop_on_error" + ) + return exit_code + except Exception as e: + logger.exception(f"Test failed with exception: {e}") + max_exit_code = 1 + if stop_on_error: + logger.error("Stopping due to stop_on_error") + return 1 + + return max_exit_code + + def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" From 798b762cca60d77eb560d7d5dcb65c175be12365 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:04:11 +0200 Subject: [PATCH 297/344] [llm_d] orchestration: improve the P/D config and rendering --- .../orchestration/config.d/deployments.yaml | 13 ++- .../orchestration/render_inference_service.py | 102 ++++++++++++------ .../llm_d/orchestration/runtime_config.py | 16 +++ 3 files changed, 97 insertions(+), 34 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index f25bdfdf8..ac2a14f55 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -40,10 +40,15 @@ profiles: templates: pd: scheduler_manifest: {} # scheduler enabled with default config - tensor_parallelism: FROM_NAME - pd_config: - prefill_pods: FROM_NAME - decode_pods: FROM_NAME + tensor_parallelism: FROM_NAME # Extract main deployment tensor parallelism from profile name + + prefill: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + decode: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + intelligentrouting: scheduler_manifest: {} # scheduler enabled with default config diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index c893c75b9..decfbaf91 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -70,7 +70,7 @@ def render_inference_service_from_parts( manifest = _load_yaml(template_path) # Check if this is a P/D deployment - is_pd_deployment = "pd_config" in deployment_profile + is_pd_deployment = "prefill" in deployment_profile name = inference_service["name"] if deployment_profile_name: @@ -128,6 +128,50 @@ def render_inference_service_from_parts( return rendered_manifest +def handle_pd_resources( + base_resources: dict[str, Any], + deployment_profile: dict[str, Any], + is_prefill: bool = False, +) -> None: + """Handle P/D resource configuration, including DRA support. + + Args: + base_resources: Base resources dict to modify in-place + deployment_profile: Deployment profile configuration + is_prefill: Whether this is for a prefill container + """ + pd_resources = config.project.get_config("deployments.pd.resources") + + if pd_resources == "composite.dra.io/gpu-nic-pair": + # Handle DRA (Dynamic Resource Allocation) case + if is_prefill: + # For prefill pods, use prefill tensor parallelism + tensor_parallelism = deployment_profile["prefill"]["tensor_parallelism"] + else: + # For decode pods, use decode tensor parallelism + tensor_parallelism = deployment_profile["decode"]["tensor_parallelism"] + + # Override GPU resources for DRA + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + # Set nvidia.com/gpu to 0 and add composite DRA resource + base_resources[bound]["nvidia.com/gpu"] = "0" + base_resources[bound][pd_resources] = str(tensor_parallelism) + elif isinstance(pd_resources, dict): + # Normal case: pd_resources is a dictionary to merge + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + base_resources[bound].update(pd_resources) + elif pd_resources is not None: + # Handle unexpected pd_resources type + raise ValueError( + f"Unexpected type for deployments.pd.resources: {type(pd_resources)}, value: {pd_resources}" + ) + # If pd_resources is None, do nothing + + def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: tensor_parallelism = deployment_profile["tensor_parallelism"] profile_resources = deployment_profile.get("resources", {}) @@ -327,13 +371,8 @@ def _build_pd_pod_template( # For decode pods, use main tensor parallelism base_resources = _build_serving_resources(deployment_profile) - # Add P/D extra resources to both requests and limits - pd_resources = config.project.get_config("deployments.pd.resources") - - for bound in ("requests", "limits"): - if bound not in base_resources: - base_resources[bound] = {} - base_resources[bound].update(pd_resources) + # Handle P/D extra resources + handle_pd_resources(base_resources, deployment_profile, is_prefill) # Build container configuration container = { @@ -393,34 +432,29 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: full_annotation_key = f"{kueue_prefix}{annotation_key}" manifest["metadata"]["annotations"][full_annotation_key] = annotation_value - # Apply Kueue annotations to router scheduler pod template if it exists + # Apply Kueue annotations to router scheduler if it exists if ( "spec" in manifest and "router" in manifest["spec"] and "scheduler" in manifest["spec"]["router"] ): - scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + scheduler = manifest["spec"]["router"]["scheduler"] - # Ensure metadata exists in scheduler template - if "metadata" not in scheduler_template: - scheduler_template["metadata"] = {} - if "annotations" not in scheduler_template["metadata"]: - scheduler_template["metadata"]["annotations"] = {} - if "labels" not in scheduler_template["metadata"]: - scheduler_template["metadata"]["labels"] = {} + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} - # Apply the same Kueue annotations to the scheduler pod template + # Apply the same Kueue annotations to the scheduler for annotation_key, annotation_value in kueue_annotations.items(): full_annotation_key = f"{kueue_prefix}{annotation_key}" - scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + scheduler["annotations"][full_annotation_key] = annotation_value - # Apply the same Kueue labels to the scheduler pod template + # Apply the same Kueue labels to the scheduler for label_key, label_value in kueue_labels.items(): full_label_key = f"{kueue_prefix}{label_key}" - scheduler_template["metadata"]["labels"][full_label_key] = label_value - - # Update the scheduler template back to the data structure - manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template + scheduler["labels"][full_label_key] = label_value # Calculate pod group total count: 1 scheduler + number of replicas replicas = manifest.get("spec", {}).get("replicas", 1) @@ -448,10 +482,18 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: pod_group_name = manifest["metadata"]["name"] manifest["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name - # Also add pod-group-name label to scheduler template if it exists + # Also add required Kueue annotations/labels to scheduler if it exists if has_scheduler: - scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) - if "metadata" in scheduler_template and "labels" in scheduler_template["metadata"]: - scheduler_template["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = ( - pod_group_name - ) + scheduler = manifest["spec"]["router"]["scheduler"] + + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} + + # Add the same pod-group annotations and labels to scheduler + scheduler["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) + scheduler["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 93c071269..1efebf46c 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -515,12 +515,28 @@ def get_pd_config() -> dict[str, Any]: def get_prefill_pod_count() -> int: """Get number of prefill pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "prefill_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails pd_config = get_pd_config() return pd_config.get("prefill_pods", 1) def get_decode_pod_count() -> int: """Get number of decode pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "decode_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails pd_config = get_pd_config() return pd_config.get("decode_pods", 1) From 4bbdbfa985e3ae4154c3775e2b931401d7e65af7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:46:50 +0200 Subject: [PATCH 298/344] [llm_d] orchestration: runtime_config: remove legacy check --- projects/llm_d/orchestration/runtime_config.py | 15 ++------------- projects/llm_d/tests/test_profiles.py | 8 -------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 1efebf46c..5dd428446 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -53,16 +53,7 @@ def ensure_artifact_directories(artifact_dir: Path) -> None: def _get_runtime_value(key: str) -> Any: - return config.project.get_config(f"runtime.{key}", None) - - -def _assert_no_legacy_model_key() -> None: - legacy_model_key = _get_runtime_value("model_key") - if legacy_model_key not in (None, ""): - raise ValueError( - "llm_d no longer supports runtime.model_key. " - "Use runtime.model_name with a literal Hugging Face model name instead." - ) + return config.project.get_config(f"runtime.{key}", None, warn=False) def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: @@ -256,7 +247,6 @@ def _load_scheduler_with_epp_config( # Load the router template router_template_path = config_dir / router_template_file if not router_template_path.exists(): - # Fallback to legacy behavior if template doesn't exist raise FileNotFoundError(f"Router template not found at {router_template_path}") # Load the EPP config content @@ -318,7 +308,7 @@ def get_namespace() -> str: def get_model_name() -> str: """Get the selected Hugging Face model name""" - _assert_no_legacy_model_key() + model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") if len(model_names) != 1: raise ValueError( @@ -560,7 +550,6 @@ def get_smoke_request() -> dict[str, Any]: def get_run_specs() -> list[RunSpec]: - _assert_no_legacy_model_key() model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") profile_names = _normalize_string_or_list( _get_runtime_value("deployment_profile"), diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 54180e8dc..6fd10e53c 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -650,14 +650,6 @@ def test_benchmark_deployment_overrides_empty_when_no_benchmark() -> None: assert overrides == {} -def test_runtime_rejects_legacy_model_key_path() -> None: - _init_project_config() - - core_config.project.config.setdefault("runtime", {})["model_key"] = "qwen3-0-6b" - with pytest.raises(ValueError, match="runtime.model_key"): - runtime_config.get_run_specs() - - def test_render_uses_sanitized_model_name_and_profile_resources() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) From b6c0e53713c1318fc8aed0a0551b63843f0bcbc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 12:23:56 +0200 Subject: [PATCH 299/344] [llm_d] orchestration: runtime_config: mute some config.project.get_config --- projects/llm_d/orchestration/runtime_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 5dd428446..7b08b0546 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -298,7 +298,9 @@ def get_job_name() -> str: def get_platform_config() -> dict[str, Any]: """Get the normalized platform configuration""" - return normalize_platform_config(copy.deepcopy(config.project.get_config("platform"))) + return normalize_platform_config( + copy.deepcopy(config.project.get_config("platform", print=False)) + ) def get_namespace() -> str: @@ -340,7 +342,7 @@ def get_served_model_name(model_name: str | None = None) -> str: def get_model_cache_config() -> dict[str, Any]: """Get the model cache configuration""" - return copy.deepcopy(config.project.get_config("model_cache")) + return copy.deepcopy(config.project.get_config("model_cache", print=False)) def get_benchmark_keys() -> list[str]: @@ -349,9 +351,9 @@ def get_benchmark_keys() -> list[str]: def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: benchmark = copy.deepcopy( - config.project.get_config(f"workloads.benchmarks['{benchmark_name}']") + config.project.get_config(f"workloads.benchmarks['{benchmark_name}']", print=False) ) - workload_defaults = copy.deepcopy(config.project.get_config("workloads")) + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") for key in default_keys: @@ -438,7 +440,7 @@ def _resolve_template_profile( def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() - deployment_config = copy.deepcopy(config.project.get_config("deployments")) + deployment_config = copy.deepcopy(config.project.get_config("deployments", print=False)) # First try direct profile lookup in profiles section profiles = deployment_config.get("profiles", {}) From fe33de4d866ecc3491e43eeb50b07bceca6999a9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:29:29 +0200 Subject: [PATCH 300/344] [llm_d] orchestration: runtime_config: use the deployment in the test dirname --- projects/llm_d/orchestration/runtime_config.py | 2 +- projects/llm_d/tests/test_profiles.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 7b08b0546..257b63502 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -594,7 +594,7 @@ def get_run_specs() -> list[RunSpec]: benchmark_key=bench_key, benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=f"llmd__{bench_key or 'default'}", + artifact_dirname=f"llmd__{bench_key or 'default'}__{profile_slug}", ) ) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 6fd10e53c..9409efce6 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -600,7 +600,7 @@ def test_single_benchmark_key_backward_compatible() -> None: assert len(run_specs) == 1 assert run_specs[0].benchmark_key == "short" assert run_specs[0].benchmark_slug == "short" - assert run_specs[0].artifact_dirname == "llmd__short" + assert run_specs[0].artifact_dirname == "llmd__short__approximate-prefix-cache" def test_activate_run_spec_sets_benchmark_key() -> None: From 50866153b232c7420ed2487b131566af39b2013a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:29:54 +0200 Subject: [PATCH 301/344] [llm_d] orchestration: test_phase: don't put the test in the llmd_test dir --- projects/llm_d/orchestration/test_phase.py | 63 +++++++++++----------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c5d87a337..15f8e3173 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -200,39 +200,36 @@ def do_test() -> int: primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None - with env.NextArtifactDir("llmd_test"): - try: - # Create test labels with actual model and profile information - create_test_labels() - - endpoint_url = deploy_inference_service() - - if dry_run: - logging.warning("Running in dry-run mode, skipping the rest of the test steps") - return 0 - - if not endpoint_url: - raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") - run_smoke_request(endpoint_url=endpoint_url) - - run_guidellm_benchmark(endpoint_url=endpoint_url) - except Exception: - primary_exc = sys.exc_info() - except SignalInterrupt: - primary_exc = sys.exc_info() - finally: - do_finalizers = config.project.get_config("runtime.run_test_finalizers") - if primary_exc and isinstance(primary_exc[1], SignalInterrupt): - logging.warning("Caught a SignalInterrupt, skipping the finalizers") - do_finalizers = False - - if dry_run: - do_finalizers = False - - if do_finalizers: - primary_exc, finalizer_exc = run_finalizers( - endpoint_url, primary_exc, finalizer_exc - ) + try: + # Create test labels with actual model and profile information + create_test_labels() + + endpoint_url = deploy_inference_service() + + if dry_run: + logging.warning("Running in dry-run mode, skipping the rest of the test steps") + return 0 + + if not endpoint_url: + raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") + run_smoke_request(endpoint_url=endpoint_url) + + run_guidellm_benchmark(endpoint_url=endpoint_url) + except Exception: + primary_exc = sys.exc_info() + except SignalInterrupt: + primary_exc = sys.exc_info() + finally: + do_finalizers = config.project.get_config("runtime.run_test_finalizers") + if primary_exc and isinstance(primary_exc[1], SignalInterrupt): + logging.warning("Caught a SignalInterrupt, skipping the finalizers") + do_finalizers = False + + if dry_run: + do_finalizers = False + + if do_finalizers: + primary_exc, finalizer_exc = run_finalizers(endpoint_url, primary_exc, finalizer_exc) if primary_exc is not None: raise primary_exc[1].with_traceback(primary_exc[2]) From e361302cd9d9f818a36b976a326ebacdfe2f5d0f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:40:33 +0200 Subject: [PATCH 302/344] [llm_d] orchestration: test_phase: capture the llmisvc state with the correct name --- projects/llm_d/orchestration/test_phase.py | 58 ++++++++++++++++------ 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 15f8e3173..581cd7621 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -134,6 +134,7 @@ def run() -> int: def run_finalizers( endpoint_url: str | None, + llmisvc_name: str | None, primary_exc: tuple[type[BaseException], BaseException, Any] | None, finalizer_exc: tuple[type[BaseException], BaseException, Any] | None, ) -> tuple[type[BaseException], BaseException, Any] | None: @@ -156,10 +157,16 @@ def _run_finalizer( platform = runtime_config.get_platform_config() capture_namespace_events = platform["artifacts"]["capture_namespace_events"] - finalizer_exc = _run_finalizer( - "capturing inference-service state", - capture_inference_service_state, - ) + # Only capture service state if we have the llmisvc_name + if llmisvc_name: + finalizer_exc = _run_finalizer( + "capturing inference-service state", + capture_inference_service_state, + llmisvc_name=llmisvc_name, + ) + else: + logging.warning("No llmisvc name received, cannot capture the llmisvc state") + finalizer_exc = _run_finalizer( "writing endpoint URL", write_endpoint_url, @@ -197,6 +204,7 @@ def do_test() -> int: ensure_kueue_local_queue() endpoint_url: str | None = None + llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None @@ -204,7 +212,20 @@ def do_test() -> int: # Create test labels with actual model and profile information create_test_labels() - endpoint_url = deploy_inference_service() + # Generate the LLMInferenceService name before deployment + # so we have it available even if deployment fails + from projects.core.dsl.utils import slugify_identifier + + platform = runtime_config.get_platform_config() + inference_service = platform["inference_service"] + base_name = inference_service["name"] + deployment_profile_name = runtime_config.get_deployment_profile_name() + llmisvc_name = ( + f"{base_name}-{deployment_profile_name}" if deployment_profile_name else base_name + ) + llmisvc_name = slugify_identifier(llmisvc_name) + + endpoint_url = deploy_inference_service(llmisvc_name) if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") @@ -229,7 +250,9 @@ def do_test() -> int: do_finalizers = False if do_finalizers: - primary_exc, finalizer_exc = run_finalizers(endpoint_url, primary_exc, finalizer_exc) + primary_exc, finalizer_exc = run_finalizers( + endpoint_url, llmisvc_name, primary_exc, finalizer_exc + ) if primary_exc is not None: raise primary_exc[1].with_traceback(primary_exc[2]) @@ -240,11 +263,14 @@ def do_test() -> int: return 0 -def deploy_inference_service() -> str: - """Deploy LLMInferenceService and return endpoint URL. +def deploy_inference_service(llmisvc_name: str) -> str: + """Deploy LLMInferenceService and return endpoint URL and service name. + + Args: + llmisvc_name: The name of the LLMInferenceService to deploy Returns: - Gateway endpoint URL for the deployed service + Gateway endpoint URL """ logger.info("Starting LLMInferenceService deployment") @@ -253,6 +279,9 @@ def deploy_inference_service() -> str: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] + inference_service = platform["inference_service"] + + # llmisvc_name is now passed as a parameter dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") @@ -289,7 +318,7 @@ def deploy_inference_service() -> str: llmisvc_manifest_path = endpoint_url logger.info("Dry-run completed: LLMInferenceService manifest prepared:") logger.info(llmisvc_manifest_path) - return llmisvc_manifest_path + return llmisvc_manifest_path, llmisvc_name logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url @@ -391,15 +420,12 @@ def run_guidellm_benchmark(*, endpoint_url: str) -> None: ) -def capture_inference_service_state() -> None: - # Load config where it's consumed - +def capture_inference_service_state(llmisvc_name: str) -> None: + """Capture inference service state for the given llmisvc name.""" namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] capture_llmisvc_state.run( - llmisvc_name=inference_service["name"], + llmisvc_name=llmisvc_name, namespace=namespace, ) From e199f0b5f72afd55d350233f789f4c7e0599d1af Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:22:37 +0200 Subject: [PATCH 303/344] [llm_d] orchestration: include VLLM args in the workload configuration --- .../orchestration/config.d/deployments.yaml | 1 - .../orchestration/config.d/workloads.yaml | 8 ++++ .../orchestration/render_inference_service.py | 45 +++++++++++++------ .../llm_d/orchestration/runtime_config.py | 34 ++++++++++++++ projects/llm_d/orchestration/test_phase.py | 2 + 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index ac2a14f55..a2b29c769 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -6,7 +6,6 @@ defaults: tensor_parallelism: 1 vllm_extra: args: - - --max-model-len=4096 - --gpu-memory-utilization=0.92 - --trust-remote-code - --no-enable-log-requests diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 5136666c9..0e29ea5eb 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -11,6 +11,10 @@ smoke_requests: temperature: 0.7 benchmarks: + default: + vllm_args: + max_model_len: 4096 + short: timeout_seconds: 900 outputs: json @@ -31,6 +35,8 @@ benchmarks: max_seconds: 600 heavy-heterogeneous: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent @@ -39,6 +45,8 @@ benchmarks: max_seconds: 600 multi-turn: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index decfbaf91..2daee9e10 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -64,6 +64,7 @@ def render_inference_service_from_parts( deployment_profile: dict[str, Any], model_cache: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render an llm_d-owned LLMInferenceService manifest from concrete runtime inputs.""" template_path = Path(config_dir) / inference_service["template"] @@ -115,11 +116,11 @@ def render_inference_service_from_parts( if is_pd_deployment: rendered_manifest = _render_pd_deployment( - manifest, deployment_profile, deployment_profile_name + manifest, deployment_profile, deployment_profile_name, workload ) else: rendered_manifest = _render_standard_deployment( - manifest, deployment_profile, deployment_profile_name + manifest, deployment_profile, deployment_profile_name, workload ) # Apply Kueue configuration if enabled @@ -212,17 +213,25 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: def _build_vllm_additional_args( deployment_profile: dict[str, Any], + workload: dict[str, Any] | None = None, ) -> str: """Build VLLM_ADDITIONAL_ARGS string based on deployment profile configuration. Args: deployment_profile: The deployment profile configuration + workload: The workload configuration (merged benchmark config) Returns: String suitable for VLLM_ADDITIONAL_ARGS environment variable """ - vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra", {}).get("args")) + vllm_extra = deployment_profile.get("vllm_extra", {}) + vllm_deploy_args = _build_vllm_args(vllm_extra.get("args", {})) + + # Add workload vllm_args if available + if workload and "vllm_args" in workload: + workload_vllm_args = _build_vllm_args(workload["vllm_args"]) + vllm_deploy_args.extend(workload_vllm_args) return " ".join(vllm_deploy_args) @@ -231,6 +240,7 @@ def _render_standard_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render standard (non-P/D) deployment configuration.""" # Check if this is intelligent routing (scheduler_manifest exists) @@ -248,7 +258,7 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - vllm_additional_args = _build_vllm_additional_args(deployment_profile) + vllm_additional_args = _build_vllm_additional_args(deployment_profile, workload) # Add environment variable (don't set generic env vars or args) if "env" not in serving_container: @@ -281,6 +291,7 @@ def _render_pd_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" from .runtime_config import get_decode_pod_count, get_prefill_pod_count @@ -295,14 +306,14 @@ def _render_pd_deployment( manifest["spec"]["prefill"] = { "replicas": get_prefill_pod_count(), "template": _build_pd_pod_template( - deployment_profile, deployment_profile_name, is_prefill=True + deployment_profile, deployment_profile_name, is_prefill=True, workload=workload ), } # Configure main template (decode) manifest["spec"]["replicas"] = get_decode_pod_count() manifest["spec"]["template"] = _build_pd_pod_template( - deployment_profile, deployment_profile_name, is_prefill=False + deployment_profile, deployment_profile_name, is_prefill=False, workload=workload ) return manifest @@ -312,6 +323,7 @@ def _build_pd_pod_template( deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, is_prefill: bool = False, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build pod template for P/D deployment.""" @@ -325,21 +337,26 @@ def _build_pd_pod_template( # For prefill pods, use prefill tensor parallelism from .runtime_config import _extract_value_from_profile_name - prefill_tp = _extract_value_from_profile_name( - deployment_profile_name, "prefill_tensor_parallelism" - ) - # Create modified profile with prefill tensor parallelism - prefill_profile = copy.deepcopy(deployment_profile) - prefill_profile["tensor_parallelism"] = prefill_tp - base_vllm_args = _build_vllm_additional_args(prefill_profile) + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_vllm_args = _build_vllm_additional_args(prefill_profile, workload) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) else: # For decode pods, use main tensor parallelism - base_vllm_args = _build_vllm_additional_args(deployment_profile) + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) # Add P/D extra args pd_extra_args = pd_vllm_extra.get("args", []) all_vllm_args = base_vllm_args.split() + pd_extra_args + vllm_additional_args = " ".join(all_vllm_args) # Build base environment variables diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 257b63502..dcfc9a7f1 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -360,6 +360,12 @@ def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: if key in workload_defaults and key not in benchmark: benchmark[key] = workload_defaults[key] + # Merge vllm_args from default benchmark if not present in specific benchmark + if "vllm_args" not in benchmark: + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) + if "vllm_args" in default_benchmark: + benchmark["vllm_args"] = copy.deepcopy(default_benchmark["vllm_args"]) + benchmark_args = benchmark.get("args", {}) workload_args = workload_defaults.get("args", {}) if workload_args: @@ -381,6 +387,34 @@ def get_benchmark_config() -> dict[str, Any] | None: return _resolve_benchmark_config(benchmark_keys[0]) +def get_workload_config() -> dict[str, Any] | None: + """Get workload configuration, falling back to default if no benchmark is specified.""" + benchmark_config = get_benchmark_config() + if benchmark_config: + return benchmark_config + + # No benchmark specified, return default workload configuration + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) + + if not default_benchmark: + return None + + # Apply same merging logic as _resolve_benchmark_config + default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") + for key in default_keys: + if key in workload_defaults and key not in default_benchmark: + default_benchmark[key] = workload_defaults[key] + + # Merge args from workloads.args to default benchmark args + benchmark_args = default_benchmark.get("args", {}) + workload_args = workload_defaults.get("args", {}) + if workload_args: + default_benchmark["args"] = deep_merge(workload_args, benchmark_args) + + return default_benchmark + + def get_benchmark_deployment_overrides() -> dict[str, Any]: """Return deployment overrides from the active benchmark config, if any.""" benchmark = get_benchmark_config() diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 581cd7621..aff9f02da 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -346,6 +346,7 @@ def _build_inference_service_manifest() -> Path: model_slug = runtime_config.get_model_slug(model_name) deployment_profile = runtime_config.get_deployment_profile() model_cache = runtime_config.get_model_cache_config() + workload = runtime_config.get_workload_config() # Get workload config with vllm_args benchmark_overrides = runtime_config.get_benchmark_deployment_overrides() if benchmark_overrides: @@ -362,6 +363,7 @@ def _build_inference_service_manifest() -> Path: deployment_profile=deployment_profile, model_cache=model_cache, deployment_profile_name=deployment_profile_name, + workload=workload, ) # Write the manifest to artifacts From a04fb6918abe4b3b191ce51ea6c5455e73d2459d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:22:55 +0200 Subject: [PATCH 304/344] [llm_d] orchestration: presets: new preset for llama3-70b --- .../llm_d/orchestration/presets.d/benchmarks.yaml | 10 ++++++++++ projects/llm_d/orchestration/presets.d/models.yaml | 14 ++++++++++++++ .../llm_d/orchestration/presets.d/presets.yaml | 8 -------- 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 projects/llm_d/orchestration/presets.d/benchmarks.yaml create mode 100644 projects/llm_d/orchestration/presets.d/models.yaml diff --git a/projects/llm_d/orchestration/presets.d/benchmarks.yaml b/projects/llm_d/orchestration/presets.d/benchmarks.yaml new file mode 100644 index 000000000..e9dc9e475 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/benchmarks.yaml @@ -0,0 +1,10 @@ +__multiple: true + +concurrent-1k-1k: + runtime.benchmark_key: concurrent-1k-1k + +heavy-heterogeneous: + runtime.benchmark_key: heavy-heterogeneous + +multi-turn: + runtime.benchmark_key: multi-turn diff --git a/projects/llm_d/orchestration/presets.d/models.yaml b/projects/llm_d/orchestration/presets.d/models.yaml new file mode 100644 index 000000000..995860310 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/models.yaml @@ -0,0 +1,14 @@ +__multiple: true + +gpt-oss: + runtime.model_name: openai/gpt-oss-120b + runtime.deployment_profile: distributed-default + runtime.smoke_request_key: default + model_cache.pvc.size: 300Gi + workloads.args: + request_type: text_completions + +llama3-70b: + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic + model_cache.pvc.size: 100Gi + runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 491296ed8..6b42bb701 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -38,14 +38,6 @@ benchmark-short: runtime.smoke_request_key: default runtime.benchmark_key: short -gpt-oss: - runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: distributed-default - runtime.smoke_request_key: default - model_cache.pvc.size: 300Gi - workloads.args: - request_type: text_completions - gpt-oss-120b-inference-scheduling-release: extends: - gpt-oss From 0655e23368a6f039f0715bb7049cfd0a0582bdde Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:23:18 +0200 Subject: [PATCH 305/344] [llm_d] tests: test_deployment_profiles: disable the post-processing --- projects/llm_d/tests/test_deployment_profiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 0663760a3..f23a2731e 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -53,6 +53,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): yaml.safe_dump( { "runtime.kserve.dry_run": True, + "caliper.postprocess.enabled": False, }, sort_keys=True, ), @@ -229,7 +230,7 @@ def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: def _find_generated_llmisvc(artifact_dir: Path) -> Path: - """Find the generated LLMISVC manifest in the artifact directory (legacy single deployment).""" + """Find the generated LLMISVC manifest in the artifact directory.""" deployments = _find_generated_llmisvcs(artifact_dir) if len(deployments) == 1: From 381ee4e4933d1278b4b3df8b5e00903c319e2185 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:10:03 +0200 Subject: [PATCH 306/344] [llm_d] orchestration: add a config flag to reuse the existing llmisvc --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 48 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 90c370119..e306b1a01 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -9,6 +9,7 @@ kserve: dry_run: false wait_readiness: true wait_long_scheduling: false + reuse_existing: false kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index aff9f02da..f99588dd6 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -279,12 +279,58 @@ def deploy_inference_service(llmisvc_name: str) -> str: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - inference_service = platform["inference_service"] # llmisvc_name is now passed as a parameter dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + # Check if we should reuse existing LLMInferenceService + if reuse_existing and not dry_run: + logger.info(f"Checking if LLMInferenceService {llmisvc_name} already exists") + + # Check if the service already exists + try: + existing_llmisvc = oc( + "get", + "llminferenceservice", + llmisvc_name, + "-n", + namespace, + check=False, + ) + + if existing_llmisvc.returncode == 0: + logger.info( + f"Found existing LLMInferenceService {llmisvc_name}, attempting to reuse" + ) + + # Import and use the toolbox function to extract URL + from projects.kserve.toolbox.deploy_llmisvc import try_resolve_endpoint_url + + endpoint_url = try_resolve_endpoint_url( + namespace=namespace, + inference_service_name=llmisvc_name, + gateway_status_address_name=gateway["status_address_name"], + ) + + if endpoint_url: + logger.info(f"Successfully reused existing LLMInferenceService: {endpoint_url}") + return endpoint_url + else: + logger.warning( + "Existing LLMInferenceService found but no endpoint URL could be resolved, proceeding with new deployment" + ) + else: + logger.info( + f"LLMInferenceService {llmisvc_name} does not exist, proceeding with new deployment" + ) + + except Exception as e: + logger.warning( + f"Error checking for existing LLMInferenceService: {e}, proceeding with new deployment" + ) # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: From c6a27ea6d338d6da167e2a38927db10444e7dadd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:49:26 +0200 Subject: [PATCH 307/344] [llm_d] orchestration: test_phase: delete the llmisvc & workloads on start --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index e306b1a01..996064747 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -10,6 +10,7 @@ kserve: wait_readiness: true wait_long_scheduling: false reuse_existing: false + delete_all_on_start: true kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f99588dd6..113665f5f 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -31,6 +31,77 @@ logger = logging.getLogger(__name__) +def _delete_resources_by_type(resource_type: str, namespace: str, description: str) -> None: + """Delete all resources of a given type in the namespace. + + Args: + resource_type: Kubernetes resource type (e.g., 'llminferenceservice', 'workload') + namespace: Target namespace + description: Human-readable description for logging + """ + logger.info("Deleting all %s in namespace %s", description, namespace) + + result = oc( + "get", + resource_type, + "-n", + namespace, + "--no-headers", + "-o", + "name", + check=False, + ) + + if result.returncode == 0 and result.stdout.strip(): + resource_names = result.stdout.strip().split("\n") + logger.info( + "Found %d %s to delete: %s", + len(resource_names), + description, + ", ".join(resource_names), + ) + + # Delete all found resources + for resource_name in resource_names: + logger.info("Deleting %s", resource_name) + oc("delete", resource_name, "-n", namespace, check=False) + + logger.info("Successfully deleted all existing %s", description) + else: + logger.info("No existing %s found in namespace %s", description, namespace) + + +def cleanup_existing_resources(namespace: str) -> None: + """Delete all existing LLMInferenceServices and Kueue workloads if configured. + + Args: + namespace: Target namespace for cleanup + """ + delete_all_on_start = config.project.get_config("runtime.kserve.delete_all_on_start", False) + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + if not delete_all_on_start: + return + + if reuse_existing: + logger.info("Skipping cleanup - reuse_existing is enabled") + return + + logger.info("Starting cleanup of existing resources in namespace %s", namespace) + + try: + # Delete LLMInferenceServices + _delete_resources_by_type("llminferenceservice", namespace, "LLMInferenceServices") + + # Also delete Kueue workloads if Kueue is enabled + enable_kueue = config.project.get_config("runtime.kueue.enabled", False) + if enable_kueue: + _delete_resources_by_type("workload", namespace, "Kueue workloads") + + except Exception as e: + logger.warning("Failed to delete existing resources: %s, continuing with test", e) + + def ensure_kueue_local_queue() -> None: """Create LocalQueue when kueue is enabled.""" enable_kueue = config.project.get_config("runtime.kueue.enabled") @@ -203,6 +274,9 @@ def do_test() -> int: # Ensure LocalQueue exists when kueue is enabled ensure_kueue_local_queue() + # Delete all existing resources if configured + cleanup_existing_resources(namespace) + endpoint_url: str | None = None llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None From 89ff380aee8a9fd50e69bb6fb04c7221bf59be57 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 19:09:29 +0200 Subject: [PATCH 308/344] [llm_d] tests: reference_deployments: update --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 6 +++--- .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 10 ++++------ .../deployment-simple-tp4-x4/llmisvc.yaml | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 5be1b330d..c4a16a88b 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -22,7 +22,7 @@ spec: containers: - name: main nodeSelector: - nvidia.com/gpu.deploy.container-toolkit: 'true' + nvidia.com/gpu.present: 'true' route: {} gateway: {} template: @@ -58,5 +58,5 @@ spec: timeoutSeconds: 30 env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index ad5475ddb..8dbe6a352 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -21,8 +21,6 @@ spec: template: containers: - name: main - nodeSelector: - nvidia.com/gpu.present: 'true' route: {} gateway: {} template: @@ -37,8 +35,8 @@ spec: rdma/ib: '1' env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - name: VLLM_NIXL_SIDE_CHANNEL_HOST valueFrom: @@ -59,8 +57,8 @@ spec: rdma/ib: '1' env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - name: VLLM_NIXL_SIDE_CHANNEL_HOST valueFrom: diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 6e3edb0be..f17b89144 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -49,5 +49,5 @@ spec: timeoutSeconds: 30 env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 From e2903884cb27da8a04614e8cc9ce856512928c27 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:33:34 +0200 Subject: [PATCH 309/344] [llm_d] orchestration: test_phase: cleanup with the right llmisvc name --- projects/llm_d/orchestration/test_phase.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 113665f5f..c50d9958c 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -251,9 +251,11 @@ def _run_finalizer( namespace=namespace, capture_namespace_events=capture_namespace_events, ) + finalizer_exc = _run_finalizer( "cleaning up runtime resources", cleanup_test_resources, + llmisvc_name=llmisvc_name or "llmisvc-name-not-available", ) return primary_exc, finalizer_exc @@ -561,8 +563,12 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: endpoint_file.write_text(f"{endpoint_url}\n", encoding="utf-8") -def cleanup_test_resources() -> None: - """Cleanup test resources using the toolbox script""" +def cleanup_test_resources(llmisvc_name: str | None) -> None: + """Cleanup test resources using the toolbox script + + Args: + llmisvc_name: The actual LLMInferenceService name that was deployed, or None if deployment failed + """ # Skip cleanup when in dry-run mode dry_run = config.project.get_config("runtime.kserve.dry_run", False) @@ -570,19 +576,17 @@ def cleanup_test_resources() -> None: logger.info("Skipping cleanup_test_resources - dry-run mode enabled") return + if not llmisvc_name: + logger.warning("No LLMInferenceService name provided, cleanup may be incomplete") + return + namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] smoke = platform["smoke"] - # Build the actual inference service name (includes deployment profile) - base_name = inference_service["name"] - deployment_profile_name = runtime_config.get_deployment_profile_name() - actual_service_name = f"{base_name}-{deployment_profile_name}" - cleanup_test_resources_command.run( namespace=namespace, - inference_service_name=actual_service_name, + inference_service_name=llmisvc_name, smoke_pod_name=smoke["pod_name"], benchmark_job_name=runtime_config.get_benchmark_job_name(), ) From 03c63d6767d2e1203c623eb19287a8fb0ddc2ae5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:37:13 +0200 Subject: [PATCH 310/344] [llm_d] orchestration: test_phase: use the status_address_name only with a scheduler --- projects/llm_d/orchestration/test_phase.py | 98 +++++++++++++++++----- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c50d9958c..4876db1e7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -339,29 +339,21 @@ def do_test() -> int: return 0 -def deploy_inference_service(llmisvc_name: str) -> str: - """Deploy LLMInferenceService and return endpoint URL and service name. +def _try_reuse_existing_service( + namespace: str, llmisvc_name: str, gateway: dict[str, str], reuse_existing: bool, dry_run: bool +) -> str | None: + """Try to reuse an existing LLMInferenceService if enabled and available. Args: - llmisvc_name: The name of the LLMInferenceService to deploy + namespace: Target namespace + llmisvc_name: Name of the LLMInferenceService + gateway: Gateway configuration + reuse_existing: Whether to attempt reuse + dry_run: Whether in dry-run mode Returns: - Gateway endpoint URL + Endpoint URL if reuse successful, None otherwise """ - logger.info("Starting LLMInferenceService deployment") - - # Load config where it's consumed - - namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - gateway = platform["gateway"] - - # llmisvc_name is now passed as a parameter - - dry_run = config.project.get_config("runtime.kserve.dry_run") - wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") - reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) - # Check if we should reuse existing LLMInferenceService if reuse_existing and not dry_run: logger.info(f"Checking if LLMInferenceService {llmisvc_name} already exists") @@ -383,12 +375,26 @@ def deploy_inference_service(llmisvc_name: str) -> str: ) # Import and use the toolbox function to extract URL + # Check if the existing service has a scheduler to determine gateway address + from projects.core.dsl.utils.k8s import oc_get_json from projects.kserve.toolbox.deploy_llmisvc import try_resolve_endpoint_url + existing_service = oc_get_json( + "llminferenceservice", name=llmisvc_name, namespace=namespace + ) + has_scheduler = ( + existing_service.get("spec", {}).get("router", {}).get("scheduler") is not None + ) + + # Use None for status_address_name when existing service has no scheduler + gateway_status_address_name = ( + gateway["status_address_name"] if has_scheduler else None + ) + endpoint_url = try_resolve_endpoint_url( namespace=namespace, inference_service_name=llmisvc_name, - gateway_status_address_name=gateway["status_address_name"], + gateway_status_address_name=gateway_status_address_name, ) if endpoint_url: @@ -408,6 +414,43 @@ def deploy_inference_service(llmisvc_name: str) -> str: f"Error checking for existing LLMInferenceService: {e}, proceeding with new deployment" ) + return None + + +def deploy_inference_service(llmisvc_name: str) -> str: + """Deploy LLMInferenceService and return endpoint URL and service name. + + Args: + llmisvc_name: The name of the LLMInferenceService to deploy + + Returns: + Gateway endpoint URL + """ + logger.info("Starting LLMInferenceService deployment") + + # Load config where it's consumed + + namespace = runtime_config.get_namespace() + platform = runtime_config.get_platform_config() + gateway = platform["gateway"] + + # llmisvc_name is now passed as a parameter + + dry_run = config.project.get_config("runtime.kserve.dry_run") + wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + # Try to reuse existing LLMInferenceService if enabled + endpoint_url = _try_reuse_existing_service( + namespace=namespace, + llmisvc_name=llmisvc_name, + gateway=gateway, + reuse_existing=reuse_existing, + dry_run=dry_run, + ) + if endpoint_url: + return endpoint_url + # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: _prepare_model_cache() @@ -422,6 +465,21 @@ def deploy_inference_service(llmisvc_name: str) -> str: # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() + # Check if the manifest has a scheduler to determine gateway status address name + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) + + # Use None for status_address_name when deploying without a scheduler + gateway_status_address_name = gateway["status_address_name"] if has_scheduler else None + # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) @@ -431,7 +489,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), - gateway_status_address_name=gateway["status_address_name"], + gateway_status_address_name=gateway_status_address_name, dry_run=dry_run, wait_pods_scheduled=wait_pods_scheduled, ) From a7e47b7005d9901f7bc8585e58f6fde05dc38b41 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 22:38:45 +0200 Subject: [PATCH 311/344] [llm_d] orchestration: test_phase: correctly extract the llmisvc name --- projects/llm_d/orchestration/test_phase.py | 70 +++++++++++++++------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 4876db1e7..dbb0d1f6a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -280,10 +280,10 @@ def do_test() -> int: cleanup_existing_resources(namespace) endpoint_url: str | None = None - llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None + actual_llmisvc_name = "llmisvc-na-not-computed" try: # Create test labels with actual model and profile information create_test_labels() @@ -296,12 +296,16 @@ def do_test() -> int: inference_service = platform["inference_service"] base_name = inference_service["name"] deployment_profile_name = runtime_config.get_deployment_profile_name() - llmisvc_name = ( + # Step 1: Build manifest and get actual truncated name + initial_llmisvc_name = ( f"{base_name}-{deployment_profile_name}" if deployment_profile_name else base_name ) - llmisvc_name = slugify_identifier(llmisvc_name) + initial_llmisvc_name = slugify_identifier(initial_llmisvc_name) - endpoint_url = deploy_inference_service(llmisvc_name) + manifest_path, actual_llmisvc_name = build_inference_service_manifest(initial_llmisvc_name) + + # Step 2: Deploy using the actual name from the manifest + endpoint_url = deploy_inference_service_from_manifest(manifest_path, actual_llmisvc_name) if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") @@ -312,7 +316,7 @@ def do_test() -> int: run_smoke_request(endpoint_url=endpoint_url) run_guidellm_benchmark(endpoint_url=endpoint_url) - except Exception: + except Exception as e: primary_exc = sys.exc_info() except SignalInterrupt: primary_exc = sys.exc_info() @@ -327,7 +331,7 @@ def do_test() -> int: if do_finalizers: primary_exc, finalizer_exc = run_finalizers( - endpoint_url, llmisvc_name, primary_exc, finalizer_exc + endpoint_url, actual_llmisvc_name, primary_exc, finalizer_exc ) if primary_exc is not None: @@ -417,25 +421,52 @@ def _try_reuse_existing_service( return None -def deploy_inference_service(llmisvc_name: str) -> str: - """Deploy LLMInferenceService and return endpoint URL and service name. +def build_inference_service_manifest(llmisvc_name: str) -> tuple[Path, str]: + """Build inference service manifest and return path and actual service name. Args: - llmisvc_name: The name of the LLMInferenceService to deploy + llmisvc_name: The initial name for the LLMInferenceService + + Returns: + Tuple of (manifest_path, actual_llmisvc_name) where actual_llmisvc_name may be truncated + """ + # Step 1: Build and write inference service manifest + manifest_path = _build_inference_service_manifest() + + # Step 2: Extract the actual deployed name from the manifest (may be truncated) + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + # Get the actual deployed name from the manifest (may be truncated) + actual_llmisvc_name = manifest["metadata"]["name"] + + logger.info( + "Built LLMInferenceService manifest: %s (actual name: %s)", + manifest_path, + actual_llmisvc_name, + ) + return manifest_path, actual_llmisvc_name + + +def deploy_inference_service_from_manifest(manifest_path: Path, actual_llmisvc_name: str) -> str: + """Deploy LLMInferenceService from pre-built manifest and return endpoint URL. + + Args: + manifest_path: Path to the pre-built manifest + actual_llmisvc_name: The actual name of the LLMInferenceService from the manifest Returns: Gateway endpoint URL """ - logger.info("Starting LLMInferenceService deployment") + logger.info("Starting LLMInferenceService deployment from manifest") # Load config where it's consumed - namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # llmisvc_name is now passed as a parameter - dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) @@ -443,7 +474,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: # Try to reuse existing LLMInferenceService if enabled endpoint_url = _try_reuse_existing_service( namespace=namespace, - llmisvc_name=llmisvc_name, + llmisvc_name=actual_llmisvc_name, gateway=gateway, reuse_existing=reuse_existing, dry_run=dry_run, @@ -462,10 +493,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: rhoai_namespace = platform["rhoai"]["namespace"] wait_kserve_ready.run(namespace=rhoai_namespace) - # Step 3: Build and write inference service manifest - manifest_path = _build_inference_service_manifest() - - # Check if the manifest has a scheduler to determine gateway status address name + # Step 3: Check manifest for scheduler to determine gateway status address name import yaml with manifest_path.open(encoding="utf-8") as f: @@ -495,10 +523,8 @@ def deploy_inference_service(llmisvc_name: str) -> str: ) if dry_run: - llmisvc_manifest_path = endpoint_url - logger.info("Dry-run completed: LLMInferenceService manifest prepared:") - logger.info(llmisvc_manifest_path) - return llmisvc_manifest_path, llmisvc_name + logger.info("Dry-run completed: LLMInferenceService manifest prepared: %s", manifest_path) + return str(manifest_path) logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url From 23ec952016c116d7a4a305a80ea40d5469da0c6f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:55:29 +0200 Subject: [PATCH 312/344] [llm_d] orchestration: config: enable the KPI export --- projects/llm_d/orchestration/config.yaml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 76bc188dc..3ced4607f 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -59,18 +59,13 @@ caliper: enabled: true output: kpis/kpis.json kpis_to_csv: - enabled: false + enabled: true output: kpis/kpis.csv include_header_comments: true artifacts_to_ai_data: enabled: true output_dir: ai_data - analyze: - enabled: false - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke @@ -93,9 +88,11 @@ caliper: include_kpis_json: true include_ai_data: true - analyse_kpis: - enabled: false - output: kpi_analysis.json + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json agentic: enabled: true From 66dbd08c6192bf7579a37e0f847943f89ac19d6f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:58:52 +0200 Subject: [PATCH 313/344] [llm_d] orchestration: test_phase: expose the test completion status in the __test_labels__.yaml --- projects/llm_d/orchestration/test_phase.py | 41 +++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index dbb0d1f6a..b8c25fb67 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -157,6 +157,38 @@ def create_test_labels() -> None: logger.info("Created test labels: %s", labels) +def update_test_labels_with_status(success: bool, message: str) -> None: + """Update __test_labels__.yaml with test execution status. + + Args: + success: True if test succeeded, False if failed + message: Status message describing the result + """ + test_labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + + # Read existing labels + if test_labels_path.exists(): + with test_labels_path.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) + else: + # Fallback if labels file doesn't exist + data = {"version": "1", "labels": {}} + + # Add completion information as separate top-level field + data["completion"] = { + "success": success, + "message": message, + } + + # Write updated labels + with test_labels_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(data, f, sort_keys=False) + + logger.info( + "Updated test labels with completion status: success=%s, message=%s", success, message + ) + + def run_all_tests(stop_on_error: bool = False) -> int: """Run tests for all run specifications without post-processing. @@ -183,6 +215,7 @@ def run_all_tests(stop_on_error: bool = False) -> int: return exit_code except Exception as e: logger.exception(f"Test failed with exception: {e}") + # Note: Status update already handled by do_test() exception handler max_exit_code = 1 if stop_on_error: logger.error("Stopping due to stop_on_error") @@ -309,6 +342,7 @@ def do_test() -> int: if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") + update_test_labels_with_status(True, "Dry-run completed successfully") return 0 if not endpoint_url: @@ -318,8 +352,10 @@ def do_test() -> int: run_guidellm_benchmark(endpoint_url=endpoint_url) except Exception as e: primary_exc = sys.exc_info() - except SignalInterrupt: + update_test_labels_with_status(False, f"Test failed with exception: {str(e)}") + except SignalInterrupt as e: primary_exc = sys.exc_info() + update_test_labels_with_status(False, f"Test interrupted: {str(e)}") finally: do_finalizers = config.project.get_config("runtime.run_test_finalizers") if primary_exc and isinstance(primary_exc[1], SignalInterrupt): @@ -340,6 +376,9 @@ def do_test() -> int: if finalizer_exc is not None: raise finalizer_exc[1].with_traceback(finalizer_exc[2]) + # Update test labels with success status + update_test_labels_with_status(True, "Test completed successfully") + return 0 From e973bdbc42807e5b6ac5c13f0605d8328611cc3a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:07:25 +0200 Subject: [PATCH 314/344] [llm_d] orchestration: disable Kueue when necessary --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../llm_d/orchestration/presets.d/cks.yaml | 1 + .../orchestration/render_inference_service.py | 50 ++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 996064747..bea48e83a 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -15,6 +15,7 @@ kueue: enabled: false queue_name: default-queue local_queue_manifest: manifests/kueue-localqueue.yaml + disable_above_n_gpus: null router: template: file: manifests/scheduler_config/__router_template__.yaml diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index e96a453c3..78cc89024 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -7,6 +7,7 @@ cks: runtime.kserve.wait_readiness: false runtime.kueue.enabled: true runtime.kueue.queue_name: llm-d-cluster-queue + runtime.kueue.disable_above_n_gpus: 8 # Use ReadWriteMany for model cache PVC in CKS environment model_cache.pvc.access_mode: ReadWriteMany diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 2daee9e10..d71e0b22c 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -2,6 +2,7 @@ import copy import hashlib +import logging from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.library import config +logger = logging.getLogger(__name__) + def _load_yaml(path: Path) -> Any: with path.open(encoding="utf-8") as handle: @@ -124,7 +127,7 @@ def render_inference_service_from_parts( ) # Apply Kueue configuration if enabled - _apply_kueue_configuration(rendered_manifest) + _apply_kueue_configuration(rendered_manifest, deployment_profile) return rendered_manifest @@ -405,11 +408,45 @@ def _build_pd_pod_template( return {"containers": [container]} -def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: +def _calculate_total_gpu_usage(deployment_profile: dict[str, Any]) -> int: + """Calculate the total GPU usage for a deployment profile. + + Args: + deployment_profile: The deployment profile configuration + + Returns: + Total number of GPUs required for this deployment + """ + # Check if this is a P/D deployment (has prefill/decode sections) + if "prefill" in deployment_profile and "decode" in deployment_profile: + # P/D deployment: sum up prefill and decode GPU usage + prefill_gpus = ( + deployment_profile["prefill"]["tensor_parallelism"] + * deployment_profile["prefill"]["replicas"] + ) + decode_gpus = ( + deployment_profile["decode"]["tensor_parallelism"] + * deployment_profile["decode"]["replicas"] + ) + return prefill_gpus + decode_gpus + else: + # Standard deployment: tensor_parallelism * replicas + tensor_parallelism = deployment_profile.get("tensor_parallelism", 1) + replicas = deployment_profile.get("replicas", 1) + return tensor_parallelism * replicas + + +def _apply_kueue_configuration( + manifest: dict[str, Any], deployment_profile: dict[str, Any] +) -> None: """Apply Kueue annotations and labels to the ISVC manifest. Based on the implementation from topsail's test_llmd.py. Can be enabled by setting runtime.kserve_use_kueue config. + + Args: + manifest: The Kubernetes manifest to modify + deployment_profile: The deployment profile configuration used to calculate GPU usage """ # Check if kueue annotations should be enabled enable_kueue = config.project.get_config("runtime.kueue.enabled") @@ -417,6 +454,15 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: if not enable_kueue: return + # Calculate total GPU usage from deployment profile + total_gpus = _calculate_total_gpu_usage(deployment_profile) + + # Check if we should skip Kueue due to high GPU usage + disable_above_n_gpus = config.project.get_config("runtime.kueue.disable_above_n_gpus") + if disable_above_n_gpus is not None and total_gpus > disable_above_n_gpus: + logger.info(f"Skipping Kueue labels: {total_gpus} GPUs > {disable_above_n_gpus} threshold") + return + # Configure kueue settings queue_name = config.project.get_config("runtime.kueue.queue_name") kueue_config = { From c25eed5a22316848bd024f09ceaa6e173719e47e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:12:47 +0200 Subject: [PATCH 315/344] [llm_d] orchestration: save the config and fjob --- projects/llm_d/orchestration/test_phase.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index b8c25fb67..7ba815fbc 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -1,12 +1,14 @@ from __future__ import annotations import logging +import shutil import sys from pathlib import Path from typing import Any import yaml +from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier from projects.core.dsl.utils.k8s import oc @@ -156,6 +158,27 @@ def create_test_labels() -> None: write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) + # Dump config.project to config.yaml + config_path = env.ARTIFACT_DIR / "config.yaml" + try: + with config_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(config.project.config, f, sort_keys=False) + logger.info("Saved project configuration to: %s", config_path) + except Exception as e: + logger.warning("Failed to save project configuration: %s", e) + + # Copy fournos_fjob.yaml if available + fournos_source = env.ARTIFACT_DIR / CI_METADATA_DIRNAME / "fournos_fjob.yaml" + if fournos_source.exists(): + fournos_dest = env.ARTIFACT_DIR / "fournos_fjob.yaml" + try: + shutil.copy2(fournos_source, fournos_dest) + logger.info("Copied fournos job file: %s -> %s", fournos_source, fournos_dest) + except Exception as e: + logger.warning("Failed to copy fournos job file: %s", e) + else: + logger.debug("No fournos job file found at: %s", fournos_source) + def update_test_labels_with_status(success: bool, message: str) -> None: """Update __test_labels__.yaml with test execution status. From 20c57bbcf5b92036287e5d4a2b578f0ccb9148ea Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 16:11:05 +0200 Subject: [PATCH 316/344] [llm_d] orchestration: render_inference_service: correctly set the TP level --- projects/llm_d/orchestration/render_inference_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index d71e0b22c..3e869d5ef 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -255,6 +255,7 @@ def _render_standard_deployment( name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name manifest["spec"]["replicas"] = deployment_profile["replicas"] + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["tensor_parallelism"]} serving_container = manifest["spec"]["template"]["containers"][0] serving_container["resources"] = _build_serving_resources(deployment_profile) @@ -308,6 +309,7 @@ def _render_pd_deployment( # Configure prefill section manifest["spec"]["prefill"] = { "replicas": get_prefill_pod_count(), + "parallelism": {"tensor": deployment_profile["prefill"]["tensor_parallelism"]}, "template": _build_pd_pod_template( deployment_profile, deployment_profile_name, is_prefill=True, workload=workload ), @@ -315,6 +317,7 @@ def _render_pd_deployment( # Configure main template (decode) manifest["spec"]["replicas"] = get_decode_pod_count() + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["decode"]["tensor_parallelism"]} manifest["spec"]["template"] = _build_pd_pod_template( deployment_profile, deployment_profile_name, is_prefill=False, workload=workload ) From 1d0b97db44635198730320bb93692f1276004cb0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:09:40 +0200 Subject: [PATCH 317/344] [llm_d] orchestration: ci: make the RHOAI vaults mandatory if the custom catalog is enabled --- projects/llm_d/orchestration/ci.py | 2 +- projects/llm_d/tests/test_profiles.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index de85604bc..cb0adc76d 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -69,7 +69,7 @@ def init_vaults_for_phase(phase: str) -> None: if phase == "prepare" and config.project.get_config( "platform.rhoai.custom_catalog.enabled", False ): - optional_vaults = [*optional_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] + mandatory_vaults = [*mandatory_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 9409efce6..1efce2f07 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -233,10 +233,10 @@ def _fake_init(**kwargs): core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) llmd_ci.init_vaults_for_phase("prepare") - assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] - assert "psap-forge-staging-image-pull" not in calls[0]["optional_vaults"] - assert "psap-rhoai-rc" in calls[1]["optional_vaults"] - assert "psap-forge-staging-image-pull" in calls[1]["optional_vaults"] + assert "psap-rhoai-rc" not in calls[0]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" not in calls[0]["mandatory_vaults"] + assert "psap-rhoai-rc" in calls[1]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" in calls[1]["mandatory_vaults"] def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( From 66eb60def1ade714e415a2a3b0d66b1d18af7b58 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:25:14 +0200 Subject: [PATCH 318/344] [llm_d] orchestration: presets: rhoai-rc: new preset for RHOAI RC --- projects/llm_d/orchestration/presets.d/rhoai-rc.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/rhoai-rc.yaml diff --git a/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml new file mode 100644 index 000000000..d1943a32c --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml @@ -0,0 +1,7 @@ +rhoai-rc: + platform.rhoai.custom_catalog.enabled: true + platform.operators.rhods-operator.channel: beta + +rhoai-3.5-ea.2: + extends: [rhoai-rc] + platform.rhoai.custom_catalog.image: quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310 From e935a3ba07706c646e5fc01c22a29b709ef8e531 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 21:54:11 +0200 Subject: [PATCH 319/344] [llm_d] tests: reference_deployments: correctly set the TP --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 2 ++ .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 4 ++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index c4a16a88b..42aa4b492 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -60,3 +60,5 @@ spec: - name: VLLM_ADDITIONAL_ARGS value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index 8dbe6a352..b9cc3dee3 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -45,6 +45,8 @@ spec: fieldPath: status.podIP prefill: replicas: 8 + parallelism: + tensor: 1 template: containers: - name: main @@ -65,3 +67,5 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index f17b89144..a9e9aeb1b 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -51,3 +51,5 @@ spec: - name: VLLM_ADDITIONAL_ARGS value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 From 865e50fdfdf6c6e93ecfd1ce8ddd00f181c37791 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 22:08:28 +0200 Subject: [PATCH 320/344] [llm_d] orchestration: presets: cpt: more presets --- projects/llm_d/orchestration/presets.d/cks.yaml | 2 ++ projects/llm_d/orchestration/presets.d/cpt.yaml | 3 +++ projects/llm_d/orchestration/presets.d/janus.yaml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index 78cc89024..1b59f614b 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -20,6 +20,7 @@ cks: cks-intelligent: extends: - cks + - cpt - llama3-70b runtime.deployment_profile: @@ -33,6 +34,7 @@ cks-intelligent: cks-pd: extends: - cks + - cpt - gpt-oss runtime.deployment_profile: diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index dfb731711..ce7ce6859 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -8,3 +8,6 @@ cpt-reference-flavors: - pd-d.x2-p.tp1-d.tp4-p.x8 - intelligentrouting-tp4-x4 - simple-tp4-x4 + +cpt: + caliper.export.backend.mlflow.config.experiment: cpt-llm-d diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml index 4026bc346..9612228ce 100644 --- a/projects/llm_d/orchestration/presets.d/janus.yaml +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -13,6 +13,7 @@ janus: janus-intelligent: extends: - janus + - cpt - llama3-70b runtime.deployment_profile: @@ -26,6 +27,7 @@ janus-intelligent: janus-pd: extends: - janus + - cpt - gpt-oss runtime.deployment_profile: From b35cc93342f51478bc834ced34889b20873ed5b6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 07:40:51 +0200 Subject: [PATCH 321/344] [llm_d] orchestration: render_inference_service: add anti-affinity for prefill/decode pods --- .../orchestration/render_inference_service.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 3e869d5ef..b27456e9c 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -408,7 +408,38 @@ def _build_pd_pod_template( if deployment_profile.get("serving_image"): container["image"] = deployment_profile["serving_image"] - return {"containers": [container]} + # Build pod template with anti-affinity for P/D deployments + component_type = "prefill" if is_prefill else "decode" + opposite_component = "decode" if is_prefill else "prefill" + + pod_template = { + "containers": [container], + "metadata": {"labels": {"app.kubernetes.io/component": component_type}}, + } + + # Add anti-affinity to prevent prefill and decode pods from landing on the same node + affinity = { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 100, + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + # Anti-affinity between prefill and decode pods + "app.kubernetes.io/component": opposite_component + } + }, + "topologyKey": "kubernetes.io/hostname", + }, + } + ] + } + } + + pod_template["affinity"] = affinity + + return pod_template def _calculate_total_gpu_usage(deployment_profile: dict[str, Any]) -> int: From 25c7083ae3871f2ad2f86e5db6e75dd922c69678 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 09:42:01 +0200 Subject: [PATCH 322/344] [llm_d] tests: reference_deployments: add the anti-affinity fields --- .../llmisvc.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index b9cc3dee3..a12df366c 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -43,6 +43,18 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: decode + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: prefill + topologyKey: kubernetes.io/hostname prefill: replicas: 8 parallelism: @@ -67,5 +79,17 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: prefill + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: decode + topologyKey: kubernetes.io/hostname parallelism: tensor: 4 From d70c86df2e2147c19a5672e2ce90ee901f504516 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 10:37:28 +0200 Subject: [PATCH 323/344] Revert Revert '[caliper] remove unnecessary reference to the hierachical format' --- projects/caliper/engine/kpi/analyze.py | 349 +++++++++++++++++++++++-- 1 file changed, 330 insertions(+), 19 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 246c317e5..47ae439ff 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,32 +2,343 @@ from __future__ import annotations +import json +import logging +import time +from pathlib import Path from typing import Any -from projects.caliper.engine.model import RegressionFinding +logger = logging.getLogger(__name__) + + +def run_kpi_analysis( + current_kpi_file: Path, + historical_data_dir: Path, + output_file: Path, + plugin_module: str, +) -> int: + """Run KPI analysis and generate output file. + + Args: + current_kpi_file: Path to current KPI JSON file + historical_data_dir: Directory containing historical KPI files + output_file: Path where analysis results will be written + plugin_module: Plugin module name for analysis + + Returns: + Exit code (0 for success, non-zero for failure) + """ + try: + logger.info(f"Running KPI analysis for: {current_kpi_file}") + logger.info(f"Historical data directory: {historical_data_dir}") + logger.info(f"Output file: {output_file}") + logger.info(f"Plugin module: {plugin_module}") + + # Check if current KPI file exists + if not current_kpi_file.exists(): + logger.error(f"Current KPI file not found: {current_kpi_file}") + return 1 + + # Check if historical data directory exists + if not historical_data_dir.exists(): + logger.error(f"Historical data directory not found: {historical_data_dir}") + return 1 + + # Find all historical KPI files + historical_kpi_files = [] + for kpi_file in historical_data_dir.rglob("kpis.json"): + historical_kpi_files.append(str(kpi_file)) + logger.info(f"Found historical KPI file: {kpi_file}") + + if not historical_kpi_files: + logger.warning("No historical KPI files found for analysis") + # Return special exit code 2 to indicate warning (no historical data) + analysis_result = { + "status": "warning", + "message": "no historical KPI found for regression testing", + "current_kpi_file": str(current_kpi_file), + "historical_kpi_files": historical_kpi_files, + "baseline_files_count": 0, + "plugin_module": plugin_module, + "completed_at": time.time(), + } + # Write warning result to output file + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis_result, f, indent=2) + return 2 # Special exit code for warning + + logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") + + # Create analysis result + analysis_result = { + "status": "success (stub)", + "current_kpi_file": str(current_kpi_file), + "historical_kpi_files": historical_kpi_files, + "baseline_files_count": len(historical_kpi_files), + "plugin_module": plugin_module, + "completed_at": time.time(), + } + + # Create output directory if needed + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Write analysis results to output file + with open(output_file, "w") as f: + json.dump(analysis_result, f, indent=2) + f.write("\n") # Add EOL at EOF + + logger.info(f"Analysis completed successfully, results written to: {output_file}") + return 0 + + except Exception as e: + logger.exception("KPI analysis failed") + + # Create failure result + failure_result = { + "status": "failed", + "error": str(e), + "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, + "completed_at": time.time(), + } + + try: + # Attempt to write failure result + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(failure_result, f, indent=2) + f.write("\n") # Add EOL at EOF + except Exception: + logger.exception("Failed to write error result to output file") + + return 1 + + +def analyze_kpis( + postprocess_config, # CaliperOrchestrationPostprocessConfig + plugin_module: str, + base_dir: Path, + output_dir: Path, + current_kpis_file: Path, +) -> dict[str, Any]: + """Run KPI analysis step and return result status. + + This is the orchestration interface for KPI analysis. + """ + if not postprocess_config.analyze.enabled: + return {"status": "disabled", "reason": "analyze disabled"} + + # Determine paths for analysis + analyze_config = postprocess_config.analyze + + # Historical KPIs directory path + historical_kpis_dir = Path(analyze_config.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Output path for analysis results + output_path = output_dir / analyze_config.output + + try: + # Check if required files exist + if not current_kpis_file.exists(): + return { + "status": "failed", + "error": f"Current KPI file not found: {current_kpis_file}", + "completed_at": time.time(), + } + + if not historical_kpis_dir.exists(): + return { + "status": "failed", + "error": f"Historical KPIs directory not found: {historical_kpis_dir}", + "completed_at": time.time(), + } + + # Call the core analysis function + exit_code = run_kpi_analysis( + current_kpi_file=current_kpis_file, + historical_data_dir=historical_kpis_dir, + output_file=output_path, + plugin_module=plugin_module, + ) + + if exit_code == 0: + # Read the results from the output file + if output_path.exists(): + with open(output_path) as f: + result_data = json.load(f) + + # Return success with output file path for notifications + return { + "status": "success", + "output_file": str(output_path), + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": "Analysis completed but no output file was generated", + "completed_at": time.time(), + } + elif exit_code == 2: + # Warning: no historical data found + if output_path.exists(): + with open(output_path) as f: + result_data = json.load(f) + + # Return warning status with message + return { + "status": "warning", + "message": result_data.get( + "message", "no historical KPI found for regression testing" + ), + "output_file": str(output_path), + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": time.time(), + } + else: + return { + "status": "warning", + "message": "no historical KPI found for regression testing", + "baseline_files_count": 0, + "completed_at": time.time(), + } + else: + # Read error from output file if it exists + error_msg = f"Analysis failed with exit code {exit_code}" + if output_path.exists(): + try: + with open(output_path) as f: + result_data = json.load(f) + error_msg = result_data.get("error", error_msg) + except Exception: + pass + + return { + "status": "failed", + "error": error_msg, + "completed_at": time.time(), + } + + except Exception as e: + error_msg = f"{type(e).__name__}: {str(e)}" + logger.exception(f"Analysis step failed with {error_msg}") + + # Don't suppress the exception - provide detailed error info and re-raise + import traceback + + full_traceback = traceback.format_exc() + logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") + + # Re-raise the original exception so the caller can see the full details + raise RuntimeError(f"KPI analysis failed: {error_msg}") from e + + +def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: + """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. + + Args: + historical_dir: Directory to search for historical kpis.json files + + Returns: + Dictionary mapping file paths to loaded KPI JSON objects + """ + baseline_kpis = {} + kpi_files = list(historical_dir.rglob("kpis.json")) + + if not kpi_files: + logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") + return baseline_kpis + + logger.info(f"Found {len(kpi_files)} historical KPI files to load") + + for kpi_file in kpi_files: + try: + with open(kpi_file) as f: + kpi_data = json.load(f) + + # Validate that it's a hierarchical format (schema_version 2) + schema_version = kpi_data.get("schema_version", "unknown") + if schema_version != "2": + logger.warning( + f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" + ) + continue + + baseline_kpis[kpi_file] = kpi_data + logger.debug(f"Loaded KPI file: {kpi_file}") + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON in {kpi_file}: {e}") + continue + except FileNotFoundError as e: + logger.error(f"KPI file not found: {kpi_file}: {e}") + continue + except Exception as e: + logger.error(f"Failed to load KPI file {kpi_file}: {e}") + continue + + logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") + return baseline_kpis def run_analyze( *, current_path: Any, - baseline_path: Any, + baseline_kpis: dict[Path, dict[str, Any]], output_path: Any, plugin: Any = None, -) -> list[RegressionFinding]: - """Run KPI analysis - hierarchical format only.""" - from pathlib import Path - - from .analyze_hierarchical import analyze_hierarchical_kpis - - result = analyze_hierarchical_kpis( - current_kpis_path=Path(current_path), - baseline_kpis_path=Path(baseline_path), - output_path=Path(output_path), - plugin=plugin, - ) - - # Convert to expected format for CLI - if result["status"] == "success": - return [] # Return empty list for now, actual findings are in the output file +) -> dict[str, Any]: + """Run KPI analysis against ALL baseline files (CLI interface).""" + # For CLI use, we need to extract the baseline directory from the dict + # This is a bit of a hack, but the CLI interface expects a directory + if baseline_kpis: + # Get the parent directory of the first baseline file + first_baseline_path = next(iter(baseline_kpis.keys())) + historical_dir = first_baseline_path.parent + + # Extract plugin module name if available + plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" + + # Call the core analysis function + exit_code = run_kpi_analysis( + current_kpi_file=Path(current_path), + historical_data_dir=historical_dir, + output_file=Path(output_path), + plugin_module=plugin_module, + ) + + if exit_code == 0: + # Read results and convert to expected format for CLI + try: + with open(Path(output_path)) as f: + result_data = json.load(f) + return { + "status": "success", + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": result_data.get("completed_at", time.time()), + } + except Exception as e: + return { + "status": "failed", + "error": f"Failed to read analysis results: {e}", + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": f"Analysis failed with exit code {exit_code}", + "completed_at": time.time(), + } else: - raise RuntimeError(f"Analysis failed: {result.get('error', 'Unknown error')}") + return { + "status": "failed", + "error": "No baseline KPI files provided", + "completed_at": time.time(), + } + + +# EOF From bee915b053340b338ff4c4a7d849ec4858480a36 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 01:06:52 -0400 Subject: [PATCH 324/344] feat(caliper): implement KPI regression analysis POC Replace the stub run_kpi_analysis with a working regression engine that: - Matches current KPIs to historical baselines via configurable comparison_keys/ignored_keys on record labels - Runs per-KPI regression tests (relative threshold, higher/lower-is-better) - Produces a structured YAML report with processed/tested/results/overall - Supports plugin-provided AnalysisConfig for per-project tuning Exit codes: 0=pass, 1=error, 2=no-baseline, 3=regression-detected. Includes end-to-end test suite (15 tests). Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 616 ++++++++++++++------- projects/caliper/tests/test_kpi_analyze.py | 442 +++++++++++++++ 2 files changed, 855 insertions(+), 203 deletions(-) create mode 100644 projects/caliper/tests/test_kpi_analyze.py diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 47ae439ff..593f8acdb 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -1,120 +1,406 @@ -"""Generic KPI analysis against historical baselines.""" +"""Generic KPI regression analysis against historical baselines.""" from __future__ import annotations import json import logging import time +from dataclasses import dataclass, field from pathlib import Path from typing import Any +import yaml + logger = logging.getLogger(__name__) +@dataclass +class AnalysisConfig: + """Configuration for KPI regression analysis. + + comparison_keys: Label keys that define what we compare against. + Records must differ on at least one comparison key to be distinct baselines. + E.g. ["version"] means we test the current version against other versions. + ignored_keys: Label keys excluded when matching current to baseline records. + E.g. ["os"] means we match across operating systems. + sorting_keys: Label keys used to order entries in the output report. + max_relative_regression: Fraction threshold for flagging regression (0.1 = 10%). + min_baseline_points: Minimum number of baseline data points required to run a test. + """ + + comparison_keys: list[str] = field(default_factory=list) + ignored_keys: list[str] = field(default_factory=list) + sorting_keys: list[str] = field(default_factory=list) + max_relative_regression: float = 0.1 + min_baseline_points: int = 1 + + +@dataclass +class KpiTestResult: + """Result of a single KPI regression test.""" + + kpi_id: str + labels: dict[str, Any] + current_value: float + baseline_mean: float + relative_change: float + higher_is_better: bool + regression: bool + baseline_count: int + + +@dataclass +class AnalysisReport: + """Full analysis report.""" + + status: str # "success", "no_regression", "regression_detected", "error" + processed: dict[str, Any] + tested: list[dict[str, Any]] + results: list[dict[str, Any]] + overall: dict[str, Any] + + +def _load_analysis_config(plugin_module: str) -> AnalysisConfig: + """Load analysis config from plugin module if available, else use defaults. + + Plugins can expose an `analysis_config` dict or `get_analysis_config()` callable. + """ + try: + mod = __import__(plugin_module, fromlist=[""]) + if hasattr(mod, "get_analysis_config"): + raw = mod.get_analysis_config() + elif hasattr(mod, "analysis_config"): + raw = mod.analysis_config + else: + return AnalysisConfig() + + if isinstance(raw, AnalysisConfig): + return raw + if isinstance(raw, dict): + return AnalysisConfig( + **{k: v for k, v in raw.items() if k in AnalysisConfig.__dataclass_fields__} + ) + except (ImportError, Exception) as exc: + logger.debug("Could not load analysis config from plugin %s: %s", plugin_module, exc) + + return AnalysisConfig() + + +def _match_key( + labels: dict[str, Any], ignored_keys: list[str], comparison_keys: list[str] +) -> tuple: + """Build a hashable match key from labels, excluding ignored and comparison keys.""" + excluded = set(ignored_keys) | set(comparison_keys) + return tuple(sorted((k, str(v)) for k, v in labels.items() if k not in excluded)) + + +def _extract_kpi_records_from_hierarchical(data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract flat KPI records from a schema_version=2 hierarchical document.""" + records = [] + for test in data.get("tests", []): + test_labels = test.get("labels", {}) + for kpi in test.get("kpis", []): + records.append( + { + "kpi_id": kpi.get("id"), + "value": kpi.get("value"), + "unit": kpi.get("unit"), + "higher_is_better": kpi.get("higher_is_better", True), + "labels": test_labels, + "run_id": test.get("run_id"), + "metadata": test.get("metadata", {}), + } + ) + return records + + +def _build_baseline_index( + baseline_kpi_data: dict[Path, dict[str, Any]], + config: AnalysisConfig, +) -> dict[tuple, list[dict[str, Any]]]: + """Index baseline records by (kpi_id, match_key) for fast lookup. + + Returns mapping from (kpi_id, match_key) -> list of baseline records. + """ + index: dict[tuple, list[dict[str, Any]]] = {} + for _path, data in baseline_kpi_data.items(): + records = _extract_kpi_records_from_hierarchical(data) + for rec in records: + kpi_id = rec.get("kpi_id") + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + index.setdefault(key, []).append(rec) + return index + + +def _run_regression_test( + current: dict[str, Any], + baselines: list[dict[str, Any]], + config: AnalysisConfig, +) -> KpiTestResult: + """Run a regression test for a single KPI record against its baselines.""" + current_value = float(current["value"]) + higher_is_better = current.get("higher_is_better", True) + baseline_values = [float(b["value"]) for b in baselines if b.get("value") is not None] + baseline_mean = sum(baseline_values) / len(baseline_values) + + if baseline_mean == 0: + relative_change = 0.0 + else: + relative_change = (current_value - baseline_mean) / abs(baseline_mean) + + # Determine regression: if higher is better, a negative change is regression. + # If lower is better, a positive change is regression. + if higher_is_better: + regression = relative_change < -config.max_relative_regression + else: + regression = relative_change > config.max_relative_regression + + return KpiTestResult( + kpi_id=current["kpi_id"], + labels=current.get("labels", {}), + current_value=current_value, + baseline_mean=round(baseline_mean, 6), + relative_change=round(relative_change, 6), + higher_is_better=higher_is_better, + regression=regression, + baseline_count=len(baseline_values), + ) + + +def _sort_results(results: list[KpiTestResult], sorting_keys: list[str]) -> list[KpiTestResult]: + """Sort results by sorting keys extracted from labels, then by kpi_id.""" + + def sort_key(r: KpiTestResult): + label_key = tuple(str(r.labels.get(k, "")) for k in sorting_keys) + return (*label_key, r.kpi_id) + + return sorted(results, key=sort_key) + + +def _build_yaml_report( + results: list[KpiTestResult], + config: AnalysisConfig, + current_source: str, + baseline_sources: list[str], + skipped: list[dict[str, Any]], +) -> dict[str, Any]: + """Build the final YAML-serializable report structure.""" + regressions = [r for r in results if r.regression] + improvements = [r for r in results if not r.regression and r.relative_change != 0] + + if regressions: + overall_status = "REGRESSION_DETECTED" + else: + overall_status = "PASS" + + report = { + "analysis": { + "status": overall_status, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "config": { + "comparison_keys": config.comparison_keys, + "ignored_keys": config.ignored_keys, + "sorting_keys": config.sorting_keys, + "max_relative_regression": config.max_relative_regression, + "min_baseline_points": config.min_baseline_points, + }, + }, + "processed": { + "current_source": current_source, + "baseline_sources": baseline_sources, + "baseline_source_count": len(baseline_sources), + }, + "tested": { + "total_kpis": len(results), + "regressions": len(regressions), + "passes": len(results) - len(regressions), + "skipped": len(skipped), + }, + "results": [ + { + "kpi_id": r.kpi_id, + "labels": r.labels, + "current_value": r.current_value, + "baseline_mean": r.baseline_mean, + "relative_change_pct": round(r.relative_change * 100, 2), + "higher_is_better": r.higher_is_better, + "verdict": "REGRESSION" if r.regression else "PASS", + "baseline_count": r.baseline_count, + } + for r in results + ], + "overall": { + "verdict": overall_status, + "regression_count": len(regressions), + "total_tested": len(results), + "total_skipped": len(skipped), + }, + } + + if skipped: + report["skipped"] = skipped + + return report + + def run_kpi_analysis( current_kpi_file: Path, historical_data_dir: Path, output_file: Path, plugin_module: str, ) -> int: - """Run KPI analysis and generate output file. + """Run KPI regression analysis and generate a YAML report. Args: - current_kpi_file: Path to current KPI JSON file - historical_data_dir: Directory containing historical KPI files - output_file: Path where analysis results will be written - plugin_module: Plugin module name for analysis + current_kpi_file: Path to current KPI JSON file (hierarchical schema v2) + historical_data_dir: Directory containing historical KPI files (kpis.json) + output_file: Path where YAML analysis report will be written + plugin_module: Plugin module name (for loading analysis config) Returns: - Exit code (0 for success, non-zero for failure) + Exit code: 0=pass, 1=error, 2=warning (no baseline), 3=regression detected """ try: - logger.info(f"Running KPI analysis for: {current_kpi_file}") - logger.info(f"Historical data directory: {historical_data_dir}") - logger.info(f"Output file: {output_file}") - logger.info(f"Plugin module: {plugin_module}") + logger.info("Running KPI regression analysis") + logger.info(" current_kpi_file: %s", current_kpi_file) + logger.info(" historical_data_dir: %s", historical_data_dir) + logger.info(" output_file: %s", output_file) + logger.info(" plugin_module: %s", plugin_module) - # Check if current KPI file exists if not current_kpi_file.exists(): - logger.error(f"Current KPI file not found: {current_kpi_file}") + logger.error("Current KPI file not found: %s", current_kpi_file) return 1 - # Check if historical data directory exists if not historical_data_dir.exists(): - logger.error(f"Historical data directory not found: {historical_data_dir}") + logger.error("Historical data directory not found: %s", historical_data_dir) return 1 - # Find all historical KPI files - historical_kpi_files = [] - for kpi_file in historical_data_dir.rglob("kpis.json"): - historical_kpi_files.append(str(kpi_file)) - logger.info(f"Found historical KPI file: {kpi_file}") + config = _load_analysis_config(plugin_module) + logger.info( + " config: comparison_keys=%s, ignored_keys=%s", + config.comparison_keys, + config.ignored_keys, + ) - if not historical_kpi_files: - logger.warning("No historical KPI files found for analysis") - # Return special exit code 2 to indicate warning (no historical data) - analysis_result = { - "status": "warning", - "message": "no historical KPI found for regression testing", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": 0, - "plugin_module": plugin_module, - "completed_at": time.time(), - } - # Write warning result to output file - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - return 2 # Special exit code for warning - - logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") - - # Create analysis result - analysis_result = { - "status": "success (stub)", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": len(historical_kpi_files), - "plugin_module": plugin_module, - "completed_at": time.time(), - } + # Load current KPIs + with open(current_kpi_file) as f: + current_data = json.load(f) - # Create output directory if needed - output_file.parent.mkdir(parents=True, exist_ok=True) + if current_data.get("schema_version") != "2": + logger.error("Current KPI file must be schema_version 2 (hierarchical)") + return 1 - # Write analysis results to output file - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - f.write("\n") # Add EOL at EOF + current_records = _extract_kpi_records_from_hierarchical(current_data) + if not current_records: + logger.warning("No KPI records found in current file") + return 1 - logger.info(f"Analysis completed successfully, results written to: {output_file}") - return 0 + # Load baseline KPIs + baseline_kpi_data = find_baseline_kpis(historical_data_dir) + if not baseline_kpi_data: + _write_warning_report(output_file, current_kpi_file, plugin_module) + return 2 - except Exception as e: - logger.exception("KPI analysis failed") + # Build baseline index + baseline_index = _build_baseline_index(baseline_kpi_data, config) - # Create failure result - failure_result = { - "status": "failed", - "error": str(e), - "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, - "completed_at": time.time(), - } + # Run regression tests + results: list[KpiTestResult] = [] + skipped: list[dict[str, Any]] = [] - try: - # Attempt to write failure result - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(failure_result, f, indent=2) - f.write("\n") # Add EOL at EOF - except Exception: - logger.exception("Failed to write error result to output file") + for rec in current_records: + kpi_id = rec.get("kpi_id") + value = rec.get("value") + # Skip non-scalar values (2D KPIs, lists, etc.) + if not isinstance(value, (int, float)): + skipped.append({"kpi_id": kpi_id, "reason": "non-scalar value"}) + continue + + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + + baselines = baseline_index.get(key, []) + # Filter to only scalar baselines + scalar_baselines = [b for b in baselines if isinstance(b.get("value"), (int, float))] + + if len(scalar_baselines) < config.min_baseline_points: + skipped.append( + { + "kpi_id": kpi_id, + "labels": labels, + "reason": f"insufficient baselines ({len(scalar_baselines)} < {config.min_baseline_points})", + } + ) + continue + + result = _run_regression_test(rec, scalar_baselines, config) + results.append(result) + + # Sort results + results = _sort_results(results, config.sorting_keys) + + # Build report + baseline_sources = [str(p) for p in baseline_kpi_data.keys()] + report = _build_yaml_report( + results=results, + config=config, + current_source=str(current_kpi_file), + baseline_sources=baseline_sources, + skipped=skipped, + ) + + # Write YAML report + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + regressions = report["overall"]["regression_count"] + total = report["overall"]["total_tested"] + logger.info( + "Analysis complete: %d/%d KPIs tested, %d regressions", + total, + len(current_records), + regressions, + ) + + return 3 if regressions > 0 else 0 + + except Exception: + logger.exception("KPI analysis failed") return 1 +def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: + """Write a warning-level report when no baselines are available.""" + report = { + "analysis": { + "status": "NO_BASELINE", + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }, + "processed": { + "current_source": str(current_kpi_file), + "baseline_sources": [], + "baseline_source_count": 0, + }, + "tested": {"total_kpis": 0, "regressions": 0, "passes": 0, "skipped": 0}, + "results": [], + "overall": { + "verdict": "NO_BASELINE", + "message": "No historical KPI files found for regression testing", + "regression_count": 0, + "total_tested": 0, + "total_skipped": 0, + }, + } + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + def analyze_kpis( postprocess_config, # CaliperOrchestrationPostprocessConfig plugin_module: str, @@ -129,19 +415,15 @@ def analyze_kpis( if not postprocess_config.analyze.enabled: return {"status": "disabled", "reason": "analyze disabled"} - # Determine paths for analysis analyze_config = postprocess_config.analyze - # Historical KPIs directory path historical_kpis_dir = Path(analyze_config.historical_kpis) if not historical_kpis_dir.is_absolute(): historical_kpis_dir = output_dir / historical_kpis_dir - # Output path for analysis results output_path = output_dir / analyze_config.output try: - # Check if required files exist if not current_kpis_file.exists(): return { "status": "failed", @@ -156,7 +438,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Call the core analysis function exit_code = run_kpi_analysis( current_kpi_file=current_kpis_file, historical_data_dir=historical_kpis_dir, @@ -165,123 +446,71 @@ def analyze_kpis( ) if exit_code == 0: - # Read the results from the output file - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return success with output file path for notifications - return { - "status": "success", - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": "Analysis completed but no output file was generated", - "completed_at": time.time(), - } + return { + "status": "success", + "output_file": str(output_path), + "completed_at": time.time(), + } elif exit_code == 2: - # Warning: no historical data found - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return warning status with message - return { - "status": "warning", - "message": result_data.get( - "message", "no historical KPI found for regression testing" - ), - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "baseline_files_count": 0, - "completed_at": time.time(), - } + return { + "status": "warning", + "message": "no historical KPI found for regression testing", + "output_file": str(output_path), + "completed_at": time.time(), + } + elif exit_code == 3: + return { + "status": "regression_detected", + "output_file": str(output_path), + "completed_at": time.time(), + } else: - # Read error from output file if it exists - error_msg = f"Analysis failed with exit code {exit_code}" - if output_path.exists(): - try: - with open(output_path) as f: - result_data = json.load(f) - error_msg = result_data.get("error", error_msg) - except Exception: - pass - return { "status": "failed", - "error": error_msg, + "error": f"Analysis failed with exit code {exit_code}", "completed_at": time.time(), } except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Analysis step failed with {error_msg}") - - # Don't suppress the exception - provide detailed error info and re-raise - import traceback - - full_traceback = traceback.format_exc() - logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") - - # Re-raise the original exception so the caller can see the full details - raise RuntimeError(f"KPI analysis failed: {error_msg}") from e + logger.exception("Analysis step failed") + raise RuntimeError(f"KPI analysis failed: {e}") from e def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: - """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. - - Args: - historical_dir: Directory to search for historical kpis.json files + """Load all kpis.json files from historical directory. - Returns: - Dictionary mapping file paths to loaded KPI JSON objects + Returns mapping of file paths to loaded hierarchical KPI data (schema v2 only). """ - baseline_kpis = {} + baseline_kpis: dict[Path, dict[str, Any]] = {} kpi_files = list(historical_dir.rglob("kpis.json")) if not kpi_files: - logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") + logger.warning("No kpis.json files found in: %s", historical_dir) return baseline_kpis - logger.info(f"Found {len(kpi_files)} historical KPI files to load") + logger.info("Found %d historical KPI files to load", len(kpi_files)) for kpi_file in kpi_files: try: with open(kpi_file) as f: kpi_data = json.load(f) - # Validate that it's a hierarchical format (schema_version 2) schema_version = kpi_data.get("schema_version", "unknown") if schema_version != "2": logger.warning( - f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" + "Skipping %s: unsupported schema version %s", kpi_file, schema_version ) continue baseline_kpis[kpi_file] = kpi_data - logger.debug(f"Loaded KPI file: {kpi_file}") + logger.debug("Loaded baseline: %s", kpi_file) except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON in {kpi_file}: {e}") - continue - except FileNotFoundError as e: - logger.error(f"KPI file not found: {kpi_file}: {e}") - continue + logger.error("Failed to parse JSON in %s: %s", kpi_file, e) except Exception as e: - logger.error(f"Failed to load KPI file {kpi_file}: {e}") - continue + logger.error("Failed to load %s: %s", kpi_file, e) - logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") + logger.info("Successfully loaded %d historical KPI files", len(baseline_kpis)) return baseline_kpis @@ -293,52 +522,33 @@ def run_analyze( plugin: Any = None, ) -> dict[str, Any]: """Run KPI analysis against ALL baseline files (CLI interface).""" - # For CLI use, we need to extract the baseline directory from the dict - # This is a bit of a hack, but the CLI interface expects a directory - if baseline_kpis: - # Get the parent directory of the first baseline file - first_baseline_path = next(iter(baseline_kpis.keys())) - historical_dir = first_baseline_path.parent - - # Extract plugin module name if available - plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=Path(current_path), - historical_data_dir=historical_dir, - output_file=Path(output_path), - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read results and convert to expected format for CLI - try: - with open(Path(output_path)) as f: - result_data = json.load(f) - return { - "status": "success", - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": result_data.get("completed_at", time.time()), - } - except Exception as e: - return { - "status": "failed", - "error": f"Failed to read analysis results: {e}", - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } - else: + if not baseline_kpis: return { "status": "failed", "error": "No baseline KPI files provided", "completed_at": time.time(), } + first_baseline_path = next(iter(baseline_kpis.keys())) + historical_dir = first_baseline_path.parent + + plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" + + exit_code = run_kpi_analysis( + current_kpi_file=Path(current_path), + historical_data_dir=historical_dir, + output_file=Path(output_path), + plugin_module=plugin_module, + ) + + if exit_code == 0: + return {"status": "success", "completed_at": time.time()} + elif exit_code == 2: + return {"status": "warning", "message": "no baselines found", "completed_at": time.time()} + elif exit_code == 3: + return {"status": "regression_detected", "completed_at": time.time()} + else: + return {"status": "failed", "error": f"exit code {exit_code}", "completed_at": time.time()} + # EOF diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py new file mode 100644 index 000000000..a71e01c3d --- /dev/null +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -0,0 +1,442 @@ +"""Tests for KPI regression analysis (analyze.py).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from projects.caliper.engine.kpi.analyze import ( + AnalysisConfig, + _build_baseline_index, + _extract_kpi_records_from_hierarchical, + _match_key, + _run_regression_test, + run_kpi_analysis, +) + + +def _make_hierarchical_kpi( + tests: list[dict], +) -> dict: + """Helper to build a schema_version=2 hierarchical KPI doc.""" + return {"schema_version": "2", "tests": tests} + + +def _make_test_entry( + run_id: str, + labels: dict, + kpis: list[dict], +) -> dict: + return { + "run_id": run_id, + "labels": labels, + "metadata": {"timestamp": "2025-01-01T00:00:00Z"}, + "kpis": kpis, + } + + +def _make_kpi(kpi_id: str, value, unit: str = "tokens/s", higher_is_better: bool = True) -> dict: + return {"id": kpi_id, "value": value, "unit": unit, "higher_is_better": higher_is_better} + + +class TestMatchKey: + def test_basic_match_key(self): + labels = {"platform": "A100", "version": "1.0", "os": "linux"} + key = _match_key(labels, ignored_keys=["os"], comparison_keys=["version"]) + assert ("os", "linux") not in key + assert ("version", "1.0") not in key + assert ("platform", "A100") in key + + def test_empty_config(self): + labels = {"a": "1", "b": "2"} + key = _match_key(labels, ignored_keys=[], comparison_keys=[]) + assert key == tuple(sorted([("a", "1"), ("b", "2")])) + + +class TestExtractRecords: + def test_extracts_flat_records(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("latency", 0.5, unit="s", higher_is_better=False), + ], + ), + ] + ) + records = _extract_kpi_records_from_hierarchical(data) + assert len(records) == 2 + assert records[0]["kpi_id"] == "throughput" + assert records[0]["labels"] == {"platform": "A100"} + assert records[1]["value"] == 0.5 + + def test_empty_tests(self): + data = _make_hierarchical_kpi([]) + assert _extract_kpi_records_from_hierarchical(data) == [] + + +class TestBuildBaselineIndex: + def test_indexes_by_kpi_and_match_key(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + _make_test_entry( + "run2", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + config = AnalysisConfig(comparison_keys=["version"]) + baseline_data = {Path("/fake/kpis.json"): data} + index = _build_baseline_index(baseline_data, config) + + mk = _match_key({"platform": "A100"}, [], ["version"]) + key = ("throughput", mk) + assert key in index + assert len(index[key]) == 2 + + +class TestRegressionTest: + def test_no_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 100.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 95.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + assert result.relative_change > 0 + + def test_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 80.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 100.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change < -0.1 + + def test_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 1.5, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}, {"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change > 0.1 + + def test_no_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 0.9, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + + +class TestEndToEnd: + """Full end-to-end test of run_kpi_analysis.""" + + def _write_hierarchical_kpi(self, path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(data, f) + + def test_pass_no_regression(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current KPIs: throughput=100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Historical: throughput=95, 98 (within 10% threshold) + baseline1 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + baseline2 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_2", + {"platform": "A100"}, + [ + _make_kpi("throughput", 98.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline1) + self._write_hierarchical_kpi(historical_dir / "run2" / "kpis.json", baseline2) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 0 + assert output_file.exists() + + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "PASS" + assert report["overall"]["verdict"] == "PASS" + assert report["overall"]["regression_count"] == 0 + assert report["tested"]["total_kpis"] == 1 + + def test_regression_detected(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: throughput dropped from 100 to 70 (30% regression) + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 70.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "REGRESSION_DETECTED" + assert report["overall"]["regression_count"] == 1 + assert report["results"][0]["verdict"] == "REGRESSION" + assert report["results"][0]["relative_change_pct"] == pytest.approx(-30.0) + + def test_no_historical_data(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + historical_dir.mkdir(parents=True) + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 2 + with open(output_file) as f: + report = yaml.safe_load(f) + assert report["overall"]["verdict"] == "NO_BASELINE" + + def test_mixed_regression_and_pass(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 2.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 1.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["overall"]["regression_count"] == 1 + latency_result = next(r for r in report["results"] if r["kpi_id"] == "latency") + assert latency_result["verdict"] == "REGRESSION" + throughput_result = next(r for r in report["results"] if r["kpi_id"] == "throughput") + assert throughput_result["verdict"] == "PASS" + + def test_skips_non_scalar_kpis(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("curve_data", [1.0, 2.0, 3.0]), # non-scalar + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 0 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["tested"]["skipped"] == 1 + assert report["tested"]["total_kpis"] == 1 + + def test_comparison_keys_separate_baselines(self, tmp_path): + """Records that differ on comparison_keys should still be matched.""" + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: version=2.0 on platform=A100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Baseline: version=1.0 on platform=A100 (different comparison key value) + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + # With version as comparison_key, both match on platform=A100 + # (plugin would expose this config; here we test the core logic directly) + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + # Without plugin config, default AnalysisConfig has no comparison_keys, + # so version must also match → no baselines found → skip + assert exit_code == 0 or exit_code == 2 + with open(output_file) as f: + report = yaml.safe_load(f) + # The version label differs, so with empty comparison_keys they don't match + assert report["tested"]["total_kpis"] == 0 or report["overall"]["verdict"] == "NO_BASELINE" From d39659f6e4544778ca1b606a4e4d0d0767e11f64 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 11:21:52 +0200 Subject: [PATCH 325/344] [caliper] orchestration: postprocess: improve the step status reporting --- projects/caliper/orchestration/postprocess.py | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a32ef2736..3e3104c75 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1009,13 +1009,14 @@ def __init__( # State tracking self.steps: list[dict[str, Any]] = [] - self.parse_failed = False - self.visualize_failed = False - self.artifacts_to_kpis_failed = False - self.ai_data_failed = False - self.s3_import_failed = False - self.analyze_failed = False - self.s3_export_failed = False + # Initialize failure flags to None (skipped), will be set to False (success) or True (failure) + self.parse_failed = None + self.visualize_failed = None + self.artifacts_to_kpis_failed = None + self.ai_data_failed = None + self.s3_import_failed = None + self.analyze_failed = None + self.s3_export_failed = None # Configuration try: @@ -1147,7 +1148,7 @@ def _add_step( self.steps.append({step_name: step_data}) def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, Any]) -> bool: - """Check step result status and set appropriate failure flag if needed. + """Check step result status and set appropriate failure flag. Args: step_name: Name of the step @@ -1157,31 +1158,40 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A True if step failed or warned, False if successful """ status = result.get("status") - if status in ("failed", "warning"): - # Map step names to their failure flags - step_failure_map = { - "artifacts_to_kpis": "artifacts_to_kpis_failed", - "artifacts_to_ai_data": "ai_data_failed", - "s3_import": "s3_import_failed", - "analyse_kpis": "analyze_failed", - "s3_export": "s3_export_failed", - } - failure_attr = step_failure_map.get(step_name) - if failure_attr: - setattr(self, failure_attr, True) + # Map step names to their failure flags + step_failure_map = { + "artifacts_to_kpis": "artifacts_to_kpis_failed", + "artifacts_to_ai_data": "ai_data_failed", + "s3_import": "s3_import_failed", + "analyse_kpis": "analyze_failed", + "s3_export": "s3_export_failed", + } - if status == "warning": - warning_msg = result.get("message", "unknown warning") - logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") - elif status == "failed": - error_msg = result.get("error", "unknown error") - traceback_msg = result.get("traceback") - logger.error(f"Step '{step_name}' failed: {error_msg}") - if traceback_msg: - logger.error("Full traceback:\n%s", traceback_msg) + failure_attr = step_failure_map.get(step_name) + if not failure_attr: + return False + + if status in ("failed", "warning"): + setattr(self, failure_attr, True) + + if status == "warning": + warning_msg = result.get("message", "unknown warning") + logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") + elif status == "failed": + error_msg = result.get("error", "unknown error") + traceback_msg = result.get("traceback") + logger.error(f"Step '{step_name}' failed: {error_msg}") + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) return True + elif status == "success": + # Set flag to False for successful steps + setattr(self, failure_attr, False) + return False + + # For other statuses (disabled, etc.), leave flag as None return False def _get_step(self, step_name: str) -> dict[str, Any]: @@ -1263,6 +1273,7 @@ def _run_parse_step(self) -> None: ) if result.returncode == 0 and status_data and status_data.get("success", False): + self.parse_failed = False self._add_step( "parse", { @@ -1381,6 +1392,7 @@ def _run_visualize_step(self) -> None: # If output_dir is not under ARTIFACT_DIR, use absolute path relative_output_dir = str(output_dir) + self.visualize_failed = False self._add_step( "visualize", { @@ -1927,7 +1939,6 @@ def _compute_final_status(self) -> str: logger.info(f" s3_import_failed: {self.s3_import_failed}") logger.info(f" analyze_failed: {self.analyze_failed}") logger.info(f" s3_export_failed: {self.s3_export_failed}") - logger.info(f" analyze_failed: {self.analyze_failed}") final_status = compute_final_postprocess_status( test_outcome=self.test_outcome, From 2f6be00ab550fcd5df2bf6cbc6713d68d28fe0a6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 11:33:18 +0200 Subject: [PATCH 326/344] [caliper] orchestration: postprocess: better configure the analyze section --- projects/caliper/orchestration/postprocess.py | 28 ++++++------------- .../orchestration/postprocess_config.py | 19 +++++++++---- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 3e3104c75..939addacb 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1736,36 +1736,26 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: ) return - # Get KPI file path from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if not artifacts_to_kpis_step or artifacts_to_kpis_step.get("status") != "success": - self._add_step( - "analyse_kpis", - { - "status": "failed", - "error": "artifacts_to_kpis step did not complete successfully", - "completed_at": time.time(), - }, - ) - self.analyze_failed = True - return + # Get current KPI file path from config + current_kpis_file = Path(self.config.analyze.current_kpis) + if not current_kpis_file.is_absolute(): + current_kpis_path = output_dir / current_kpis_file + else: + current_kpis_path = current_kpis_file - current_kpis_file = artifacts_to_kpis_step.get("output_file") - if not current_kpis_file: + # Check if current KPI file exists + if not current_kpis_path.exists(): self._add_step( "analyse_kpis", { "status": "failed", - "error": "No output file found in artifacts_to_kpis step", + "error": f"Current KPI file not found: {current_kpis_path}", "completed_at": time.time(), }, ) self.analyze_failed = True return - # Convert relative path back to absolute path - current_kpis_path = env.ARTIFACT_DIR / current_kpis_file - result = _run_analyse_kpis( postprocess_config=self.config, output_dir=output_dir, diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index b30ccb222..13f789a8a 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -115,6 +115,10 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False + current_kpis: str | None = Field( + default="kpis.json", + description="Current KPI file path (relative to postprocess output dir unless absolute).", + ) historical_kpis: str | None = Field( default=None, description="Directory containing historical KPI JSON files (relative → postprocess output dir unless absolute).", @@ -125,11 +129,16 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): ) @model_validator(mode="after") - def _historical_kpis_when_enabled(self) -> Self: - if self.enabled and not (self.historical_kpis and str(self.historical_kpis).strip()): - raise ValueError( - "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." - ) + def _validate_when_enabled(self) -> Self: + if self.enabled: + if not (self.current_kpis and str(self.current_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty current_kpis file path." + ) + if not (self.historical_kpis and str(self.historical_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." + ) return self From 21fab2d9b5596fbbaaf5e2228095a8d1e61f040d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:05:04 +0200 Subject: [PATCH 327/344] [caliper] orchestration: postprocess: re-enable the hierarchical KPI format --- projects/caliper/orchestration/postprocess.py | 164 ------------------ 1 file changed, 164 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 939addacb..8fd31e2b3 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -323,170 +323,6 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() -def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: - """ - Transform flat KPI list into hierarchical JSON structure. - - Groups KPIs by test (run_id), extracting common labels and organizing - KPI metadata (name, help, unit, etc.) for improved readability. - - Args: - kpis: List of flat KPI records from compute_kpis - model: Unified model for accessing plugin metadata - - Returns: - Hierarchical JSON structure organized by test - """ - from collections import defaultdict - - if not kpis: - return {"schema_version": "2", "tests": []} - - # Group KPIs by test (run_id) - tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) - - # Get KPI function metadata from the plugin module - from projects.caliper.engine.kpi.decorators import get_kpi_functions - - try: - plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) - kpi_functions = get_kpi_functions(plugin_module_obj) - except (ImportError, AttributeError): - kpi_functions = {} - - # First pass: determine which labels are common (same value for all KPIs in a run) - # vs per-KPI (vary across KPIs, e.g. rate_index, intended_concurrency) - run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - for k, v in kpi.get("labels", {}).items(): - if k == "higher_is_better": - continue - run_label_values[run_id][k].add(str(v)) - - # Labels with more than one distinct value per run are per-KPI - per_kpi_label_keys: dict[str, set[str]] = {} - for run_id, label_vals in run_label_values.items(): - per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} - - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - test_data = tests_data[run_id] - varying_keys = per_kpi_label_keys.get(run_id, set()) - - kpi_labels = kpi.get("labels", {}) - test_labels = { - k: v - for k, v in kpi_labels.items() - if k not in ("higher_is_better",) and k not in varying_keys - } - - if not test_data["labels"]: - test_data["labels"] = test_labels - - # Store test metadata from first KPI - if not test_data["metadata"]: - test_data["metadata"] = { - "timestamp": kpi.get("timestamp"), - "source": kpi.get("source", {}), - "run_id": run_id, - } - - # Per-KPI labels that vary across rate points - kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} - - # Create KPI record with metadata - kpi_id = kpi.get("kpi_id") - raw_value = kpi.get("value") - - # Transform 2D KPI values to structured format - if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: - # Check if this looks like 2D data: list of tuples/lists with 2 elements - first_item = raw_value[0] - if isinstance(first_item, list | tuple) and len(first_item) == 2: - try: - # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format - structured_value = { - "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], - "count": len(raw_value), - } - final_value = structured_value - except (ValueError, TypeError, IndexError): - # If conversion fails, keep original value - final_value = raw_value - else: - final_value = raw_value - else: - final_value = raw_value - - kpi_record: dict[str, Any] = { - "id": kpi_id, - "value": final_value, - "unit": kpi.get("unit"), - "higher_is_better": kpi_labels.get("higher_is_better", True), - } - if kpi_specific_labels: - kpi_record["labels"] = kpi_specific_labels - - # Add KPI metadata from function decorator if available - if kpi_id in kpi_functions: - func = kpi_functions[kpi_id] - kpi_record.update( - { - "name": ( - func.__doc__.replace(" KPI.", "") - if func.__doc__ - else kpi_id.replace("_", " ").title() - ), - "help": getattr(func, "_kpi_help", ""), - } - ) - - # Add 2D-specific metadata if present - if getattr(func, "_kpi_is_2d", False): - kpi_record.update( - { - "is_2d": True, - "x_unit": getattr(func, "_kpi_x_unit", ""), - "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], - "y_help": getattr(func, "_kpi_y_help", None) - or getattr(func, "_kpi_help", ""), - } - ) - else: - kpi_record["is_2d"] = False - - # Add formatting info if available - if hasattr(func, "_kpi_format"): - kpi_record["format"] = func._kpi_format - else: - # Fallback if no function metadata available - kpi_record.update( - { - "name": kpi_id.replace("_", " ").title(), - "help": f"KPI: {kpi_id}", - "is_2d": isinstance(kpi.get("value"), list), - } - ) - - test_data["kpis"].append(kpi_record) - - # Convert to final structure - tests_list = [] - for run_id, test_data in tests_data.items(): - tests_list.append( - { - "run_id": run_id, - "labels": test_data["labels"], - "metadata": test_data["metadata"], - "kpis": test_data["kpis"], - } - ) - - return {"schema_version": "2", "tests": tests_list} - - def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, plugin, From 975112c5fba54a20585cef6f32ff56ff3da2b0db Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:27:21 +0200 Subject: [PATCH 328/344] [caliper] orchestration: postprocess: move logging code to postprocess_logging --- projects/caliper/orchestration/postprocess.py | 209 +----------------- .../orchestration/postprocess_logging.py | 165 ++++++++++++++ 2 files changed, 166 insertions(+), 208 deletions(-) create mode 100644 projects/caliper/orchestration/postprocess_logging.py diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 8fd31e2b3..a5a1f6661 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -11,7 +11,6 @@ import json import logging -import subprocess import time import traceback from pathlib import Path @@ -27,11 +26,11 @@ build_s3_export_command, build_s3_import_command, build_visualize_command, - log_caliper_start_banner, ) from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) +from projects.caliper.orchestration.postprocess_logging import _execute_caliper_command from projects.caliper.orchestration.postprocess_outcome import ( FINAL_KPI_PIPELINE_FAILED, FINAL_SUCCESS, @@ -46,212 +45,6 @@ logger = logging.getLogger(__name__) -def _format_command_for_log(command: list[str]) -> str: - """Format command for log file display with only --arguments on new lines. - - Args: - command: Command arguments list - - Returns: - Formatted command string for log file - """ - if not command: - return "" - - # Start with base command parts until we hit options - lines = [] - base_parts = [] - i = 0 - - # Collect all non-flag arguments into base_parts - while i < len(command) and not command[i].startswith("--"): - base_parts.append(command[i]) - i += 1 - - if base_parts: - lines.append(" ".join(base_parts)) - - # Handle --flag arguments on separate lines - while i < len(command): - if command[i].startswith("--"): - if i + 1 < len(command) and not command[i + 1].startswith("--"): - # Flag with value - lines.append(f"{command[i]} {command[i + 1]}") - i += 2 - else: - # Boolean flag - lines.append(command[i]) - i += 1 - else: - # Standalone argument - lines.append(command[i]) - i += 1 - - return " \\\n ".join(lines) - - -def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: - """Write step header to log file. - - Args: - log_file: Path to log file - command: Command that will be executed - step_name: Name of the step (e.g., "CALIPER PARSE") - """ - formatted_command = _format_command_for_log(command) - header = f"""{"=" * 60} -🚀 STARTING {step_name} (fork/exec) -📄 Command: - {formatted_command} -{"=" * 60} - -""" - with open(log_file, "w", encoding="utf-8") as f: - f.write(header) - - -def _write_step_footer_to_log_file( - log_file: Path, result: subprocess.CompletedProcess, step_name: str -) -> None: - """Write step footer to log file. - - Args: - log_file: Path to log file - result: Subprocess result - step_name: Name of the step (e.g., "CALIPER PARSE") - """ - footer = f""" - -{"=" * 60} -🏁 {step_name} COMPLETED -📊 Exit code: {result.returncode} -📄 Log file: {log_file} -{"=" * 60} -""" - with open(log_file, "a", encoding="utf-8") as f: - f.write(footer) - - -def _handle_caliper_output_and_completion_with_header( - result: subprocess.CompletedProcess, - log_file: Path, - status_file: Path, - step_name: str, -) -> dict: - """Handle Caliper output writing, status parsing, and log completion banner. - - This version appends to the log file instead of overwriting it, since we've - already written a header. - - Args: - result: Subprocess result from caliper execution - log_file: Path to write the step log file - status_file: Path to the status file to read and parse - step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") - - Returns: - Parsed status data dictionary - """ - import yaml - - # Append output to log file (don't overwrite since we've written header) - with open(log_file, "a", encoding="utf-8") as log_f: - if result.stdout: - log_f.write(result.stdout) - - # Also display output in main orchestration logs - if result.stdout: - prefix = step_name.upper() - for line in result.stdout.splitlines(): - logger.info("%s: %s", prefix, line) - - # Log banner indicating Caliper execution completed - logger.info("=" * 60) - logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) - logger.info("📊 Exit code: %s", result.returncode) - logger.info("📄 Log file: %s", log_file) - - # Read and parse status file - try: - with open(status_file, encoding="utf-8") as f: - status_data = yaml.safe_load(f) - # Handle case where YAML file is empty or invalid - if status_data is None: - status_data = { - "success": False, - "error": "Status file is empty or contains no valid YAML data", - } - except Exception as e: - status_data = {"success": False, "error": f"Failed to read status file: {e}"} - - # Log status file content for visibility - logger.info("📄 Status file content:") - if isinstance(status_data, dict): - for key, value in status_data.items(): - logger.info(" %s: %s", key, value) - else: - logger.info(" %s", status_data) - - logger.info("=" * 60) - - return status_data - - -def _execute_caliper_command( - command: list[str], - step_name: str, - status_file: Path, - step_logs_dir: Path, -) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: - """Execute a Caliper CLI command and return result and status data. - - Args: - command: CLI command arguments list - step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") - status_file: Path to read status YAML from - step_logs_dir: Directory to create log file in - - Returns: - Tuple of (subprocess result, parsed status data, log file path) - """ - # Create log file path with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(step_logs_dir) - step_name_safe = step_name.replace(" ", "_") - log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Create script file for debugging in step_scripts directory with index prefix - step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" - step_scripts_dir.mkdir(parents=True, exist_ok=True) - script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" - - # Log start banner and save script - log_caliper_start_banner(command, script_file, step_name.upper()) - - # Write header to log file - _write_step_header_to_log_file(log_file, command, step_name.upper()) - - # Execute command with combined stdout/stderr - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = _handle_caliper_output_and_completion_with_header( - result, log_file, status_file, step_name.upper() - ) - - # Write footer to log file - _write_step_footer_to_log_file(log_file, result, step_name.upper()) - - return result, status_data, log_file - - def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. diff --git a/projects/caliper/orchestration/postprocess_logging.py b/projects/caliper/orchestration/postprocess_logging.py new file mode 100644 index 000000000..d4ea1704f --- /dev/null +++ b/projects/caliper/orchestration/postprocess_logging.py @@ -0,0 +1,165 @@ +""" +Command logging and execution utilities for Caliper postprocessing. + +Handles formatting commands for logs, writing log headers/footers, and executing +commands with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _format_command_for_log(command: list[str]) -> str: + """Format command for log file display with only --arguments on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for log file + """ + if not command: + return "" + + # Start with base command parts until we hit options + lines = [] + base_parts = [] + i = 0 + + # Collect all non-flag arguments into base_parts + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle --flag arguments on separate lines + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return " \\\n ".join(lines) + + +def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: + """Write step header to log file. + + Args: + log_file: Path to log file + command: Command that will be executed + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + formatted_command = _format_command_for_log(command) + header = f"""{"=" * 60} +🚀 STARTING {step_name} (fork/exec) +📄 Command: + {formatted_command} +{"=" * 60} + +""" + with open(log_file, "w", encoding="utf-8") as f: + f.write(header) + + +def _write_step_footer_to_log_file( + log_file: Path, result: subprocess.CompletedProcess, step_name: str +) -> None: + """Write step footer to log file. + + Args: + log_file: Path to log file + result: Subprocess result + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + footer = f""" + +{"=" * 60} +🏁 {step_name} COMPLETED +📊 Exit code: {result.returncode} +📄 Log file: {log_file} +{"=" * 60} +""" + with open(log_file, "a", encoding="utf-8") as f: + f.write(footer) + + +def _handle_caliper_output_and_completion_with_header( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + This version appends to the log file instead of overwriting it, since we've + already written a header. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Append output to log file (don't overwrite since we've written header) + with open(log_file, "a", encoding="utf-8") as log_f: + if result.stdout: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + if isinstance(status_data, dict): + for key, value in status_data.items(): + logger.info(" %s: %s", key, value) + else: + logger.info(" %s", status_data) + + logger.info("=" * 60) + + return status_data From b06798523256a2332a8c9255ccfd5f2ca3a2e872 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 16:38:03 +0200 Subject: [PATCH 329/344] [caliper] orchestration: postprocess: move code to caliper_invocation --- .../orchestration/caliper_invocation.py | 741 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 437 +++++------ .../orchestration/postprocess_logging.py | 7 + 3 files changed, 925 insertions(+), 260 deletions(-) create mode 100644 projects/caliper/orchestration/caliper_invocation.py diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py new file mode 100644 index 000000000..8acad1b07 --- /dev/null +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -0,0 +1,741 @@ +""" +Caliper CLI command invocation functions. + +This module contains functions that build and execute caliper CLI commands +using subprocess execution with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +import tempfile +import time +import traceback +from pathlib import Path +from typing import Any + +from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_analyse_kpis_command, + build_kpi_csv_export_command, + build_kpi_generate_command, + build_parse_command, + build_s3_export_command, + build_s3_import_command, + build_visualize_command, +) +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) +from projects.core.library import env + +logger = logging.getLogger(__name__) + + +def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: + """Convert absolute path to relative path from base directory. + + Args: + file_path: Absolute or relative file path + base_dir: Base directory to make path relative to + + Returns: + Relative path as string + """ + try: + path_obj = Path(file_path) + if path_obj.is_absolute() and base_dir: + base_obj = Path(base_dir) + if base_obj.is_absolute(): + try: + relative = path_obj.relative_to(base_obj) + return str(relative) + except ValueError: + # Path is not relative to base_dir, return as-is + return str(file_path) + return str(file_path) + except Exception: + return str(file_path) + + +def _execute_caliper_command( + command: list[str], + step_name: str, + status_file: Path, + step_logs_dir: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: + """Execute a Caliper CLI command and return result and status data. + + Args: + command: CLI command arguments list + step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") + status_file: Path to read status YAML from + step_logs_dir: Directory to create log file in + + Returns: + Tuple of (subprocess result, parsed status data, log file path) + """ + from projects.caliper.orchestration.cli_builder import log_caliper_start_banner + from projects.caliper.orchestration.postprocess_logging import ( + _handle_caliper_output_and_completion_with_header, + _write_step_footer_to_log_file, + _write_step_header_to_log_file, + ) + from projects.caliper.orchestration.step_logging import _get_next_step_index + + # Create log file path with proper step index + step_index = _get_next_step_index(step_logs_dir) + step_name_safe = step_name.replace(" ", "_") + log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Create script file for debugging in step_scripts directory with index prefix + step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" + step_scripts_dir.mkdir(parents=True, exist_ok=True) + script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" + + # Log start banner and save script + log_caliper_start_banner(command, script_file, step_name.upper()) + + # Write header to log file + _write_step_header_to_log_file(log_file, command, step_name.upper()) + + # Execute command with combined stdout/stderr + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = _handle_caliper_output_and_completion_with_header( + result, log_file, status_file, step_name.upper() + ) + + # Write footer to log file + _write_step_footer_to_log_file(log_file, result, step_name.upper()) + + return result, status_data, log_file + + +def run_artifacts_to_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Generate KPI JSON using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.enabled: + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } + if not postprocess_config.kpi.artifacts_to_kpis.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_kpis disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi generate", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_artifacts_to_ai_data( + postprocess_config, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export AI evaluation data using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.artifacts_to_ai_data.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_ai_data disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "ai_data_dir": relative_path, # Expected by postprocess.py + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"AI evaluation export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_kpis_to_csv( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export KPI JSON to CSV using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.csv.enabled: + return { + "status": "disabled", + "reason": "kpi.csv disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.csv.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" + + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + kpi_json_path=kpi_json_path, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return { + "status": "disabled", + "reason": "analyze disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("KPI analysis failed in run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_parse_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the parse step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.parse.enabled: + return True, {"status": "disabled", "reason": "parse disabled", "completed_at": time.time()} + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Parse completed successfully", + "test_directories": status_data.get("test_directories", []), + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper parse failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Parse step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during parse: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_visualize_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, + output_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the visualize step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.visualize.enabled: + return True, { + "status": "disabled", + "reason": "visualize disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_visualize_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Visualize completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper visualize failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Visualize step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during visualize: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_import_step( + config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the S3 import step. + + Returns: + Tuple of (success, step_result) + """ + if not config.s3.import_.enabled: + return True, { + "status": "disabled", + "reason": "s3_import disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_import_command( + config=config, + status_file=status_file, + output_dir=output_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-import", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "S3 import completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 import failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 import step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during s3-import: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_export_step( + config: CaliperOrchestrationPostprocessConfig, + step_logs_dir: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> dict[str, Any]: + """Execute the S3 export step. + + Returns: + Step result dictionary + """ + if not config.s3.export.enabled: + return { + "status": "disabled", + "reason": "s3_export disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_export_command( + config=config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return { + "status": "success", + "detail": "S3 export completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 export failed: %s", error_msg) + return { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 export step execution failed") + return { + "status": "failed", + "detail": f"Exception during s3-export: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a5a1f6661..70430ebd5 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -18,6 +18,10 @@ from pydantic import ValidationError +from projects.caliper.orchestration.caliper_invocation import ( + _execute_caliper_command, + run_analyse_kpis, +) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, build_kpi_csv_export_command, @@ -30,7 +34,6 @@ from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) -from projects.caliper.orchestration.postprocess_logging import _execute_caliper_command from projects.caliper.orchestration.postprocess_outcome import ( FINAL_KPI_PIPELINE_FAILED, FINAL_SUCCESS, @@ -115,11 +118,122 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() + # _transform_kpis_to_hierarchical_format function removed - CLI commands now handle format conversion + + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + test_data = tests_data[run_id] + + # Extract common labels (excluding KPI-specific ones) + kpi_labels = kpi.get("labels", {}) + test_labels = { + k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] + } # Exclude KPI-specific labels + + # Merge test labels (they should be the same for all KPIs in a test) + if not test_data["labels"]: + test_data["labels"] = test_labels + + # Store test metadata from first KPI + if not test_data["metadata"]: + test_data["metadata"] = { + "timestamp": kpi.get("timestamp"), + "source": kpi.get("source", {}), + "run_id": run_id, + } + + # Create KPI record with metadata + kpi_id = kpi.get("kpi_id") + raw_value = kpi.get("value") + + # Transform 2D KPI values to structured format + if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: + # Check if this looks like 2D data: list of tuples/lists with 2 elements + first_item = raw_value[0] + if isinstance(first_item, list | tuple) and len(first_item) == 2: + try: + # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format + structured_value = { + "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], + "count": len(raw_value), + } + final_value = structured_value + except (ValueError, TypeError, IndexError): + # If conversion fails, keep original value + final_value = raw_value + else: + final_value = raw_value + else: + final_value = raw_value + + kpi_record = { + "id": kpi_id, + "value": final_value, + "unit": kpi.get("unit"), + "higher_is_better": kpi_labels.get("higher_is_better", True), + } + + # Add KPI metadata from function decorator if available + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + kpi_record.update( + { + "name": ( + func.__doc__.replace(" KPI.", "") + if func.__doc__ + else kpi_id.replace("_", " ").title() + ), + "help": getattr(func, "_kpi_help", ""), + } + ) + + # Add 2D-specific metadata if present + if getattr(func, "_kpi_is_2d", False): + kpi_record.update( + { + "is_2d": True, + "x_unit": getattr(func, "_kpi_x_unit", ""), + "x_help": getattr(func, "_kpi_x_help", ""), + "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_help": getattr(func, "_kpi_y_help", None) + or getattr(func, "_kpi_help", ""), + } + ) + else: + kpi_record["is_2d"] = False + + # Add formatting info if available + if hasattr(func, "_kpi_format"): + kpi_record["format"] = func._kpi_format + else: + # Fallback if no function metadata available + kpi_record.update( + { + "name": kpi_id.replace("_", " ").title(), + "help": f"KPI: {kpi_id}", + "is_2d": isinstance(kpi.get("value"), list), + } + ) + + test_data["kpis"].append(kpi_record) + + # Convert to final structure + tests_list = [] + for run_id, test_data in tests_data.items(): + tests_list.append( + { + "run_id": run_id, + "labels": test_data["labels"], + "metadata": test_data["metadata"], + "kpis": test_data["kpis"], + } + ) + + return {"schema_version": "2", "tests": tests_list} + def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, @@ -181,43 +295,6 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): - # Transform JSONL (schema v1) to hierarchical JSON (schema v2) - try: - logger.info(f"Transforming KPI output to hierarchical format: {output_file}") - - # Read the generated JSONL file - kpis = [] - with open(output_file, encoding="utf-8") as f: - for line in f: - line = line.strip() - if line: - import json - - kpis.append(json.loads(line)) - - if kpis: - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) - - import json - - with open(output_file, "w", encoding="utf-8") as f: - json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) - - logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format" - ) - else: - logger.warning("No KPI records found in output file") - - except Exception as transform_error: - logger.error(f"Failed to transform KPIs to hierarchical format: {transform_error}") - return { - "status": "failed", - "error": f"KPI transformation failed: {transform_error}", - "completed_at": time.time(), - "log_file": log_file, - } - relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) logger.info( f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" @@ -251,8 +328,6 @@ def _run_artifacts_to_kpis( def _run_artifacts_to_ai_data( postprocess_config, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, @@ -346,94 +421,8 @@ def _load_test_labels(test_dir: Path) -> dict[str, Any]: return {} -def _export_test_entries_with_artifacts( - model, ai_data_dir: Path, base_dir: Path, plugin -) -> list[dict]: - """ - Export test entries by creating directories and copying specific artifacts. - - Args: - model: Unified model containing test results - ai_data_dir: Directory where test entries should be exported - base_dir: Base directory of the test artifacts (test directory) - plugin: Plugin instance to get artifact file list - - Returns: - List of exported test entry information - """ - import shutil - - exported_entries = [] - - for idx, record in enumerate(model.unified_result_records): - # Create directory for this test entry - test_entry_dir = ai_data_dir / f"test_entry_{idx:03d}" - test_entry_dir.mkdir(parents=True, exist_ok=True) - - # Load test labels from __test_labels__.yaml if available - test_dir = base_dir / record.test_base_path - test_labels = _load_test_labels(test_dir) - - # Record test entry metadata - entry_info = { - "entry_id": f"test_entry_{idx:03d}", - "test_base_path": str(record.test_base_path), - "distinguishing_labels": record.distinguishing_labels, - "test_labels": test_labels, - "copied_files": [], - "missing_files": [], - } - - # Get artifact files specific to this test directory only (plugin is scoped to test directory) - relevant_files = plugin.get_ai_data_artifact_files_for_test(test_dir) - - logger.debug( - f"Test entry {idx}: found {len(relevant_files)} artifact files in test directory {test_dir}" - ) - - # Copy relevant files for this test entry (preserving directory structure) - for target_file in relevant_files: - # source is test_dir + test_relative_path, target is test_entry_dir + test_relative_path - source_file = test_dir / target_file - target_path = test_entry_dir / target_file - - if source_file.exists(): - target_path.parent.mkdir(parents=True, exist_ok=True) - - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "relative_path": target_file, - "size_bytes": source_file.stat().st_size, - } - ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) - else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) - - # Write entry metadata - entry_metadata_file = test_entry_dir / "entry_metadata.json" - - with open(entry_metadata_file, "w") as f: - json.dump(entry_info, f, indent=2) - - exported_entries.append(entry_info) - - return exported_entries - - def _run_kpis_to_csv( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, kpi_json_path: Path, base_dir: Path, @@ -552,35 +541,23 @@ def _run_analyse_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "analyse_kpis_status.yaml" - # Note: CLI command building removed in favor of direct function call - - # Load plugin for KPI definitions - from projects.caliper.engine.load_plugin import load_plugin - - plugin = load_plugin(plugin_module) - - # Find the most recent baseline file - from projects.caliper.engine.kpi.analyze_hierarchical import ( - analyze_hierarchical_kpis, - find_most_recent_baseline, + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, ) - baseline_file = find_most_recent_baseline(historical_kpis_dir) - if not baseline_file: - return { - "status": "failed", - "error": f"No baseline KPI files found in {historical_kpis_dir}", - "completed_at": time.time(), - "log_file": None, - } - - # Run hierarchical KPI analysis using core engine - logger.info(f"Running KPI analysis: {current_kpis_file} vs {baseline_file}") - result = analyze_hierarchical_kpis( - current_kpis_path=current_kpis_file, - baseline_kpis_path=baseline_file, - output_path=output_file, - plugin=plugin, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, ) # Clean up temporary status file @@ -589,19 +566,16 @@ def _run_analyse_kpis( except FileNotFoundError: pass - # Handle the analysis result - if result["status"] == "success": - logger.info( - f"Analysis completed: {result['regressions_count']} regressions, {result['improvements_count']} improvements" - ) + # Convert to expected format + if status_data.get("success"): return { "status": "success", "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), - "findings_count": result["findings_count"], - "regressions_count": result["regressions_count"], - "improvements_count": result["improvements_count"], + "findings_count": status_data.get("findings_count", 0), + "regressions_count": status_data.get("regressions_count", 0), + "improvements_count": status_data.get("improvements_count", 0), "completed_at": time.time(), - "log_file": None, + "log_file": log_file, } else: return { @@ -1077,117 +1051,66 @@ def _run_kpi_and_ai_data_steps(self) -> None: if not self.config.kpi.enabled: return + # Setup output directory and module string with focused error handling try: # Determine output directory for KPI/AI data steps - use base artifact directory output_dir = env.ARTIFACT_DIR logger.info(f"KPI steps using base artifact directory: {output_dir}") output_dir.mkdir(parents=True, exist_ok=True) - # Load plugin and model - from projects.caliper.engine.load_plugin import load_plugin - from projects.caliper.engine.parse import run_parse - from projects.caliper.engine.plugin_config import resolve_plugin_module_string - - # Load plugin - mod_str, _manifest = resolve_plugin_module_string( - base_dir=self.tree_root, - postprocess_config=self.manifest_path, - cli_plugin=self.config.plugin_module, - ) - plugin = load_plugin(mod_str) - - # Parse model - model = run_parse( - base_dir=self.tree_root, - plugin_module=mod_str, - plugin=plugin, - use_cache=not self.config.parse.no_cache, - ) - - # KPI JSON generation - self._run_artifacts_to_kpis_step(plugin, model, output_dir, mod_str) - - # KPI CSV export - self._run_kpis_to_csv_step(plugin, model, output_dir) - - # AI evaluation export - self._run_artifacts_to_ai_data_step(plugin, model, output_dir, mod_str) - - # S3 import (historical data) - self._run_s3_import_step(output_dir) - - # Analyze KPIs (current vs historical) - moved before S3 export - self._run_analyse_kpis_step(output_dir, mod_str) - - # S3 export - self._run_s3_export_step(output_dir) - + # Resolve plugin module string (this is just string manipulation, not engine access) + mod_str = self.config.plugin_module or "unknown" except Exception as e: completion_time = time.time() - logger.error(f"Failed to run KPI/AI eval operations: {e}") - self._add_step( + logger.error(f"Failed to setup KPI/AI eval operations: {e}") + # Only mark all steps as failed if basic setup fails + for step_name in [ "artifacts_to_kpis", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "kpis_to_csv", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "artifacts_to_ai_data", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "s3_import", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "analyse_kpis", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "s3_export", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) + ]: + self._add_step( + step_name, + { + "status": "failed", + "error": f"Setup failed: {e}", + "completed_at": completion_time, + }, + ) self.artifacts_to_kpis_failed = True self.ai_data_failed = True self.s3_import_failed = True self.analyze_failed = True self.s3_export_failed = True + return - def _run_artifacts_to_kpis_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + # Run each step independently - each has its own error handling + # KPI JSON generation + self._run_artifacts_to_kpis_step(output_dir, mod_str) + + # KPI CSV export + self._run_kpis_to_csv_step(output_dir) + + # AI evaluation export + self._run_artifacts_to_ai_data_step(output_dir, mod_str) + + # S3 import (historical data) + self._run_s3_import_step(output_dir) + + # Analyze KPIs (current vs historical) - moved before S3 export + self._run_analyse_kpis_step(output_dir, mod_str) + + # S3 export + self._run_s3_export_step(output_dir) + + def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: result = _run_artifacts_to_kpis( self.config, - plugin, - model, output_dir, mod_str, self.tree_root, @@ -1207,7 +1130,7 @@ def _run_artifacts_to_kpis_step( }, ) - def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> None: + def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" if not self.config.kpi.kpis_to_csv.enabled: self._add_step( @@ -1223,8 +1146,6 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv( self.config, - plugin, - model, output_dir, kpi_json_path, self.tree_root, @@ -1237,9 +1158,7 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No # CSV export failure doesn't affect overall status - it's supplementary logger.warning("KPI CSV export failed but continuing execution") - def _run_artifacts_to_ai_data_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + def _run_artifacts_to_ai_data_step(self, output_dir: Path, mod_str: str) -> None: """Execute the AI evaluation export step.""" if not self.config.kpi.artifacts_to_ai_data.enabled: self._add_step( @@ -1255,8 +1174,6 @@ def _run_artifacts_to_ai_data_step( try: result = _run_artifacts_to_ai_data( self.config, - plugin, - model, output_dir, mod_str, self.tree_root, @@ -1385,7 +1302,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - result = _run_analyse_kpis( + result = run_analyse_kpis( postprocess_config=self.config, output_dir=output_dir, plugin_module=plugin_module, diff --git a/projects/caliper/orchestration/postprocess_logging.py b/projects/caliper/orchestration/postprocess_logging.py index d4ea1704f..fd4e15daa 100644 --- a/projects/caliper/orchestration/postprocess_logging.py +++ b/projects/caliper/orchestration/postprocess_logging.py @@ -149,6 +149,13 @@ def _handle_caliper_output_and_completion_with_header( "success": False, "error": "Status file is empty or contains no valid YAML data", } + # Normalize non-mapping payloads (lists, strings, integers) into failure dict + elif not isinstance(status_data, dict): + status_data = { + "success": False, + "error": f"Status file contains non-mapping data: {type(status_data).__name__}", + "raw_data": status_data, + } except Exception as e: status_data = {"success": False, "error": f"Failed to read status file: {e}"} From 0a43c5da9fe48e9c51b6d8c2683d07ace1e97339 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:29:09 +0200 Subject: [PATCH 330/344] [llm_d] orchestration: test_phase: save extra KPI labels --- projects/llm_d/orchestration/test_phase.py | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 7ba815fbc..6e8455b2b 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -140,6 +140,32 @@ def ensure_kueue_local_queue() -> None: logger.info("LocalQueue %s created successfully", queue_name) +def extract_kpi_labels_from_config() -> dict[str, str]: + """Extract kpi_labels from project configuration. + + Returns: + Dictionary of kpi_labels for system context + """ + kpi_labels = {} + + # Extract platform name + platform_name = config.project.get_config("cpt.kpi.labels.platform_name") + if platform_name: + kpi_labels["platform"] = platform_name + + # Extract gpu_type + gpu_type = config.project.get_config("cpt.kpi.labels.gpu_type") + if gpu_type: + kpi_labels["gpu_type"] = gpu_type + + # Extract test_harness + test_harness = config.project.get_config("cpt.kpi.labels.test_harness") + if test_harness: + kpi_labels["test_harness"] = test_harness + + return kpi_labels + + def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" @@ -155,7 +181,10 @@ def create_test_labels() -> None: if benchmark_keys: labels["guidellm_loadshape"] = benchmark_keys[0] - write_test_labels(env.ARTIFACT_DIR, labels) + # Extract kpi_labels from config + kpi_labels = extract_kpi_labels_from_config() + + write_test_labels(env.ARTIFACT_DIR, labels, kpi_labels=kpi_labels if kpi_labels else None) logger.info("Created test labels: %s", labels) # Dump config.project to config.yaml From a2c5a8642f01ff13287c21e909656060d8b70665 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:29:53 +0200 Subject: [PATCH 331/344] [core] library: postprocess: allow saving kpi_labels in the test_labels file --- projects/core/library/postprocess.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 5fe2c100a..8eccb5af4 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -33,7 +33,12 @@ def write_test_labels( - directory: Path, labels: dict[str, str], *, version: str = "1", dump_config: bool = True + directory: Path, + labels: dict[str, str], + *, + version: str = "1", + dump_config: bool = True, + kpi_labels: dict[str, str] | None = None, ) -> Path: """Write a __test_labels__.yaml file to mark a directory as a Caliper test base. @@ -42,6 +47,7 @@ def write_test_labels( labels: Dictionary of label key-value pairs version: Version string for the test labels format (default: "1") dump_config: Whether to save project configuration to config.yaml (default: True) + kpi_labels: Optional dictionary of KPI labels for system context Returns: Path to the created __test_labels__.yaml file @@ -53,6 +59,10 @@ def write_test_labels( "model": "llama-3", "deployment": "single-zone", "rate": "10" + }, + kpi_labels={ + "platform": "CKS", + "gpu_type": "H100" } ) """ @@ -62,6 +72,9 @@ def write_test_labels( "labels": labels, } + if kpi_labels: + payload["kpi_labels"] = kpi_labels + # Create directory and write YAML test_labels_path.parent.mkdir(parents=True, exist_ok=True) with test_labels_path.open("w", encoding="utf-8") as handle: From 6201c537897c5f570166090454497790f453f7ae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:31:13 +0200 Subject: [PATCH 332/344] [guidellm] postprocess: guidellm: generate the KPI labels --- .../postprocess/guidellm/parsing/kpis.py | 47 +++-- .../postprocess/guidellm/parsing/parsers.py | 190 ++++++++++++++++++ 2 files changed, 219 insertions(+), 18 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index 2eb4ac959..35ecf5a85 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -13,7 +13,6 @@ LowerBetter, TwoDimensional, build_catalog_from_functions, - create_label_extractor, get_kpi_functions, is_2d_kpi, ) @@ -220,13 +219,34 @@ def guidellm_tpot_p99(unified_record) -> list[tuple[float, float]]: class GuideLLMKpiHandler: """Handles KPI catalog and computation for GuideLLM benchmarks.""" - # Define label extractor for all GuideLLM test conditions - LABEL_EXTRACTOR = create_label_extractor( - { - "strategy": "metrics.strategy", - "duration": "metrics.duration", + # Define custom label extractor for GuideLLM with fallbacks for missing kpi_labels + @staticmethod + def _extract_labels(record) -> dict[str, Any]: + """Extract labels from record with fallbacks for missing kpi_labels.""" + + from projects.caliper.engine.kpi.decorators import _extract_value_by_path + + labels = {} + + # Extract artifact-based labels (graceful handling) + artifact_fields = { + "product_version": "metrics.product_version", + "cluster": "metrics.cluster", + "deployment_profile": "metrics.deployment_profile", + "model_name": "metrics.model_name", + "load_shape": "metrics.benchmark_key", } - ) + + for label_key, path in artifact_fields.items(): + value = _extract_value_by_path(record, path) + if value is not None: + labels[label_key] = str(value) + + labels |= record.metrics.get("kpi_labels", {}) or {} + + return labels + + LABEL_EXTRACTOR = type("TestLabelExtractor", (), {"extract": _extract_labels})() # Metadata fields to include in KPI records but not as labels @staticmethod @@ -284,7 +304,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: # Generate scalar KPIs for each record for r in valid_records: - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -303,9 +322,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if value is None: continue - # Merge base labels, test condition labels, and system labels + # Use only extracted KPI labels, not test labels all_labels = { - **base_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } @@ -338,7 +356,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not curves or not request_rates: continue - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -357,14 +374,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not value or value is None: continue - # Remove rate-specific labels for aggregated KPIs - aggregated_labels = { - k: v - for k, v in base_labels.items() - if k not in ["concurrency", "rate", "max_concurrency"] - } + # Use only extracted KPI labels, not test labels all_labels = { - **aggregated_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index 4208b2bcf..3853ef99f 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -7,6 +7,9 @@ from pathlib import Path from typing import Any +import jsonpath_ng +import yaml + from projects.caliper.engine.model import ( ParseResult, TestBaseNode, @@ -27,6 +30,77 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: return {"facet": "default"} +def _kpi_labels_from_node(node: TestBaseNode) -> dict[str, Any]: + """Extract kpi_labels from a test node.""" + raw = node.labels + kpi_labels = raw.get("kpi_labels") + if isinstance(kpi_labels, dict): + return dict(kpi_labels) + return {} + + +def extract_field_by_jsonpath(data: dict[str, Any], jsonpath: str, default: Any = None) -> Any: + """ + Extract a field from nested dictionary using JSONPath notation. + + Args: + data: The dictionary to extract from + jsonpath: JSONPath like "spec.model.name" or "spec.template.containers[0].name" + default: Default value if field not found + + Returns: + The extracted value or default if not found + + Examples: + extract_field_by_jsonpath(yaml_data, "spec.model.name") + extract_field_by_jsonpath(yaml_data, "spec.template.containers[0].name") + """ + try: + return jsonpath_ng.parse(jsonpath).find(data)[0].value + except IndexError: + return default + + +def parse_product_version_from_annotation(annotation_value: str) -> str | None: + """ + Parse product version from kserve config annotation. + + Args: + annotation_value: Raw annotation like "v3-5-0-ea-2-kserve-config-llm-decode-template" + + Returns: + Cleaned version like "v3.5.0-ea.2" or None if parsing fails + + Examples: + parse_product_version_from_annotation("v3-5-0-ea-2-kserve-config-llm-decode-template") + # Returns: "v3.5.0-ea.2" + """ + if not annotation_value: + return None + + # Remove the kserve suffix + suffix = "-kserve-config-llm-decode-template" + if annotation_value.endswith(suffix): + version_part = annotation_value[: -len(suffix)] + else: + version_part = annotation_value + + # Transform v3-5-0-ea-2 -> v3.5.0-ea.2 + # Replace hyphens with dots in the version numbers, keep -ea- as is + if version_part.startswith("v") and "-ea-" in version_part: + # Split on -ea- to handle the pre-release part separately + base_version, ea_part = version_part.split("-ea-", 1) + + # Replace hyphens with dots in base version (v3-5-0 -> v3.5.0) + base_version = base_version.replace("-", ".") + + # Reconstruct with -ea. format + cleaned_version = f"{base_version}-ea.{ea_part}" + return cleaned_version + + return None + + class GuideLLMParser: """Parser for GuideLLM benchmark JSON artifacts.""" @@ -36,6 +110,99 @@ def _is_benchmarks_artifact(path: Path) -> bool: path.name.startswith("benchmarks-rate-") and path.suffix == ".json" ) + @staticmethod + def _is_llmisvc_artifact(path: Path) -> bool: + """Check if path is an LLMInferenceService YAML artifact.""" + return path.name in [ + "llminferenceservice.yaml", + "llminferenceservice.yml", + ] and "__capture_llmisvc_state" in str(path) + + @staticmethod + def _is_config_artifact(path: Path) -> bool: + """Check if path is a config.yaml artifact.""" + return path.name == "config.yaml" + + def extract_fields_from_llmisvc(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from LLMInferenceService YAML file. + + Args: + file_path: Path to llminferenceservice.yaml file + + Returns: + Dictionary with extracted fields (product_version, deployment_profile, model_name) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract product version from kserve annotation + annotation_value = extract_field_by_jsonpath( + yaml_data, 'status.annotations["serving.kserve.io/config-llm-decode-template"]' + ) + if annotation_value: + product_version = parse_product_version_from_annotation(annotation_value) + if product_version: + result["product_version"] = product_version + logging.info(f"Extracted product_version '{product_version}' from {file_path}") + + # Extract deployment profile from forge annotation + deployment_profile = extract_field_by_jsonpath( + yaml_data, 'metadata.annotations["forge.openshift.io/deployment-profile"]' + ) + if deployment_profile: + result["deployment_profile"] = deployment_profile + logging.info( + f"Extracted deployment_profile '{deployment_profile}' from {file_path}" + ) + + # Extract model name from spec + model_name = extract_field_by_jsonpath(yaml_data, "spec.model.name") + if model_name: + result["model_name"] = model_name + logging.info(f"Extracted model_name '{model_name}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + + def extract_fields_from_config(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from config.yaml file. + + Args: + file_path: Path to config.yaml file + + Returns: + Dictionary with extracted fields (cluster, benchmark_key) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract cluster name + cluster = extract_field_by_jsonpath(yaml_data, "ci_job.cluster") + if cluster: + result["cluster"] = cluster + logging.info(f"Extracted cluster '{cluster}' from {file_path}") + + # Extract benchmark key + benchmark_key = extract_field_by_jsonpath(yaml_data, "runtime.benchmark_key") + if benchmark_key: + result["benchmark_key"] = benchmark_key + logging.info(f"Extracted benchmark_key '{benchmark_key}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + def parse_benchmarks_json( self, file_path: Path ) -> tuple[list[GuideLLMBenchmark], GuideLLMConfiguration | None, list[str]]: @@ -376,6 +543,10 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: benchmarks_files = [p for p in node.artifact_paths if self._is_benchmarks_artifact(p)] benchmarks_files.sort(key=lambda path: path.name) + # Also look for LLMInferenceService YAML files and config.yaml files for system information + llmisvc_files = [p for p in node.artifact_paths if self._is_llmisvc_artifact(p)] + config_files = [p for p in node.artifact_paths if self._is_config_artifact(p)] + if not benchmarks_files: # No benchmark result JSON found for this node, create empty record labels = _labels_from_node(node) @@ -408,6 +579,25 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: if node_config: metrics["configuration"] = node_config.to_dict() + # Extract fields from LLMInferenceService YAML if available + for llmisvc_file in llmisvc_files: + llmisvc_fields = self.extract_fields_from_llmisvc(llmisvc_file) + for field_name, field_value in llmisvc_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract fields from config.yaml if available + for config_file in config_files: + config_fields = self.extract_fields_from_config(config_file) + for field_name, field_value in config_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract kpi_labels from test labels + kpi_labels = _kpi_labels_from_node(node) + if kpi_labels: + metrics["kpi_labels"] = kpi_labels + records.append( UnifiedResultRecord( test_base_path=str(node.test_path), From 1cafc9eabf3f1391da711adc0eef348715368524 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:33:30 +0200 Subject: [PATCH 333/344] [llm_d] orchestration: config: xks CPT labels --- projects/llm_d/orchestration/config.d/cpt.yaml | 5 +++++ projects/llm_d/orchestration/presets.d/cks.yaml | 4 ++++ projects/llm_d/orchestration/presets.d/janus.yaml | 4 ++++ 3 files changed, 13 insertions(+) create mode 100644 projects/llm_d/orchestration/config.d/cpt.yaml diff --git a/projects/llm_d/orchestration/config.d/cpt.yaml b/projects/llm_d/orchestration/config.d/cpt.yaml new file mode 100644 index 000000000..bfb9b06c8 --- /dev/null +++ b/projects/llm_d/orchestration/config.d/cpt.yaml @@ -0,0 +1,5 @@ +kpi: + labels: + platform_name: null + gpu_type: null + test_harness: null diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index 1b59f614b..b03451fce 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -17,6 +17,10 @@ cks: platform.inference_service.scheduler.node_selector: gpu.nvidia.com/class: H200 + cpt.kpi.labels.platform_name: CKS + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + cks-intelligent: extends: - cks diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml index 9612228ce..ed629be57 100644 --- a/projects/llm_d/orchestration/presets.d/janus.yaml +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -10,6 +10,10 @@ janus: platform.gateway.status_address_name: null platform.gateway.status_address_name: gateway-internal + cpt.kpi.labels.platform_name: OCP + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + janus-intelligent: extends: - janus From 60b9c09c8be027312a42dd60bbc68fbe774d8282 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:56:19 +0200 Subject: [PATCH 334/344] [caliper] KPI analyze --- projects/caliper/cli/commands.py | 278 +++++------- projects/caliper/cli/kpi_analysis.py | 27 +- projects/caliper/cli/main.py | 4 +- projects/caliper/engine/kpi/analyze.py | 425 ++++++++++++++---- .../engine/kpi/analyze_hierarchical.py | 153 +++++-- projects/caliper/engine/kpi/format.py | 1 + projects/caliper/engine/kpi/generate.py | 22 +- projects/caliper/orchestration/postprocess.py | 112 +---- 8 files changed, 610 insertions(+), 412 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 4ae1710be..22be577ec 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -13,7 +13,6 @@ from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml -from projects.caliper.engine.kpi.analyze import run_analyze from projects.caliper.engine.kpi.generate import run_kpi_generate from projects.caliper.engine.kpi.import_export import ( import_kpis_snapshot, @@ -447,6 +446,13 @@ def ai_eval_export( @click.command("generate") @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option( + "--format", + "format_type", + type=click.Choice(["hierarchical", "jsonl"], case_sensitive=False), + default="hierarchical", + help="Output format: hierarchical JSON (default) or JSONL", +) @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) @@ -454,6 +460,7 @@ def ai_eval_export( def kpi_generate( ctx: click.Context, output: Path, + format_type: str, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -478,6 +485,7 @@ def kpi_generate( output=output, use_cache=True, cache_path=None, + format_type=format_type, ) status_data = {"success": True, "output_file": str(output)} click.echo(f"Generated {output}") @@ -602,97 +610,6 @@ def kpi_import(ctx: click.Context, snapshot: Path) -> None: click.echo(f"Imported {snapshot}") -@click.command("analyze") -@click.option( - "--current", type=click.Path(path_type=Path), required=True, help="Current KPI file to analyze" -) -@click.option( - "--baseline-dir", - type=click.Path(path_type=Path), - required=True, - help="Directory containing historical KPI files (will use most recent)", -) -@click.option( - "--output", - type=click.Path(path_type=Path), - required=True, - help="Output file for analysis results", -) -@click.option( - "--plugin", - "plugin_module", - metavar="MODULE", - required=True, - help="Plugin module for KPI definitions and analysis rules", -) -@click.pass_context -def kpi_analyze( - ctx: click.Context, - current: Path, - baseline_dir: Path, - output: Path, - plugin_module: str, -) -> None: - """Analyze KPIs for regressions using hierarchical format.""" - try: - from projects.caliper.engine.kpi.analyze_hierarchical import find_most_recent_baseline - from projects.caliper.engine.load_plugin import load_plugin - - # Load plugin for KPI definitions and analysis rules - plugin = load_plugin(plugin_module) - if not plugin: - click.echo(f"❌ Failed to load plugin: {plugin_module}", err=True) - sys.exit(1) - - click.echo(f"🔌 Using plugin: {plugin_module}") - - baseline_kpis = find_most_recent_baseline(baseline_dir) - if not baseline_kpis: - click.echo( - f"❌ No kpis.json files found in baseline directory: {baseline_dir}", err=True - ) - sys.exit(1) - - click.echo(f"📊 Found {len(baseline_kpis)} baseline files to process") - - # Run analysis with ALL baseline files (not just the most recent) - result = run_analyze( - current_path=current, baseline_kpis=baseline_kpis, output_path=output, plugin=plugin - ) - - # Check result status at CLI level - if result["status"] == "success": - click.echo(f"✅ Analysis complete. Results written to: {output}") - elif result["status"] == "skipped": - click.echo(f"⚠️ Analysis skipped: {result.get('reason', 'Unknown reason')}") - click.echo(f"📝 Output written to: {output}") - else: - click.echo(f"❌ Analysis failed: {result.get('error', 'Unknown error')}", err=True) - sys.exit(1) - - # Show the analysis results on screen - if output.exists(): - try: - import json - - with open(output) as f: - result_data = json.load(f) - - # Convert to YAML for better readability - yaml_output = yaml.dump(result_data, default_flow_style=False, indent=2) - click.echo("\n📊 Analysis Results:") - click.echo("=" * 50) - click.echo(yaml_output) - except Exception as e: - click.echo(f"⚠️ Could not display results: {e}") - else: - click.echo("⚠️ Output file not found") - except Exception as e: # noqa: BLE001 - raise e - click.echo(f"❌ kpi analyze failed: {e}", err=True) - sys.exit(3) - - @click.command("analyse-kpis") @click.option( "--output", @@ -702,13 +619,13 @@ def kpi_analyze( ) @click.option( "--current-kpis-file", - type=click.Path(path_type=Path, exists=True), + type=click.Path(path_type=Path), required=True, help="Path to current KPIs JSON file", ) @click.option( "--historical-kpis-dir", - type=click.Path(path_type=Path, exists=True), + type=click.Path(path_type=Path), required=True, help="Directory containing historical KPI files", ) @@ -730,85 +647,37 @@ def analyse_kpis_cmd( status_file: Path | None, ) -> None: """Analyze KPIs for orchestration (fork/exec).""" - import yaml + import time - status_data = {"success": False} - - try: - # Create a minimal postprocess config for analysis - from projects.caliper.orchestration.postprocess_config import ( - CaliperOrchestrationPostprocessConfig, - ) + import yaml - # Create config with analyze enabled and using provided directories - postprocess_config_dict = { - "analyze": { - "enabled": True, - "output": output.name, # Use the provided output filename - "historical_kpis": str(historical_kpis_dir), # Use provided historical dir - } + # Validate paths exist before proceeding + if not current_kpis_file.exists(): + status_data = { + "success": False, + "error": f"Current KPI file not found: {current_kpis_file}", + "completed_at": time.time(), } + click.echo(f"❌ Current KPI file not found: {current_kpis_file}", err=True) - postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( - postprocess_config_dict - ) - - output_dir = output.parent - - # Run analysis - from projects.caliper.engine.kpi.analyze import analyze_kpis - - try: - result = analyze_kpis( - postprocess_config=postprocess_config_obj, - plugin_module=plugin_module, - base_dir=output_dir, # Use output dir as base since we have absolute paths - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) - except Exception as e: - # Don't suppress the exception - let it bubble up with full traceback - import sys - import traceback - - print(f"❌ Exception in analyze_kpis: {e}", file=sys.stderr) - print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) - raise - - # Check if the analysis function returned a proper result - if not isinstance(result, dict) or "status" not in result: - raise ValueError(f"Invalid result from analyze_kpis: {result}") + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + sys.exit(1) + if not historical_kpis_dir.exists(): status_data = { - "success": result.get("status") == "success", - "output_file": result.get("output_file"), + "success": False, + "error": f"Historical KPI directory not found: {historical_kpis_dir}", + "completed_at": time.time(), } + click.echo(f"❌ Historical KPI directory not found: {historical_kpis_dir}", err=True) - if result.get("status") == "success": - click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") - else: - # Show the full result for debugging if no clear error message - if "error" in result: - error_msg = result["error"] - else: - error_msg = f"Analysis failed with result: {result}" - - status_data["error"] = error_msg - status_data.update(result) # Include all result data for debugging - click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) - if not status_file: - sys.exit(1) - - except Exception as e: # noqa: BLE001 - import traceback - - full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} - click.echo(f"❌ analyse-kpis failed: {e}", err=True) - click.echo(f"Full traceback:\n{full_traceback}", err=True) - if not status_file: - sys.exit(2) - finally: # Write status file if requested if status_file: try: @@ -817,6 +686,85 @@ def analyse_kpis_cmd( except Exception as status_err: click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) sys.exit(4) + sys.exit(2) + + # Delegate to engine layer + from projects.caliper.engine.kpi.analyze import analyze_kpis + + result = analyze_kpis( + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + output_file=output, + plugin_module=plugin_module, + ) + + # Format status data for CLI + status_data = { + "success": result.get("success", False), + "completed_at": result.get("completed_at"), + } + + if result.get("output_file"): + status_data["output_file"] = result["output_file"] + + if result.get("error"): + status_data["error"] = result["error"] + + # Extract detailed analysis results for status file + if result.get("success") and result.get("output_file"): + try: + import yaml + + with open(result["output_file"]) as f: + analysis_report = yaml.safe_load(f) + + # Extract key fields from the analysis report + if analysis_report: + processed = analysis_report.get("processed", {}) + tested = analysis_report.get("tested", {}) + overall = analysis_report.get("overall", {}) + results = analysis_report.get("results", []) + + status_data.update( + { + "baseline_source_count": processed.get("baseline_source_count", 0), + "tested": { + "total_kpis": tested.get("total_kpis", 0), + "regressions": tested.get("regressions", 0), + "passes": tested.get("passes", 0), + "skipped": tested.get("skipped", 0), + }, + "results": results, + "overall": { + "verdict": overall.get("verdict", "UNKNOWN"), + "regression_count": overall.get("regression_count", 0), + "total_tested": overall.get("total_tested", 0), + "total_skipped": overall.get("total_skipped", 0), + }, + } + ) + except Exception as e: + # If we can't read the analysis report, just include basic info + status_data["analysis_report_error"] = f"Could not read analysis report: {e}" + + if result.get("result"): + status_data.update(result["result"]) # Include analysis result for debugging + + # Display result + if result.get("success"): + click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") + else: + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) + + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) # Exit with error code if operation failed if not status_data.get("success", False): diff --git a/projects/caliper/cli/kpi_analysis.py b/projects/caliper/cli/kpi_analysis.py index 7acc5ca84..4eeee8d77 100644 --- a/projects/caliper/cli/kpi_analysis.py +++ b/projects/caliper/cli/kpi_analysis.py @@ -33,21 +33,34 @@ ) def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module: str) -> None: """CLI entrypoint for KPI analysis.""" - from projects.caliper.engine.kpi.analyze import run_kpi_analysis + from projects.caliper.engine.kpi.analyze import ( + analyze_kpis_with_format_conversion, + status_dict_to_exit_code, + ) try: - # Call the core analysis function from engine - exit_code = run_kpi_analysis( - current_kpi_file=current, - historical_data_dir=historical_dir, + # Call the core analysis function from engine (returns status dict) + result = analyze_kpis_with_format_conversion( + current_kpis_file=current, + historical_kpis_dir=historical_dir, output_file=output, plugin_module=plugin_module, ) - if exit_code == 0: + # Convert status dict to exit code for CLI + exit_code = status_dict_to_exit_code(result) + + if result.get("success"): click.echo(f"✅ Analysis completed successfully. Results written to: {output}") + if result.get("regressions_detected"): + click.echo("⚠️ Regressions detected in analysis results") + elif result.get("message"): + click.echo(f"ℹ️ {result.get('message')}") else: - click.echo(f"❌ Analysis failed with exit code {exit_code}", err=True) + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ Analysis failed: {error_msg}", err=True) + + if exit_code != 0: sys.exit(exit_code) except Exception as e: diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 0cccaeca4..f3e16902d 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -14,7 +14,6 @@ analyse_kpis_cmd, artifacts_export, artifacts_import, - kpi_analyze, kpi_csv_export, kpi_generate, kpi_import, @@ -151,7 +150,7 @@ def main( @main.group("kpi") @click.pass_context def kpi_group(ctx: click.Context) -> None: - """KPI generate/import/export/analyze.""" + """KPI generate/import/export/.""" @main.group("artifacts") @@ -211,7 +210,6 @@ def run_cli() -> None: kpi_group.add_command(kpi_generate) kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) -kpi_group.add_command(kpi_analyze) kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 593f8acdb..a6430af31 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -188,7 +188,6 @@ def _build_yaml_report( ) -> dict[str, Any]: """Build the final YAML-serializable report structure.""" regressions = [r for r in results if r.regression] - improvements = [r for r in results if not r.regression and r.relative_change != 0] if regressions: overall_status = "REGRESSION_DETECTED" @@ -250,7 +249,7 @@ def run_kpi_analysis( historical_data_dir: Path, output_file: Path, plugin_module: str, -) -> int: +) -> dict[str, Any]: """Run KPI regression analysis and generate a YAML report. Args: @@ -260,7 +259,7 @@ def run_kpi_analysis( plugin_module: Plugin module name (for loading analysis config) Returns: - Exit code: 0=pass, 1=error, 2=warning (no baseline), 3=regression detected + Status dictionary with success/error information and exit_code for CLI compatibility """ try: logger.info("Running KPI regression analysis") @@ -271,11 +270,23 @@ def run_kpi_analysis( if not current_kpi_file.exists(): logger.error("Current KPI file not found: %s", current_kpi_file) - return 1 + return { + "status": "failed", + "success": False, + "error": f"Current KPI file not found: {current_kpi_file}", + "exit_code": 1, + "completed_at": time.time(), + } if not historical_data_dir.exists(): logger.error("Historical data directory not found: %s", historical_data_dir) - return 1 + return { + "status": "failed", + "success": False, + "error": f"Historical data directory not found: {historical_data_dir}", + "exit_code": 1, + "completed_at": time.time(), + } config = _load_analysis_config(plugin_module) logger.info( @@ -290,18 +301,37 @@ def run_kpi_analysis( if current_data.get("schema_version") != "2": logger.error("Current KPI file must be schema_version 2 (hierarchical)") - return 1 + return { + "status": "failed", + "success": False, + "error": "Current KPI file must be schema_version 2 (hierarchical)", + "exit_code": 1, + "completed_at": time.time(), + } current_records = _extract_kpi_records_from_hierarchical(current_data) if not current_records: logger.warning("No KPI records found in current file") - return 1 + return { + "status": "failed", + "success": False, + "error": "No KPI records found in current file", + "exit_code": 1, + "completed_at": time.time(), + } # Load baseline KPIs baseline_kpi_data = find_baseline_kpis(historical_data_dir) if not baseline_kpi_data: _write_warning_report(output_file, current_kpi_file, plugin_module) - return 2 + return { + "status": "warning", + "success": True, + "message": "no historical KPI found for regression testing", + "output_file": str(output_file), + "exit_code": 2, + "completed_at": time.time(), + } # Build baseline index baseline_index = _build_baseline_index(baseline_kpi_data, config) @@ -367,11 +397,33 @@ def run_kpi_analysis( regressions, ) - return 3 if regressions > 0 else 0 + if regressions > 0: + return { + "status": "regression_detected", + "success": True, + "regressions_detected": True, + "output_file": str(output_file), + "exit_code": 3, + "completed_at": time.time(), + } + else: + return { + "status": "success", + "success": True, + "output_file": str(output_file), + "exit_code": 0, + "completed_at": time.time(), + } - except Exception: + except Exception as e: logger.exception("KPI analysis failed") - return 1 + return { + "status": "failed", + "success": False, + "error": f"KPI analysis failed: {e}", + "exit_code": 1, + "completed_at": time.time(), + } def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: @@ -401,81 +453,6 @@ def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_modu yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) -def analyze_kpis( - postprocess_config, # CaliperOrchestrationPostprocessConfig - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Run KPI analysis step and return result status. - - This is the orchestration interface for KPI analysis. - """ - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled"} - - analyze_config = postprocess_config.analyze - - historical_kpis_dir = Path(analyze_config.historical_kpis) - if not historical_kpis_dir.is_absolute(): - historical_kpis_dir = output_dir / historical_kpis_dir - - output_path = output_dir / analyze_config.output - - try: - if not current_kpis_file.exists(): - return { - "status": "failed", - "error": f"Current KPI file not found: {current_kpis_file}", - "completed_at": time.time(), - } - - if not historical_kpis_dir.exists(): - return { - "status": "failed", - "error": f"Historical KPIs directory not found: {historical_kpis_dir}", - "completed_at": time.time(), - } - - exit_code = run_kpi_analysis( - current_kpi_file=current_kpis_file, - historical_data_dir=historical_kpis_dir, - output_file=output_path, - plugin_module=plugin_module, - ) - - if exit_code == 0: - return { - "status": "success", - "output_file": str(output_path), - "completed_at": time.time(), - } - elif exit_code == 2: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "output_file": str(output_path), - "completed_at": time.time(), - } - elif exit_code == 3: - return { - "status": "regression_detected", - "output_file": str(output_path), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } - - except Exception as e: - logger.exception("Analysis step failed") - raise RuntimeError(f"KPI analysis failed: {e}") from e - - def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: """Load all kpis.json files from historical directory. @@ -505,8 +482,21 @@ def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: baseline_kpis[kpi_file] = kpi_data logger.debug("Loaded baseline: %s", kpi_file) - except json.JSONDecodeError as e: - logger.error("Failed to parse JSON in %s: %s", kpi_file, e) + except json.JSONDecodeError: + # Try parsing as JSONL (v1 format) fallback + try: + v1_records = [] + with open(kpi_file) as f: + for line in f: + if line.strip(): + v1_records.append(json.loads(line)) + + if v1_records: + v2_data = _convert_v1_to_v2(v1_records) + baseline_kpis[kpi_file] = v2_data + logger.debug("Loaded baseline (converted from v1): %s", kpi_file) + except Exception: + logger.error("Failed to parse %s as JSON or JSONL", kpi_file) except Exception as e: logger.error("Failed to load %s: %s", kpi_file, e) @@ -534,21 +524,264 @@ def run_analyze( plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - exit_code = run_kpi_analysis( + result = run_kpi_analysis( current_kpi_file=Path(current_path), historical_data_dir=historical_dir, output_file=Path(output_path), plugin_module=plugin_module, ) - if exit_code == 0: - return {"status": "success", "completed_at": time.time()} - elif exit_code == 2: - return {"status": "warning", "message": "no baselines found", "completed_at": time.time()} - elif exit_code == 3: - return {"status": "regression_detected", "completed_at": time.time()} + # Extract just the relevant fields for this legacy interface + return { + "status": result.get("status"), + "message": result.get("message"), + "completed_at": result.get("completed_at"), + } + + +def _convert_v1_to_v2(v1_records: list[dict]) -> dict: + """Convert v1 JSONL records to v2 hierarchical format.""" + metrics = {} + for record in v1_records: + # Extract metric name and value + metric_name = record.get("name", "unknown_metric") + value = record.get("value") + unit = record.get("unit") + labels = record.get("labels", {}) + + # Create metric entry in v2 format + metric_entry = { + "value": value, + "unit": unit, + "labels": labels, + } + + # Determine if higher is better from labels or default to false + higher_is_better = labels.get("higher_is_better", False) + metric_entry["higher_is_better"] = higher_is_better + + metrics[metric_name] = metric_entry + + return { + "schema_version": "2", + "metrics": metrics, + "metadata": { + "converted_from_v1": True, + "original_record_count": len(v1_records), + }, + } + + +def ensure_kpi_file_v2_format(file_path: Path) -> tuple[Path, bool]: + """Convert KPI file to v2 hierarchical format if it's in v1 JSONL format. + + Args: + file_path: Path to KPI file (v1 or v2 format) + + Returns: + Tuple of (converted_file_path, is_temporary_file) + - converted_file_path: Path to v2 format file + - is_temporary_file: True if a temporary file was created and needs cleanup + """ + import json + import tempfile + + logger.debug("Checking format of KPI file: %s", file_path) + + try: + with open(file_path) as f: + content = f.read().strip() + if not content: + raise ValueError(f"KPI file {file_path} is empty") + + # Try to parse as v2 (hierarchical JSON) first + try: + data = json.loads(content) + if isinstance(data, dict) and "schema_version" in data: + # Already in v2 format + logger.debug("KPI file %s is already in v2 format", file_path) + return file_path, False + elif isinstance(data, list): + raise ValueError(f"KPI file {file_path} appears to be in unexpected list format") + else: + # Single dict but missing schema_version - might be v1 single record + logger.debug( + "KPI file %s appears to be single record without schema_version", file_path + ) + except json.JSONDecodeError: + # Not valid single JSON, might be JSONL (v1 format) + logger.debug("KPI file %s not valid single JSON, checking JSONL format", file_path) + + # Try to parse as v1 (JSONL) format + lines = content.split("\n") + non_empty_lines = [line.strip() for line in lines if line.strip()] + + if len(non_empty_lines) == 0: + raise ValueError(f"KPI file {file_path} contains no data") + + # Try to parse first line to validate v1 format + try: + first_record = json.loads(non_empty_lines[0]) + if not isinstance(first_record, dict): + raise ValueError(f"KPI file {file_path} first line is not a JSON object") + + # Parse all lines as v1 JSONL records + v1_records = [] + for i, line in enumerate(non_empty_lines): + try: + record = json.loads(line) + v1_records.append(record) + except json.JSONDecodeError as line_err: + raise ValueError( + f"KPI file {file_path} line {i + 1} has invalid JSON: {line_err}" + ) from line_err + + # Convert v1 records to v2 hierarchical format + logger.info( + "Converting KPI file %s from v1 (JSONL) to v2 (hierarchical) format", file_path + ) + + # Transform to v2 + v2_data = _convert_v1_to_v2(v1_records) + + # Write v2 format to temporary file + temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + try: + json.dump(v2_data, temp_file, indent=2, ensure_ascii=False) + temp_file.flush() + temp_path = Path(temp_file.name) + logger.info( + "Converted %s (%d records) to v2 format at %s", + file_path, + len(v1_records), + temp_path, + ) + return temp_path, True + except Exception as write_err: + temp_file.close() + Path(temp_file.name).unlink(missing_ok=True) + raise ValueError( + f"Failed to write converted v2 format file: {write_err}" + ) from write_err + finally: + temp_file.close() + + except Exception as v1_err: + raise ValueError( + f"KPI file {file_path} is not in valid v1 (JSONL) or v2 (hierarchical) format: {v1_err}" + ) from v1_err + + except Exception as format_err: + raise ValueError( + f"Failed to process KPI file format for {file_path}: {format_err}" + ) from format_err + + +def analyze_kpis( + current_kpis_file: Path, + historical_kpis_dir: Path, + output_file: Path, + plugin_module: str, +) -> dict[str, Any]: + """Analyze KPIs with automatic v1/v2 format conversion. + + This function handles: + 1. Format detection and conversion from v1 (JSONL) to v2 (hierarchical) + 2. Plugin loading + 3. Analysis execution + 4. Temporary file cleanup + 5. Result formatting + + Args: + current_kpis_file: Path to current KPIs file (v1 or v2 format) + historical_kpis_dir: Directory containing historical KPI files + output_file: Path where analysis results will be written + plugin_module: Plugin module name for KPI definitions and analysis rules + + Returns: + Dictionary with status information: + - success: bool - whether analysis completed successfully + - output_file: str - path to analysis results (if successful) + - error: str - error message (if failed) + """ + + try: + # Use directory-based analysis (supports multiple baseline files) + result = run_kpi_analysis( + current_kpi_file=current_kpis_file, + historical_data_dir=historical_kpis_dir, + output_file=output_file, + plugin_module=plugin_module, + ) + + # Convert to the expected format for orchestration and CLI + if result.get("status") == "success": + return { + "status": "success", + "success": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "warning": + return { + "status": "warning", + "success": True, # Warning is still success + "message": result.get("message"), + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "regression_detected": + return { + "status": "regression_detected", + "success": True, # Regression detection is successful analysis + "regressions_detected": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + else: + return { + "status": "failed", + "success": False, + "error": result.get("error", "Analysis failed"), + "completed_at": result.get("completed_at"), + } + + except Exception as e: + logger.exception("KPI analysis with format conversion failed") + return { + "success": False, + "error": str(e), + "completed_at": time.time(), + } + + +def status_dict_to_exit_code(status_dict: dict[str, Any]) -> int: + """Convert a status dictionary to an exit code for CLI use. + + Args: + status_dict: Status dictionary from engine analysis functions + + Returns: + Exit code: 0=success, 1=error, 2=warning (no baseline), 3=regression detected + """ + # If the status dict includes an explicit exit_code, use it + if "exit_code" in status_dict: + return status_dict["exit_code"] + + # Otherwise, determine exit code from status fields + if not status_dict.get("success", False): + return 1 # General error + + status = status_dict.get("status", "") + + if status == "success": + return 0 + elif status == "warning" or "no baseline" in status_dict.get("message", "").lower(): + return 2 # Warning (no baseline data) + elif status == "regression_detected" or status_dict.get("regressions_detected"): + return 3 # Regression detected else: - return {"status": "failed", "error": f"exit code {exit_code}", "completed_at": time.time()} + return 0 # Default to success if unclear # EOF diff --git a/projects/caliper/engine/kpi/analyze_hierarchical.py b/projects/caliper/engine/kpi/analyze_hierarchical.py index c76a9514d..88d56ee92 100644 --- a/projects/caliper/engine/kpi/analyze_hierarchical.py +++ b/projects/caliper/engine/kpi/analyze_hierarchical.py @@ -11,6 +11,59 @@ logger = logging.getLogger(__name__) +def _load_kpis_any_format(kpi_file_path: Path) -> dict[str, Any]: + """Load KPIs from file, handling both v1 (JSONL) and v2 (hierarchical) formats. + + Returns: + Dictionary in hierarchical format with schema_version and metrics + """ + with open(kpi_file_path) as f: + content = f.read().strip() + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Already hierarchical format + return data + else: + # Unknown JSON format, convert list to v1 JSONL handling + raise ValueError("Unknown JSON format") + + except (json.JSONDecodeError, ValueError): + # Try to parse as JSONL (v1 format) + kpis = [] + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + # Convert v1 format to hierarchical format + metrics = {} + for kpi in kpis: + kpi_id = kpi.get("kpi_id") + if kpi_id: + value = kpi.get("value") + labels = kpi.get("labels", {}) + higher_is_better = labels.get("higher_is_better", False) + + metrics[kpi_id] = { + "value": value, + "higher_is_better": higher_is_better, + "labels": labels, + "unit": kpi.get("unit"), + "run_id": kpi.get("run_id"), + "timestamp": kpi.get("timestamp"), + } + + return {"schema_version": "2", "metrics": metrics, "converted_from": "v1_jsonl"} + + def analyze_hierarchical_kpis( current_kpis_path: Path, baseline_kpis_path: Path, @@ -30,31 +83,15 @@ def analyze_hierarchical_kpis( Analysis result dictionary with status, findings, etc. """ try: - # Load KPI files - with open(current_kpis_path) as f: - current_data = json.load(f) - - with open(baseline_kpis_path) as f: - baseline_data = json.load(f) - - # Validate format - if not isinstance(current_data, dict) or not isinstance(baseline_data, dict): - return { - "status": "failed", - "error": "KPI files must be in hierarchical format (dict), not list format", - "completed_at": time.time(), - } - - # Extract schema version and validate - current_schema = current_data.get("schema_version", "unknown") - baseline_schema = baseline_data.get("schema_version", "unknown") - - if current_schema != "2" or baseline_schema != "2": - return { - "status": "failed", - "error": f"Only schema_version 2 supported (current: {current_schema}, baseline: {baseline_schema})", - "completed_at": time.time(), - } + # Load KPI files (supports both v1 JSONL and v2 hierarchical formats) + current_data = _load_kpis_any_format(current_kpis_path) + baseline_data = _load_kpis_any_format(baseline_kpis_path) + + # Both files are now guaranteed to be in hierarchical format with schema_version 2 + logger.info(f"Loaded current KPIs: {current_data.get('converted_from', 'v2_hierarchical')}") + logger.info( + f"Loaded baseline KPIs: {baseline_data.get('converted_from', 'v2_hierarchical')}" + ) # Analyze metrics using plugin-aware logic current_metrics = current_data.get("metrics", {}) @@ -172,8 +209,12 @@ def analyze_hierarchical_kpis( output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: json.dump(analysis_result, f, indent=2) + f.write("\n") - logger.info(f"Analysis completed: {regressions} regressions, {improvements} improvements") + metrics_tested = len([m for m in current_metrics if m in baseline_metrics]) + logger.info( + f"Analysis completed: {metrics_tested} verifications, {regressions} regressions, {improvements} improvements" + ) return { "status": "success", @@ -210,5 +251,63 @@ def find_most_recent_baseline(historical_dir: Path) -> Path | None: """Find the most recently modified kpis.json file in historical directory.""" kpi_files = list(historical_dir.rglob("kpis.json")) if not kpi_files: + logger.warning(f"No kpis.json files found in: {historical_dir}") return None - return max(kpi_files, key=lambda p: p.stat().st_mtime) + + logger.info(f"Discovered {len(kpi_files)} baseline candidate files:") + + # Check format of each file and log results + v1_files = [] + v2_files = [] + failed_files = [] + + for kpi_file in kpi_files: + try: + with open(kpi_file) as f: + content = f.read().strip() + + try: + data = json.loads(content) + if isinstance(data, dict) and data.get("schema_version") == "2": + v2_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v2_hierarchical") + else: + # Try parsing as JSONL (v1) + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (unknown format)" + ) + except (json.JSONDecodeError, ValueError): + # Try as JSONL + try: + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (invalid JSONL)" + ) + except Exception: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (parse error)" + ) + except Exception as e: + failed_files.append(kpi_file) + logger.error(f" {kpi_file.relative_to(historical_dir)} → FAILED ({e})") + + logger.info( + f"Summary: {len(v2_files)} v2_hierarchical, {len(v1_files)} v1_jsonl, {len(failed_files)} failed" + ) + + most_recent = max(kpi_files, key=lambda p: p.stat().st_mtime) + logger.info(f"Selected most recent: {most_recent.relative_to(historical_dir)}") + + return most_recent diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index ee7ea1409..009cb0e94 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -291,6 +291,7 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: } run_id = test_metadata.get("run_id") + flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index 27dd575c2..99abacada 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -2,10 +2,10 @@ from __future__ import annotations -import json from pathlib import Path from typing import Any +from projects.caliper.engine.kpi.format import write_kpis_in_format from projects.caliper.engine.parse import run_parse from projects.caliper.engine.validation import load_schema, schema_path, validate_instance @@ -18,7 +18,22 @@ def run_kpi_generate( output: Path | None, use_cache: bool, cache_path: Path | None, + format_type: str = "hierarchical", ) -> list[dict[str, Any]]: + """Generate KPI output in specified format. + + Args: + base_dir: Directory containing test artifacts + plugin_module: Name of the plugin module + plugin: Plugin instance + output: Path to write output file + use_cache: Whether to use cached parse results + cache_path: Path to cache file (not used currently) + format_type: Output format - "hierarchical" (default) or "jsonl" + + Returns: + List of KPI records + """ model = run_parse( base_dir=base_dir, plugin_module=plugin_module, @@ -30,7 +45,8 @@ def run_kpi_generate( kpi_schema = load_schema(schema_path("kpi_record.schema.json")) for row in rows: validate_instance(row, kpi_schema, "KPI record") - text = "\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + ("\n" if rows else "") + if output: - output.write_text(text, encoding="utf-8") + write_kpis_in_format(rows, output, format_type, model) + return rows diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 70430ebd5..a46eb906a 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -24,6 +24,7 @@ ) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, + build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -120,117 +121,6 @@ def _resolve_visualize_config_path( # _transform_kpis_to_hierarchical_format function removed - CLI commands now handle format conversion - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - test_data = tests_data[run_id] - - # Extract common labels (excluding KPI-specific ones) - kpi_labels = kpi.get("labels", {}) - test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels - - # Merge test labels (they should be the same for all KPIs in a test) - if not test_data["labels"]: - test_data["labels"] = test_labels - - # Store test metadata from first KPI - if not test_data["metadata"]: - test_data["metadata"] = { - "timestamp": kpi.get("timestamp"), - "source": kpi.get("source", {}), - "run_id": run_id, - } - - # Create KPI record with metadata - kpi_id = kpi.get("kpi_id") - raw_value = kpi.get("value") - - # Transform 2D KPI values to structured format - if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: - # Check if this looks like 2D data: list of tuples/lists with 2 elements - first_item = raw_value[0] - if isinstance(first_item, list | tuple) and len(first_item) == 2: - try: - # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format - structured_value = { - "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], - "count": len(raw_value), - } - final_value = structured_value - except (ValueError, TypeError, IndexError): - # If conversion fails, keep original value - final_value = raw_value - else: - final_value = raw_value - else: - final_value = raw_value - - kpi_record = { - "id": kpi_id, - "value": final_value, - "unit": kpi.get("unit"), - "higher_is_better": kpi_labels.get("higher_is_better", True), - } - - # Add KPI metadata from function decorator if available - if kpi_id in kpi_functions: - func = kpi_functions[kpi_id] - kpi_record.update( - { - "name": ( - func.__doc__.replace(" KPI.", "") - if func.__doc__ - else kpi_id.replace("_", " ").title() - ), - "help": getattr(func, "_kpi_help", ""), - } - ) - - # Add 2D-specific metadata if present - if getattr(func, "_kpi_is_2d", False): - kpi_record.update( - { - "is_2d": True, - "x_unit": getattr(func, "_kpi_x_unit", ""), - "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], - "y_help": getattr(func, "_kpi_y_help", None) - or getattr(func, "_kpi_help", ""), - } - ) - else: - kpi_record["is_2d"] = False - - # Add formatting info if available - if hasattr(func, "_kpi_format"): - kpi_record["format"] = func._kpi_format - else: - # Fallback if no function metadata available - kpi_record.update( - { - "name": kpi_id.replace("_", " ").title(), - "help": f"KPI: {kpi_id}", - "is_2d": isinstance(kpi.get("value"), list), - } - ) - - test_data["kpis"].append(kpi_record) - - # Convert to final structure - tests_list = [] - for run_id, test_data in tests_data.items(): - tests_list.append( - { - "run_id": run_id, - "labels": test_data["labels"], - "metadata": test_data["metadata"], - "kpis": test_data["kpis"], - } - ) - - return {"schema_version": "2", "tests": tests_list} - def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, From 86b6523bea8d1ed7a165f8b5d4d0c2245408c98b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 22:20:58 +0200 Subject: [PATCH 335/344] [caliper] tests: test_kpi_analyze: update to use the test status dict --- projects/caliper/tests/test_kpi_analyze.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py index a71e01c3d..2ca06e4de 100644 --- a/projects/caliper/tests/test_kpi_analyze.py +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -197,14 +197,14 @@ def test_pass_no_regression(self, tmp_path): self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline1) self._write_hierarchical_kpi(historical_dir / "run2" / "kpis.json", baseline2) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 0 + assert test_status["exit_code"] == 0 assert output_file.exists() with open(output_file) as f: @@ -247,14 +247,14 @@ def test_regression_detected(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 3 + assert test_status["exit_code"] == 3 with open(output_file) as f: report = yaml.safe_load(f) @@ -282,14 +282,14 @@ def test_no_historical_data(self, tmp_path): ) self._write_hierarchical_kpi(current_dir / "kpis.json", current) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 2 + assert test_status["exit_code"] == 2 with open(output_file) as f: report = yaml.safe_load(f) assert report["overall"]["verdict"] == "NO_BASELINE" @@ -327,14 +327,14 @@ def test_mixed_regression_and_pass(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 3 + assert test_status["exit_code"] == 3 with open(output_file) as f: report = yaml.safe_load(f) @@ -376,14 +376,14 @@ def test_skips_non_scalar_kpis(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 0 + assert test_status["exit_code"] == 0 with open(output_file) as f: report = yaml.safe_load(f) @@ -426,7 +426,7 @@ def test_comparison_keys_separate_baselines(self, tmp_path): # With version as comparison_key, both match on platform=A100 # (plugin would expose this config; here we test the core logic directly) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, @@ -435,7 +435,7 @@ def test_comparison_keys_separate_baselines(self, tmp_path): # Without plugin config, default AnalysisConfig has no comparison_keys, # so version must also match → no baselines found → skip - assert exit_code == 0 or exit_code == 2 + assert test_status["exit_code"] in (0, 2) with open(output_file) as f: report = yaml.safe_load(f) # The version label differs, so with empty comparison_keys they don't match From 376e78c83f77453b340d4c71e093b901ed9e7ad3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 22:35:28 +0200 Subject: [PATCH 336/344] [skeleton] orchestration: config: fix the analyze config --- projects/skeleton/orchestration/config.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 9ee43c738..d0662ae62 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -77,11 +77,6 @@ caliper: enabled: true output_dir: ai_data - analyze: - enabled: true - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke @@ -104,6 +99,13 @@ caliper: include_kpis_json: true include_ai_data: true + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json + + agentic: enabled: false model_key: qwen-3-6-35b From 823ca895395e7e55c0f5689f66a18b079bf4d46f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 1 Aug 2026 07:54:18 +0200 Subject: [PATCH 337/344] [caliper] orchestration: cli_builder: make the command easy to copy-and-paste --- projects/caliper/orchestration/cli_builder.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 7e12fd4f4..51cc03498 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -516,9 +516,7 @@ def log_caliper_start_banner(command: list[str], script_path: Path, step_name: s # Log banner indicating Caliper is starting logger.info("=" * 60) logger.info("🚀 STARTING CALIPER %s (fork/exec)", step_name.upper()) - logger.info("📄 Command:") - for line in formatted_command.splitlines(): - logger.info(" %s", line) + logger.info(f"📄 Command:\n{formatted_command}") logger.info("=" * 60) @@ -563,7 +561,7 @@ def _format_command_for_display(command: list[str]) -> str: lines.append(command[i]) i += 1 - return "\n".join(lines) + return f" \\\n{len('caliper ') * ' '}".join(lines).strip() def handle_caliper_output_and_completion( From de3a282e8224b45dc3b95ee7de9d74dda81d4cac Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 1 Aug 2026 08:00:40 +0200 Subject: [PATCH 338/344] [kserve] toolbox: capture_llmisvc_state: capture the nodes details --- .../toolbox/capture_llmisvc_state/main.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index c921d45be..2daafbddc 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -312,5 +312,126 @@ def capture_pods_describe(args, context): return f"Pod describe output captured for {len(pod_names)} pods" +@task +def determine_used_nodes(args, context): + """Determine which nodes are used by LLMInferenceService pods""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].spec.nodeName}}"', + check=False, + log_stdout=False, + ) + + node_names = list(set(result.stdout.strip().split())) # Remove duplicates + if not node_names or not result.stdout.strip(): + context.used_nodes = [] + return "No nodes found for LLMInferenceService pods" + + context.used_nodes = [node for node in node_names if node] # Filter out empty strings + return f"Found {len(context.used_nodes)} nodes used by LLMInferenceService pods: {', '.join(context.used_nodes)}" + + +@task +def capture_pod_node_mapping(args, context): + """Capture pod->node mapping as YAML""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o custom-columns=POD:.metadata.name,NODE:.spec.nodeName --no-headers', + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return "No pod->node mapping found" + + mapping_file = args.artifact_dir / "artifacts/pod_node_mapping.yaml" + pod_node_mapping = {} + + for line in result.stdout.strip().split("\n"): + if line.strip(): + parts = line.split() + if len(parts) >= 2: + pod_name = parts[0] + node_name = parts[1] if parts[1] != "" else None + pod_node_mapping[pod_name] = node_name + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"pod_node_mapping": pod_node_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Pod->node mapping captured to {mapping_file} ({len(pod_node_mapping)} pods)" + + +@task +def capture_node_gpu_mapping(args, context): + """Capture node->GPU type mapping as YAML""" + if not context.used_nodes: + return "No nodes to capture GPU mapping for" + + node_gpu_mapping = {} + + for node_name in context.used_nodes: + # Try nvidia.com/gpu.product first + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.nvidia\\.com/gpu\\.product}}"', + check=False, + log_stdout=False, + ) + gpu_type = result.stdout.strip() + + # If not found, try gpu.nvidia.com/class as fallback + if not gpu_type: + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.gpu\\.nvidia\\.com/class}}"', + check=False, + log_stdout=False, + ) + gpu_class = result.stdout.strip() + if gpu_class: + gpu_type = f"NVIDIA-{gpu_class}" + else: + gpu_type = "unknown" + + node_gpu_mapping[node_name] = gpu_type + + mapping_file = args.artifact_dir / "artifacts/node_gpu_mapping.yaml" + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"node_gpu_mapping": node_gpu_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Node->GPU mapping captured to {mapping_file} ({len(node_gpu_mapping)} nodes)" + + +@task +def capture_used_nodes_yaml(args, context): + """Capture YAML definitions for nodes used by LLMInferenceService pods""" + if not context.used_nodes: + return "No nodes to capture YAML for" + + nodes_dir = args.artifact_dir / "artifacts/nodes" + shell.mkdir("artifacts/nodes") + + captured_count = 0 + for node_name in context.used_nodes: + node_file = nodes_dir / f"{node_name}.yaml" + shell.run( + f"oc get node {node_name} -oyaml", + stdout_dest=node_file, + check=False, + ) + captured_count += 1 + + return f"Captured YAML for {captured_count} nodes in {nodes_dir}" + + if __name__ == "__main__": run.main() From a9b04840ceffe9555a5f56d8c59acf98dc619333 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:24:54 +0200 Subject: [PATCH 339/344] [caliper] orchestration: notification: include the post-processing message status --- .../caliper/orchestration/notification.py | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 8901afdf6..9a8158842 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -89,31 +89,21 @@ def format_postprocess_status_notification( else: step_name_display = f"**{step_name}**" - lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") - - # Add step message if available (but not for warning/failed steps to avoid duplication) - if step_result.message and step_result.status not in ("warning", "failed", "failure"): - lines.append(f" > {step_result.message}") - - # Add reason for skipped steps - if step_result.status in ("skipped", "disabled") and step_result.reason: - lines.append(f" > {step_result.reason}") - - # Add error message for failed steps - if ( - step_result.status in ("failed", "failure") - and hasattr(step_result, "error") - and step_result.error - ): - lines.append(f" > ❌ {step_result.error}") + # Format step with message if available + if step_result.message: + lines.append( + f"- {step_emoji} {step_name_display}: `{step_result.status}` * {step_result.message}" + ) + else: + lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") - # Add warning message for warning steps + # Add reason for skipped steps (only if no message already shown) if ( - step_result.status == "warning" - and hasattr(step_result, "message") - and step_result.message + step_result.status in ("skipped", "disabled") + and step_result.reason + and not step_result.message ): - lines.append(f" > ⚠️ {step_result.message}") + lines.append(f" > {step_result.reason}") # Add specific file links for certain steps if step_result.status == "success" and get_file_link: From dd3a6e33818d75d4e115ee444bdede0a10b976c8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:25:31 +0200 Subject: [PATCH 340/344] [caliper] orchestration: postprocess: include a message in the post-processing status --- projects/caliper/orchestration/postprocess.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a46eb906a..9fd7b1927 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -210,7 +210,12 @@ def _run_artifacts_to_kpis( full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } # _run_s3_import function removed - now using fork/exec subprocess execution directly @@ -282,7 +287,12 @@ def _run_artifacts_to_ai_data( full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } def _load_test_labels(test_dir: Path) -> dict[str, Any]: @@ -396,7 +406,12 @@ def _run_kpis_to_csv( full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } def _run_analyse_kpis( @@ -477,7 +492,12 @@ def _run_analyse_kpis( except Exception as e: logger.exception("KPI analysis failed in _run_analyse_kpis") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } class CaliperPostprocessOrchestrator: @@ -672,7 +692,7 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A warning_msg = result.get("message", "unknown warning") logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") elif status == "failed": - error_msg = result.get("error", "unknown error") + error_msg = result.get("message", "unknown error") traceback_msg = result.get("traceback") logger.error(f"Step '{step_name}' failed: {error_msg}") if traceback_msg: From 2769596bc3b48528333187842d7631fe79d6c04f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:38:23 +0200 Subject: [PATCH 341/344] fournos: gitops: base/workflows/pipeline-replot: use 0x instead of 00x --- fournos/gitops/base/workflows/pipeline-replot.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fournos/gitops/base/workflows/pipeline-replot.yaml b/fournos/gitops/base/workflows/pipeline-replot.yaml index 952407f02..07fdc5b92 100644 --- a/fournos/gitops/base/workflows/pipeline-replot.yaml +++ b/fournos/gitops/base/workflows/pipeline-replot.yaml @@ -17,7 +17,7 @@ spec: - name: kubeconfig-secret type: string tasks: - - name: 000-replot + - name: 00-replot taskRef: name: forge-step kind: Task @@ -34,10 +34,10 @@ spec: - name: job-step value: replot - name: job-step-name - value: 000__replot + value: 00__replot finally: - - name: 001-export-artifacts + - name: 01-export-artifacts taskRef: name: forge-step kind: Task @@ -54,4 +54,4 @@ spec: - name: job-step value: export-artifacts - name: job-step-name - value: 001__export-artifacts + value: 01__export-artifacts From 9cc931b55149f0eeadde4766e400873f213fa594 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:49:23 +0200 Subject: [PATCH 342/344] [caliper] cli: commands: fail early if no data is parsed --- projects/caliper/cli/commands.py | 50 ++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 22be577ec..849d2f0d7 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -478,27 +478,59 @@ def kpi_generate( status_data = {"success": False} try: - run_kpi_generate( + # First run parse to check test directory count + model = run_parse( base_dir=artifact_root, plugin_module=mod, plugin=plugin, - output=output, use_cache=True, - cache_path=None, - format_type=format_type, ) - status_data = {"success": True, "output_file": str(output)} - click.echo(f"Generated {output}") + + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + test_dir_count = len(test_directories) + + # Check if any test directories were found + if test_dir_count == 0: + status_data = { + "success": False, + "message": "No test directories found - nothing to process for KPI generation", + "test_directories_count": 0, + "test_directories": [], + } + click.echo("❌ No test directories found - KPI generation failed", err=True) + click.echo(" No __test_labels__.yaml files found in artifact directory", err=True) + + sys.exit(3) + else: + # Proceed with KPI generation + run_kpi_generate( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, + format_type=format_type, + ) + status_data = { + "success": True, + "output_file": str(output), + "test_directories_count": test_dir_count, + "test_directories": test_directories, + } + click.echo(f"Generated {output}") + click.echo(f"📁 Processed {test_dir_count} test directories") + except Exception as e: # noqa: BLE001 import traceback full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} + status_data = {"success": False, "message": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) click.echo(f"Full traceback:\n{full_traceback}", err=True) - if not status_file: - sys.exit(3) + sys.exit(3) finally: # Write status file if requested if status_file: From d4c1e14613aef41b6321b81f497a64477a8ca4a2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:49:44 +0200 Subject: [PATCH 343/344] [caliper] orchestration: replot: raise the warning level for visibility --- projects/caliper/orchestration/replot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/replot.py b/projects/caliper/orchestration/replot.py index 86b6382a4..fd2132327 100644 --- a/projects/caliper/orchestration/replot.py +++ b/projects/caliper/orchestration/replot.py @@ -187,7 +187,7 @@ def run_replot_from_orchestration_config( if skip_download: keep_replot_dir = True - logger.info("No replot URL provided, skipping artifact download") + logger.warning("No replot URL provided, skipping artifact download") logger.info("Automatically setting keep_replot_dir=True to preserve existing files") logger.info(f"Download directory: {replot_download_dir}") logger.info(f"Output directory: {artifact_directory}") From c66dfd672ba77910ed1ce36b84f231ea2a54a7cc Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 13:57:52 +0300 Subject: [PATCH 344/344] [caliper] kpi: generic kpis.json to metrics.json conversion for MLflow Move metrics.json generation from project-specific parsers into caliper as a generic mechanism. After kpis.json is produced, caliper now automatically writes per-run metrics.json and parameters.json files using KPI ids as metric keys. The MLflow export backend picks these up via _log_metrics_and_params_from_tree without any project-specific code. - Add caliper/engine/kpi/metrics_from_kpis.py with generate_metrics_from_kpis() - Wire into postprocess pipeline after kpis.json generation step - Remove mcp_gateway-specific metrics.json/parameters.json writing - Simplify mcp_gateway notifications MLflow metric key lookup - Update tests to reflect new ownership Co-authored-by: Cursor --- .../caliper/engine/kpi/metrics_from_kpis.py | 144 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 36 +++++ .../orchestration/notifications.py | 17 +-- .../mcp_gateway/parsing/parsers.py | 18 +-- .../tests/test_mcp_gateway_plugin.py | 27 +--- 5 files changed, 190 insertions(+), 52 deletions(-) create mode 100644 projects/caliper/engine/kpi/metrics_from_kpis.py diff --git a/projects/caliper/engine/kpi/metrics_from_kpis.py b/projects/caliper/engine/kpi/metrics_from_kpis.py new file mode 100644 index 000000000..0ba22f833 --- /dev/null +++ b/projects/caliper/engine/kpi/metrics_from_kpis.py @@ -0,0 +1,144 @@ +"""Generic kpis.json -> metrics.json + parameters.json conversion. + +Reads a hierarchical kpis.json (schema v2) and writes per-test-run +metrics.json and parameters.json files into the matching artifact tree +directories. The MLflow export backend picks these up automatically via +``_log_metrics_and_params_from_tree``. + +This replaces project-specific metrics.json generation (e.g. in +mcp_gateway parsers) with a single generic caliper mechanism that works +for every project producing a kpis.json. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" + + +def _build_run_dir_index(artifact_tree: Path) -> dict[str, Path]: + """Map run directory names to their paths using __test_labels__.yaml markers.""" + index: dict[str, Path] = {} + for marker in sorted(artifact_tree.rglob(TEST_LABELS_MARKER)): + if marker.is_file(): + run_dir = marker.parent + try: + rel = run_dir.relative_to(artifact_tree) + except ValueError: + rel = Path(run_dir.name) + index[str(rel)] = run_dir + index[run_dir.name] = run_dir + return index + + +def _is_scalar(value: Any) -> bool: + """Check if a KPI value is a scalar number (not 2D data).""" + return isinstance(value, int | float) and not isinstance(value, bool) + + +def _write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + + +def generate_metrics_from_kpis( + kpis_json_path: Path, + artifact_tree: Path, +) -> dict[str, Any]: + """Convert kpis.json into per-run metrics.json and parameters.json files. + + For each test entry in kpis.json, finds the matching directory under + ``artifact_tree`` (via ``__test_labels__.yaml`` markers) and writes: + + - ``metrics.json``: ``{kpi_id: value}`` for all scalar KPIs + - ``parameters.json``: test-level labels as string key-value pairs + + Args: + kpis_json_path: Path to the kpis.json file (schema v2). + artifact_tree: Root of the caliper artifact tree containing + test run directories with ``__test_labels__.yaml`` markers. + + Returns: + Status dict with counts and any warnings. + """ + if not kpis_json_path.is_file(): + return {"status": "skipped", "reason": f"kpis.json not found: {kpis_json_path}"} + + try: + with kpis_json_path.open(encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to read kpis.json at %s: %s", kpis_json_path, e) + return {"status": "failed", "error": str(e)} + + if not isinstance(data, dict) or data.get("schema_version") != "2": + return {"status": "skipped", "reason": "Not a schema v2 kpis.json"} + + tests = data.get("tests", []) + if not tests: + return {"status": "skipped", "reason": "No tests in kpis.json"} + + run_dir_index = _build_run_dir_index(artifact_tree) + if not run_dir_index: + logger.warning("No test run directories found under %s", artifact_tree) + return {"status": "skipped", "reason": "No run directories with __test_labels__.yaml found"} + + written = 0 + warnings: list[str] = [] + + for test_entry in tests: + run_id = test_entry.get("run_id", "") + test_base_path = test_entry.get("metadata", {}).get("source", {}).get( + "test_base_path", run_id + ) + + run_dir = run_dir_index.get(test_base_path) or run_dir_index.get(run_id) + if run_dir is None: + warnings.append(f"No matching directory for run_id={run_id!r}") + continue + + kpis = test_entry.get("kpis", []) + metrics: dict[str, float] = {} + for kpi in kpis: + kpi_id = kpi.get("id", "") + value = kpi.get("value") + if kpi_id and _is_scalar(value): + metrics[kpi_id] = value + + if metrics: + _write_json(run_dir / METRICS_FILE, metrics) + + labels = test_entry.get("labels", {}) + if labels: + params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} + _write_json(run_dir / PARAMETERS_FILE, params) + + written += 1 + + result: dict[str, Any] = { + "status": "success", + "tests_processed": written, + "total_tests": len(tests), + } + if warnings: + result["warnings"] = warnings + for w in warnings: + logger.warning("kpis-to-metrics: %s", w) + + logger.info( + "Generated metrics.json for %d/%d test(s) from %s", + written, + len(tests), + kpis_json_path.name, + ) + return result diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 9fd7b1927..41f1306af 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1001,6 +1001,9 @@ def _run_kpi_and_ai_data_steps(self) -> None: # KPI JSON generation self._run_artifacts_to_kpis_step(output_dir, mod_str) + # Generate per-run metrics.json + parameters.json from kpis.json + self._run_kpis_to_metrics_step(output_dir) + # KPI CSV export self._run_kpis_to_csv_step(output_dir) @@ -1040,6 +1043,39 @@ def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: }, ) + def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: + """Generate per-run metrics.json + parameters.json from kpis.json. + + Runs automatically after kpis.json generation succeeds. Writes + files into each test run directory so the MLflow export backend + picks them up via ``_log_metrics_and_params_from_tree``. + """ + kpi_step = self._get_step("artifacts_to_kpis") + if not kpi_step or kpi_step.get("status") != "success": + return + + kpis_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + if not kpis_json_path.is_file(): + logger.warning("kpis.json not found at %s, skipping metrics generation", kpis_json_path) + return + + try: + from projects.caliper.engine.kpi.metrics_from_kpis import generate_metrics_from_kpis + + result = generate_metrics_from_kpis(kpis_json_path, self.tree_root) + status = result.get("status", "unknown") + if status == "success": + logger.info( + "kpis-to-metrics: wrote metrics.json for %d test(s)", + result.get("tests_processed", 0), + ) + elif status == "skipped": + logger.info("kpis-to-metrics: skipped — %s", result.get("reason", "")) + else: + logger.warning("kpis-to-metrics: %s", result.get("error", "unknown error")) + except Exception as e: + logger.warning("kpis-to-metrics conversion failed (non-fatal): %s", e) + def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" if not self.config.kpi.kpis_to_csv.enabled: diff --git a/projects/mcp_gateway/orchestration/notifications.py b/projects/mcp_gateway/orchestration/notifications.py index 297eb9aa8..0c4683960 100644 --- a/projects/mcp_gateway/orchestration/notifications.py +++ b/projects/mcp_gateway/orchestration/notifications.py @@ -37,13 +37,9 @@ ("mcp_gw_failure_rate", "Failure rate", "%"), ] -# Mapping from kpis.jsonl IDs to MLflow metrics.json keys (logged via mlflow.log_metric) -_KPI_TO_MLFLOW_METRIC = { - "mcp_gw_requests_per_second": "requests_per_second", - "mcp_gw_p95_ms": "p95_ms", - "mcp_gw_p99_ms": "p99_ms", - "mcp_gw_failure_rate": "failure_rate", -} +# KPI IDs are now logged directly as MLflow metric keys via the generic +# caliper kpis-to-metrics conversion — no mapping needed. +_KPI_IDS = {kpi_id for kpi_id, _, _ in TARGET_KPIS} _RUN_NAME_PREFIX = "forge-mcp-gateway-" @@ -376,12 +372,7 @@ def _load_previous_kpis_from_mlflow( return {}, "", True raw_metrics = previous_run.data.metrics or {} - mlflow_to_kpi = {v: k for k, v in _KPI_TO_MLFLOW_METRIC.items()} - metrics = {} - for mlflow_key, value in raw_metrics.items(): - kpi_id = mlflow_to_kpi.get(mlflow_key) - if kpi_id: - metrics[kpi_id] = value + metrics = {k: v for k, v in raw_metrics.items() if k in _KPI_IDS} return metrics, prev_name, False diff --git a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py index 107f60ae5..6fd9e2cdf 100644 --- a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py +++ b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py @@ -2,9 +2,7 @@ from __future__ import annotations -import json import logging -from pathlib import Path from typing import Any from projects.agentic_tools.locust.helpers.parse_results import RunMetrics, parse_stats_csv @@ -17,8 +15,6 @@ logger = logging.getLogger(__name__) STATS_CSV = "stats.csv" -METRICS_FILE = "metrics.json" -PARAMETERS_FILE = "parameters.json" def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: @@ -33,7 +29,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: - """Convert RunMetrics to a flat dictionary suitable for metrics.json.""" + """Convert RunMetrics to a flat dictionary for unified result records.""" return { "total_requests": metrics.total_requests, "total_failures": metrics.total_failures, @@ -48,13 +44,6 @@ def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: } -def _write_json(path: Path, data: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write("\n") - - class MCPGatewayParser: """Parser for Locust stats.csv artifacts from MCP Gateway tests.""" @@ -89,11 +78,6 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: labels = _labels_from_node(node) metrics_dict = _run_metrics_to_dict(run_metrics) - _write_json(node.directory / METRICS_FILE, metrics_dict) - - params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} - _write_json(node.directory / PARAMETERS_FILE, params) - records.append( UnifiedResultRecord( test_base_path=str(node.test_path), diff --git a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py index 78f45ab95..d45287aab 100644 --- a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py +++ b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path import pytest @@ -66,30 +65,14 @@ def test_parse_creates_records(self, tmp_path: Path): assert record.metrics["requests_per_second"] == 31.5 assert record.metrics["p95_ms"] == 80.0 - def test_parse_writes_metrics_json(self, tmp_path: Path): + def test_parse_does_not_write_metrics_json(self, tmp_path: Path): + """Parser no longer writes metrics.json — caliper handles it generically.""" node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) parser = MCPGatewayParser() parser.parse(tmp_path, [node]) - metrics_file = tmp_path / "run-a" / "metrics.json" - assert metrics_file.exists() - data = json.loads(metrics_file.read_text()) - assert data["total_requests"] == 1000 - assert data["requests_per_second"] == 31.5 - - def test_parse_writes_parameters_json(self, tmp_path: Path): - node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) - parser = MCPGatewayParser() - - parser.parse(tmp_path, [node]) - - params_file = tmp_path / "run-a" / "parameters.json" - assert params_file.exists() - data = json.loads(params_file.read_text()) - assert data["preset"] == "smoke" - assert data["target"] == "gateway" - assert data["users"] == "16" + assert not (tmp_path / "run-a" / "metrics.json").exists() def test_parse_no_stats_csv(self, tmp_path: Path): node_dir = tmp_path / "run-empty" @@ -123,8 +106,8 @@ def test_parse_multiple_nodes(self, tmp_path: Path): assert "run-b" in paths for node_name in ["run-a", "run-b"]: - assert (tmp_path / node_name / "metrics.json").exists() - assert (tmp_path / node_name / "parameters.json").exists() + assert not (tmp_path / node_name / "metrics.json").exists() + assert not (tmp_path / node_name / "parameters.json").exists() # ---------------------------------------------------------------------------