From 214c234fcc4eb0dfcf0583177311e4466c107684 Mon Sep 17 00:00:00 2001 From: morpheuslord Date: Sun, 25 Oct 2026 12:29:36 +0530 Subject: [PATCH 01/21] Add a public exception hierarchy and strict mode for the scanner wrappers cloudg.exceptions collects every error cloudg raises on purpose under CloudGError, keeping the old built-in bases so existing except clauses still match. The scanner wrappers record each failure as a ScannerError in .failures, raise it at once with strict=True, and raise_for_errors() raises after a run. parse_report takes strict=True and raises ReportParseError for unreadable reports. Credential, SDK and collection failures now raise CredentialError, ProviderSDKMissingError and CollectionError; ResilienceError and LiveOperationRejected join the hierarchy. --- cloudg/__init__.py | 26 ++++ cloudg/api_scanners.py | 17 +-- cloudg/collectors/gcp.py | 7 +- cloudg/credentials.py | 37 +++--- cloudg/exceptions.py | 212 +++++++++++++++++++++++++++++++ cloudg/ingest.py | 3 +- cloudg/inventory/organization.py | 8 +- cloudg/resilience/errors.py | 4 +- cloudg/resilience/guard.py | 4 +- cloudg/scanners/_errors.py | 87 +++++++++++++ cloudg/scanners/checkov.py | 50 +++++--- cloudg/scanners/iam_linter.py | 18 ++- cloudg/scanners/prowler.py | 34 +++-- cloudg/scanners/scoutsuite.py | 37 ++++-- cloudg/scanners/trivy.py | 55 +++++--- tests/test_exceptions.py | 168 ++++++++++++++++++++++++ 16 files changed, 659 insertions(+), 108 deletions(-) create mode 100644 cloudg/exceptions.py create mode 100644 cloudg/scanners/_errors.py create mode 100644 tests/test_exceptions.py diff --git a/cloudg/__init__.py b/cloudg/__init__.py index 22da0bb..a77078a 100644 --- a/cloudg/__init__.py +++ b/cloudg/__init__.py @@ -9,6 +9,20 @@ PipelineResult, ) from cloudg.config import CloudGConfig, load_config +from cloudg.exceptions import ( + CloudGError, + CollectionError, + ConfigurationError, + CredentialError, + ProviderSDKMissingError, + ReportParseError, + ScannerError, + ScannerExecutionError, + ScannerFailed, + ScannerNotInstalledError, + ScannerOutputError, + ScannerTimeoutError, +) from cloudg.inventory import InventoryMapper, InventoryResult __all__ = [ @@ -16,6 +30,18 @@ "AnalysisResult", "CloudGEngine", "CloudGConfig", + "CloudGError", + "CollectionError", + "ConfigurationError", + "CredentialError", + "ProviderSDKMissingError", + "ReportParseError", + "ScannerError", + "ScannerExecutionError", + "ScannerFailed", + "ScannerNotInstalledError", + "ScannerOutputError", + "ScannerTimeoutError", "CollectionResult", "InventoryMapper", "InventoryResult", diff --git a/cloudg/api_scanners.py b/cloudg/api_scanners.py index ed26b30..39f9c3c 100644 --- a/cloudg/api_scanners.py +++ b/cloudg/api_scanners.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Callable +from cloudg.exceptions import ScannerFailed from cloudg.schema.models import CloudAsset, Finding, NetworkEdge __all__ = [ @@ -64,24 +65,12 @@ def resolve_iac_dirs( # --------------------------------------------------------------------------- -class ScannerFailed(RuntimeError): - """A scanner started but did not finish (timeout, launch failure). - - ``findings`` holds whatever it produced before that (for example the - images Trivy did scan), so callers can keep partial results. - """ - - def __init__(self, scanner: str, errors: list[str], findings: list[Finding]) -> None: - super().__init__(f"{scanner} " + "; ".join(errors)) - self.errors = list(errors) - self.findings = findings - - def _finish(label: str, scanner: Any, findings: list[Finding]) -> list[Finding]: """Return ``findings``, or raise ScannerFailed when the wrapper recorded errors.""" errors = list(getattr(scanner, "errors", None) or []) if errors: - raise ScannerFailed(label, errors, findings) + failures = [f for f in getattr(scanner, "failures", None) or [] if f.detail in errors] + raise ScannerFailed(label, errors, findings, failures=failures) return findings diff --git a/cloudg/collectors/gcp.py b/cloudg/collectors/gcp.py index e891ee6..8ecb4b1 100644 --- a/cloudg/collectors/gcp.py +++ b/cloudg/collectors/gcp.py @@ -43,6 +43,7 @@ ) from cloudg.config import GCP_DEFAULT_SKIP_ASSET_TYPES from cloudg.coverage import CollectionCoverage, ServiceStatus +from cloudg.exceptions import CollectionError, ProviderSDKMissingError from cloudg.resilience.errors import describe_error from cloudg.resilience.gcp import RetryCounter, gcp_retry, gcp_scope, paced from cloudg.resilience.governor import get_governor @@ -186,9 +187,7 @@ def _get_client(self) -> Any: try: from google.cloud import asset_v1 except ImportError as exc: - raise RuntimeError( - "google-cloud-asset is not installed; install cloudg[gcp] to collect GCP" - ) from exc + raise ProviderSDKMissingError("gcp", missing="google-cloud-asset") from exc self._client = asset_v1.AssetServiceClient(credentials=self._credentials) return self._client @@ -301,7 +300,7 @@ async def _collect_from_search(self, error: Exception, start: float) -> list[dic error=describe_error(search_error), start=search_start, ) - raise RuntimeError( + raise CollectionError( f"GCP asset enumeration failed for {self._scope}: " f"ListAssets: {describe_error(error)}; " f"SearchAllResources: {describe_error(search_error)}" diff --git a/cloudg/credentials.py b/cloudg/credentials.py index ed51aa6..4454eb8 100644 --- a/cloudg/credentials.py +++ b/cloudg/credentials.py @@ -39,6 +39,8 @@ from dataclasses import dataclass from typing import Any +from cloudg.exceptions import CredentialError, ProviderSDKMissingError + logger = logging.getLogger(__name__) _GCP_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] @@ -94,7 +96,9 @@ def _aws_session_from_web_identity( WebIdentityToken=token, ) except Exception as exc: - raise RuntimeError(f"AssumeRoleWithWebIdentity failed for {role_arn}: {exc}") from exc + raise CredentialError( + "aws", f"AssumeRoleWithWebIdentity failed for {role_arn}: {exc}" + ) from exc creds = resp["Credentials"] return boto3.Session( aws_access_key_id=creds["AccessKeyId"], @@ -162,7 +166,7 @@ def _aws_assume_role( try: assumed = session.client("sts").assume_role(**assume_kwargs) except Exception as exc: - raise RuntimeError(f"AssumeRole failed for {target_role}: {exc}") from exc + raise CredentialError("aws", f"AssumeRole failed for {target_role}: {exc}") from exc creds = assumed["Credentials"] return boto3.Session( aws_access_key_id=creds["AccessKeyId"], @@ -191,16 +195,14 @@ def build_aws_session( boto3.Session ready for API calls. Raises: - ImportError: If boto3 is not installed. - RuntimeError: If a requested role assumption fails, or account_id + ProviderSDKMissingError: If boto3 is not installed (an ImportError). + CredentialError: If a requested role assumption fails, or account_id is given without cfg.role_name (there is no role to reach it). """ try: import boto3 except ImportError as exc: - raise ImportError( - "boto3 is required for AWS. Install with: pip install cloudg[aws]" - ) from exc + raise ProviderSDKMissingError("aws", missing="boto3") from exc session_name = getattr(cfg, "role_session_name", None) or "cloudg-scan" role_arn = getattr(cfg, "role_arn", None) @@ -237,9 +239,10 @@ def _aws_target_roles( if account_id: role_name = getattr(cfg, "role_name", None) if not role_name: - raise RuntimeError( + raise CredentialError( + "aws", f"Cannot reach account {account_id}: no role to assume there. " - "Set aws.role_name (--role-name) to the role that exists in each account." + "Set aws.role_name (--role-name) to the role that exists in each account.", ) member_role = f"arn:aws:iam::{account_id}:role/{role_name}" if member_role not in roles: @@ -295,7 +298,7 @@ def aws_scanner_auth(cfg: Any, region: str | None = None) -> tuple[str | None, d them for (one hour). Raises: - RuntimeError: assuming role_arn failed. + CredentialError: assuming role_arn failed (a RuntimeError). """ if not region or region.upper() == "ALL": region = "us-east-1" @@ -307,7 +310,7 @@ def aws_scanner_auth(cfg: Any, region: str | None = None) -> tuple[str | None, d session = build_aws_session(cfg, region) credentials = session.get_credentials() if credentials is None: - raise RuntimeError(f"No credentials resolved for role {role_arn}") + raise CredentialError("aws", f"No credentials resolved for role {role_arn}") frozen = credentials.get_frozen_credentials() return None, _aws_key_env(frozen.access_key, frozen.secret_key, frozen.token) if access_key and secret_key: @@ -353,14 +356,12 @@ def build_azure_credential(cfg: Any) -> Any: A TokenCredential usable with all Azure mgmt SDK clients. Raises: - ImportError: If azure-identity is not installed. + ProviderSDKMissingError: If azure-identity is not installed (an ImportError). """ try: from azure import identity except ImportError as exc: - raise ImportError( - "azure-identity is required for Azure. Install with: pip install cloudg[azure]" - ) from exc + raise ProviderSDKMissingError("azure", missing="azure-identity") from exc explicit = _azure_service_principal(identity, cfg) if explicit is not None: @@ -443,14 +444,12 @@ def build_gcp_credentials(cfg: Any) -> tuple[Any, str | None]: auth method carries no project (e.g. impersonation). Raises: - ImportError: If google-auth is not installed. + ProviderSDKMissingError: If google-auth is not installed (an ImportError). """ try: import google.auth except ImportError as exc: - raise ImportError( - "google-auth is required for GCP. Install with: pip install cloudg[gcp]" - ) from exc + raise ProviderSDKMissingError("gcp", missing="google-auth") from exc credentials_file = getattr(cfg, "credentials_file", None) or os.environ.get( "GOOGLE_APPLICATION_CREDENTIALS" diff --git a/cloudg/exceptions.py b/cloudg/exceptions.py new file mode 100644 index 0000000..2ce20cb --- /dev/null +++ b/cloudg/exceptions.py @@ -0,0 +1,212 @@ +"""Every exception cloudg raises on purpose, in one importable place. + +All of them derive from :class:`CloudGError`, so one ``except CloudGError`` +catches anything cloudg itself raised, while SDK and OS errors keep passing +through untouched. The narrower classes also keep the built-in base they +always had (``ValueError``, ``RuntimeError``, ``ImportError``), so existing +``except ValueError`` code goes on working. + +:: + + CloudGError + ├── ConfigurationError (ValueError) + ├── ProviderSDKMissingError (ImportError) + ├── CollectionError (RuntimeError) + │ ├── CredentialError + │ └── OrganizationScopeError (ValueError) + ├── ScannerError + │ ├── ScannerNotInstalledError + │ ├── ScannerTimeoutError + │ ├── ScannerExecutionError + │ ├── ScannerOutputError + │ │ └── ReportParseError + │ └── ScannerFailed (RuntimeError) + ├── ResilienceError (cloudg.resilience) + └── LiveOperationRejected (cloudg.resilience) + +Scanner wrappers record failures instead of raising, so one broken scanner +never stops a pipeline. To get exceptions instead, pass ``strict=True`` to +the wrapper, or call :meth:`raise_for_errors` after a run:: + + from cloudg.exceptions import ScannerError, ScannerTimeoutError + from cloudg.scanners.checkov import CheckovScanner + + scanner = CheckovScanner(target_dir="infra/", timeout_seconds=600) + findings = scanner.run() + try: + scanner.raise_for_errors() + except ScannerTimeoutError as exc: + print(f"{exc.scanner} gave up after {exc.timeout_seconds}s") + except ScannerError as exc: + print(f"{exc.scanner} failed: {exc.detail}") + +This module imports nothing from the rest of cloudg, so any module can +import it without a cycle. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "CloudGError", + "CollectionError", + "ConfigurationError", + "CredentialError", + "OrganizationScopeError", + "ProviderSDKMissingError", + "ReportParseError", + "ScannerError", + "ScannerExecutionError", + "ScannerFailed", + "ScannerNotInstalledError", + "ScannerOutputError", + "ScannerTimeoutError", +] + + +class CloudGError(Exception): + """Base of every exception cloudg raises on purpose.""" + + +class ConfigurationError(CloudGError, ValueError): + """A configuration value is missing, malformed or contradicts another.""" + + +class ProviderSDKMissingError(CloudGError, ImportError): + """A provider's SDK is not installed. + + ``provider`` is ``"aws"``, ``"azure"`` or ``"gcp"`` and ``extra`` the pip + extra that installs it, so the message can say ``pip install cloudg[gcp]``. + """ + + def __init__(self, provider: str, extra: str | None = None, missing: str | None = None) -> None: + self.provider = provider + self.extra = extra or provider + self.missing = missing + what = f" ({missing})" if missing else "" + super().__init__( + f"the {provider} SDK is not installed{what}; run: pip install 'cloudg[{self.extra}]'" + ) + + +class CollectionError(CloudGError, RuntimeError): + """Collecting inventory from a cloud provider failed.""" + + +class CredentialError(CollectionError): + """No usable credentials were found for a provider, or they were refused.""" + + def __init__(self, provider: str, detail: str) -> None: + self.provider = provider + self.detail = detail + super().__init__(f"{provider}: {detail}") + + +class OrganizationScopeError(CollectionError, ValueError): + """The requested organization scope (an OU filter) names nothing in the org. + + Raised instead of quietly mapping only the caller's account. + """ + + +# --------------------------------------------------------------------------- +# Scanners +# --------------------------------------------------------------------------- + + +class ScannerError(CloudGError): + """A security scanner could not produce a complete result. + + ``scanner`` is the wrapper's short name (``"prowler"``, ``"checkov"``, + ...), ``detail`` the reason in a few words and ``target`` what was being + scanned (an image, a directory, a report file), when there is one. + """ + + def __init__(self, scanner: str, detail: str, *, target: str | None = None) -> None: + self.scanner = scanner + self.detail = detail + self.target = target + super().__init__(f"{scanner}: {detail}") + + +class ScannerNotInstalledError(ScannerError): + """The scanner's executable is not on ``PATH``.""" + + def __init__(self, scanner: str, executable: str, install_hint: str = "") -> None: + self.executable = executable + self.install_hint = install_hint + detail = f"{executable!r} is not on PATH" + if install_hint: + detail += f"; install with: {install_hint}" + super().__init__(scanner, detail) + + +class ScannerTimeoutError(ScannerError): + """The scanner ran longer than its ``timeout_seconds`` and was killed.""" + + def __init__(self, scanner: str, timeout_seconds: float, *, target: str | None = None) -> None: + self.timeout_seconds = timeout_seconds + detail = f"timed out after {timeout_seconds:g}s" + if target: + detail += f" scanning {target}" + super().__init__(scanner, detail, target=target) + + +class ScannerExecutionError(ScannerError): + """The scanner process could not be started, or crashed. + + ``returncode`` is the exit status when the process ran; ``output`` the + tail of what it printed, when that was captured. + """ + + def __init__( + self, + scanner: str, + detail: str, + *, + target: str | None = None, + returncode: int | None = None, + output: str = "", + ) -> None: + self.returncode = returncode + self.output = output + super().__init__(scanner, detail, target=target) + + +class ScannerOutputError(ScannerError): + """The scanner ran, but its output could not be read or parsed.""" + + def __init__( + self, scanner: str, detail: str, *, target: str | None = None, path: str | None = None + ) -> None: + self.path = path + super().__init__(scanner, detail, target=target or path) + + +class ReportParseError(ScannerOutputError): + """An existing report handed to ``parse_report`` or ``cloudg ingest`` is unreadable.""" + + +class ScannerFailed(ScannerError, RuntimeError): + """A scanner started but did not finish (timeout, launch failure). + + ``findings`` holds whatever it produced before that (for example the + images Trivy did scan), so callers can keep partial results. ``errors`` + are the recorded reasons as text and ``failures`` the matching + :class:`ScannerError` objects, when the wrapper recorded them. + """ + + def __init__( + self, + scanner: str, + errors: list[str], + findings: list[Any], + failures: list[ScannerError] | None = None, + ) -> None: + self.errors = list(errors) + self.findings = findings + self.failures = list(failures or []) + ScannerError.__init__(self, scanner, "; ".join(self.errors)) + # Keep the message format callers already match on: " " + self.args = (f"{scanner} " + "; ".join(self.errors),) diff --git a/cloudg/ingest.py b/cloudg/ingest.py index b61b30a..3f00a77 100644 --- a/cloudg/ingest.py +++ b/cloudg/ingest.py @@ -20,6 +20,7 @@ import logging from pathlib import Path +from cloudg.exceptions import ConfigurationError from cloudg.normaliser import FindingList from cloudg.schema.models import Finding @@ -70,7 +71,7 @@ def parse_report(tool: str, path: str | Path) -> list[Finding]: return TrivyScanner.parse_report(str(p)) - raise ValueError(f"Unsupported tool '{tool}'. Supported: {', '.join(SUPPORTED_TOOLS)}") + raise ConfigurationError(f"Unsupported tool '{tool}'. Supported: {', '.join(SUPPORTED_TOOLS)}") def ingest_reports(reports: dict[str, list[str | Path]]) -> list[Finding]: diff --git a/cloudg/inventory/organization.py b/cloudg/inventory/organization.py index cafde4c..73484a9 100644 --- a/cloudg/inventory/organization.py +++ b/cloudg/inventory/organization.py @@ -25,6 +25,7 @@ from typing import Any # account_ref is re-exported: it is part of this module's public API. +from cloudg.exceptions import OrganizationScopeError from cloudg.inventory.organization_assets import account_ref, topology_assets from cloudg.schema.models import CloudAsset @@ -33,13 +34,6 @@ DEFAULT_MEMBER_ROLE = "AWSControlTowerExecution" -class OrganizationScopeError(ValueError): - """The requested organization scope (an OU filter) names nothing in the org. - - Raised instead of quietly mapping only the caller's account. - """ - - _POLICY_TYPES = [ "SERVICE_CONTROL_POLICY", "RESOURCE_CONTROL_POLICY", diff --git a/cloudg/resilience/errors.py b/cloudg/resilience/errors.py index dac635f..b4d7a26 100644 --- a/cloudg/resilience/errors.py +++ b/cloudg/resilience/errors.py @@ -30,6 +30,8 @@ from enum import Enum from typing import Any, Iterable, Mapping +from cloudg.exceptions import CloudGError + __all__ = [ "AWS_THROTTLE_CODES", "AWS_TRANSIENT_CODES", @@ -167,7 +169,7 @@ class ErrorKind(str, Enum): _LIMIT_EXCEEDED_RATE_RE = re.compile(r"(rate|throttl|too many|per second|tps)", re.IGNORECASE) -class ResilienceError(Exception): +class ResilienceError(CloudGError): """Base of the errors raised by :mod:`cloudg.resilience` itself.""" #: Seconds after which trying again makes sense (None: unknown) diff --git a/cloudg/resilience/guard.py b/cloudg/resilience/guard.py index 067041b..3ad7296 100644 --- a/cloudg/resilience/guard.py +++ b/cloudg/resilience/guard.py @@ -52,6 +52,8 @@ from dataclasses import dataclass, fields, replace from typing import Any, Awaitable, Callable, Iterable, Mapping, TypeVar +from cloudg.exceptions import CloudGError + __all__ = [ "CallerQuotaExceeded", "CooldownActive", @@ -66,7 +68,7 @@ T = TypeVar("T") -class LiveOperationRejected(Exception): +class LiveOperationRejected(CloudGError): """A live operation was refused; ``retry_after`` says when to try again.""" reason = "rejected" diff --git a/cloudg/scanners/_errors.py b/cloudg/scanners/_errors.py new file mode 100644 index 0000000..6c68e8d --- /dev/null +++ b/cloudg/scanners/_errors.py @@ -0,0 +1,87 @@ +"""How the scanner wrappers record failures, and raise them on request.""" + +from __future__ import annotations + +import logging +from typing import ClassVar, Self + +from cloudg.exceptions import ( + ReportParseError, + ScannerError, + ScannerFailed, + ScannerNotInstalledError, + ScannerOutputError, + ScannerTimeoutError, +) + +logger = logging.getLogger(__name__) + + +class ScannerErrorLog: + """Failure bookkeeping shared by every scanner wrapper. + + ``errors`` lists, as text, why the last run did not finish (timeouts, + launch failures). ``failures`` holds every :class:`ScannerError` the + wrapper met, including output it could not parse, which does not stop a + run and so is not in ``errors``. With ``strict`` set, the first failure + is raised at once instead. + """ + + #: Short name used in messages and on the exceptions + scanner_name: ClassVar[str] = "scanner" + #: Executable looked up on PATH, and how to install it + executable: ClassVar[str] = "" + install_hint: ClassVar[str] = "" + + strict: bool = False + #: Raised for unparseable output; parse_report switches it to ReportParseError + _output_error: type[ScannerOutputError] = ScannerOutputError + + def _init_errors(self, strict: bool = False) -> None: + self.strict = bool(strict) + self.errors: list[str] = [] + self.failures: list[ScannerError] = [] + + def _fail(self, exc: ScannerError, *, stops_run: bool = True) -> None: + """Record ``exc``; raise it when ``strict``.""" + if not hasattr(self, "failures"): + self._init_errors(self.strict) + self.failures.append(exc) + if stops_run: + self.errors.append(exc.detail) + if self.strict: + raise exc + + def _timed_out(self, timeout_seconds: float, target: str | None = None) -> None: + self._fail(ScannerTimeoutError(self.scanner_name, timeout_seconds, target=target)) + + def _bad_output(self, detail: str, path: str | None = None) -> None: + self._fail(self._output_error(self.scanner_name, detail, path=path), stops_run=False) + + @classmethod + def _report_reader(cls, strict: bool, **kwargs: object) -> Self: + """An instance for ``parse_report``: output problems become ReportParseError.""" + scanner = cls(strict=strict, **kwargs) # type: ignore[call-arg] + scanner._output_error = ReportParseError + return scanner + + def _missing(self) -> None: + """Called when the executable is not installed: a warning, or a raise when strict.""" + exc = ScannerNotInstalledError(self.scanner_name, self.executable, self.install_hint) + if self.strict: + raise exc + logger.warning("%s. Skipping the %s scan.", exc, self.scanner_name) + + def raise_for_errors(self) -> None: + """Raise when the last run recorded a failure; do nothing otherwise. + + A single failure is raised as it is (a :class:`ScannerTimeoutError`, + :class:`ScannerOutputError`, ...). Several are raised together as a + :class:`ScannerFailed` whose ``failures`` lists each one. + """ + failures = list(getattr(self, "failures", None) or []) + if not failures: + return + if len(failures) == 1: + raise failures[0] + raise ScannerFailed(self.scanner_name, [f.detail for f in failures], [], failures=failures) diff --git a/cloudg/scanners/checkov.py b/cloudg/scanners/checkov.py index 430c8a9..c92df76 100644 --- a/cloudg/scanners/checkov.py +++ b/cloudg/scanners/checkov.py @@ -8,6 +8,8 @@ import subprocess # nosec B404 - runs the wrapped scanner CLIs with argv lists, never a shell from typing import Any +from cloudg.exceptions import ReportParseError, ScannerExecutionError +from cloudg.scanners._errors import ScannerErrorLog from cloudg.schema.models import Finding, Severity logger = logging.getLogger(__name__) @@ -22,25 +24,31 @@ } -class CheckovScanner: +class CheckovScanner(ScannerErrorLog): """Wraps Checkov CLI to scan IaC directories for misconfigurations. Supports Terraform, CloudFormation, ARM templates, Kubernetes manifests. + With ``strict=True`` a failure raises a :mod:`cloudg.exceptions` + ``ScannerError`` instead of being recorded in ``errors``. """ + scanner_name = "checkov" + executable = "checkov" + install_hint = "pip install checkov" + def __init__( self, target_dir: str = ".", frameworks: list[str] | None = None, extra_args: list[str] | None = None, timeout_seconds: int = 1800, + strict: bool = False, ) -> None: self._target_dir = target_dir self._frameworks = frameworks or [] self._extra_args = extra_args or [] self._timeout_seconds = timeout_seconds - #: Why the last run did not finish (timeout, launch failure); empty on success - self.errors: list[str] = [] + self._init_errors(strict) @staticmethod def is_available() -> bool: @@ -56,10 +64,7 @@ def run(self) -> list[Finding]: infrastructure scanning. """ if not self.is_available(): - logger.warning( - "Checkov is not installed. Install with: pip install checkov. " - "Skipping Checkov scan." - ) + self._missing() return [] cmd = [ @@ -91,24 +96,26 @@ def run(self) -> list[Finding]: ) except subprocess.TimeoutExpired: logger.error("Checkov scan timed out after %ds", self._timeout_seconds) - self.errors.append( - f"timed out after {self._timeout_seconds}s scanning {self._target_dir}" - ) + self._timed_out(self._timeout_seconds, self._target_dir) return [] except Exception as exc: logger.error("Failed to run Checkov: %s", exc) - self.errors.append(f"could not run: {exc}") + self._fail( + ScannerExecutionError("checkov", f"could not run: {exc}", target=self._target_dir) + ) return [] return self._parse_output(result.stdout) @classmethod - def parse_report(cls, path: str) -> list[Finding]: + def parse_report(cls, path: str, *, strict: bool = False) -> list[Finding]: """Parse existing Checkov JSON output without running Checkov. Accepts the JSON report file (``checkov --output json``, saved to a file, or ``--output-file-path``'s ``results_json.json``) or a - directory containing it. + directory containing it. With ``strict=True`` an unreadable or + malformed file raises :class:`~cloudg.exceptions.ReportParseError` + instead of being skipped with a warning. """ from pathlib import Path @@ -116,22 +123,30 @@ def parse_report(cls, path: str) -> list[Finding]: if p.is_dir(): candidates = list(p.rglob("results_json.json")) or list(p.rglob("*.json")) if not candidates: + if strict: + raise ReportParseError("checkov", "no JSON results found", path=str(path)) logger.warning("No Checkov JSON results found in %s", path) return [] else: candidates = [p] - scanner = cls() + scanner = cls._report_reader(strict) findings: list[Finding] = [] for file_path in candidates: try: - findings.extend(scanner._parse_output(file_path.read_text())) + text = file_path.read_text() except OSError as exc: + if strict: + raise ReportParseError( + "checkov", f"could not read: {exc}", path=str(file_path) + ) from exc logger.warning("Failed to read Checkov output %s: %s", file_path, exc) + continue + findings.extend(scanner._parse_output(text, source=str(file_path))) return findings - def _parse_output(self, stdout: str) -> list[Finding]: - """Parse Checkov JSON output from stdout.""" + def _parse_output(self, stdout: str, source: str | None = None) -> list[Finding]: + """Parse Checkov JSON output from stdout (or from the file ``source``).""" findings: list[Finding] = [] if not stdout.strip(): @@ -142,6 +157,7 @@ def _parse_output(self, stdout: str) -> list[Finding]: data = json.loads(stdout) except json.JSONDecodeError as exc: logger.warning("Failed to parse Checkov JSON output: %s", exc) + self._bad_output(f"output is not JSON: {exc}", path=source) return [] # Checkov output can be a list (multiple frameworks) or single dict diff --git a/cloudg/scanners/iam_linter.py b/cloudg/scanners/iam_linter.py index beda415..28d4809 100644 --- a/cloudg/scanners/iam_linter.py +++ b/cloudg/scanners/iam_linter.py @@ -5,19 +5,27 @@ import json import logging +from cloudg.exceptions import ScannerExecutionError +from cloudg.scanners._errors import ScannerErrorLog from cloudg.schema.models import CloudAsset, AssetType, Finding, Severity logger = logging.getLogger(__name__) -class IAMLinter: +class IAMLinter(ScannerErrorLog): """Analyses IAM policies using Parliament for linting and policy-sentry for least-privilege scoring. Parliament: Detects wildcard abuse, logical errors, unused conditions. Policy-sentry: Scores how over-permissioned policies are. + + A policy Parliament cannot analyse is recorded in ``failures`` (or + raised, with ``strict=True``); the wildcard checks still run on it. """ - def __init__(self) -> None: + scanner_name = "parliament" + + def __init__(self, strict: bool = False) -> None: + self._init_errors(strict) self._parliament_available = False self._policy_sentry_available = False self._check_availability() @@ -105,6 +113,12 @@ def _lint_with_parliament(self, policy_str: str, asset: CloudAsset) -> list[Find ) except Exception as exc: logger.debug("Parliament analysis failed for %s: %s", asset.name, exc) + self._fail( + ScannerExecutionError( + "parliament", f"could not analyse {asset.name}: {exc}", target=asset.name + ), + stops_run=False, + ) return findings diff --git a/cloudg/scanners/prowler.py b/cloudg/scanners/prowler.py index 28db0fe..4ec3750 100644 --- a/cloudg/scanners/prowler.py +++ b/cloudg/scanners/prowler.py @@ -17,6 +17,8 @@ ocsf_status, parse_ocsf_record, ) +from cloudg.exceptions import ScannerExecutionError +from cloudg.scanners._errors import ScannerErrorLog from cloudg.schema.models import Finding, Severity logger = logging.getLogger(__name__) @@ -59,18 +61,26 @@ def _get(item: dict[str, Any], *keys: str) -> Any: "aws_regions", "timeout_seconds", "env", + "strict", } ) -class ProwlerScanner: +class ProwlerScanner(ScannerErrorLog): """Wraps Prowler CLI to run cloud security checks and parse ASFF JSON output. + With ``strict=True`` a failure raises a :mod:`cloudg.exceptions` + ``ScannerError`` instead of being recorded in ``errors``. + Usage: scanner = ProwlerScanner(provider="aws", profile="default") findings = scanner.run() """ + scanner_name = "prowler" + executable = "prowler" + install_hint = "pip install prowler" + def __init__( self, provider: str = "aws", @@ -97,6 +107,8 @@ def __init__( timeout_seconds: How long Prowler may run before it is killed (default 3600). env: Extra environment variables for the process. + strict: Raise a :class:`~cloudg.exceptions.ScannerError` on the + first failure instead of recording it (default False). Raises: TypeError: an unknown setting is passed. @@ -119,8 +131,7 @@ def __init__( for r in (settings.get("aws_regions") or []) if r and r.strip() and r.strip().upper() != "ALL" ] - #: Why the last run did not finish (timeout, launch failure); empty on success - self.errors: list[str] = [] + self._init_errors(bool(settings.get("strict", False))) @staticmethod def is_available() -> bool: @@ -138,10 +149,7 @@ def run(self) -> list[Finding]: import time if not self.is_available(): - logger.warning( - "Prowler is not installed. Install with: pip install prowler. " - "Skipping Prowler scan." - ) + self._missing() return [] cmd = self._build_command() @@ -257,11 +265,11 @@ def _execute(self, cmd: list[str], env: dict[str, str], start_time: float) -> bo "[Prowler] Scan timed out after %ds; killing process", self._timeout_seconds ) proc.kill() - self.errors.append(f"timed out after {self._timeout_seconds}s") + self._timed_out(self._timeout_seconds) return False except Exception as exc: logger.error("[Prowler] Failed to run: %s", exc) - self.errors.append(f"could not run: {exc}") + self._fail(ScannerExecutionError("prowler", f"could not run: {exc}")) return False return True @@ -271,7 +279,7 @@ def _execute(self, cmd: list[str], env: dict[str, str], start_time: float) -> bo passed_checks: frozenset[str] = frozenset() @classmethod - def parse_report(cls, path: str) -> list[Finding]: + def parse_report(cls, path: str, *, strict: bool = False) -> list[Finding]: """Parse existing Prowler output without running Prowler. Accepts a single ASFF (``-M json-asff``) or OCSF (``-M json-ocsf``, @@ -279,11 +287,12 @@ def parse_report(cls, path: str) -> list[Finding]: searched recursively for ``*.json`` files (Prowler's ``-o`` output directory works as-is). Returns a :class:`~cloudg.normaliser.FindingList` of the failing checks whose ``passed_checks`` names the checks that - passed. + passed. With ``strict=True`` an unreadable or malformed file raises + :class:`~cloudg.exceptions.ReportParseError` instead of being skipped. """ import os - scanner = cls() + scanner = cls._report_reader(strict) if os.path.isdir(path): files = glob.glob(f"{path}/**/*.json", recursive=True) else: @@ -324,6 +333,7 @@ def _parse_files(self, output_files: list[str]) -> list[Finding]: except (json.JSONDecodeError, OSError) as exc: logger.warning("Failed to parse Prowler output %s: %s", file_path, exc) + self._bad_output(f"could not parse: {exc}", path=str(file_path)) self.passed_checks = frozenset(findings.passed_checks) logger.info( diff --git a/cloudg/scanners/scoutsuite.py b/cloudg/scanners/scoutsuite.py index 39bc523..d683b4f 100644 --- a/cloudg/scanners/scoutsuite.py +++ b/cloudg/scanners/scoutsuite.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import Any +from cloudg.exceptions import ReportParseError, ScannerExecutionError +from cloudg.scanners._errors import ScannerErrorLog from cloudg.schema.models import Finding, Severity logger = logging.getLogger(__name__) @@ -44,14 +46,19 @@ def _compliance_frameworks(finding_data: dict[str, Any]) -> list[str]: return frameworks -class ScoutSuiteScanner: +class ScoutSuiteScanner(ScannerErrorLog): """Wraps ScoutSuite CLI: runs `scout` and parses the results JS file. ScoutSuite produces a `scoutsuite_results.js` file containing a JSON object assigned to a JS variable. We strip the variable assignment - and parse the JSON. + and parse the JSON. With ``strict=True`` a failure raises a + :mod:`cloudg.exceptions` ``ScannerError`` instead of being recorded. """ + scanner_name = "scoutsuite" + executable = "scout" + install_hint = "pip install scoutsuite" + def __init__( self, provider: str = "aws", @@ -61,6 +68,7 @@ def __init__( timeout_seconds: int = 3600, env: dict[str, str] | None = None, regions: list[str] | None = None, + strict: bool = False, ) -> None: """``timeout_seconds`` is how long ScoutSuite may run; ``env`` adds environment variables (for example resolved AWS credentials) to the @@ -76,8 +84,7 @@ def __init__( self._regions = [ r for r in (regions or []) if r and r.strip() and r.strip().upper() != "ALL" ] - #: Why the last run did not finish (timeout, launch failure); empty on success - self.errors: list[str] = [] + self._init_errors(strict) def _build_env(self) -> dict[str, str] | None: """Process environment: the parent's plus ``env`` (None when unchanged).""" @@ -119,10 +126,7 @@ def is_available() -> bool: def run(self) -> list[Finding]: """Execute ScoutSuite scan and return normalised findings.""" if not self.is_available(): - logger.warning( - "ScoutSuite is not installed. Install with: pip install scoutsuite. " - "Skipping ScoutSuite scan." - ) + self._missing() return [] cmd = self._build_command() @@ -148,31 +152,35 @@ def run(self) -> list[Finding]: ) except subprocess.TimeoutExpired: logger.error("ScoutSuite scan timed out after %ds", self._timeout_seconds) - self.errors.append(f"timed out after {self._timeout_seconds}s") + self._timed_out(self._timeout_seconds) return [] except Exception as exc: logger.error("Failed to run ScoutSuite: %s", exc) - self.errors.append(f"could not run: {exc}") + self._fail(ScannerExecutionError("scoutsuite", f"could not run: {exc}")) return [] return self._parse_output() @classmethod - def parse_report(cls, path: str) -> list[Finding]: + def parse_report(cls, path: str, *, strict: bool = False) -> list[Finding]: """Parse existing ScoutSuite results without running ScoutSuite. Accepts the ``scoutsuite_results_*.js`` file itself or a report directory that is searched recursively for it (ScoutSuite's - ``--report-dir`` output directory works as-is). + ``--report-dir`` output directory works as-is). With ``strict=True`` + an unreadable or malformed file raises + :class:`~cloudg.exceptions.ReportParseError` instead of being skipped. """ p = Path(path) if p.is_dir(): files = list(p.rglob("scoutsuite_results*.js")) if not files: + if strict: + raise ReportParseError("scoutsuite", "no results file found", path=str(path)) logger.warning("No ScoutSuite results found in %s", path) else: files = [p] - return cls()._parse_files(files) + return cls._report_reader(strict)._parse_files(files) def _parse_output(self) -> list[Finding]: """Parse ScoutSuite results JS file.""" @@ -182,6 +190,7 @@ def _parse_output(self) -> list[Finding]: if not results_files: logger.warning("No ScoutSuite results found in %s", self._report_dir) + self._bad_output("produced no results file", path=self._report_dir) return [] return self._parse_files(results_files) @@ -199,6 +208,7 @@ def _parse_files(self, results_files: list[Path]) -> list[Finding]: match = re.search(r"=\s*({.*})\s*;?\s*$", content, re.DOTALL) if not match: logger.warning("Could not extract JSON from %s", results_file) + self._bad_output("no JSON object in the results file", path=str(results_file)) continue data = json.loads(match.group(1)) @@ -206,6 +216,7 @@ def _parse_files(self, results_files: list[Path]) -> list[Finding]: except (json.JSONDecodeError, OSError) as exc: logger.warning("Failed to parse ScoutSuite results: %s", exc) + self._bad_output(f"could not parse: {exc}", path=str(results_file)) logger.info("Parsed %d findings from ScoutSuite", len(findings)) return findings diff --git a/cloudg/scanners/trivy.py b/cloudg/scanners/trivy.py index 92caa3b..510882f 100644 --- a/cloudg/scanners/trivy.py +++ b/cloudg/scanners/trivy.py @@ -8,6 +8,8 @@ import subprocess # nosec B404 - runs the wrapped scanner CLIs with argv lists, never a shell from typing import Any +from cloudg.exceptions import ReportParseError, ScannerExecutionError +from cloudg.scanners._errors import ScannerErrorLog from cloudg.schema.models import Finding, Severity logger = logging.getLogger(__name__) @@ -21,22 +23,28 @@ } -class TrivyScanner: +class TrivyScanner(ScannerErrorLog): """Wraps Trivy CLI for container image CVE and secret scanning. Scans container images from ECR/ACR/GCR and maps CVEs to findings. + With ``strict=True`` a failure raises a :mod:`cloudg.exceptions` + ``ScannerError`` instead of being recorded in ``errors``. """ + scanner_name = "trivy" + executable = "trivy" + install_hint = "see https://trivy.dev/" + def __init__( self, extra_args: list[str] | None = None, timeout_seconds: int = 1800, + strict: bool = False, ) -> None: """``timeout_seconds`` applies to each image or directory scan.""" self._extra_args = extra_args or [] self._timeout_seconds = timeout_seconds - #: Why scans did not finish (timeouts, launch failures); empty on success - self.errors: list[str] = [] + self._init_errors(strict) @staticmethod def is_available() -> bool: @@ -53,9 +61,7 @@ def scan_image(self, image: str) -> list[Finding]: List of Finding objects for discovered CVEs. """ if not self.is_available(): - logger.warning( - "Trivy is not installed. Install from: https://trivy.dev/. Skipping Trivy scan." - ) + self._missing() return [] cmd = [ @@ -84,11 +90,13 @@ def scan_image(self, image: str) -> list[Finding]: logger.error( "Trivy scan timed out after %ds for image %s", self._timeout_seconds, image ) - self.errors.append(f"timed out after {self._timeout_seconds}s scanning {image}") + self._timed_out(self._timeout_seconds, image) return [] except Exception as exc: logger.error("Failed to run Trivy for %s: %s", image, exc) - self.errors.append(f"could not scan {image}: {exc}") + self._fail( + ScannerExecutionError("trivy", f"could not scan {image}: {exc}", target=image) + ) return [] return self._parse_output(result.stdout, image) @@ -120,10 +128,7 @@ def scan_filesystem(self, directories: list[str]) -> list[Finding]: Combined list of Finding objects. """ if not self.is_available(): - logger.warning( - "Trivy is not installed. Install from: https://trivy.dev/. " - "Skipping Trivy filesystem scan." - ) + self._missing() return [] all_findings: list[Finding] = [] @@ -158,11 +163,15 @@ def scan_filesystem(self, directories: list[str]) -> list[Finding]: self._timeout_seconds, directory, ) - self.errors.append(f"timed out after {self._timeout_seconds}s scanning {directory}") + self._timed_out(self._timeout_seconds, directory) continue except Exception as exc: logger.error("Failed to run Trivy fs for %s: %s", directory, exc) - self.errors.append(f"could not scan {directory}: {exc}") + self._fail( + ScannerExecutionError( + "trivy", f"could not scan {directory}: {exc}", target=directory + ) + ) continue findings = self._parse_fs_output(result.stdout, directory) @@ -171,12 +180,14 @@ def scan_filesystem(self, directories: list[str]) -> list[Finding]: return all_findings @classmethod - def parse_report(cls, path: str) -> list[Finding]: + def parse_report(cls, path: str, *, strict: bool = False) -> list[Finding]: """Parse existing Trivy JSON output without running Trivy. Accepts a ``trivy image|fs --format json`` report file or a directory of them. Image scans and filesystem/repository scans are - told apart by the report's ``ArtifactType`` field. + told apart by the report's ``ArtifactType`` field. With + ``strict=True`` an unreadable or malformed file raises + :class:`~cloudg.exceptions.ReportParseError` instead of being skipped. """ from pathlib import Path @@ -184,21 +195,29 @@ def parse_report(cls, path: str) -> list[Finding]: if p.is_dir(): candidates = sorted(p.rglob("*.json")) if not candidates: + if strict: + raise ReportParseError("trivy", "no JSON results found", path=str(path)) logger.warning("No Trivy JSON results found in %s", path) return [] else: candidates = [p] - scanner = cls() + scanner = cls._report_reader(strict) findings: list[Finding] = [] for file_path in candidates: try: content = file_path.read_text() data = json.loads(content) except (OSError, json.JSONDecodeError) as exc: + if strict: + raise ReportParseError( + "trivy", f"could not read: {exc}", path=str(file_path) + ) from exc logger.warning("Failed to read Trivy output %s: %s", file_path, exc) continue if not isinstance(data, dict): + if strict: + raise ReportParseError("trivy", "not a Trivy JSON report", path=str(file_path)) logger.warning("Unrecognised Trivy output structure in %s", file_path) continue @@ -223,6 +242,7 @@ def _parse_fs_output(self, stdout: str, directory: str) -> list[Finding]: data = json.loads(stdout) except json.JSONDecodeError as exc: logger.warning("Failed to parse Trivy fs JSON: %s", exc) + self._bad_output(f"fs output for {directory} is not JSON: {exc}") return [] results = data.get("Results", []) @@ -300,6 +320,7 @@ def _parse_output(self, stdout: str, image: str) -> list[Finding]: data = json.loads(stdout) except json.JSONDecodeError as exc: logger.warning("Failed to parse Trivy JSON: %s", exc) + self._bad_output(f"output for {image} is not JSON: {exc}") return [] results = data.get("Results", []) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py new file mode 100644 index 0000000..dc42b4f --- /dev/null +++ b/tests/test_exceptions.py @@ -0,0 +1,168 @@ +"""The public exception hierarchy and how the scanner wrappers use it.""" + +from __future__ import annotations + +import subprocess + +import pytest + +import cloudg +from cloudg import exceptions as exc +from cloudg.scanners.checkov import CheckovScanner +from cloudg.scanners.iam_linter import IAMLinter +from cloudg.scanners.prowler import ProwlerScanner +from cloudg.scanners.scoutsuite import ScoutSuiteScanner +from cloudg.scanners.trivy import TrivyScanner + + +class TestHierarchy: + def test_everything_derives_from_cloudg_error(self): + for name in exc.__all__: + assert issubclass(getattr(exc, name), exc.CloudGError), name + + def test_builtin_bases_are_kept(self): + assert issubclass(exc.ConfigurationError, ValueError) + assert issubclass(exc.ProviderSDKMissingError, ImportError) + assert issubclass(exc.CollectionError, RuntimeError) + assert issubclass(exc.CredentialError, RuntimeError) + assert issubclass(exc.OrganizationScopeError, ValueError) + assert issubclass(exc.ScannerFailed, RuntimeError) + assert issubclass(exc.ReportParseError, exc.ScannerOutputError) + + def test_exported_from_the_package(self): + assert cloudg.ScannerError is exc.ScannerError + assert cloudg.CloudGError is exc.CloudGError + + def test_old_import_paths_still_work(self): + from cloudg.api_scanners import ScannerFailed + from cloudg.inventory.organization import OrganizationScopeError + + assert ScannerFailed is exc.ScannerFailed + assert OrganizationScopeError is exc.OrganizationScopeError + + def test_resilience_errors_join_the_hierarchy(self): + from cloudg.resilience import CircuitOpenError, LiveOperationRejected + + assert issubclass(CircuitOpenError, exc.CloudGError) + assert issubclass(LiveOperationRejected, exc.CloudGError) + + def test_scanner_failed_keeps_its_message(self): + failed = exc.ScannerFailed("trivy", ["a", "b"], ["partial"]) + assert str(failed) == "trivy a; b" + assert failed.findings == ["partial"] + assert failed.scanner == "trivy" + + def test_sdk_missing_message_names_the_extra(self): + err = exc.ProviderSDKMissingError("gcp", missing="google-auth") + assert "pip install 'cloudg[gcp]'" in str(err) + assert err.missing == "google-auth" + + def test_timeout_detail_matches_the_recorded_text(self): + err = exc.ScannerTimeoutError("checkov", 1800, target="infra") + assert err.detail == "timed out after 1800s scanning infra" + assert err.timeout_seconds == 1800 + + +def _no_binary(monkeypatch): + monkeypatch.setattr("shutil.which", lambda _name: None) + + +def _binary(monkeypatch): + monkeypatch.setattr("shutil.which", lambda name: f"/usr/bin/{name}") + + +class TestScannerRecording: + @pytest.mark.parametrize( + "make, run", + [ + (lambda **kw: CheckovScanner(**kw), lambda s: s.run()), + (lambda **kw: ProwlerScanner(**kw), lambda s: s.run()), + (lambda **kw: ScoutSuiteScanner(**kw), lambda s: s.run()), + (lambda **kw: TrivyScanner(**kw), lambda s: s.scan_image("app:1")), + ], + ) + def test_missing_binary(self, monkeypatch, make, run): + _no_binary(monkeypatch) + lenient = make() + assert run(lenient) == [] + assert lenient.errors == [] + lenient.raise_for_errors() # nothing recorded, nothing raised + + with pytest.raises(exc.ScannerNotInstalledError) as info: + run(make(strict=True)) + assert info.value.executable + + def test_timeout_is_recorded_then_raised(self, monkeypatch): + _binary(monkeypatch) + + def boom(*_a, **_k): + raise subprocess.TimeoutExpired("checkov", 5) + + monkeypatch.setattr(subprocess, "run", boom) + scanner = CheckovScanner(target_dir="infra", timeout_seconds=5) + assert scanner.run() == [] + assert scanner.errors == ["timed out after 5s scanning infra"] + assert isinstance(scanner.failures[0], exc.ScannerTimeoutError) + with pytest.raises(exc.ScannerTimeoutError): + scanner.raise_for_errors() + + with pytest.raises(exc.ScannerTimeoutError): + CheckovScanner(target_dir="infra", timeout_seconds=5, strict=True).run() + + def test_launch_failure(self, monkeypatch): + _binary(monkeypatch) + + def boom(*_a, **_k): + raise OSError("permission denied") + + monkeypatch.setattr(subprocess, "run", boom) + scanner = TrivyScanner() + scanner.scan_images(["a:1", "b:2"]) + assert len(scanner.errors) == 2 + with pytest.raises(exc.ScannerFailed) as info: + scanner.raise_for_errors() + assert all(isinstance(f, exc.ScannerExecutionError) for f in info.value.failures) + + def test_bad_output_is_a_failure_but_not_an_error(self): + scanner = CheckovScanner() + assert scanner._parse_output("{not json") == [] + assert scanner.errors == [] + assert isinstance(scanner.failures[0], exc.ScannerOutputError) + with pytest.raises(exc.ScannerOutputError): + scanner.raise_for_errors() + + def test_parliament_failure_is_recorded(self, monkeypatch): + linter = IAMLinter() + linter._parliament_available = True + + class Asset: + name = "role/x" + + import sys + import types + + fake = types.ModuleType("parliament") + + def analyze(_doc): + raise RuntimeError("bad policy") + + fake.analyze_policy_string = analyze + monkeypatch.setitem(sys.modules, "parliament", fake) + linter._lint_with_parliament("{}", Asset()) + assert isinstance(linter.failures[0], exc.ScannerExecutionError) + + +class TestParseReport: + @pytest.mark.parametrize( + "scanner", [CheckovScanner, TrivyScanner, ProwlerScanner, ScoutSuiteScanner] + ) + def test_strict_parse_report_raises_on_garbage(self, tmp_path, scanner): + bad = tmp_path / ("scoutsuite_results_x.js" if scanner is ScoutSuiteScanner else "r.json") + bad.write_text("this is not a report") + assert scanner.parse_report(str(bad)) == [] + with pytest.raises(exc.ReportParseError): + scanner.parse_report(str(bad), strict=True) + + def test_strict_parse_report_raises_on_empty_dir(self, tmp_path): + with pytest.raises(exc.ReportParseError): + CheckovScanner.parse_report(str(tmp_path), strict=True) From 74b17350df5301c6c9aa0bbdde3a8d680ab6f3e9 Mon Sep 17 00:00:00 2001 From: morpheuslord Date: Sun, 25 Oct 2026 12:53:15 +0530 Subject: [PATCH 02/21] Let several GCP extractors register for one asset type EXTRACTORS[t] = fn replaced the earlier extractor when a second module registered the same Cloud Asset Inventory type. The registry now keeps a list per type, like the Azure registry, and extract() runs every entry in order. A failing extractor is recorded in extraction_error and the rest still run. --- cloudg/inventory/gcp_relations/context.py | 22 ++++++++--- tests/test_gcp_registry.py | 48 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 tests/test_gcp_registry.py diff --git a/cloudg/inventory/gcp_relations/context.py b/cloudg/inventory/gcp_relations/context.py index e1b9bf9..c99cd46 100644 --- a/cloudg/inventory/gcp_relations/context.py +++ b/cloudg/inventory/gcp_relations/context.py @@ -102,28 +102,38 @@ def expose( Extractor = Callable[[GCPContext, Extracted], None] -EXTRACTORS: dict[str, Extractor] = {} +# Asset type -> extractors, run in registration order. Several modules can +# register for the same type; each one adds to the same Extracted. +EXTRACTORS: dict[str, list[Extractor]] = {} def _extractor(*asset_types: str) -> Callable[[Extractor], Extractor]: def register(fn: Extractor) -> Extractor: for t in asset_types: - EXTRACTORS[t] = fn + fns = EXTRACTORS.setdefault(t, []) + if fn not in fns: + fns.append(fn) return fn return register def extract(ctx: GCPContext) -> Extracted: - """Run the extractor for ``ctx.asset_type`` plus the generic CMEK scan.""" + """Run every extractor registered for ``ctx.asset_type``, then the generic CMEK scan. + + A failing extractor is recorded in ``extraction_error`` and does not stop + the others registered for the same type. + """ out = Extracted() - fn = EXTRACTORS.get(ctx.asset_type) - if fn is not None: + errors: list[str] = [] + for fn in EXTRACTORS.get(ctx.asset_type, ()): try: fn(ctx, out) except Exception as exc: # one malformed resource must not stop the sweep logger.debug("GCP extractor failed for %s: %s", ctx.name, exc, exc_info=True) - out.metadata["extraction_error"] = f"{type(exc).__name__}: {exc}" + errors.append(f"{type(exc).__name__}: {exc}") + if errors: + out.metadata["extraction_error"] = "; ".join(errors) if asset_type_service(ctx.asset_type) == "cloudkms": return out keys = kms_refs(ctx.data) diff --git a/tests/test_gcp_registry.py b/tests/test_gcp_registry.py new file mode 100644 index 0000000..c257a26 --- /dev/null +++ b/tests/test_gcp_registry.py @@ -0,0 +1,48 @@ +"""The GCP extractor registry runs every extractor registered for a type.""" + +from __future__ import annotations + +from cloudg.inventory.gcp_relations import EXTRACTORS, GCPContext, extract +from cloudg.inventory.gcp_relations.context import _extractor +from cloudg.schema.models import EdgeType + +_TYPE = "widgets.googleapis.com/Widget" + + +def test_second_extractor_appends_instead_of_replacing(): + calls: list[str] = [] + + @_extractor(_TYPE) + def first(ctx, out): + calls.append("first") + out.add("//widgets.googleapis.com/a", EdgeType.REFERENCES, "DEPENDS_ON") + + @_extractor(_TYPE) + def second(ctx, out): + calls.append("second") + out.metadata["second"] = True + + @_extractor(_TYPE) + def broken(ctx, out): + raise ValueError("bad shape") + + @_extractor(_TYPE) + def after_broken(ctx, out): + out.alias("widget-alias") + + try: + _extractor(_TYPE)(first) # registering twice is a no-op + assert EXTRACTORS[_TYPE] == [first, second, broken, after_broken] + out = extract(GCPContext(name="//widgets.googleapis.com/w", asset_type=_TYPE, data={})) + assert calls == ["first", "second"] + assert out.relations[0]["target"] == "//widgets.googleapis.com/a" + assert out.metadata["second"] is True + assert out.aliases == ["widget-alias"] + assert "ValueError: bad shape" in out.metadata["extraction_error"] + finally: + EXTRACTORS.pop(_TYPE, None) + + +def test_registry_values_are_lists(): + assert EXTRACTORS + assert all(isinstance(v, list) and v for v in EXTRACTORS.values()) From 7e4cd08979f14601a6764f1c7b4d2899260e630b Mon Sep 17 00:00:00 2001 From: morpheuslord Date: Sun, 25 Oct 2026 13:04:23 +0530 Subject: [PATCH 03/21] Redesign report.html around an animated isometric estate map The report opens on a posture overview (severity bar, share of assets with no open findings, top risks) and gains an Assets tab, CSV export, light and dark themes, print styles and a phone layout. The map is laid out in Python (cloudg/renderers/iso_scene.py): a base plate per provider, platforms per account and region, slabs for networks, tiles for subnets and a solid per asset, with identity and governance assets on a control deck held above the plate. Every relation is routed orthogonally in 3D from its own source port, on a lane no other trace shares, to its own target port, so each one can be followed end to end. The page projects the scene with the Anatomy iso-kit, paints it back to front, animates it in, runs a dot along each lit trace, turns, zooms and pans, and traces paths from the internet. The D3 tree and force views stay one toggle away, and are used alone above 2500 assets or without the map engines. The Anatomy iso-kit and the Hairline kernel (both MIT) are vendored and always inlined. The empty states use a Hairline figure of an empty tray. --- cloudg/renderers/html_report.py | 94 +- cloudg/renderers/iso_scene.py | 793 ++++ cloudg/templates/figures/empty-tray.js | 138 + cloudg/templates/report.html.j2 | 2167 +++++++--- cloudg/templates/vendor/LICENSE-anatomy.txt | 21 + cloudg/templates/vendor/LICENSE-hairline.txt | 21 + cloudg/templates/vendor/NOTICE | 25 +- cloudg/templates/vendor/hairline-kernel.js | 714 ++++ cloudg/templates/vendor/iso-kit.js | 684 ++++ site/static/sample-report.html | 3709 +++++++++++++++--- tests/test_html_report.py | 6 +- tests/test_iso_scene.py | 314 ++ 12 files changed, 7391 insertions(+), 1295 deletions(-) create mode 100644 cloudg/renderers/iso_scene.py create mode 100644 cloudg/templates/figures/empty-tray.js create mode 100644 cloudg/templates/vendor/LICENSE-anatomy.txt create mode 100644 cloudg/templates/vendor/LICENSE-hairline.txt create mode 100644 cloudg/templates/vendor/hairline-kernel.js create mode 100644 cloudg/templates/vendor/iso-kit.js create mode 100644 tests/test_iso_scene.py diff --git a/cloudg/renderers/html_report.py b/cloudg/renderers/html_report.py index b894d89..a64a8b0 100644 --- a/cloudg/renderers/html_report.py +++ b/cloudg/renderers/html_report.py @@ -10,6 +10,7 @@ from jinja2 import Environment, FileSystemLoader +from cloudg.renderers.iso_scene import build_scene from cloudg.schema.models import ScanResult logger = logging.getLogger(__name__) @@ -43,6 +44,24 @@ } +# The isometric map's drawing engines and the Hairline figures built on them, +# as paths under the templates directory. They have no CDN build, so they are +# always inlined; without them the report shows the 2D map only. +MAP_ENGINES: dict[str, str] = { + "isokit": "vendor/iso-kit.js", # Anatomy iso-kit (MIT): camera, solids, shading + "hairline": "vendor/hairline-kernel.js", # Hairline kernel (MIT): line figures, springs + "figures": "figures/empty-tray.js", # cloudg's empty-state tray, a Hairline figure +} + +# Figures end with hairline({...}); the report collects them instead of mounting them +_FIGURE_HOST = ( + "var hairline = function (spec) {" + " (window.HAIRLINE_FIGURES = window.HAIRLINE_FIGURES || {})[spec.name] = spec; };\n" +) + +_SEVERITY_RANK = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} + + def script_json(value: Any) -> str: """JSON for embedding in an HTML `` + + +{% endif %} - - -
- -
-
-

☁️ CloudG Security Report

-

Scan: {{ scan_id }} · Provider: {{ provider }} · Account: {{ account_id }} · {{ started_at }}

-
- -
+ - -
-
Total Assets
{{ total_assets }}
-
Total Findings
{{ total_findings }}
-
Critical
{{ severity_counts.CRITICAL }}
-
High
{{ severity_counts.HIGH }}
-
Medium
{{ severity_counts.MEDIUM }}
-
Low
{{ severity_counts.LOW }}
+
+
+ + cloudgSecurity report
- - -
- - - - -
- - -
-
-
- - - - +
+ + + + +
+ +
+
+
+
+
Security report · {{ provider }}
+

Estate posture

+
+

{{ total_findings }} findings across {{ total_assets }} assets.

+
+ Scan {{ scan_id }} + Account: {{ account_id }} + {% if summary.providers %}Providers {{ summary.providers|join(', ') }}{% endif %} + {% if summary.regions %}Regions {{ summary.regions|length }}{% endif %} + Started {{ started_at }}
-
-
Legend
-
Account
-
Region
-
VPC
-
Subnet
-
Compute
-
Storage
-
Security
-
Other
-
Has Findings
-
Internet Exposed
-
Ontology Entities
-
Security Finding (▲)
-
Compliance Framework (◆)
-
─ ─ FINDING_AFFECTS
-
─ ─ COMPLIANCE_GOVERNS
+
+
Assets
{{ total_assets }}
{{ summary.edges }} relations
+
Findings
{{ total_findings }}
{{ source_tools|length }} sources
+
Critical
{{ severity_counts.CRITICAL }}
High {{ severity_counts.HIGH }}
+
Exposed
{{ summary.exposed }}
reachable from the internet
+
+
+ +
-
+
+
+ + + + 0% + +
+

Assets with no open findings

+

{{ summary.clean }} of {{ total_assets }} assets have nothing open against them. The rest carry at least one finding.

+
+
+
+
Top risks
+
    +
    +
    +
    + + + + +
    +
    +
    +
    + + +
    + + + + + + + + + + + +
    +
    +
    +
    + Fig 1 + Estate map + Drag to pan · scroll to zoom · click an asset + rest + +
    +
    + +
    +
    +
    Each provider is a base plate. Accounts and regions stand on it as platforms, networks as slabs and subnets as tiles; every asset stands where it was found. Identity and governance assets sit on the control deck behind and above the plate. Trace lines follow each relation from its source port to its target port on a lane of their own, and a beacon above an asset shows its worst open finding.
    +
    +
    -
    -
    - + - + - + +
    - - - - - - - - - - - -
    SeverityTitleResourceSourceRisk Score
    -
    +
    + + + + + +
    SeverityFindingResourceSourceRisk
    + +
    + + + +
    +
    + + + +
    +
    + + + + + +
    AssetTypeProviderRegionFindings
    +
    +
    -
    +
    -
    -

    Severity Distribution

    - -
    -
    -

    Resource Types

    - -
    +

    Severity distribution {{ total_findings }} findings

    +

    Resource types top 12

    -
    -

    Findings by Source

    - -
    -
    -

    Compliance Overview

    - -
    +

    Findings by source {{ source_tools|length }} sources

    +

    Compliance overview pass and fail

    -
    + -
    +
    -
    -
    + + + +
    + Generated by cloudg {{ summary.version }} · scan {{ scan_id }} + Map drawn with Anatomy and Hairline (MIT) · charts by Chart.js (MIT) and D3 (ISC) +
    + - -
    - +
    + @@ -352,29 +528,32 @@ const graphData = {{ graph_json|safe }}; const hierarchyData = {{ hierarchy_json|safe }}; const findingsData = {{ findings_json|safe }}; + const assetsData = {{ assets_json|safe }}; const findingsPerResource = {{ findings_per_resource_json|safe }}; const severityCounts = {{ severity_counts_json|safe }}; const resourceTypes = {{ resource_types_json|safe }}; const sourceTools = {{ source_tools_json|safe }}; const complianceSummary = {{ compliance_summary_json|safe }}; + const reportSummary = {{ summary_json|safe }}; + const topRisks = {{ top_risks_json|safe }}; + const isoScene = {{ scene_json|safe }}; let currentLayout = 'tree'; // 'tree' or 'force' // ── Tab Switching ── function switchTab(tabId) { document.querySelectorAll('.tab-content').forEach(el => el.classList.remove('active')); - document.querySelectorAll('.tab').forEach(el => el.classList.remove('active')); + document.querySelectorAll('.tab').forEach(el => el.classList.toggle('active', el.dataset.tab === tabId)); document.getElementById('tab-' + tabId).classList.add('active'); - event.target.classList.add('active'); - if (tabId === 'topology') { - setTimeout(() => renderTopology(), 50); - } + if (tabId === 'topology') setTimeout(() => renderTopology(), 30); + if (tabId === 'charts') setTimeout(() => renderCharts(), 30); } // ── HTML escaping ── // Finding, asset and compliance values come from scanner output and // cloud metadata. Markup is only ever built with the html`` tag below, - // which escapes every interpolated value. + // which escapes every interpolated value. The map builds its SVG with + // createElementNS and textContent, so it never parses markup at all. function escapeHtml(value) { if (value === null || value === undefined) return ''; return String(value) @@ -395,6 +574,16 @@ function severityClass(severity) { return 'severity-' + (SEVERITIES.includes(severity) ? severity : 'INFO'); } + function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + } + + // Asset names for resource columns, by id and by ARN + const assetById = new Map(); + assetsData.forEach(a => { assetById.set(a.id, a); if (a.arn) assetById.set(a.arn, a); }); + function assetOf(f) { + return assetById.get(f.resource_id) || assetById.get(f.resource_arn) || null; + } // ── Findings Table ── // Rows keep the finding's position in findingsData, so the detail panel @@ -403,22 +592,24 @@ function renderFindings(data) { const tbody = document.getElementById('findings-body'); - tbody.innerHTML = data.map(f => html` + tbody.innerHTML = data.map(f => { + const asset = assetOf(f); + return html` ${f.severity} - ${f.title} - ${f.resource_arn || f.resource_id || ''} - ${f.source_tool} - ${f.risk_score || '-'} +
    ${f.title}
    + ${asset ? asset.name : ''}
    ${f.resource_arn || f.resource_id || ''}
    + ${f.source_tool} + ${f.risk_score || '-'} - `).join(''); + `; + }).join(''); + const count = document.getElementById('findings-count'); + count.textContent = data.length + ' of ' + findingsData.length; + const empty = document.getElementById('findings-empty'); + empty.hidden = data.length > 0; } - document.getElementById('findings-body').addEventListener('click', (e) => { - const row = e.target.closest('tr[data-index]'); - if (row) showDetail(Number(row.dataset.index)); - }); - function filterFindings() { const severity = document.getElementById('filter-severity').value; const source = document.getElementById('filter-source').value; @@ -429,219 +620,840 @@ if (search) filtered = filtered.filter(f => String(f.title || '').toLowerCase().includes(search) || String(f.resource_arn || '').toLowerCase().includes(search) || + String(f.resource_id || '').toLowerCase().includes(search) || String(f.description || '').toLowerCase().includes(search) ); renderFindings(filtered); } - // Populate source filter - const sourceSelect = document.getElementById('filter-source'); - Object.keys(sourceTools).forEach(tool => { - const opt = document.createElement('option'); - opt.value = tool; opt.textContent = tool; - sourceSelect.appendChild(opt); - }); - // ── Detail Panel ── function showDetail(index) { const f = findingsData[index]; if (!f) return; const panel = document.getElementById('detail-panel'); const content = document.getElementById('detail-content'); + const asset = assetOf(f); content.innerHTML = html` + ${f.severity}

    ${f.title}

    -
    Severity
    ${f.severity}
    -
    Resource
    ${f.resource_arn || f.resource_id || 'N/A'}
    +
    Resource
    ${asset ? asset.name : ''} ${f.resource_arn || f.resource_id || 'N/A'}
    Description
    ${f.description || 'N/A'}
    Evidence
    ${f.evidence || 'N/A'}
    Remediation
    ${f.remediation || 'N/A'}
    -
    Source Tool
    ${f.source_tool}
    -
    CVSS Score
    ${f.cvss_score || 'N/A'}
    -
    Compliance Frameworks
    ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
    -
    Risk Score
    ${f.risk_score || 'N/A'}
    +
    Source tool
    ${f.source_tool}
    +
    CVSS score
    ${f.cvss_score || 'N/A'}
    +
    Compliance frameworks
    ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
    +
    Risk score
    ${f.risk_score || 'N/A'}
    `; panel.classList.add('open'); + document.getElementById('scrim').classList.add('open'); + } + function closeDetail() { + document.getElementById('detail-panel').classList.remove('open'); + document.getElementById('scrim').classList.remove('open'); + } + + // ── Compliance Grid ── + function renderCompliance() { + const grid = document.getElementById('compliance-grid'); + const entries = Object.entries(complianceSummary); + grid.innerHTML = entries.map(([fw, data]) => { + const pass = Number(data.pass) || 0; + const fail = Number(data.fail) || 0; + const total = pass + fail; + const pct = total > 0 ? Math.round((pass / total) * 100) : 0; + return html` +
    +

    ${fw}

    +
    ${pct}%
    +
    +
    ${pass} pass${fail} fail${total} controls
    +
    + `; + }).join(''); + document.getElementById('compliance-empty').hidden = entries.length > 0; + } + + // ── Overview ── + function renderOverview() { + const total = SEVERITIES.reduce((n, s) => n + (severityCounts[s] || 0), 0); + const bar = document.getElementById('sevbar'); + bar.innerHTML = SEVERITIES.filter(s => severityCounts[s]).map(s => + html``).join(''); + document.getElementById('sevlegend').innerHTML = SEVERITIES.map(s => + html`${s.charAt(0) + s.slice(1).toLowerCase()} ${severityCounts[s] || 0}`).join(''); + const assets = assetsData.length; + const pct = assets ? Math.round((reportSummary.clean / assets) * 100) : 100; + const ring = document.getElementById('ring-val'); + ring.style.strokeDashoffset = String(251.3 * (1 - pct / 100)); + if (pct < 50) ring.style.stroke = cssVar('--high'); + document.getElementById('ring-text').textContent = pct + '%'; + const worst = SEVERITIES.find(s => severityCounts[s]); + const providers = reportSummary.providers.length; + document.getElementById('lede').innerHTML = html`${total} findings across ${assets} assets in ${providers} ${providers === 1 ? 'provider' : 'providers'}. ${worst ? 'The worst open finding is ' + worst.toLowerCase() + '.' : 'Nothing is open.'} ${reportSummary.exposed} ${reportSummary.exposed === 1 ? 'asset is' : 'assets are'} reachable from the internet.`; + const list = document.getElementById('top-risks'); + list.innerHTML = topRisks.length ? topRisks.map(i => { + const f = findingsData[i]; + const asset = assetOf(f); + return html`
    • `; + }).join('') : html`
  • No open findings.
    • `; + } + + // ── Assets Table ── + function renderAssets() { + const provider = document.getElementById('asset-provider').value; + const search = document.getElementById('asset-search').value.toLowerCase(); + let rows = assetsData; + if (provider) rows = rows.filter(a => a.provider === provider); + if (search) rows = rows.filter(a => [a.name, a.asset_type, a.region, a.arn, a.id, a.account_id] + .some(v => String(v || '').toLowerCase().includes(search))); + const shown = rows.slice(0, 2000); + document.getElementById('assets-body').innerHTML = shown.map(a => { + const counts = findingsPerResource[a.id] || findingsPerResource[a.arn || ''] || null; + const worst = counts ? SEVERITIES.find(s => counts[s]) : null; + // Escaped fragments are joined, never nested: html`` would escape them again + const exposed = a.is_internet_exposed ? html` exposed` : ''; + const badge = worst ? html`${counts.total}` : html`none`; + return html` +
    ${a.name}
    ${a.arn || a.id}
    + ${a.asset_type}` + exposed + html` + ${a.provider} + ${a.region} + ` + badge + html` + `; + }).join(''); + document.getElementById('assets-count').textContent = rows.length + ' of ' + assetsData.length + (rows.length > shown.length ? ' (first 2000 shown)' : ''); } - function closeDetail() { document.getElementById('detail-panel').classList.remove('open'); } - - // ── Hierarchical Topology ── - const typeColors = { - account: '#6366F1', region: '#22C55E', vpc: '#4A90D9', vnet: '#4A90D9', - subnet: '#7EC8E3', global: '#8B5CF6', - EC2: '#FF9900', VIRTUAL_MACHINE: '#FF9900', GCE_INSTANCE: '#FF9900', - VPC: '#4A90D9', VNET: '#4A90D9', SUBNET: '#7EC8E3', - RDS_INSTANCE: '#3B48CC', AURORA_CLUSTER: '#3B48CC', AZURE_SQL: '#3B48CC', CLOUD_SQL: '#3B48CC', - S3_BUCKET: '#569A31', BLOB_STORAGE: '#569A31', GCS_BUCKET: '#569A31', - SECURITY_GROUP: '#DD4B39', NSG: '#DD4B39', NACL: '#DD4B39', - LOAD_BALANCER: '#8C4FFF', CLOUDFRONT: '#8C4FFF', - IAM_USER: '#F472B6', IAM_ROLE: '#F472B6', IAM_POLICY: '#F472B6', - LAMBDA_FUNCTION: '#FF9900', CLOUD_FUNCTION: '#FF9900', - KMS_KEY: '#14B8A6', SECRET: '#14B8A6', - EBS_VOLUME: '#569A31', DYNAMODB_TABLE: '#3B48CC', - ECS_CLUSTER: '#FF9900', EKS_CLUSTER: '#FF9900', - SECURITY_FINDING: '#EF4444', COMPLIANCE_FRAMEWORK: '#14B8A6', - }; - const typeRadius = { - account: 22, region: 18, vpc: 16, vnet: 16, subnet: 14, global: 18, + // ── Isometric map ── + // The scene (cloudg/renderers/iso_scene.py) holds every position in world + // units. This code projects it with the Anatomy iso-kit camera, paints it + // back to front, and animates it. Nothing here parses markup: elements are + // made with createElementNS and names go in with textContent. + const SVGNS = 'http://www.w3.org/2000/svg'; + const FAMILIES = [ + ['exposure', 'Internet exposure'], ['network', 'Network path'], ['identity', 'Identity and access'], + ['data', 'Data and invocation'], ['governance', 'Governance and protection'], + ]; + const MAP = { + az: 45, el: 30, P: null, view: null, base: null, labels: false, + families: new Set(FAMILIES.map(f => f[0])), selected: null, focusTrace: null, + traceEls: new Map(), nodeEls: new Map(), byId: new Map(), pulses: [], raf: 0, + built: false, flat: false, turning: false, }; + // The drawing box: 1200 wide, as tall as the stage's aspect asks (set in buildMap) + let W = 1200, H = 760; + + function svgEl(tag, attrs, parent) { + const el = document.createElementNS(SVGNS, tag); + if (attrs) for (const k in attrs) if (attrs[k] !== undefined && attrs[k] !== null && attrs[k] !== '') el.setAttribute(k, attrs[k]); + if (parent) parent.appendChild(el); + return el; + } + function reducedMotion() { + return window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + } - let zoomBehavior, svgG; + // A kit solid as DOM: halo, fill, four side tones, top, crease, bevel, edge + function solid(parent, paths, opts) { + const o = opts || {}; + const g = svgEl('g', { class: 'iso-solid' + (o.cls ? ' ' + o.cls : '') }, parent); + if (o.lit) g.setAttribute('data-lit', ''); + if (o.halo) svgEl('path', { class: 'iso-halo', d: paths.fill }, g); + svgEl('path', { class: 'iso-fill', d: paths.fill }, g); + if (!o.flat) { + paths.shades.forEach((d, i) => { if (d) svgEl('path', { class: 'iso-shade', 'data-shade': i, d }, g); }); + svgEl('path', { class: 'iso-top', d: paths.top }, g); + } + if (paths.crease) svgEl('path', { class: 'iso-line iso-crease', 'data-tone': o.crease || 'faint', d: paths.crease }, g); + if (paths.bevel) svgEl('path', { class: 'iso-line iso-bevel', 'data-tone': o.crease || 'faint', d: paths.bevel }, g); + if (o.inner) o.inner(g); + svgEl('path', { class: 'iso-line iso-edge', 'data-tone': o.tone || 'hi', d: paths.outline }, g); + return g; + } + function line(parent, d, tone, cls) { + if (!d) return null; + return svgEl('path', { class: 'iso-line' + (cls ? ' ' + cls : ''), 'data-tone': tone || 'lo', d }, parent); + } + + // A matrix that lays text flat on a top face and keeps it reading left to right + function faceMatrix(at, z, P) { + const o = IsoKit.iso([at[0], at[1], z], P); + const vx = IsoKit.iso([at[0] + 1, at[1], z], P), vy = IsoKit.iso([at[0], at[1] + 1, z], P); + const ex = [vx[0] - o[0], vx[1] - o[1]], ey = [vy[0] - o[0], vy[1] - o[1]]; + const cands = [[ex, ey], [[-ex[0], -ex[1]], ey], [ey, ex], [[-ey[0], -ey[1]], ex]]; + let best = cands[0]; + for (const c of cands) if (c[0][0] > best[0][0]) best = c; + let [u, v] = best; + if (v[1] < 0) v = [-v[0], -v[1]]; + const f = n => Math.round(n * 1000) / 1000; + return 'matrix(' + [u[0], u[1], v[0], v[1], o[0], o[1]].map(f).join(' ') + ')'; + } + + // Draw order for siblings with disjoint footprints: a before b when b is + // nearer along the one axis that separates them (a topological sort, with + // centre depth breaking ties), so each platform's contents paint in order. + function orderSiblings(items, P) { + const [vx, vy] = IsoKit.towardViewer(P); + const n = items.length; + if (n < 2) return items.slice(); + const depth = it => (it.cx * vx + it.cy * vy); + if (n > 60) return items.slice().sort((a, b) => depth(a) - depth(b)); + const after = items.map(() => []), deg = new Array(n).fill(0); + for (let i = 0; i < n; i++) for (let j = i + 1; j < n; j++) { + const a = items[i], b = items[j]; + const sx = a.x1 <= b.x0 + 1e-6 ? 1 : (b.x1 <= a.x0 + 1e-6 ? -1 : 0); + const sy = a.y1 <= b.y0 + 1e-6 ? 1 : (b.y1 <= a.y0 + 1e-6 ? -1 : 0); + let aFirst = null; + if (sx && !sy) aFirst = sx * vx > 0; + else if (sy && !sx) aFirst = sy * vy > 0; + if (aFirst === null) continue; + if (aFirst) { after[i].push(j); deg[j]++; } else { after[j].push(i); deg[i]++; } + } + const out = [], ready = []; + for (let i = 0; i < n; i++) if (!deg[i]) ready.push(i); + while (ready.length) { + ready.sort((p, q) => depth(items[q]) - depth(items[p])); + const i = ready.pop(); + out.push(items[i]); + for (const j of after[i]) if (!--deg[j]) ready.push(j); + } + if (out.length < n) return items.slice().sort((a, b) => depth(a) - depth(b)); + return out; + } + function rectOf(x, y, w, d) { return { x0: x, y0: y, x1: x + w, y1: y + d, cx: x + w / 2, cy: y + d / 2 }; } + + // ── Parts ── + function drawPlatform(parent, p, P) { + const K = IsoKit; + const radius = { provider: 3.2, region: 2.4, network: 1.8, subnet: 1.2, zone: 1.2, deck: 2.4 }[p.kind] || 1.2; + const plan = { x: p.x, y: p.y, w: p.w, d: p.d, r: radius }; + const g = svgEl('g', { class: 'platform pl-' + p.kind }, parent); + if (p.kind === 'provider') { + // Feet under the plate, and a soft halo on the bench + for (const [fx, fy] of K.corners(plan, 7)) solid(g, K.cylinder(fx, fy, 2.6, -3, 3, P, 18), { tone: 'mid' }); + } + const top = p.z + p.h; + const tones = { provider: 'mid', region: 'mid', network: 'mid', subnet: 'lo', zone: 'lo', deck: 'mid' }; + solid(g, K.slabOf(plan, p.z, p.h, P, 8, p.kind === 'provider' ? 1.4 : 0.8), { + tone: tones[p.kind] || 'lo', halo: p.kind === 'provider' || p.kind === 'deck', + inner: s => { + const inset = K.insetPlan(plan, Math.min(3.5, Math.min(p.w, p.d) / 6)); + line(s, K.planOutline(inset, top, P), 'faint'); + if (p.kind === 'provider' || p.kind === 'deck') { + // Ruler ticks along the front edge and a screw at each corner + const t = K.topTicks(p.x + 6, p.x + p.w - 6, 4, 5, p.y + p.d - 2.2, top, [1.0, 1.9], P, 'x'); + line(s, t.minor, 'faint'); line(s, t.major, 'lo'); + for (const [sx, sy] of K.corners(plan, 3.2)) { + const c = K.iso([sx, sy, top], P); + svgEl('circle', { class: 'screw', cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 1.1 }, s); + } + } + if (p.kind === 'network') line(s, K.sideSeam(plan, p.z + p.h * 0.5, P), 'faint'); + }, + }); + if (p.kind !== 'zone' && p.w > 30) { + const label = svgEl('text', { + class: 'pl-label' + (p.kind === 'provider' || p.kind === 'deck' ? ' major' : ''), + transform: faceMatrix([p.x + 4.5, p.y + 3.2], top, P), 'font-size': p.kind === 'provider' ? 4.2 : 3.2, + 'dominant-baseline': 'hanging', + }, g); + const max = Math.max(6, Math.floor((p.w - 10) / (p.kind === 'provider' ? 2.6 : 2.0))); + const text = p.label.length > max ? p.label.slice(0, max - 1) + '…' : p.label; + label.textContent = text; + } + return g; + } + + function drawNode(parent, n, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node c-' + n.category, 'data-node': n.id, tabindex: '-1' }, parent); + const title = svgEl('title', null, g); + title.textContent = n.name + ' · ' + n.type + (n.region ? ' · ' + n.region : ''); + const { x, y, z, w, d, h } = n; + const plan = { x: x - w / 2, y: y - d / 2, w, d, r: 1.2 }; + const top = z + h; + if (n.exposed) line(g, K.ring(x, y, Math.max(w, d) / 2 + 2.4, z + 0.05, P, 36), null, 'exposed-ring'); + const box = (pl, z0, hh, opt) => solid(g, K.slabOf(pl, z0, hh, P, 6, 0.5), opt || {}); + const cyl = (r, z0, hh, steps, opt) => solid(g, K.cylinder(x, y, r, z0, hh, P, steps || 28, 0.4), opt || {}); + switch (n.shape) { + case 'server': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.sideSeam(plan, z + (h * i) / 4, P), 'faint'); + for (const dot of K.dotGrid(x - 3, y - 3, 3, 3, 3, top, P)) svgEl('circle', { class: 'iso-dot-vent', cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .45, fill: 'var(--anatomy-lo)' }, s); + } }); + break; + case 'cluster': { + box({ x: plan.x, y: plan.y, w, d, r: 1.4 }, z, 3); + const q = w / 2 - 0.8; + for (const [ox, oy] of [[-1, -1], [1, -1], [-1, 1], [1, 1]].sort((a, b) => (a[0] + a[1]) - (b[0] + b[1]))) + box({ x: x + (ox < 0 ? -q - 0.4 : 0.4), y: y + (oy < 0 ? -q - 0.4 : 0.4), w: q, d: q, r: 0.8 }, z + 3, h - 3); + break; + } + case 'function': + box(plan, z, h, { inner: s => line(s, K.sideSeam(plan, z + h * 0.7, P), 'faint') }); + box({ x: x - w / 2 - 0.6, y: y - d / 2 - 0.6, w: w + 1.2, d: d + 1.2, r: 1 }, top, 1.2); + break; + case 'database': + cyl(w / 2, z, h, 30, { inner: s => { + line(s, K.sideArc(x, y, w / 2, z + h / 3, P), 'faint'); + line(s, K.sideArc(x, y, w / 2, z + (2 * h) / 3, P), 'faint'); + } }); + break; + case 'bucket': + cyl(w / 2 - 0.6, z, h, 30); + line(g, K.ring(x, y, w / 2 - 2, top, P, 30), 'lo'); + break; + case 'queue': { + box(plan, z, h, { inner: s => { + for (let i = 1; i < 4; i++) line(s, K.lineOnTop([plan.x + (w * i) / 4, plan.y + 1], [plan.x + (w * i) / 4, plan.y + d - 1], top, P), 'faint'); + } }); + break; + } + case 'gateway': + box(plan, z, h, { inner: s => { + for (const dot of K.dotGrid(x - 4, y, 3, 1, 4, top, P)) svgEl('circle', { class: 'led' + (n.exposed ? ' exposed' : ''), cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .9 }, s); + } }); + break; + case 'shield': + cyl(w / 2, z, h, 8, { inner: s => line(s, K.ring(x, y, w / 2 - 2, top, P, 8), 'faint') }); + break; + case 'key': + cyl(w / 2, z, h * 0.6, 24); + cyl(w / 4, z + h * 0.6, h * 0.4, 18); + break; + case 'log': + for (let i = 0; i < 3; i++) box({ x: plan.x + i * 0.4, y: plan.y + i * 0.4, w: w - i * 0.8, d: d - i * 0.8, r: 1 }, z + i * (h / 3), h / 3 - 0.3); + break; + case 'core': + cyl(w / 2, z, h * 0.7, 6); + cyl(w / 4, z + h * 0.7, h * 0.3, 20, { lit: true }); + break; + case 'principal': + cyl(w / 2 - 1, z, h * 0.65, 22); + cyl(w / 2, z + h * 0.65, h * 0.35, 22); + break; + case 'plate': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.lineOnTop([plan.x + 2.5, plan.y + (d * i) / 4.4], [plan.x + w - 2.5, plan.y + (d * i) / 4.4], top, P), 'faint'); + } }); + break; + default: + box(plan, z, h); + } + return g; + } + + function drawBeacon(parent, n, P) { + if (!n.worst) return null; + const K = IsoKit; + const rank = SEVERITIES.indexOf(n.worst); + const mastTop = n.z + n.h + 7 + (4 - rank) * 2.6; + const g = svgEl('g', { class: 'beacon-group sev-' + n.worst, 'data-node': n.id }, parent); + svgEl('path', { class: 'mast', d: K.segment([n.x, n.y, n.z + n.h], [n.x, n.y, mastTop], P) }, g); + solid(g, K.cylinder(n.x, n.y, 1.9, mastTop, 2.4, P, 18), { cls: 'beacon', tone: 'hi' }); + const c = K.iso([n.x, n.y, mastTop + 2.4], P); + const halo = svgEl('circle', { class: 'beacon-halo' + (rank <= 1 ? ' pulse' : ''), cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 3.2 }, g); + halo.style.animationDelay = ((n.x * 7 + n.y * 13) % 2400) + 'ms'; + return g; + } + + function drawExternal(parent, e, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node external ' + (e.kind === 'internet' ? 'internet' : ''), 'data-node': e.id }, parent); + const t = svgEl('title', null, g); + t.textContent = e.kind === 'internet' ? 'The internet (' + e.name + ')' : 'Outside the estate: ' + e.name; + if (e.kind === 'internet') { + // An uplink mast: a weighted foot, a tapering column, three dish rings + solid(g, K.cylinder(e.x, e.y, 9, 0, 3, P, 36, 0.6), { tone: 'mid', halo: true }); + solid(g, K.cylinder(e.x, e.y, 3.4, 3, e.h - 9, P, 24), { tone: 'hi' }); + for (let i = 0; i < 3; i++) line(g, K.sideArc(e.x, e.y, 3.4, 8 + i * 6, P), 'faint'); + solid(g, K.cylinder(e.x, e.y, 6.5, e.h - 6, 2, P, 32, 0.4), { tone: 'hi' }); + solid(g, K.cylinder(e.x, e.y, 2.2, e.h - 4, 4, P, 20), { tone: 'hi', lit: true }); + } else { + solid(g, K.slabOf({ x: e.x - e.w / 2, y: e.y - e.d / 2, w: e.w, d: e.d, r: 1.4 }, 0, e.h, P, 6, 0.5), { tone: 'mid' }); + } + return g; + } + + function roundedPath(pts, r) { + if (pts.length < 3) return 'M' + pts.map(p => p[0].toFixed(1) + ' ' + p[1].toFixed(1)).join('L'); + let d = 'M' + pts[0][0].toFixed(1) + ' ' + pts[0][1].toFixed(1); + for (let i = 1; i < pts.length - 1; i++) { + const [ax, ay] = pts[i - 1], [bx, by] = pts[i], [cx, cy] = pts[i + 1]; + const l1 = Math.hypot(bx - ax, by - ay), l2 = Math.hypot(cx - bx, cy - by); + const rr = Math.min(r, l1 / 2, l2 / 2); + if (rr < 0.3) { d += 'L' + bx.toFixed(1) + ' ' + by.toFixed(1); continue; } + const p1 = [bx - ((bx - ax) / l1) * rr, by - ((by - ay) / l1) * rr]; + const p2 = [bx + ((cx - bx) / l2) * rr, by + ((cy - by) / l2) * rr]; + d += 'L' + p1[0].toFixed(1) + ' ' + p1[1].toFixed(1) + 'Q' + bx.toFixed(1) + ' ' + by.toFixed(1) + ' ' + p2[0].toFixed(1) + ' ' + p2[1].toFixed(1); + } + const last = pts[pts.length - 1]; + return d + 'L' + last[0].toFixed(1) + ' ' + last[1].toFixed(1); + } + + function drawTrace(parent, t, P) { + const g = svgEl('g', { class: 'trace f-' + t.family + (MAP.families.has(t.family) ? '' : ' hidden-family'), 'data-trace': t.id }, parent); + const pts = t.points.map(p => IsoKit.iso(p, P)).filter((p, i, a) => i === 0 || Math.hypot(p[0] - a[i - 1][0], p[1] - a[i - 1][1]) > 0.05); + const d = roundedPath(pts, 4); + const title = svgEl('title', null, g); + title.textContent = nameOf(t.source) + ' → ' + nameOf(t.target) + ' · ' + t.label + (t.ports ? ' · ' + t.ports : ''); + svgEl('path', { class: 'trace-hit', d }, g); + const lineEl = svgEl('path', { class: 'trace-line', d }, g); + svgEl('path', { class: 'trace-flow', d }, g); + const a = pts[0], b = pts[pts.length - 1]; + svgEl('circle', { class: 'trace-end', cx: a[0].toFixed(1), cy: a[1].toFixed(1), r: 1.5 }, g); + svgEl('circle', { class: 'trace-end to', cx: b[0].toFixed(1), cy: b[1].toFixed(1), r: 1.7 }, g); + MAP.traceEls.set(t.id, { g, line: lineEl, t }); + return g; + } + + function nameOf(id) { + const n = MAP.byId.get(id); + return n ? n.name : id; + } + + // ── Build ── + function buildMap() { + const svg = document.getElementById('iso-svg'); + while (svg.firstChild) svg.removeChild(svg.firstChild); + MAP.traceEls.clear(); MAP.nodeEls.clear(); + const s = isoScene; + const b = s.bounds; + const stageEl = document.getElementById('stage'); + const sw = stageEl.clientWidth || 1200, sh = stageEl.clientHeight || 760; + const nextH = Math.round(W * Math.min(1.6, Math.max(0.45, sh / sw))); + if (nextH !== H) { H = nextH; MAP.view = null; } + const corners = []; + for (const x of b.x) for (const y of b.y) for (const z of b.z) corners.push([x, y, z]); + corners.push([b.x[0], b.y[0], -3]); + const P = IsoKit.fitProjection(corners, W, H, { pad: 46, azimuth: MAP.az, elevation: MAP.el }); + MAP.P = P; + if (!MAP.view) MAP.view = [0, 0, W, H]; + svg.setAttribute('viewBox', MAP.view.join(' ')); + svg.setAttribute('preserveAspectRatio', 'xMidYMid meet'); + svg.classList.add('iso-svg'); + + const kids = new Map(); + const push = (key, item) => { if (!kids.has(key)) kids.set(key, []); kids.get(key).push(item); }; + s.platforms.forEach(p => push(p.parent || '', { kind: 'platform', p, ...rectOf(p.x, p.y, p.w, p.d) })); + s.nodes.forEach(n => push(n.parent, { kind: 'node', n, ...rectOf(n.x - 9, n.y - 9, 18, 18) })); + + const layers = { decks: svgEl('g', { class: 'layer-decks' }), ground: svgEl('g', { class: 'layer-ground' }), + groundTraces: svgEl('g', { class: 'layer-traces' }), deckTraces: svgEl('g', { class: 'layer-traces deck' }), + labels: svgEl('g', { class: 'layer-labels' }) }; + const rising = []; + const paint = (parent, item, depthIndex) => { + if (item.kind === 'platform') { + const p = item.p; + const g = svgEl('g', { class: 'stack' }, parent); + if (p.kind === 'deck') { + // Four posts carry the control deck above the plate + const posts = IsoKit.corners({ x: p.x, y: p.y, w: p.w, d: p.d }, 4).map(([px, py]) => ({ px, py, ...rectOf(px - 1, py - 1, 2, 2) })); + for (const post of orderSiblings(posts, P)) solid(g, IsoKit.cylinder(post.px, post.py, 1.5, 0, p.z, P, 16), { cls: 'post', tone: 'lo' }); + } + drawPlatform(g, p, P); + rising.push(g); + for (const child of orderSiblings(kids.get(p.id) || [], P)) paint(g, child, depthIndex + 1); + } else { + const n = item.n; + const g = drawNode(parent, n, P); + MAP.nodeEls.set(n.id, g); + drawBeacon(parent, n, P); + } + }; + const roots = orderSiblings(kids.get('') || [], P); + for (const root of roots) paint(root.p.kind === 'deck' ? layers.decks : layers.ground, root, 0); + const externals = s.externals.map(e => ({ e, ...rectOf(e.x - e.w, e.y - e.d, e.w * 2, e.d * 2) })); + for (const ext of orderSiblings(externals, P)) { const g = drawExternal(layers.ground, ext.e, P); MAP.nodeEls.set(ext.e.id, g); rising.push(g); } + + const deckIds = new Set(s.nodes.filter(n => n.parent && n.parent.startsWith('d:')).map(n => n.id)); + for (const t of s.traces) drawTrace(deckIds.has(t.source) || deckIds.has(t.target) ? layers.deckTraces : layers.groundTraces, t, P); + + // The deck is behind the plate (toward -y): paint it first unless the camera looks from behind + const deckInFront = IsoKit.towardViewer(P)[1] < 0; + const order = deckInFront + ? [layers.ground, layers.groundTraces, layers.decks, layers.deckTraces, layers.labels] + : [layers.decks, layers.ground, layers.groundTraces, layers.deckTraces, layers.labels]; + order.forEach(l => svg.appendChild(l)); + MAP.layers = layers; + MAP.rising = rising; + renderLabels(); + applyFocus(); + MAP.built = true; + } + + function renderLabels() { + const layer = MAP.layers && MAP.layers.labels; + if (!layer) return; + while (layer.firstChild) layer.removeChild(layer.firstChild); + const ids = new Set(); + if (MAP.labels) isoScene.nodes.forEach(n => ids.add(n.id)); + if (MAP.selected) { + ids.add(MAP.selected); + for (const t of isoScene.traces) if (t.source === MAP.selected || t.target === MAP.selected) { ids.add(t.source); ids.add(t.target); } + } + if (MAP.hover) ids.add(MAP.hover); + // Labels are placed top to bottom; one that would overlap a placed label + // moves up in steps, and a leader then ties it to its asset's top + const anchors = [...ids].map(id => MAP.byId.get(id)).filter(Boolean).map(n => { + const top = (n.z || 0) + (n.h || 0) + (n.worst ? 18 : 5); + return { n, a: IsoKit.iso([n.x, n.y, top], MAP.P) }; + }).sort((p, q) => p.a[1] - q.a[1]); + const placed = []; + const hits = (r) => placed.some(q => r.x0 < q.x1 && q.x0 < r.x1 && r.y0 < q.y1 && q.y0 < r.y1); + for (const { n, a } of anchors) { + const name = n.name.length > 34 ? n.name.slice(0, 33) + '…' : n.name; + const half = Math.max(name.length * 3.1, String(n.type || '').length * 2.7) + 4; + let dy = 0, rect; + for (let k = 0; k < 14; k++) { + rect = { x0: a[0] - half, x1: a[0] + half, y0: a[1] - 16 - dy, y1: a[1] + 9 - dy }; + if (!hits(rect)) break; + dy += 13; + } + placed.push(rect); + if (dy) svgEl('path', { class: 'iso-line', 'data-tone': 'mid', d: 'M' + a[0].toFixed(1) + ' ' + (a[1] + 2).toFixed(1) + 'V' + (a[1] - dy + 10).toFixed(1) }, layer); + if (dy) svgEl('circle', { cx: a[0].toFixed(1), cy: (a[1] + 2).toFixed(1), r: 1.2, fill: 'var(--anatomy-mid)' }, layer); + const text = svgEl('text', { class: 'node-label', x: a[0].toFixed(1), y: (a[1] - 4 - dy).toFixed(1), 'text-anchor': 'middle' }, layer); + text.appendChild(document.createTextNode(name)); + const sub = svgEl('tspan', { x: a[0].toFixed(1), dy: '11' }, text); + sub.textContent = n.type || n.kind || ''; + } + } + + // ── Intro: plates rise, assets settle, traces draw in, then flow ── + function playIntro() { + const svg = document.getElementById('iso-svg'); + svg.classList.remove('flowing'); + if (reducedMotion()) { svg.classList.add('flowing'); return; } + const groups = MAP.rising || []; + groups.forEach(g => { g.classList.remove('risen'); g.classList.add('rise-in'); }); + const traces = [...MAP.traceEls.values()]; + traces.forEach(({ line: l }) => { + const len = l.getTotalLength ? l.getTotalLength() : 0; + l.style.transition = 'none'; + l.style.strokeDasharray = len + ' ' + len; + l.style.strokeDashoffset = String(len); + }); + svg.getBoundingClientRect(); + const step = Math.min(70, 900 / Math.max(1, groups.length)); + groups.forEach((g, i) => setTimeout(() => { g.classList.remove('rise-in'); g.classList.add('risen'); }, 80 + i * step)); + const start = 260 + groups.length * step; + const famDelay = { network: 0, data: 220, identity: 440, governance: 660, exposure: 880 }; + traces.forEach(({ line: l, t }, i) => { + const delay = start + (famDelay[t.family] || 0) + (i % 12) * 30; + setTimeout(() => { + l.style.transition = 'stroke-dashoffset 1.1s cubic-bezier(.32,.72,0,1)'; + l.style.strokeDashoffset = '0'; + }, delay); + }); + clearTimeout(MAP.flowTimer); + MAP.flowTimer = setTimeout(() => { + traces.forEach(({ line: l }) => { l.style.strokeDasharray = ''; l.style.strokeDashoffset = ''; l.style.transition = ''; }); + svg.classList.add('flowing'); + }, start + 2200); + } + + // ── Focus: a selected asset, a traced path ── + function connectedTraces(id) { + return isoScene.traces.filter(t => t.source === id || t.target === id); + } + function applyFocus(traceIds) { + const svg = document.getElementById('iso-svg'); + const on = new Set(), onNodes = new Set(); + if (traceIds) traceIds.forEach(id => on.add(id)); + else if (MAP.selected) connectedTraces(MAP.selected).forEach(t => on.add(t.id)); + for (const id of on) { const t = MAP.traceEls.get(id); if (t) { onNodes.add(t.t.source); onNodes.add(t.t.target); } } + if (MAP.selected) onNodes.add(MAP.selected); + svg.classList.toggle('focus', on.size > 0 || !!MAP.selected); + MAP.traceEls.forEach(({ g }, id) => g.classList.toggle('on', on.has(id))); + svg.querySelectorAll('[data-node]').forEach(el => { + const id = el.getAttribute('data-node'); + el.classList.toggle('on', onNodes.has(id)); + el.classList.toggle('sel', id === MAP.selected); + }); + startPulses([...on]); + } + + // A dot runs along each lit trace from source to target, so the eye can follow it + function startPulses(ids, hops) { + cancelAnimationFrame(MAP.raf); + MAP.pulses.forEach(p => p.el.remove()); + MAP.pulses = []; + if (!ids.length || reducedMotion()) return; + const svg = document.getElementById('iso-svg'); + ids.slice(0, 40).forEach((id, i) => { + const t = MAP.traceEls.get(id); + if (!t || t.g.classList.contains('hidden-family')) return; + const el = svgEl('circle', { class: 'pulse-dot', r: 2.3 }, t.g); + MAP.pulses.push({ el, path: t.line, len: t.line.getTotalLength(), delay: hops ? (hops.get(id) || 0) * 420 : i * 90 }); + }); + const t0 = performance.now(); + const tick = now => { + for (const p of MAP.pulses) { + const e = now - t0 - p.delay; + if (e < 0) { p.el.setAttribute('opacity', '0'); continue; } + const dur = Math.max(900, p.len * 9); + const f = (e % (dur + 500)) / dur; + if (f > 1) { p.el.setAttribute('opacity', '0'); continue; } + const eased = f < .5 ? 2 * f * f : 1 - Math.pow(-2 * f + 2, 2) / 2; + const pt = p.path.getPointAtLength(eased * p.len); + p.el.setAttribute('cx', pt.x.toFixed(1)); + p.el.setAttribute('cy', pt.y.toFixed(1)); + p.el.setAttribute('opacity', String(Math.min(1, Math.sin(f * Math.PI) * 1.6))); + } + MAP.raf = requestAnimationFrame(tick); + }; + MAP.raf = requestAnimationFrame(tick); + } + + function selectNode(id, opts) { + MAP.selected = id && MAP.byId.has(id) ? id : null; + id = MAP.selected; + MAP.focusTrace = null; + applyFocus(); + renderLabels(); + renderInspector(); + if (id && opts && opts.center) centerOn(id); + const n = MAP.byId.get(id); + setReadout(n ? n.name + ' · ' + (n.type || n.kind) + ' · ' + connectedTraces(id).length + ' relations' : null); + } + + function focusTrace(traceId) { + const t = MAP.traceEls.get(traceId); + if (!t) return; + MAP.focusTrace = traceId; + applyFocus([traceId]); + document.querySelectorAll('.conn-item').forEach(el => el.classList.toggle('on', el.dataset.trace === traceId)); + setReadout(nameOf(t.t.source) + ' → ' + nameOf(t.t.target) + ' · ' + t.t.label); + } + + function traceFromInternet() { + const starts = isoScene.externals.filter(e => e.kind === 'internet').map(e => e.id); + if (!starts.length) { setReadout('nothing in this scan is reachable from the internet'); return; } + const out = new Map(); + isoScene.traces.forEach(t => { if (!out.has(t.source)) out.set(t.source, []); out.get(t.source).push(t); }); + const hops = new Map(), seen = new Set(starts); + let frontier = starts.slice(), depth = 0; + while (frontier.length && depth < 5) { + const next = []; + for (const id of frontier) for (const t of out.get(id) || []) { + if (!MAP.families.has(t.family)) continue; + if (!hops.has(t.id)) hops.set(t.id, depth); + if (!seen.has(t.target)) { seen.add(t.target); next.push(t.target); } + } + frontier = next; depth++; + } + const maxHop = hops.size ? Math.max(...hops.values()) + 1 : 0; + MAP.selected = null; + applyFocus([...hops.keys()]); + startPulses([...hops.keys()], hops); + renderLabels(); + renderInspector({ internet: { reached: seen.size - starts.length, hops: maxHop, traces: hops.size } }); + setReadout('the internet reaches ' + (seen.size - starts.length) + ' assets in up to ' + maxHop + (maxHop === 1 ? ' hop' : ' hops')); + } + + function setReadout(text) { + document.getElementById('readout').textContent = text || (isoScene.nodes ? isoScene.nodes.length + ' assets · ' + isoScene.traces.length + ' traces' : 'rest'); + } + + function renderInspector(extra) { + const box = document.getElementById('inspector'); + const n = MAP.selected ? MAP.byId.get(MAP.selected) : null; + if (extra && extra.internet) { + const x = extra.internet; + box.innerHTML = html`
    Attack surface

    Reachable from the internet

    +
    Assets${x.reached}Hops${x.hops}Traces${x.traces}
    +

    The lit traces start at the internet mast and follow each relation's direction, one hop after another. Click an asset to see its own connections.

    `; + return; + } + if (!n) { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + box.innerHTML = html`
    Inspector

    Pick an asset

    +

    Click any asset to light its relations; each lit trace carries a dot from its source to its target. Click a relation in this panel to follow it alone.

    +
    Assets${(isoScene.nodes || []).length}Platforms${(isoScene.platforms || []).length}Traces${(isoScene.traces || []).length}
    +

    [ ] turn · + - zoom · 0 fit · Esc clear

    `; + return; + } + const conns = connectedTraces(n.id); + const connItems = conns.map(t => html` + `).join(''); + const f = n.findings || {}; + const fsum = SEVERITIES.filter(s => f[s]).map(s => s.charAt(0) + s.slice(1).toLowerCase() + ' ' + f[s]).join(', '); + box.innerHTML = html`
    ${n.type || n.kind}

    ${n.name}

    +
    + Provider${n.provider || ''} + Region${n.region || ''} + Account${n.account || ''} + Identifier${n.arn || n.id} + Exposure${n.exposed ? 'Reachable from the internet' : 'Not exposed'} + Findings${fsum || 'None open'} +
    +
    Relations ${conns.length}
    ` + connItems + html`
    `; + } + + // ── Pan, zoom, turn ── + function setView(v) { + MAP.view = v; + document.getElementById('iso-svg').setAttribute('viewBox', v.map(n => n.toFixed(1)).join(' ')); + } + function zoomBy(factor, cx, cy) { + const [x, y, w, h] = MAP.view; + const nw = Math.min(W * 4, Math.max(W / 14, w / factor)), nh = nw * (h / w); + const px = cx === undefined ? x + w / 2 : cx, py = cy === undefined ? y + h / 2 : cy; + setView([px - (px - x) * (nw / w), py - (py - y) * (nh / h), nw, nh]); + } + function fitView() { setView([0, 0, W, H]); } + function centerOn(id) { + const n = MAP.byId.get(id); + if (!n || !MAP.P) return; + const p = IsoKit.iso([n.x, n.y, (n.z || 0) + (n.h || 0)], MAP.P); + const [, , w, h] = MAP.view; + const nw = Math.min(w, W / 2.2), nh = nw * (h / w); + animateView([p[0] - nw / 2, p[1] - nh / 2, nw, nh]); + } + function animateView(target) { + if (reducedMotion()) { setView(target); return; } + const from = MAP.view.slice(), t0 = performance.now(), dur = 520; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = 1 - Math.pow(1 - k, 3); + setView(from.map((v, i) => v + (target[i] - v) * e)); + if (k < 1) requestAnimationFrame(step); + }; + requestAnimationFrame(step); + } + function turn(by) { + if (MAP.turning) return; + const from = MAP.az, to = MAP.az + by; + const heavy = (isoScene.nodes || []).length > 260 || reducedMotion(); + const finish = () => { MAP.az = ((to % 360) + 360) % 360; MAP.turning = false; buildMap(); document.getElementById('iso-svg').classList.add('flowing'); }; + if (heavy) { finish(); return; } + MAP.turning = true; + const t0 = performance.now(), dur = 640; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = k < .5 ? 4 * k * k * k : 1 - Math.pow(-2 * k + 2, 3) / 2; + MAP.az = from + (to - from) * e; + buildMap(); + if (k < 1) requestAnimationFrame(step); else finish(); + }; + requestAnimationFrame(step); + } + + function setFlat(flat) { + MAP.flat = flat; + document.getElementById('stage').classList.toggle('flat', flat); + document.getElementById('view-3d').setAttribute('aria-pressed', String(!flat)); + document.getElementById('view-2d').setAttribute('aria-pressed', String(flat)); + document.getElementById('layout-2d').hidden = !flat; + ['rot-left', 'rot-right', 'labels-btn', 'trace-internet', 'replay-btn'].forEach(id => { document.getElementById(id).disabled = flat; }); + renderTopology(); + } + + function renderFamilies() { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + document.getElementById('families').innerHTML = FAMILIES.map(([key, label]) => + html``).join(''); + } + + function renderLegend() { + const legend = document.getElementById('legend'); + legend.innerHTML = FAMILIES.map(([key, label]) => html`${label}`).join('') + + SEVERITIES.slice(0, 4).map(s => html`${s.charAt(0) + s.slice(1).toLowerCase()} beacon`).join('') + + html`Internet exposed`; + } + // ── Topology entry point: the 3D map, or the 2D tree and force views ── function renderTopology() { const container = document.getElementById('topology-container'); - // Remove existing SVG only - container.querySelectorAll('svg').forEach(s => s.remove()); + const stage = document.getElementById('stage'); + const can3d = typeof IsoKit !== 'undefined' && isoScene && isoScene.nodes && !isoScene.too_large && !isoScene.error && isoScene.nodes.length > 0; + if (!can3d && !MAP.flat) { + MAP.flat = true; + stage.classList.add('flat'); + document.getElementById('view-3d').disabled = true; + document.getElementById('view-3d').setAttribute('aria-pressed', 'false'); + document.getElementById('view-2d').setAttribute('aria-pressed', 'true'); + document.getElementById('layout-2d').hidden = false; + const hint = document.getElementById('map-hint'); + if (isoScene && isoScene.too_large) hint.textContent = 'The 3D map is drawn for up to ' + isoScene.limit + ' assets; this scan has ' + isoScene.asset_count; + else hint.textContent = 'Scroll to zoom · drag to pan'; + } + if (!MAP.flat) { + if (!MAP.built) { buildMap(); setReadout(); renderInspector(); playIntro(); } + return; + } + container.querySelectorAll('svg, p').forEach(s => s.remove()); const width = container.clientWidth || 1200; - const height = 600; + const height = container.clientHeight || 640; if (!hierarchyData || !hierarchyData.children || hierarchyData.children.length === 0) { - // Fallback to force-directed if no hierarchy if (graphData.nodes && graphData.nodes.length > 0) { renderForceTopology(container, width, height); } else { const msg = document.createElement('p'); - msg.style.cssText = 'padding:40px;text-align:center;color:var(--text-muted)'; - msg.textContent = 'No topology data available. Run a collection first.'; + msg.className = 'map-note'; + msg.textContent = 'No topology data in this scan. Run cloudg collect or cloudg map first.'; container.appendChild(msg); } return; } + if (currentLayout === 'force') renderForceTopology(container, width, height); + else renderTreeTopology(container, width, height); + } - if (currentLayout === 'force') { - renderForceTopology(container, width, height); - } else { - renderTreeTopology(container, width, height); - } + // ── 2D views (D3) ── + const CATEGORY_COLORS = { + account: '--accent', region: '--f-governance', vpc: '--f-network', vnet: '--f-network', subnet: '--f-network', global: '--f-data', + }; + function typeColor(type) { + if (CATEGORY_COLORS[type]) return cssVar(CATEGORY_COLORS[type]); + if (type === 'SECURITY_FINDING') return cssVar('--critical'); + if (type === 'COMPLIANCE_FRAMEWORK') return cssVar('--f-governance'); + return cssVar('--muted'); } + let zoomBehavior, svgG; function renderTreeTopology(container, width, height) { const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); - const g = svg.append('g').attr('transform', 'translate(80, 20)'); + const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.1, 4]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - const root = d3.hierarchy(hierarchyData); - root.descendants().forEach(d => { d._children = d.children; }); - - const treeLayout = d3.tree().nodeSize([28, 220]); - treeLayout(root); - - // Center vertically + d3.tree().nodeSize([26, 230])(root); const yExtent = d3.extent(root.descendants(), d => d.x); - const yOffset = height / 2 - (yExtent[0] + yExtent[1]) / 2; - g.attr('transform', `translate(80, ${yOffset})`); - - // Links + svg.call(zoomBehavior.transform, d3.zoomIdentity.translate(80, height / 2 - (yExtent[0] + yExtent[1]) / 2)); g.selectAll('.link').data(root.links()).join('path') - .attr('class', 'link') .attr('d', d3.linkHorizontal().x(d => d.y).y(d => d.x)) - .attr('fill', 'none') - .attr('stroke', '#475569') - .attr('stroke-width', 1.5) - .attr('stroke-opacity', 0.6); - - // Nodes + .attr('fill', 'none').attr('stroke', cssVar('--line-strong')).attr('stroke-width', 1.2); const node = g.selectAll('.node').data(root.descendants()).join('g') - .attr('class', 'node') - .attr('transform', d => `translate(${d.y},${d.x})`) - .style('cursor', d => d.children || d._children ? 'pointer' : 'default') - .on('click', (e, d) => { - if (d.children) { d._children = d.children; d.children = null; } - else if (d._children) { d.children = d._children; d._children = null; } - renderTopology(); - }); - - // Node circles + .attr('transform', d => `translate(${d.y},${d.x})`); node.append('circle') - .attr('r', d => { - const nodeData = d.data; - return typeRadius[nodeData.type] || 10; - }) - .attr('fill', d => typeColors[d.data.type] || '#95A5A6') + .attr('r', d => d.children ? 7 : 5) + .attr('fill', d => d.children ? cssVar('--surface') : typeColor(d.data.type)) .attr('stroke', d => { - const nodeData = d.data; - if (nodeData.findings && nodeData.findings.total > 0) { - if (nodeData.findings.CRITICAL > 0) return '#EF4444'; - if (nodeData.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - } - if (nodeData.internet_exposed) return '#FF9900'; - return 'rgba(255,255,255,0.3)'; + const f = d.data.findings; + if (f && f.total > 0) return cssVar(f.CRITICAL ? '--critical' : f.HIGH ? '--high' : '--medium'); + if (d.data.internet_exposed) return cssVar('--f-exposure'); + return typeColor(d.data.type); }) - .attr('stroke-width', d => { - const nodeData = d.data; - if ((nodeData.findings && nodeData.findings.total > 0) || nodeData.internet_exposed) return 3; - return 1.5; - }); - - // Findings badge - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('circle') - .attr('cx', d => (typeRadius[d.data.type] || 10) - 2) - .attr('cy', d => -(typeRadius[d.data.type] || 10) + 2) - .attr('r', 8) - .attr('fill', d => { - if (d.data.findings.CRITICAL > 0) return '#EF4444'; - if (d.data.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - }); - - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) - 2) - .attr('y', d => -(typeRadius[d.data.type] || 10) + 6) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '9px') - .attr('font-weight', '700') - .text(d => d.data.findings.total); - - // Collapse indicator - node.filter(d => d._children && !d.children) - .append('text') - .attr('x', 0).attr('y', 4) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '14px') - .attr('font-weight', '700') - .text('+'); - - // Labels + .attr('stroke-width', d => (d.data.findings && d.data.findings.total) || d.data.internet_exposed ? 2.5 : 1.4); node.append('text') - .attr('x', d => (d.children || d._children ? -(typeRadius[d.data.type] || 10) - 6 : (typeRadius[d.data.type] || 10) + 6)) - .attr('dy', '0.35em') - .attr('text-anchor', d => d.children || d._children ? 'end' : 'start') - .attr('fill', '#CBD5E1') - .attr('font-size', d => ['account','region','vpc'].includes(d.data.type) ? '13px' : '11px') - .attr('font-weight', d => ['account','region','vpc'].includes(d.data.type) ? '600' : '400') - .text(d => { - const name = d.data.name || d.data.id || ''; - return name.length > 35 ? name.substring(0, 32) + '…' : name; - }); - - // Type badge for leaf nodes - node.filter(d => d.data.type && !['account','region','vpc','subnet','global','vnet'].includes(d.data.type)) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) + 6) - .attr('dy', '1.8em') - .attr('fill', '#64748B') - .attr('font-size', '9px') - .text(d => d.data.type); - - // Tooltips + .attr('x', d => d.children ? -12 : 10).attr('dy', '0.32em') + .attr('text-anchor', d => d.children ? 'end' : 'start') + .attr('fill', cssVar('--text')).attr('font-size', d => d.children ? '12px' : '11px') + .attr('font-weight', d => d.children ? 600 : 400) + .text(d => { const n = d.data.name || d.data.id || ''; return n.length > 36 ? n.slice(0, 33) + '…' : n; }); + node.filter(d => !d.children).append('text') + .attr('x', 10).attr('dy', '1.7em').attr('fill', cssVar('--muted')).attr('font-size', '9px') + .attr('font-family', 'ui-monospace, monospace').text(d => d.data.type); node.append('title').text(d => { let tip = `${d.data.type}: ${d.data.name || d.data.id || ''}`; if (d.data.arn) tip += `\nARN: ${d.data.arn}`; - if (d.data.findings && d.data.findings.total > 0) { - tip += `\n\nFindings: ${d.data.findings.total}`; - if (d.data.findings.CRITICAL) tip += `\n Critical: ${d.data.findings.CRITICAL}`; - if (d.data.findings.HIGH) tip += `\n High: ${d.data.findings.HIGH}`; - if (d.data.findings.MEDIUM) tip += `\n Medium: ${d.data.findings.MEDIUM}`; - } - if (d.data.internet_exposed) tip += '\n\n⚠ INTERNET EXPOSED'; + if (d.data.findings && d.data.findings.total > 0) tip += `\nFindings: ${d.data.findings.total}`; + if (d.data.internet_exposed) tip += '\nReachable from the internet'; return tip; }); } @@ -650,225 +1462,296 @@ const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.2, 5]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - if (!graphData.nodes || graphData.nodes.length === 0) return; - - const sim = d3.forceSimulation(graphData.nodes) - .force('link', d3.forceLink(graphData.links).id(d => d.id).distance(100)) - .force('charge', d3.forceManyBody().strength(-200)) + const nodes = graphData.nodes.map(n => Object.assign({}, n)); + const links = graphData.links.map(l => Object.assign({}, l)); + const sim = d3.forceSimulation(nodes) + .force('link', d3.forceLink(links).id(d => d.id).distance(80)) + .force('charge', d3.forceManyBody().strength(-180)) .force('center', d3.forceCenter(width / 2, height / 2)) - .force('collision', d3.forceCollide().radius(30)); - - const link = g.append('g').selectAll('line').data(graphData.links).join('line') - .attr('stroke', d => { - if (d.relation === 'FINDING_AFFECTS') { - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB'}; - return sevColors[d.severity] || '#EF4444'; - } - if (d.relation === 'COMPLIANCE_GOVERNS') return '#14B8A6'; - return d.cidr === '0.0.0.0/0' ? '#E74C3C' : '#555'; - }) - .attr('stroke-width', d => { - if (d.relation === 'FINDING_AFFECTS') return 2; - if (d.relation === 'COMPLIANCE_GOVERNS') return 1.5; - return d.cidr === '0.0.0.0/0' ? 2 : 1; - }) - .attr('stroke-dasharray', d => { - if (d.relation === 'FINDING_AFFECTS' || d.relation === 'COMPLIANCE_GOVERNS') return '6,3'; - return null; - }) - .attr('stroke-opacity', d => d.relation ? 0.7 : 0.5); - - const node = g.append('g').selectAll('g').data(graphData.nodes).join('g') + .force('collision', d3.forceCollide().radius(22)); + const sevVar = { CRITICAL: '--critical', HIGH: '--high', MEDIUM: '--medium', LOW: '--low' }; + const link = g.append('g').selectAll('line').data(links).join('line') + .attr('stroke', d => d.relation === 'FINDING_AFFECTS' ? cssVar(sevVar[d.severity] || '--critical') + : d.relation === 'COMPLIANCE_GOVERNS' ? cssVar('--f-governance') : cssVar('--line-strong')) + .attr('stroke-width', d => d.relation ? 1.4 : 1) + .attr('stroke-dasharray', d => d.relation ? '5,3' : null); + const node = g.append('g').selectAll('g').data(nodes).join('g') .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended)); - - // Node shape: findings get triangles, compliance gets diamonds, rest get circles - node.each(function(d) { - const el = d3.select(this); - const r = d.is_external ? 8 : 16; - if (d.type === 'SECURITY_FINDING') { - // Triangle for findings - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB',INFO:'#64748B'}; - el.append('polygon') - .attr('points', `0,${-r} ${r},${r} ${-r},${r}`) - .attr('fill', sevColors[d.severity] || '#EF4444') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else if (d.type === 'COMPLIANCE_FRAMEWORK') { - // Diamond for compliance - el.append('polygon') - .attr('points', `0,${-r} ${r},0 0,${r} ${-r},0`) - .attr('fill', '#14B8A6') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else { - el.append('circle').attr('r', r) - .attr('fill', typeColors[d.type] || '#95A5A6') - .attr('stroke', d.is_internet_exposed ? '#FF0000' : '#fff') - .attr('stroke-width', d.is_internet_exposed ? 3 : 1.5); - } - }); - - node.append('text').text(d => d.name ? d.name.substring(0, 15) : d.id.substring(0, 10)) - .attr('dy', 28).attr('text-anchor', 'middle') - .attr('fill', '#CBD5E1').attr('font-size', '10px'); - - node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region}\nARN: ${d.arn || 'N/A'}`); - + node.append('circle').attr('r', d => d.is_external ? 6 : d.type === 'SECURITY_FINDING' ? 5 : 9) + .attr('fill', d => d.type === 'SECURITY_FINDING' ? cssVar(sevVar[d.severity] || '--critical') : typeColor(d.type)) + .attr('stroke', d => d.is_internet_exposed ? cssVar('--f-exposure') : cssVar('--surface')) + .attr('stroke-width', d => d.is_internet_exposed ? 2.5 : 1.5); + node.append('text').text(d => (d.name || d.id || '').slice(0, 18)) + .attr('dy', 20).attr('text-anchor', 'middle').attr('fill', cssVar('--muted')).attr('font-size', '10px'); + node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region || ''}\nARN: ${d.arn || 'N/A'}`); sim.on('tick', () => { - link.attr('x1', d => d.source.x).attr('y1', d => d.source.y) - .attr('x2', d => d.target.x).attr('y2', d => d.target.y); + link.attr('x1', d => d.source.x).attr('y1', d => d.source.y).attr('x2', d => d.target.x).attr('y2', d => d.target.y); node.attr('transform', d => `translate(${d.x},${d.y})`); }); - function dragstarted(e, d) { if (!e.active) sim.alphaTarget(0.3).restart(); d.fx = d.x; d.fy = d.y; } function dragged(e, d) { d.fx = e.x; d.fy = e.y; } function dragended(e, d) { if (!e.active) sim.alphaTarget(0); d.fx = null; d.fy = null; } } - function toggleLayout() { - currentLayout = currentLayout === 'tree' ? 'force' : 'tree'; - renderTopology(); - } - function resetZoom() { + if (!MAP.flat) { animateView([0, 0, W, H]); return; } const svg = d3.select('#topology-container svg'); - if (svg.node() && zoomBehavior) { - svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity); - } - } - - function expandAll() { - function expand(d) { if (d._children) { d.children = d._children; d._children = null; } if (d.children) d.children.forEach(expand); } - if (hierarchyData) { const root = d3.hierarchy(hierarchyData); expand(root); } - renderTopology(); - } - - function collapseAll() { - // Re-render with collapsed tree (only top 2 levels open) - renderTopology(); + if (svg.node() && zoomBehavior) svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity.translate(80, 20)); } // ── Chart.js Charts ── + let chartsBuilt = false; + const charts = []; function renderCharts() { - // Severity Donut - new Chart(document.getElementById('chart-severity'), { + if (chartsBuilt || typeof Chart === 'undefined') return; + chartsBuilt = true; + charts.splice(0).forEach(c => c.destroy()); + const text = cssVar('--text'), muted = cssVar('--muted'), grid = cssVar('--line'); + const sev = SEVERITIES.map(s => cssVar('--' + s.toLowerCase())); + Chart.defaults.font.family = getComputedStyle(document.body).fontFamily; + Chart.defaults.color = muted; + const legend = { position: 'bottom', labels: { color: text, usePointStyle: true, pointStyle: 'circle', boxWidth: 8, padding: 14 } }; + charts.push(new Chart(document.getElementById('chart-severity'), { type: 'doughnut', - data: { - labels: Object.keys(severityCounts), - datasets: [{ - data: Object.values(severityCounts), - backgroundColor: ['#EF4444', '#F97316', '#EAB308', '#3498DB', '#64748B'], - borderWidth: 0, hoverOffset: 8, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Resource Types Bar - new Chart(document.getElementById('chart-resources'), { + data: { labels: SEVERITIES, datasets: [{ data: SEVERITIES.map(s => severityCounts[s] || 0), backgroundColor: sev, borderColor: cssVar('--surface'), borderWidth: 3, hoverOffset: 6 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); + const types = Object.entries(resourceTypes).sort((a, b) => b[1] - a[1]).slice(0, 12); + charts.push(new Chart(document.getElementById('chart-resources'), { type: 'bar', - data: { - labels: Object.keys(resourceTypes), - datasets: [{ - data: Object.values(resourceTypes), - backgroundColor: '#6366F1', borderRadius: 6, - }] - }, - options: { - responsive: true, indexAxis: 'y', - plugins: { legend: { display: false } }, - scales: { - x: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' } }, - y: { ticks: { color: '#CBD5E1' }, grid: { display: false } }, - }, - } - }); - - // Sources Donut - new Chart(document.getElementById('chart-sources'), { + data: { labels: types.map(t => t[0]), datasets: [{ data: types.map(t => t[1]), backgroundColor: cssVar('--accent'), borderRadius: 4, barThickness: 12 }] }, + options: { responsive: true, maintainAspectRatio: false, indexAxis: 'y', plugins: { legend: { display: false } }, + scales: { x: { ticks: { color: muted }, grid: { color: grid } }, y: { ticks: { color: text, font: { family: 'ui-monospace, monospace', size: 10 } }, grid: { display: false } } } }, + })); + const fam = ['--f-network', '--f-data', '--f-identity', '--f-governance', '--accent', '--high'].map(cssVar); + charts.push(new Chart(document.getElementById('chart-sources'), { type: 'doughnut', - data: { - labels: Object.keys(sourceTools), - datasets: [{ - data: Object.values(sourceTools), - backgroundColor: ['#6366F1', '#8B5CF6', '#EC4899', '#14B8A6', '#F97316', '#EAB308'], - borderWidth: 0, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Compliance + data: { labels: Object.keys(sourceTools), datasets: [{ data: Object.values(sourceTools), backgroundColor: fam, borderColor: cssVar('--surface'), borderWidth: 3 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); const fws = Object.keys(complianceSummary); - const passData = fws.map(fw => complianceSummary[fw]?.pass || 0); - const failData = fws.map(fw => complianceSummary[fw]?.fail || 0); - new Chart(document.getElementById('chart-compliance'), { + charts.push(new Chart(document.getElementById('chart-compliance'), { type: 'bar', - data: { - labels: fws, - datasets: [ - { label: 'Pass', data: passData, backgroundColor: '#22C55E', borderRadius: 4 }, - { label: 'Fail', data: failData, backgroundColor: '#EF4444', borderRadius: 4 }, - ] - }, - options: { - responsive: true, - plugins: { legend: { labels: { color: '#CBD5E1' } } }, - scales: { - x: { ticks: { color: '#CBD5E1' }, grid: { display: false }, stacked: true }, - y: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' }, stacked: true }, - }, - } - }); + data: { labels: fws, datasets: [ + { label: 'Pass', data: fws.map(fw => complianceSummary[fw]?.pass || 0), backgroundColor: cssVar('--pass'), borderRadius: 3 }, + { label: 'Fail', data: fws.map(fw => complianceSummary[fw]?.fail || 0), backgroundColor: cssVar('--fail'), borderRadius: 3 }, + ] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, + scales: { x: { ticks: { color: text }, grid: { display: false }, stacked: true }, y: { ticks: { color: muted }, grid: { color: grid }, stacked: true } } }, + })); } - // ── Compliance Grid ── - function renderCompliance() { - const grid = document.getElementById('compliance-grid'); - grid.innerHTML = Object.entries(complianceSummary).map(([fw, data]) => { - const pass = Number(data.pass) || 0; - const fail = Number(data.fail) || 0; - const total = pass + fail; - const pct = total > 0 ? Math.round((pass / total) * 100) : 0; - const barColor = pct >= 80 ? 'var(--success)' : pct >= 50 ? 'var(--medium)' : 'var(--critical)'; - return html` -
    -

    ${fw}

    -
    - Pass: ${pass}Fail: ${fail}${pct}% -
    -
    -
    -
    -
    - `; - }).join(''); + // ── Empty states: Hairline figures ── + // cloudg/templates/figures/empty-tray.js is a figure in Hairline's own + // format; this is the small host that mounts it (the stage, its svg and a + // read-out the figure writes to) in place of Hairline's bench page. + function mountEmptyTray(host) { + const spec = window.HAIRLINE_FIGURES && window.HAIRLINE_FIGURES.tray; + if (typeof HL === 'undefined' || !spec || !host || host.dataset.mounted) return; + host.dataset.mounted = '1'; + HL.inject(document); + const svg = HL.mk('svg', { viewBox: '0 0 400 320', 'aria-hidden': 'true' }, host); + const read = document.createElement('span'); + read.setAttribute('data-hairline-live', ''); + host.appendChild(read); + spec.mount({ stage: host, svg, read }, spec.range[1]); } // ── Export ── function exportJSON() { - const blob = new Blob([JSON.stringify({findings: findingsData, graph: graphData, hierarchy: hierarchyData}, null, 2)], {type: 'application/json'}); + const blob = new Blob([JSON.stringify({ findings: findingsData, assets: assetsData, graph: graphData, hierarchy: hierarchyData, scene: isoScene }, null, 2)], { type: 'application/json' }); const a = document.createElement('a'); a.href = URL.createObjectURL(blob); - a.download = 'findings.json'; + a.download = 'cloudg-report.json'; + a.click(); + } + function exportCSV() { + const cols = ['severity', 'title', 'resource_id', 'resource_arn', 'source_tool', 'risk_score', 'cvss_score', 'compliance_frameworks', 'remediation']; + const cell = v => { + let s = Array.isArray(v) ? v.join('; ') : (v === null || v === undefined ? '' : String(v)); + if (/^[=+\-@\t\r]/.test(s)) s = "'" + s; // keep spreadsheets from evaluating it + return '"' + s.replace(/"/g, '""') + '"'; + }; + const rows = [cols.join(',')].concat(findingsData.map(f => cols.map(c => cell(f[c])).join(','))); + const blob = new Blob([rows.join('\r\n')], { type: 'text/csv' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = 'cloudg-findings.csv'; a.click(); } // ── Init ── (tests/test_html_report.py runs the script up to this line) - renderFindings(findingsData); - renderTopology(); - renderCharts(); - renderCompliance(); + (function init() { + const root = document.documentElement; + let theme = null; + try { theme = localStorage.getItem('cloudg-report-theme'); } catch (e) { theme = null; } + if (!theme) theme = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; + root.dataset.theme = theme; + document.body.dataset.theme = theme; + + // The kit's tones, scoped to the report's body.iso + if (typeof IsoKit !== 'undefined') { + const style = document.createElement('style'); + style.textContent = IsoKit.ISO_CSS; + document.head.insertBefore(style, document.head.firstChild); + } + (isoScene.nodes || []).forEach(n => MAP.byId.set(n.id, n)); + (isoScene.externals || []).forEach(e => MAP.byId.set(e.id, e)); + + document.getElementById('main-tabs').addEventListener('click', e => { + const tab = e.target.closest('[data-tab]'); + if (tab) switchTab(tab.dataset.tab); + }); + document.getElementById('theme-toggle').addEventListener('click', () => { + const next = root.dataset.theme === 'light' ? 'dark' : 'light'; + root.dataset.theme = next; document.body.dataset.theme = next; + try { localStorage.setItem('cloudg-report-theme', next); } catch (e) { /* private mode */ } + if (chartsBuilt) { chartsBuilt = false; if (document.getElementById('tab-charts').classList.contains('active')) renderCharts(); } + if (MAP.flat) renderTopology(); + }); + document.getElementById('print-btn').addEventListener('click', () => window.print()); + document.getElementById('csv-btn').addEventListener('click', exportCSV); + document.getElementById('scrim').addEventListener('click', closeDetail); + document.addEventListener('keydown', e => { if (e.key === 'Escape') closeDetail(); }); + + document.getElementById('findings-body').addEventListener('click', (e) => { + const row = e.target.closest('tr[data-index]'); + if (row) showDetail(Number(row.dataset.index)); + }); + document.getElementById('top-risks').addEventListener('click', e => { + const b = e.target.closest('[data-index]'); + if (b) showDetail(Number(b.dataset.index)); + }); + ['filter-severity', 'filter-source'].forEach(id => document.getElementById(id).addEventListener('change', filterFindings)); + document.getElementById('filter-search').addEventListener('input', filterFindings); + const sourceSelect = document.getElementById('filter-source'); + Object.keys(sourceTools).forEach(tool => { + const opt = document.createElement('option'); + opt.value = tool; opt.textContent = tool; + sourceSelect.appendChild(opt); + }); + const providerSelect = document.getElementById('asset-provider'); + [...new Set(assetsData.map(a => a.provider))].sort().forEach(p => { + const opt = document.createElement('option'); + opt.value = p; opt.textContent = p; + providerSelect.appendChild(opt); + }); + providerSelect.addEventListener('change', renderAssets); + document.getElementById('asset-search').addEventListener('input', renderAssets); + document.getElementById('assets-body').addEventListener('click', e => { + const row = e.target.closest('tr[data-asset]'); + if (!row || !MAP.byId.has(row.dataset.asset)) return; + switchTab('topology'); + if (MAP.flat && !document.getElementById('view-3d').disabled) setFlat(false); + setTimeout(() => selectNode(row.dataset.asset, { center: true }), 80); + }); + + // Map controls + const stage = document.getElementById('stage'); + const svg = document.getElementById('iso-svg'); + document.getElementById('view-3d').addEventListener('click', () => setFlat(false)); + document.getElementById('view-2d').addEventListener('click', () => setFlat(true)); + document.getElementById('layout-tree').addEventListener('click', () => { currentLayout = 'tree'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'true'); document.getElementById('layout-force').setAttribute('aria-pressed', 'false'); renderTopology(); }); + document.getElementById('layout-force').addEventListener('click', () => { currentLayout = 'force'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'false'); document.getElementById('layout-force').setAttribute('aria-pressed', 'true'); renderTopology(); }); + document.getElementById('rot-left').addEventListener('click', () => turn(-90)); + document.getElementById('rot-right').addEventListener('click', () => turn(90)); + document.getElementById('zoom-in').addEventListener('click', () => MAP.flat ? null : zoomBy(1.35)); + document.getElementById('zoom-out').addEventListener('click', () => MAP.flat ? null : zoomBy(1 / 1.35)); + document.getElementById('zoom-fit').addEventListener('click', resetZoom); + document.getElementById('labels-btn').addEventListener('click', e => { MAP.labels = !MAP.labels; e.currentTarget.setAttribute('aria-pressed', String(MAP.labels)); renderLabels(); }); + document.getElementById('trace-internet').addEventListener('click', traceFromInternet); + document.getElementById('replay-btn').addEventListener('click', () => { MAP.selected = null; applyFocus(); renderInspector(); playIntro(); }); + document.getElementById('families').addEventListener('click', e => { + const b = e.target.closest('[data-family]'); + if (!b) return; + const f = b.dataset.family; + if (MAP.families.has(f)) MAP.families.delete(f); else MAP.families.add(f); + b.setAttribute('aria-pressed', String(MAP.families.has(f))); + MAP.traceEls.forEach(({ g, t }) => g.classList.toggle('hidden-family', !MAP.families.has(t.family))); + applyFocus(MAP.focusTrace ? [MAP.focusTrace] : undefined); + }); + document.getElementById('inspector').addEventListener('click', e => { + const b = e.target.closest('[data-trace]'); + if (b) focusTrace(b.dataset.trace); + }); + svg.addEventListener('click', e => { + if (MAP.dragMoved) return; + const tr = e.target.closest('[data-trace]'); + if (tr && !e.target.closest('[data-node]')) { const t = MAP.traceEls.get(tr.dataset.trace); if (t) { MAP.selected = t.t.source; renderInspector(); focusTrace(tr.dataset.trace); renderLabels(); } return; } + const node = e.target.closest('[data-node]'); + selectNode(node ? node.getAttribute('data-node') : null); + }); + svg.addEventListener('pointerover', e => { + const node = e.target.closest('[data-node]'); + const id = node ? node.getAttribute('data-node') : null; + if (id === MAP.hover) return; + MAP.hover = id; renderLabels(); + const n = id && MAP.byId.get(id); + if (n) setReadout(n.name + ' · ' + (n.type || n.kind)); else if (!MAP.selected) setReadout(); + }); + // Pan with a drag, zoom with the wheel about the pointer + const toView = (cx, cy) => { + const r = svg.getBoundingClientRect(), [x, y, w, h] = MAP.view; + const s = Math.min(r.width / w, r.height / h); + const ox = (r.width - w * s) / 2, oy = (r.height - h * s) / 2; + return [x + (cx - r.left - ox) / s, y + (cy - r.top - oy) / s, s]; + }; + stage.addEventListener('wheel', e => { + if (MAP.flat) return; + e.preventDefault(); + const [px, py] = toView(e.clientX, e.clientY); + zoomBy(Math.exp(-e.deltaY * 0.0015), px, py); + }, { passive: false }); + let drag = null; + stage.addEventListener('pointerdown', e => { + if (MAP.flat || e.button !== 0) return; + drag = { x: e.clientX, y: e.clientY, view: MAP.view.slice(), s: toView(e.clientX, e.clientY)[2] }; + MAP.dragMoved = false; + }); + window.addEventListener('pointermove', e => { + if (!drag) return; + const dx = e.clientX - drag.x, dy = e.clientY - drag.y; + if (!MAP.dragMoved && Math.hypot(dx, dy) < 4) return; + MAP.dragMoved = true; stage.classList.add('dragging'); + setView([drag.view[0] - dx / drag.s, drag.view[1] - dy / drag.s, drag.view[2], drag.view[3]]); + }); + window.addEventListener('pointerup', () => { drag = null; stage.classList.remove('dragging'); setTimeout(() => { MAP.dragMoved = false; }, 0); }); + stage.addEventListener('keydown', e => { + if (MAP.flat) return; + const ids = (isoScene.nodes || []).map(n => n.id); + if (e.key === '[') turn(-90); + else if (e.key === ']') turn(90); + else if (e.key === '+' || e.key === '=') zoomBy(1.35); + else if (e.key === '-') zoomBy(1 / 1.35); + else if (e.key === '0') resetZoom(); + else if (e.key === 'Escape') selectNode(null); + else if (e.key === 'ArrowRight' || e.key === 'ArrowLeft' || e.key === 'ArrowDown' || e.key === 'ArrowUp') { + const i = ids.indexOf(MAP.selected), d = e.key === 'ArrowRight' || e.key === 'ArrowDown' ? 1 : -1; + selectNode(ids[(i + d + ids.length) % ids.length], { center: true }); + } else return; + e.preventDefault(); + }); + const list = document.getElementById('map-assets'); + (isoScene.nodes || []).slice(0, 3000).forEach(n => { const o = document.createElement('option'); o.value = n.name; list.appendChild(o); }); + document.getElementById('map-search').addEventListener('change', e => { + const q = e.target.value.trim().toLowerCase(); + const n = (isoScene.nodes || []).find(x => x.name.toLowerCase() === q) || (isoScene.nodes || []).find(x => x.name.toLowerCase().includes(q) || (x.arn || '').toLowerCase().includes(q)); + if (n) { if (MAP.flat) setFlat(false); selectNode(n.id, { center: true }); } + }); + + renderOverview(); + renderFindings(findingsData); + renderAssets(); + renderCompliance(); + renderFamilies(); + renderLegend(); + renderTopology(); + mountEmptyTray(document.getElementById('findings-empty-figure')); + mountEmptyTray(document.getElementById('compliance-empty-figure')); + })(); diff --git a/cloudg/templates/vendor/LICENSE-anatomy.txt b/cloudg/templates/vendor/LICENSE-anatomy.txt new file mode 100644 index 0000000..aa20733 --- /dev/null +++ b/cloudg/templates/vendor/LICENSE-anatomy.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ryan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cloudg/templates/vendor/LICENSE-hairline.txt b/cloudg/templates/vendor/LICENSE-hairline.txt new file mode 100644 index 0000000..dec0685 --- /dev/null +++ b/cloudg/templates/vendor/LICENSE-hairline.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Lucas Marques + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cloudg/templates/vendor/NOTICE b/cloudg/templates/vendor/NOTICE index c7da8ab..b14bb49 100644 --- a/cloudg/templates/vendor/NOTICE +++ b/cloudg/templates/vendor/NOTICE @@ -1,10 +1,11 @@ Third-party JavaScript bundled with cloudg ========================================== -The HTML report (report.html) inlines these two libraries when +The HTML report (report.html) inlines Chart.js and D3 when report.inline_js is true (the default), so the report works without -network access. The files are the unmodified builds published on npm; -their own licence headers are kept. +network access. Those two files are the unmodified builds published on +npm; their own licence headers are kept. The isometric map's two drawing +engines below have no CDN build and are always inlined. chart.umd.js Chart.js 4.4.0, file dist/chart.umd.js of the npm package chart.js@4.4.0 @@ -20,6 +21,24 @@ d3.min.js ISC License, see LICENSE-d3.txt sha256 f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539 +iso-kit.js + Anatomy iso-kit, file kit/iso-kit.mjs of https://github.com/wheresryan22/anatomy + Copyright (c) 2026 Ryan + MIT License, see LICENSE-anatomy.txt + The module is wrapped as IsoKit for a classic script element: its + "export" keywords are removed, and the two script tags inside one + string are written "\x3cscript" and "<\/script>", which are the same + characters at run time. Source sha256 + 6d3ecc725d30c3b0346da309cf61dfa5a65017c23620bbd4093009d4fb104018 + +hairline-kernel.js + Hairline kernel, the src/core build of @lucasmarkes/hairline 0.5.1 + https://hairline.lucasmarkes.com + Copyright (c) 2026 Lucas Marques + MIT License, see LICENSE-hairline.txt + Unmodified. sha256 + 47ebec7647174e04e4f59c0305277c2a1dab83a763676946b17fa1694e7952fe + To update a library, replace the file with the build from the new npm release, update this notice, and update the version, CDN URL and SRI hash in cloudg/renderers/html_report.py (tests/test_html_report.py checks that diff --git a/cloudg/templates/vendor/hairline-kernel.js b/cloudg/templates/vendor/hairline-kernel.js new file mode 100644 index 0000000..336d4b6 --- /dev/null +++ b/cloudg/templates/vendor/hairline-kernel.js @@ -0,0 +1,714 @@ +/* hairline kernel sha256:346b48bdef2cdfafd5343eba01494255cf75f9c078e6fcd1aafc62bab62980a1 */ +/* + * HL: everything a figure may call. Read this index; the code under it is the + * package's src/core, unchanged, and a figure should not need to read it. + * + * Every figure is drawn in a 400 × 320 viewBox. World space is x/y on the + * ground and z up. Plates are filled with the ground colour and painted back + * to front, so a nearer one covers a farther one: append in that order. At + * the default camera, Cam(45, 0.5, S), +x runs down to the right and +y down + * to the left, so the corner with the largest x and y is nearest the viewer: + * append by ascending x + y, from the far corner (smallest x and y) to it. + * + * Camera + * Cam(azDeg, k, S) a camera: azimuth in degrees, k = sin(elevation) (0.5 is the 2:1 view), S = scale + * fit(C, points, cx, cy) centres the box of [x, y, z] points on (cx, cy); call it once, before proj + * it only centres, it never scales: choose S by trying values, with the most extreme pose in points. + * The six figures use S 1.42 to 2.12; the boxes they fit come out 230 to 310 wide and 180 to 245 tall + * proj(C) returns P(x, y, z), which gives [sx, sy] + * unproj(C, sx, sy, z) the world [x, y] under a screen point, on the plane at height z + * facing(C) returns front(sample): whether a ring sample faces the camera + * Rounded solids + * rrect(u0, v0, u1, v1, r, n) a rounded rectangle, as a ring of samples {u, v, nu, nv} + * circ(R, n) a circle, as a ring + * rings(x0, y0, x1, y1, r, b) [ring, inner]: a rounded footprint and its crease ring, inset by b + * r, the corner radius, is cut to half the shorter side. b, the crease's inset in world units + * (0.6 to 2.2 in the figures), must stay under half the shorter side or the crease turns inside out + * prism(P, front, ring, inner, z0, z1) {sil, crease}: a solid standing from z0 to z1, as two path strings + * ringAt(P, ring, z) the ring's points, projected at height z + * run(ring, keep) the one cyclic run of samples that pass keep + * hull(points) the convex hull of screen points + * extremes(P, ring) [left, right, nearest] samples: where dashed drops fall from + * fillet(points, radii, n) rounds every vertex of a closed polygon; returns the new points + * radii is an array, one radius per vertex, each cut to half its shorter edge; a single number gives NaN. + * n is the steps round each corner, default 4: each vertex becomes n + 1 points + * ghost(P, front, ring, z0, depth) a reflection's path {d, y0, y1}; reflect() draws it for you + * Paths and numbers + * poly(points) a closed path string + * open(points) an open polyline string + * seg(a, b) one segment between two screen points [sx, sy], as its own subpath; project world points with P first + * clamp(v, a, b) + * lerp(a, b, t) + * rad(deg) + * r2(n) two decimals + * The continuous clock: one spring per moving number + * spring(x, opts) at rest on x; write .t to retarget; opts {k, c, m, eps}, default k 100, c 18, m 1 + * stepS(sp, dt) advances by dt seconds; returns whether it is still moving + * The discrete clock: a 700ms tween on (.32, .72, 0, 1) + * tween(v, dur) at rest on v; dur defaults to 700 (ms) + * tset(tw, to, now, delay) retargets from where it is, after delay ms: the stagger + * the same target again does nothing, so calling it on every pointer move is safe + * tval(tw, now) its value at now + * tdone(tw, now) whether it has landed + * bezier(x1, y1, x2, y2) a CSS cubic-bezier, as a function of progress + * EASE_LIFT the lift curve itself + * reducedMotion() true when the reader asked for less motion; springs and tweens already land at once + * setReducedMotion(on) the loop's business, not a figure's + * Drawing + * mk(tag, attrs, parent) one svg element: the only way a figure makes a node + * solid(parent) {g, sil, cr}: a group holding a silhouette path and a crease path + * put(solid, paths) writes prism()'s {sil, crease} into solid()'s {sil, cr}: sil into sil, crease into cr + * flatDot(parent, C, r, cls) a dot lying on the ground plane; cls is "dot", "dot m" or "dot off" + * place(el, point) moves a dot or a circle to [sx, sy] + * reflect(svg, parent, P, front, ring, z0, depth) a fading mirror under a solid + * fade(svg, y0, y1, a0) a vertical fade, as a mask; returns the value for a mask attribute + * Life + * register(stage, tick) joins the one frame loop; tick(dt in seconds, now in ms) returns true to ask for another frame; gives {wake, unregister} + * pointer(stage, handlers) {move(point), down(point), leave()}, points in viewBox units; returns its disposer + * disposer() {add, on, dispose}: collects tear-down, so destroy is bag.dispose + * The bench's business, not a figure's + * css(lightDark) + * inject(root) + * tour(stage, stops, onStop) an unseen pointer that walks the stops, [x, y] in viewBox units or null to leave, and gives way to a real one; onStop(i) on each arrival; returns {stop} + * LAP the default stops: a diamond around the centre, then a leave + * + * Classes, on path, polygon, ellipse and line. They are the whole palette; a + * figure sets no colour, width or fill of its own. + * (none) filled with the ground colour, medium stroke + * sil the silhouette's stroke hi the bright stroke: the only highlight + * lo the dim stroke nf no fill fo fill only, no stroke + * dash a dashed guide + * dot a bright dot dot m a medium dot dot off a dim dot + */ +var HL = (() => { + var __defProp = Object.defineProperty; + var __getOwnPropDesc = Object.getOwnPropertyDescriptor; + var __getOwnPropNames = Object.getOwnPropertyNames; + var __hasOwnProp = Object.prototype.hasOwnProperty; + var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); + }; + var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; + }; + var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + + // packages/hairline/src/core/kernel.ts + var kernel_exports = {}; + __export(kernel_exports, { + Cam: () => Cam, + EASE_LIFT: () => EASE_LIFT, + LAP: () => LAP, + bezier: () => bezier, + circ: () => circ, + clamp: () => clamp, + css: () => css, + disposer: () => disposer, + extremes: () => extremes, + facing: () => facing, + fade: () => fade, + fillet: () => fillet, + fit: () => fit, + flatDot: () => flatDot, + ghost: () => ghost, + hull: () => hull, + inject: () => inject, + lerp: () => lerp, + mk: () => mk, + open: () => open, + place: () => place, + pointer: () => pointer, + poly: () => poly, + prism: () => prism, + proj: () => proj, + put: () => put, + r2: () => r2, + rad: () => rad, + reducedMotion: () => reducedMotion, + reflect: () => reflect, + register: () => register, + ringAt: () => ringAt, + rings: () => rings, + rrect: () => rrect, + run: () => run, + seg: () => seg, + setReducedMotion: () => setReducedMotion, + solid: () => solid, + spring: () => spring, + stepS: () => stepS, + tdone: () => tdone, + tour: () => tour, + tset: () => tset, + tval: () => tval, + tween: () => tween, + unproj: () => unproj + }); + + // packages/hairline/src/core/iso.ts + var clamp = (v, a, b) => Math.max(a, Math.min(b, v)); + var lerp = (a, b, t) => a + (b - a) * t; + var rad = (d) => d * Math.PI / 180; + var r2 = (n) => Math.round(n * 100) / 100; + var poly = (pts) => "M" + pts.map((p) => r2(p[0]) + " " + r2(p[1])).join("L") + "Z"; + var seg = (a, b) => `M${r2(a[0])} ${r2(a[1])}L${r2(b[0])} ${r2(b[1])}`; + var open = (pts) => pts.length < 2 ? "" : "M" + pts.map((p) => r2(p[0]) + " " + r2(p[1])).join("L"); + var Cam = (azDeg, k, S) => ({ az: rad(azDeg), k, S, ox: 0, oy: 0 }); + function proj(C) { + const c = Math.cos(C.az), s = Math.sin(C.az), zf = Math.sqrt(1 - C.k * C.k); + return (x, y, z) => { + const X = x * c - y * s, Y = x * s + y * c; + return [C.ox + C.S * X, C.oy + C.S * (Y * C.k - z * zf)]; + }; + } + function unproj(C, sx, sy, z) { + const c = Math.cos(C.az), s = Math.sin(C.az), zf = Math.sqrt(1 - C.k * C.k); + const X = (sx - C.ox) / C.S, Y = ((sy - C.oy) / C.S + z * zf) / C.k; + return [X * c + Y * s, -X * s + Y * c]; + } + function fit(C, pts, cx, cy) { + C.ox = 0; + C.oy = 0; + const P = proj(C); + let a = 1e9, b = -1e9, c = 1e9, d = -1e9; + for (const p of pts) { + const q = P(p[0], p[1], p[2]); + a = Math.min(a, q[0]); + b = Math.max(b, q[0]); + c = Math.min(c, q[1]); + d = Math.max(d, q[1]); + } + C.ox = cx - (a + b) / 2; + C.oy = cy - (c + d) / 2; + } + function rrect(u0, v0, u1, v1, r, n = 4) { + r = Math.max(0, Math.min(r, (u1 - u0) / 2, (v1 - v0) / 2)); + const out = []; + for (const [cu, cv, a0] of [[u1 - r, v1 - r, 0], [u0 + r, v1 - r, 90], [u0 + r, v0 + r, 180], [u1 - r, v0 + r, 270]]) + for (let k = 0; k <= n; k++) { + const a = rad(a0 + 90 * k / n), ca = Math.cos(a), sa = Math.sin(a); + out.push({ u: cu + r * ca, v: cv + r * sa, nu: ca, nv: sa }); + } + return out; + } + function circ(R, n = 96) { + const out = []; + for (let k = 0; k < n; k++) { + const a = k / n * Math.PI * 2, ca = Math.cos(a), sa = Math.sin(a); + out.push({ u: R * ca, v: R * sa, nu: ca, nv: sa }); + } + return out; + } + function hull(input) { + const pts = input.slice().sort((a, b) => a[0] - b[0] || a[1] - b[1]); + const x = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lo = [], up = []; + for (const p of pts) { + while (lo.length > 1 && x(lo[lo.length - 2], lo[lo.length - 1], p) <= 0) lo.pop(); + lo.push(p); + } + for (let i = pts.length - 1; i >= 0; i--) { + const p = pts[i]; + while (up.length > 1 && x(up[up.length - 2], up[up.length - 1], p) <= 0) up.pop(); + up.push(p); + } + lo.pop(); + up.pop(); + return lo.concat(up); + } + var ringAt = (P, ring, z) => ring.map((q) => P(q.u, q.v, z)); + var facing = (C) => { + const s = Math.sin(C.az), c = Math.cos(C.az); + return (q) => q.nu * s + q.nv * c >= -1e-6; + }; + function run(ring, keep) { + const n = ring.length; + let s = -1; + for (let i = 0; i < n; i++) if (keep(ring[i]) && !keep(ring[(i + n - 1) % n])) { + s = i; + break; + } + if (s < 0) return keep(ring[0]) ? ring.slice() : []; + const out = []; + for (let k = 0; k < n && keep(ring[(s + k) % n]); k++) out.push(ring[(s + k) % n]); + return out; + } + function prism(P, front, ring, inner, z0, z1) { + return { + sil: poly(hull(ringAt(P, ring, z1).concat(ringAt(P, ring, z0)))), + crease: inner ? open(ringAt(P, run(inner, front), z1)) : "" + }; + } + var rings = (x0, y0, x1, y1, r, b) => [ + rrect(x0, y0, x1, y1, r), + rrect(x0 + b, y0 + b, x1 - b, y1 - b, Math.max(0.3, r - b)) + ]; + function extremes(P, ring) { + const pr = ring.map((q) => P(q.u, q.v, 0)); + let a = 0, b = 0, c = 0; + pr.forEach((p, k) => { + if (p[0] < pr[a][0]) a = k; + if (p[0] > pr[b][0]) b = k; + if (p[1] > pr[c][1]) c = k; + }); + return [ring[a], ring[b], ring[c]]; + } + function fillet(pts, rs, n = 4) { + const m = pts.length, out = []; + for (let i = 0; i < m; i++) { + const a = pts[(i + m - 1) % m], p = pts[i], b = pts[(i + 1) % m]; + const la = Math.hypot(a[0] - p[0], a[1] - p[1]), lb = Math.hypot(b[0] - p[0], b[1] - p[1]); + const t = Math.min(rs[i], la / 2, lb / 2); + const p1 = [p[0] + (a[0] - p[0]) / la * t, p[1] + (a[1] - p[1]) / la * t]; + const p2 = [p[0] + (b[0] - p[0]) / lb * t, p[1] + (b[1] - p[1]) / lb * t]; + for (let k = 0; k <= n; k++) { + const s = k / n, w = 1 - s; + out.push([w * w * p1[0] + 2 * w * s * p[0] + s * s * p2[0], w * w * p1[1] + 2 * w * s * p[1] + s * s * p2[1]]); + } + } + return out; + } + function ghost(P, front, ring, z0, depth) { + const f = run(ring, front), lowP = ringAt(P, f, z0 - depth); + return { + d: open(lowP) + [f[0], f[f.length - 1]].map((q) => seg(P(q.u, q.v, z0), P(q.u, q.v, z0 - depth))).join(""), + y0: Math.min(...ringAt(P, f, z0).map((p) => p[1])), + y1: Math.max(...lowP.map((p) => p[1])) + 2 + }; + } + + // packages/hairline/src/core/motion.ts + var reduced = false; + var setReducedMotion = (on) => { + reduced = on; + }; + var reducedMotion = () => reduced; + function spring(x, o = {}) { + return { x, v: 0, t: x, k: o.k ?? 100, c: o.c ?? 18, m: o.m ?? 1, eps: o.eps ?? 0.01 }; + } + function stepS(sp, dt) { + if (reduced) { + sp.x = sp.t; + sp.v = 0; + return false; + } + const n = Math.max(1, Math.ceil(dt * 240)), h = dt / n; + for (let i = 0; i < n; i++) { + const a = (-sp.k * (sp.x - sp.t) - sp.c * sp.v) / sp.m; + sp.v += a * h; + sp.x += sp.v * h; + } + if (Math.abs(sp.x - sp.t) < sp.eps && Math.abs(sp.v) < sp.eps * 10) { + sp.x = sp.t; + sp.v = 0; + return false; + } + return true; + } + function bezier(x1, y1, x2, y2) { + const cx = 3 * x1, bx = 3 * (x2 - x1) - cx, ax = 1 - cx - bx; + const cy = 3 * y1, by = 3 * (y2 - y1) - cy, ay = 1 - cy - by; + const X = (u) => ((ax * u + bx) * u + cx) * u; + const Y = (u) => ((ay * u + by) * u + cy) * u; + const dX = (u) => (3 * ax * u + 2 * bx) * u + cx; + return (t) => { + if (t <= 0) return 0; + if (t >= 1) return 1; + let u = t; + for (let i = 0; i < 8; i++) { + const e = X(u) - t; + if (Math.abs(e) < 1e-5) break; + const d = dX(u); + if (Math.abs(d) < 1e-6) break; + u -= e / d; + } + if (!(u >= 0 && u <= 1) || Math.abs(X(u) - t) > 1e-4) { + let lo = 0, hi = 1; + u = t; + for (let i = 0; i < 24; i++) { + if (X(u) < t) lo = u; + else hi = u; + u = (lo + hi) / 2; + } + } + return Y(u); + }; + } + var EASE_LIFT = bezier(0.32, 0.72, 0, 1); + var tween = (v, dur = 700) => ({ from: v, to: v, t0: -1e9, dur }); + var tval = (tw, now) => { + const p = clamp((now - tw.t0) / tw.dur, 0, 1); + return tw.from + (tw.to - tw.from) * (reduced ? 1 : EASE_LIFT(p)); + }; + var tset = (tw, to, now, delay) => { + if (tw.to === to) return; + tw.from = tval(tw, now); + tw.to = to; + tw.t0 = now + delay; + }; + var tdone = (tw, now) => reduced || now >= tw.t0 + tw.dur; + + // packages/hairline/src/core/stage.ts + var NS = "http://www.w3.org/2000/svg"; + function mk(tag, attrs, parent) { + const e = document.createElementNS(NS, tag); + if (attrs) for (const k in attrs) e.setAttribute(k, String(attrs[k])); + if (parent) parent.appendChild(e); + return e; + } + function solid(parent) { + const g = mk("g", {}, parent); + return { g, sil: mk("path", { class: "sil" }, g), cr: mk("path", { class: "nf lo" }, g) }; + } + var put = (el, s) => { + el.sil.setAttribute("d", s.sil); + el.cr.setAttribute("d", s.crease); + }; + var flatDot = (parent, C, r, cls) => mk("ellipse", { rx: r2(r * C.S), ry: r2(r * C.S * C.k), class: cls }, parent); + var place = (el, q) => { + el.setAttribute("cx", String(r2(q[0]))); + el.setAttribute("cy", String(r2(q[1]))); + }; + var fid = 0; + function fade(svg, y0, y1, a0 = 0.7) { + const id = "hl-fd" + ++fid, defs = mk("defs", {}, svg); + const lg = mk("linearGradient", { id: id + "g", gradientUnits: "userSpaceOnUse", x1: 0, y1: r2(y0), x2: 0, y2: r2(y1) }, defs); + mk("stop", { offset: 0, "stop-color": "#fff", "stop-opacity": a0 }, lg); + mk("stop", { offset: 1, "stop-color": "#fff", "stop-opacity": 0 }, lg); + const m = mk("mask", { id, maskUnits: "userSpaceOnUse", x: 0, y: 0, width: 400, height: 320 }, defs); + mk("rect", { x: 0, y: 0, width: 400, height: 320, fill: `url(#${id}g)` }, m); + return `url(#${id})`; + } + function reflect(svg, parent, P, front, ring, z0, depth) { + const r = ghost(P, front, ring, z0, depth); + const gh = mk("g", { class: "ghost", mask: fade(svg, r.y0, r.y1) }, parent); + mk("path", { d: r.d }, gh); + } + var boards = []; + var byStage = /* @__PURE__ */ new Map(); + var raf = 0; + var last = 0; + var io = null; + var rm = null; + function frame() { + const now = performance.now(); + const dt = Math.min(0.05, Math.max(0, (now - last) / 1e3)); + last = now; + let any = false; + for (const b of boards.slice()) if (b.vis && b.awake) { + try { + b.awake = !!b.tick(dt, now); + } catch (err) { + b.awake = false; + setTimeout(() => { + throw err; + }); + } + any = any || b.awake; + } + raf = any ? requestAnimationFrame(frame) : 0; + } + function wake(b) { + b.awake = true; + if (!raf) { + last = performance.now(); + raf = requestAnimationFrame(frame); + } + } + var onMotion = () => { + setReducedMotion(!!rm?.matches); + boards.forEach(wake); + }; + function start() { + if (io) return; + io = new IntersectionObserver((es) => { + for (const e of es) { + const set = byStage.get(e.target); + if (!set) continue; + for (const b of set) { + b.vis = e.isIntersecting; + if (b.vis) wake(b); + } + } + }, { rootMargin: "80px" }); + rm = matchMedia("(prefers-reduced-motion: reduce)"); + setReducedMotion(rm.matches); + rm.addEventListener("change", onMotion); + } + function stop() { + if (raf) cancelAnimationFrame(raf); + raf = 0; + io?.disconnect(); + io = null; + rm?.removeEventListener("change", onMotion); + rm = null; + } + function register(stage, tick) { + start(); + const b = { stage, tick, vis: false, awake: true }; + boards.push(b); + const peers = byStage.get(stage); + if (peers) { + for (const p of peers) b.vis = p.vis; + peers.add(b); + if (b.vis) wake(b); + } else { + byStage.set(stage, /* @__PURE__ */ new Set([b])); + io.observe(stage); + } + tick(0, performance.now()); + let gone = false; + return { + wake: () => { + if (!gone) wake(b); + }, + unregister: () => { + if (gone) return; + gone = true; + boards = boards.filter((x) => x !== b); + const set = byStage.get(stage); + if (set) { + set.delete(b); + if (!set.size) { + byStage.delete(stage); + io?.unobserve(stage); + } + } + if (!boards.length) stop(); + } + }; + } + var handlers = /* @__PURE__ */ new WeakMap(); + var touring = /* @__PURE__ */ new WeakMap(); + function pointer(stage, on) { + handlers.set(stage, on); + let tm = 0; + const pt = (e) => { + const r = stage.getBoundingClientRect(); + return [(e.clientX - r.left) / r.width * 400, (e.clientY - r.top) / r.height * 320]; + }; + const move = (e) => { + clearTimeout(tm); + touring.get(stage)?.hold(); + on.move(pt(e), e); + }; + const down = (e) => { + clearTimeout(tm); + touring.get(stage)?.hold(); + if (e.pointerType !== "mouse") stage.releasePointerCapture?.(e.pointerId); + if (on.down) on.down(pt(e), e); + else on.move(pt(e), e); + }; + const leave = (e) => { + clearTimeout(tm); + tm = window.setTimeout(() => { + on.leave(e); + touring.get(stage)?.release(); + }, e.pointerType === "mouse" ? 0 : 1400); + }; + stage.addEventListener("pointermove", move); + stage.addEventListener("pointerdown", down); + stage.addEventListener("pointerleave", leave); + return () => { + clearTimeout(tm); + if (handlers.get(stage) === on) handlers.delete(stage); + stage.removeEventListener("pointermove", move); + stage.removeEventListener("pointerdown", down); + stage.removeEventListener("pointerleave", leave); + }; + } + var LAP = [[128, 150], [200, 118], [272, 150], [200, 206], null]; + var TRAVEL = 900; + var DWELL = 1200; + var REST = 1800; + var RESUME = 1200; + var STAGGER = 450; + var GHOST = { pointerType: "ghost" }; + var started = 0; + function entry([x, y]) { + const dx = x - 200, dy = y - 160; + if (!dx && !dy) return [200, 320]; + const k = Math.min(dx ? (dx > 0 ? 200 : -200) / dx : Infinity, dy ? (dy > 0 ? 160 : -160) / dy : Infinity); + return [200 + dx * k, 160 + dy * k]; + } + function tour(stage, stops, onStop) { + let i = 0; + let wait = RESUME + STAGGER * (started++ % 4); + let t = -1; + let at = null; + let origin = [200, 320]; + let hand = false, keys = stage.contains(stage.ownerDocument.activeElement); + let gone = false; + const leave = () => { + at = null; + t = -1; + handlers.get(stage)?.leave(GHOST); + }; + const tick = (dt) => { + if (hand || keys || !stops.length) return false; + if (reducedMotion()) { + if (at) leave(); + return false; + } + if (wait > 0) { + wait -= dt * 1e3; + return true; + } + const stop2 = stops[i]; + if (stop2 === null) { + if (at) leave(); + onStop?.(i); + i = (i + 1) % stops.length; + wait = REST; + return true; + } + const h = handlers.get(stage); + if (!h) return true; + if (t < 0) { + origin = at ?? entry(stop2); + t = 0; + } + t = Math.min(TRAVEL, t + dt * 1e3); + const k = EASE_LIFT(t / TRAVEL); + at = [origin[0] + (stop2[0] - origin[0]) * k, origin[1] + (stop2[1] - origin[1]) * k]; + h.move(at, GHOST); + if (t < TRAVEL) return true; + onStop?.(i); + i = (i + 1) % stops.length; + t = -1; + wait = DWELL; + return true; + }; + const take = () => { + at = null; + t = -1; + }; + const give = () => { + if (hand || keys) return; + i = 0; + t = -1; + wait = RESUME; + board.wake(); + }; + const me = { + hold: () => { + hand = true; + take(); + }, + release: () => { + if (!hand) return; + hand = false; + give(); + } + }; + const focusIn = () => { + keys = true; + take(); + }; + const focusOut = (e) => { + if (keys && !stage.contains(e.relatedTarget)) { + keys = false; + give(); + } + }; + stage.addEventListener("focusin", focusIn); + stage.addEventListener("focusout", focusOut); + touring.set(stage, me); + const board = register(stage, tick); + return { + stop: () => { + if (gone) return; + gone = true; + board.unregister(); + stage.removeEventListener("focusin", focusIn); + stage.removeEventListener("focusout", focusOut); + if (touring.get(stage) === me) touring.delete(stage); + if (at) leave(); + } + }; + } + function disposer() { + let fns = []; + return { + add: (fn) => { + fns.push(fn); + }, + on: (target, type, fn, opts) => { + const h = fn; + target.addEventListener(type, h, opts); + fns.push(() => target.removeEventListener(type, h, opts)); + }, + dispose: () => { + const run2 = fns; + fns = []; + for (let i = run2.length - 1; i >= 0; i--) run2[i](); + } + }; + } + + // packages/hairline/src/core/styles.ts + var LIGHT = { plate: "#ffffff", hi: "#232327", edge: "#a4a4ac", mid: "#c3c3c9", lo: "#e0e0e4" }; + var DARK = { plate: "#08090a", hi: "#d0d6e0", edge: "#5b5d64", mid: "#3e3e44", lo: "#29292d" }; + var KEYS = ["plate", "hi", "edge", "mid", "lo"]; + var vars = (p) => KEYS.map((k) => `--hl-${k}:var(--hairline-${k},${p[k]});`).join(""); + var EASE = "cubic-bezier(0.5,0,0.1,1)"; + var SVG = ":where([data-hairline]>svg)"; + function css(lightDark) { + const both = Object.fromEntries(KEYS.map((k) => [k, `light-dark(${LIGHT[k]},${DARK[k]})`])); + return [ + // the box, and the palette: light unless something below says otherwise + `:where([data-hairline]){display:block;position:relative;aspect-ratio:5/4;touch-action:pan-y;user-select:none;-webkit-user-select:none;--hl-sw:var(--hairline-stroke,0.9);${vars(LIGHT)}}`, + // the page's color-scheme + lightDark ? `:where([data-hairline]){${vars(both)}}` : "", + // an ancestor that says dark + `:where(.dark,[data-theme="dark"]) :where([data-hairline]){${vars(DARK)}}`, + // the figure's own theme option + `:where([data-hairline][data-hairline-theme="light"]){${vars(LIGHT)}}`, + `:where([data-hairline][data-hairline-theme="dark"]){${vars(DARK)}}`, + `:where([data-hairline]:focus-visible){outline:1.5px solid var(--hl-hi);outline-offset:2px}`, + `${SVG}{position:absolute;inset:0;width:100%;height:100%;display:block}`, + // Riffle's live region: read, not seen + `:where([data-hairline]>[data-hairline-live]){position:absolute;width:1px;height:1px;margin:-1px;padding:0;border:0;overflow:hidden;clip-path:inset(50%);white-space:nowrap}`, + // the drawing: plates are filled with the plate colour and painted back to front + `${SVG} :where(path,polygon,ellipse,line){fill:var(--hl-plate);stroke:var(--hl-mid);stroke-width:var(--hl-sw);vector-effect:non-scaling-stroke;stroke-linejoin:round;stroke-linecap:round;transition:stroke 260ms ${EASE}}`, + `${SVG} :where(.nf){fill:none}`, + `${SVG} :where(.fo){stroke:none}`, + `${SVG} :where(.sil){stroke:var(--hl-edge)}`, + `${SVG} :where(.hi){stroke:var(--hl-hi)}`, + `${SVG} :where(.lo){stroke:var(--hl-lo)}`, + `${SVG} :where(.dash){stroke-dasharray:1 3}`, + `${SVG} :where(.dot){stroke:none;fill:var(--hl-hi);transition:fill 260ms ${EASE}}`, + `${SVG} :where(.dot.m){fill:var(--hl-edge)}`, + `${SVG} :where(.dot.off){fill:var(--hl-lo)}`, + `${SVG} :where(.ghost path){fill:none;stroke:var(--hl-mid)}` + ].join(""); + } + var done = /* @__PURE__ */ new WeakSet(); + function inject(root) { + if (done.has(root)) return; + done.add(root); + const doc = root.nodeType === 9 ? root : root.ownerDocument; + const win = doc.defaultView; + const text = css(!!win?.CSS?.supports?.("color", "light-dark(#000,#fff)")); + if (win && "adoptedStyleSheets" in root) { + try { + const sheet = new win.CSSStyleSheet(); + sheet.replaceSync(text); + root.adoptedStyleSheets = [...root.adoptedStyleSheets, sheet]; + return; + } catch { + } + } + const style = doc.createElement("style"); + style.setAttribute("data-hairline-style", ""); + style.textContent = text; + (root.nodeType === 9 ? doc.head ?? doc.documentElement : root).appendChild(style); + } + return __toCommonJS(kernel_exports); +})(); +/* /hairline kernel */ diff --git a/cloudg/templates/vendor/iso-kit.js b/cloudg/templates/vendor/iso-kit.js new file mode 100644 index 0000000..45c297b --- /dev/null +++ b/cloudg/templates/vendor/iso-kit.js @@ -0,0 +1,684 @@ +/* Anatomy iso-kit (kit/iso-kit.mjs), MIT License, Copyright (c) 2026 Ryan. + https://github.com/wheresryan22/anatomy . The module source is unchanged except that + its 'export' keywords are removed, the two script tags inside one string are written + '\x3cscript' and '<\/script' (the same characters at run time) so the file can sit + inside a script element, and it is wrapped so its names live on IsoKit. */ +var IsoKit = (function () { +"use strict"; +const AZIMUTH = 45; +const ELEVATION = 30; +const SHADES = 4; +const RADIANS = Math.PI / 180; +const UNIT = Math.sqrt(1.6); +function cameraOf({ scale, azimuth = AZIMUTH, elevation = ELEVATION }) { + const a = azimuth * RADIANS; + const e = elevation * RADIANS; + return { sinA: Math.sin(a), cosA: Math.cos(a), sinE: Math.sin(e), cosE: Math.cos(e), k: scale * UNIT }; +} +function iso([x, y, z], projection) { + const { origin } = projection; + const { sinA, cosA, sinE, cosE, k } = cameraOf(projection); + return [origin[0] + (x * sinA - y * cosA) * k, origin[1] + ((x * cosA + y * sinA) * sinE - z * cosE) * k]; +} +function towardViewer(projection) { + const { sinA, cosA } = cameraOf(projection); + return [cosA, sinA]; +} +function depthOf([x, y, z], projection) { + const { sinA, cosA, sinE, cosE } = cameraOf(projection); + return (x * cosA + y * sinA) * cosE + z * sinE; +} +const tenth = (value) => { + const text = (Math.round(value * 10) / 10).toFixed(1).replace(/\.0$/, ""); + return text === "-0" ? "0" : text; +}; +function pathOf(points, closed = false) { + if (!points.length) + return ""; + return points.map(([x, y], at) => `${at ? "L" : "M"}${tenth(x)} ${tenth(y)}`).join("") + (closed ? "Z" : ""); +} +const joinPaths = (paths) => paths.filter(Boolean).join(""); +function roundedPlan({ x, y, w, d, r }, steps = 8) { + const radius = Math.max(0, Math.min(r, w / 2, d / 2)); + const corners = [ + [x + w - radius, y + radius, -90], + [x + w - radius, y + d - radius, 0], + [x + radius, y + d - radius, 90], + [x + radius, y + radius, 180], + ]; + const out = []; + for (const [cx, cy, start] of corners) { + const count = radius > 0 ? steps : 0; + for (let step = 0; step <= count; step++) { + const angle = (start + (count ? (90 * step) / count : 45)) * RADIANS; + out.push([cx + radius * Math.cos(angle), cy + radius * Math.sin(angle)]); + } + } + return out; +} +function signedArea(points) { + let area = 0; + for (let index = 0; index < points.length; index++) { + const [ax, ay] = points[index]; + const [bx, by] = points[(index + 1) % points.length]; + area += ax * by - bx * ay; + } + return area / 2; +} +const oriented = (points) => (signedArea(points) < 0 ? [...points].reverse() : points); +function convexHull(points) { + const sorted = [...points].sort((a, b) => a[0] - b[0] || a[1] - b[1]); + if (sorted.length < 3) + return sorted; + const cross = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lower = []; + for (const point of sorted) { + while (lower.length >= 2 && cross(lower[lower.length - 2], lower[lower.length - 1], point) <= 0) + lower.pop(); + lower.push(point); + } + const upper = []; + for (let index = sorted.length - 1; index >= 0; index--) { + const point = sorted[index]; + while (upper.length >= 2 && cross(upper[upper.length - 2], upper[upper.length - 1], point) <= 0) + upper.pop(); + upper.push(point); + } + return [...lower.slice(0, -1), ...upper.slice(0, -1)]; +} +function runsOf(flags, want) { + const count = flags.length; + const runs = []; + const start = flags.findIndex((flag, index) => flag !== flags[(index + count - 1) % count]); + if (start < 0) + return flags[0] === want ? [Array.from({ length: count + 1 }, (_, index) => index % count)] : []; + let run = null; + for (let step = 0; step < count; step++) { + const index = (start + step) % count; + if (flags[index] === want) { + if (!run) + run = [index]; + run.push((index + 1) % count); + } + else if (run) { + runs.push(run); + run = null; + } + } + if (run) + runs.push(run); + return runs; +} +function extrude(ring, z, height, projection, options = {}) { + const convex = options.convex ?? true; + const count = ring.length; + const counter = signedArea(ring) > 0; + const [vx, vy] = towardViewer(projection); + const { sinA, cosA } = cameraOf(projection); + const normals = ring.map(([ax, ay], index) => { + const [bx, by] = ring[(index + 1) % count]; + const dx = bx - ax; + const dy = by - ay; + const length = Math.hypot(dx, dy) || 1; + return (counter ? [dy / length, -dx / length] : [-dy / length, dx / length]); + }); + const facing = normals.map(([nx, ny]) => nx * vx + ny * vy > 1e-9); + const top = ring.map(([x, y]) => iso([x, y, z + height], projection)); + const bottom = ring.map(([x, y]) => iso([x, y, z], projection)); + const outline = []; + const crease = []; + for (const run of runsOf(facing, false)) + outline.push(pathOf(run.map((index) => top[index]))); + for (const run of runsOf(facing, true)) { + crease.push(pathOf(run.map((index) => top[index]))); + if (height > 0) + outline.push(pathOf(run.map((index) => bottom[index]))); + } + if (height > 0) { + for (let index = 0; index < count; index++) { + if (facing[(index + count - 1) % count] !== facing[index]) + outline.push(pathOf([top[index], bottom[index]])); + } + } + let fill; + if (convex) { + fill = pathOf(convexHull([...top, ...bottom]), true); + } + else { + const parts = [pathOf(oriented(top), true), pathOf(oriented(bottom), true)]; + for (let index = 0; index < count; index++) { + if (!facing[index]) + continue; + const next = (index + 1) % count; + parts.push(pathOf(oriented([top[index], top[next], bottom[next], bottom[index]]), true)); + } + fill = parts.join(""); + } + const buckets = Array.from({ length: SHADES }, () => []); + if (height > 0) { + const [lx, ly] = [-sinA, cosA]; + for (let index = 0; index < count; index++) { + if (!facing[index]) + continue; + const [nx, ny] = normals[index]; + const lit = (nx * lx + ny * ly + 1) / 2; + const shade = Math.min(SHADES - 1, Math.max(0, Math.floor(lit * SHADES))); + const next = (index + 1) % count; + buckets[shade].push(pathOf(oriented([top[index], top[next], bottom[next], bottom[index]]), true)); + } + } + let bevel; + if (options.bevel && options.bevel > 0) { + const by = options.bevel; + const inset = ring.map(([x, y], index) => { + const [ax, ay] = normals[(index + count - 1) % count]; + const [bx, by2] = normals[index]; + const mx = ax + bx; + const my = ay + by2; + const length = Math.hypot(mx, my) || 1; + return [x - (mx / length) * by, y - (my / length) * by]; + }); + const insetTop = inset.map(([x, y]) => iso([x, y, z + height], projection)); + bevel = runsOf(facing, true) + .map((run) => pathOf([top[run[0]], ...run.slice(1, -1).map((index) => insetTop[index]), top[run[run.length - 1]]])) + .join(""); + } + return { + fill, + outline: outline.join(""), + crease: crease.join(""), + top: pathOf(oriented(top), true), + shades: buckets.map((paths) => paths.join("")), + bevel, + }; +} +const ringOf = (plan, steps = 8) => roundedPlan(plan, steps); +function slabOf(plan, z, height, projection, steps = 8, bevel = 0) { + return extrude(ringOf(plan, steps), z, height, projection, { convex: true, bevel }); +} +function circleRing(cx, cy, radius, steps = 40) { + return Array.from({ length: steps }, (_, step) => { + const angle = (step / steps) * Math.PI * 2; + return [cx + radius * Math.cos(angle), cy + radius * Math.sin(angle)]; + }); +} +function cylinder(cx, cy, radius, z, height, projection, steps = 32, bevel = 0) { + return extrude(circleRing(cx, cy, radius, steps), z, height, projection, { convex: true, bevel }); +} +const insetPlan = ({ x, y, w, d, r }, by) => ({ x: x + by, y: y + by, w: w - by * 2, d: d - by * 2, r: Math.max(0, r - by) }); +function onTop(points, z, projection, closed = false) { + return pathOf(points.map(([x, y]) => iso([x, y, z], projection)), closed); +} +const planOutline = (plan, z, projection, steps = 8) => onTop(ringOf(plan, steps), z, projection, true); +const lineOnTop = (from, to, z, projection) => onTop([from, to], z, projection); +const segment = (from, to, projection) => pathOf([iso(from, projection), iso(to, projection)]); +const rise = (at, from, to, projection) => segment([at[0], at[1], from], [at[0], at[1], to], projection); +const ring = (cx, cy, radius, z, projection, steps = 40) => onTop(circleRing(cx, cy, radius, steps), z, projection, true); +const pointsAt = (points, projection) => points.map((point) => iso(point, projection)); +function dotGrid(x, y, columns, rows, gap, z, projection) { + const dots = []; + for (let row = 0; row < rows; row++) { + for (let column = 0; column < columns; column++) + dots.push(iso([x + column * gap, y + row * gap, z], projection)); + } + return dots; +} +function corners({ x, y, w, d }, inset = 0) { + return [ + [x + inset, y + inset], + [x + w - inset, y + inset], + [x + inset, y + d - inset], + [x + w - inset, y + d - inset], + ]; +} +function boxCorners({ x, y, w, d }, z0, z1) { + const out = []; + for (const px of [x, x + w]) + for (const py of [y, y + d]) + for (const z of [z0, z1]) + out.push([px, py, z]); + return out; +} +function knurl(cx, cy, radius, z0, z1, count, projection, inset = 0.4) { + const [vx, vy] = towardViewer(projection); + const lines = []; + for (let index = 0; index < count; index++) { + const angle = (index / count) * Math.PI * 2; + const nx = Math.cos(angle); + const ny = Math.sin(angle); + if (nx * vx + ny * vy < 0.15) + continue; + lines.push(segment([cx + radius * nx, cy + radius * ny, z0 + inset], [cx + radius * nx, cy + radius * ny, z1 - inset], projection)); + } + return lines.join(""); +} +function sideArc(cx, cy, radius, z, projection, steps = 24) { + const [vx, vy] = towardViewer(projection); + const facing = Math.atan2(vy, vx); + const points = []; + for (let step = 0; step <= steps; step++) { + const angle = facing - Math.PI / 2 + (step / steps) * Math.PI; + points.push(iso([cx + radius * Math.cos(angle), cy + radius * Math.sin(angle), z], projection)); + } + return pathOf(points); +} +function ringSeam(ring, z, projection) { + const count = ring.length; + const counter = signedArea(ring) > 0; + const [vx, vy] = towardViewer(projection); + const facing = ring.map(([ax, ay], index) => { + const [bx, by] = ring[(index + 1) % count]; + const dx = bx - ax; + const dy = by - ay; + const [nx, ny] = counter ? [dy, -dx] : [-dy, dx]; + return nx * vx + ny * vy > 1e-9; + }); + return runsOf(facing, true).map((run) => pathOf(run.map((index) => iso([ring[index][0], ring[index][1], z], projection)))).join(""); +} +const sideSeam = (plan, z, projection, steps = 8) => ringSeam(ringOf(plan, steps), z, projection); +function sideRing(center, radius, axis, projection, steps = 20) { + const [px, py, pz] = center; + const points = []; + for (let step = 0; step < steps; step++) { + const angle = (step / steps) * Math.PI * 2; + const u = radius * Math.cos(angle); + const v = radius * Math.sin(angle); + points.push(iso(axis === "x" ? [px, py + u, pz + v] : [px + u, py, pz + v], projection)); + } + return pathOf(points, true); +} +function coil(from, to, center, radius, turns, axis, projection, perTurn = 16) { + const count = Math.max(1, Math.round(turns * perTurn)); + const back = []; + const front = []; + let run = []; + let near = null; + for (let index = 0; index <= count; index++) { + const share = index / count; + const angle = share * turns * Math.PI * 2; + const u = radius * Math.cos(angle); + const v = radius * Math.sin(angle); + const along = from + (to - from) * share; + const point = axis === "x" ? [along, center[0] + u, center[1] + v] : [center[0] + u, along, center[1] + v]; + const normal = axis === "x" ? [0, Math.cos(angle), Math.sin(angle)] : [Math.cos(angle), 0, Math.sin(angle)]; + const facing = depthOf(normal, projection) > 0; + const flat = iso(point, projection); + if (near === null) + near = facing; + if (facing !== near) { + run.push(flat); + (near ? front : back).push(pathOf(run)); + run = [flat]; + near = facing; + continue; + } + run.push(flat); + } + if (run.length > 1) + (near ? front : back).push(pathOf(run)); + return { back: back.join(""), front: front.join("") }; +} +function topTicks(from, to, step, every, edge, z, lengths, projection, axis = "x") { + const minor = []; + const major = []; + const first = Math.ceil(from / step) * step; + for (let at = first, index = Math.round(first / step); at <= to + 1e-9; at += step, index++) { + const big = index % every === 0; + const length = big ? lengths[1] : lengths[0]; + const a = axis === "x" ? [at, edge] : [edge, at]; + const b = axis === "x" ? [at, edge + length] : [edge + length, at]; + (big ? major : minor).push(lineOnTop(a, b, z, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +function sideTicks(from, to, step, every, face, top, lengths, projection, axis = "x") { + const minor = []; + const major = []; + const first = Math.ceil(from / step) * step; + for (let at = first, index = Math.round(first / step); at <= to + 1e-9; at += step, index++) { + const big = index % every === 0; + const length = big ? lengths[1] : lengths[0]; + const a = axis === "x" ? [at, face, top] : [face, at, top]; + const b = axis === "x" ? [at, face, top - length] : [face, at, top - length]; + (big ? major : minor).push(segment(a, b, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +function radialTicks(cx, cy, radius, count, every, z, lengths, projection, from = 0, span = Math.PI * 2) { + const minor = []; + const major = []; + const closed = Math.abs(span - Math.PI * 2) < 1e-9; + const steps = closed ? count : count + 1; + for (let index = 0; index < steps; index++) { + const angle = from + (index / count) * span; + const big = index % every === 0; + const inner = radius - (big ? lengths[1] : lengths[0]); + const sx = Math.sin(angle); + const sy = -Math.cos(angle); + (big ? major : minor).push(lineOnTop([cx + sx * inner, cy + sy * inner], [cx + sx * radius, cy + sy * radius], z, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +const haloOf = (plan, z, height, projection) => slabOf(plan, z, height, projection).fill; +function axisVector(projection, [dx, dy, dz]) { + const origin = iso([0, 0, 0], projection); + const moved = iso([dx, dy, dz], projection); + return [moved[0] - origin[0], moved[1] - origin[1]]; +} +const fine = (value) => Math.round(value * 100) / 100; +function translateAlong(projection, by) { + const [x, y] = axisVector(projection, by); + return `translate(${fine(x)} ${fine(y)})`; +} +const thousandth = (value) => Math.round(value * 1000) / 1000; +function axesAt(at, z, projection) { + const origin = iso([at[0], at[1], z], projection); + const along = (point) => { + const [px, py] = iso(point, projection); + return [px - origin[0], py - origin[1]]; + }; + return { origin, x: along([at[0] + 1, at[1], z]), y: along([at[0], at[1] + 1, z]), z: along([at[0], at[1], z + 1]) }; +} +const matrixOf = (u, v, origin) => `matrix(${thousandth(u[0])} ${thousandth(u[1])} ${thousandth(v[0])} ${thousandth(v[1])} ${thousandth(origin[0])} ${thousandth(origin[1])})`; +function holdStill(element, [dx, dy]) { + const text = dx || dy ? `translate(${-dx} ${-dy})` : ""; + element.querySelectorAll("[data-hold]").forEach((node) => (text ? node.setAttribute("transform", text) : node.removeAttribute("transform"))); +} +function topMatrix(at, z, projection, along = "x") { + const { origin, x, y } = axesAt(at, z, projection); + return along === "x" ? matrixOf(x, y, origin) : matrixOf([-y[0], -y[1]], x, origin); +} +function sideMatrix(at, z, projection, face) { + const { origin, x, y, z: up } = axesAt(at, z, projection); + const down = [-up[0], -up[1]]; + return face === "left" ? matrixOf(x, down, origin) : matrixOf([-y[0], -y[1]], down, origin); +} +function screenRise(height, projection) { + const { cosE, k } = cameraOf(projection); + return height * cosE * k; +} +function fitProjection(points, width, height, { pad = 24, headroom = 0, azimuth = AZIMUTH, elevation = ELEVATION } = {}) { + const unit = { origin: [0, 0], scale: 1, azimuth, elevation }; + const flat = points.map((point) => iso(point, unit)); + const xs = flat.map(([x]) => x); + const ys = flat.map(([, y]) => y); + const left = Math.min(...xs); + const right = Math.max(...xs); + const up = Math.min(...ys); + const down = Math.max(...ys); + const scale = Math.min((width - pad * 2) / (right - left), (height - pad * 2 - headroom) / (down - up)); + return { + origin: [width / 2 - ((left + right) / 2) * scale, headroom + (height - headroom) / 2 - ((up + down) / 2) * scale], + scale, + azimuth, + elevation, + }; +} +const escape = (text) => text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +const attrs = (pairs) => Object.entries(pairs) + .filter(([, value]) => value !== undefined && value !== false) + .map(([key, value]) => ` ${key}="${escape(String(value))}"`) + .join(""); +function solidSvg(paths, { tone = "hi", crease = "faint", halo = false, lit = false, flat = false, className, inner = "" } = {}) { + const parts = []; + if (halo) + parts.push(``); + parts.push(``); + if (!flat) { + paths.shades.forEach((d, shade) => { + if (d) + parts.push(``); + }); + parts.push(``); + } + if (crease !== "none" && paths.crease) + parts.push(``); + if (crease !== "none" && paths.bevel) + parts.push(``); + if (inner) + parts.push(inner); + parts.push(``); + return `${parts.join("")}`; +} +function lineSvg(d, { tone = "lo", dotted = false, dashed = false, flow = false, className, free } = {}) { + if (!d) + return ""; + return ``; +} +function dotsSvg(points, { size = 0.5, tone = "mid", pulse = false } = {}) { + const dots = points + .map(([x, y, alpha], index) => ``) + .join(""); + return `${dots}`; +} +function fadedSvg(list, style = {}) { + const groups = new Map(); + for (const entry of list) { + const [d, alpha] = Array.isArray(entry) ? entry : [entry.d, entry.alpha]; + if (!d || !(alpha > 0.02)) + continue; + const level = Math.min(1, Math.round(alpha * 20) / 20); + groups.set(level, (groups.get(level) ?? "") + d); + } + return [...groups].map(([alpha, d]) => (alpha >= 1 ? lineSvg(d, style) : `${lineSvg(d, style)}`)).join(""); +} +function fadeLineSvg(points, { fade = [0, 0], tone = "lo", className, stops = 6 } = {}) { + if (points.length < 2) + return ""; + const ease = (t) => t * t * (3 - 2 * t); + const hashOf = (text) => { + let hash = 2166136261; + for (let index = 0; index < text.length; index++) + hash = Math.imul(hash ^ text.charCodeAt(index), 16777619); + return (hash >>> 0).toString(36); + }; + const lengths = [0]; + for (let index = 1; index < points.length; index++) + lengths.push(lengths[index - 1] + Math.hypot(points[index][0] - points[index - 1][0], points[index][1] - points[index - 1][1])); + const at = (length) => { + let index = 1; + while (index < points.length - 1 && lengths[index] < length) + index++; + const span = lengths[index] - lengths[index - 1] || 1; + const t = Math.max(0, Math.min(1, (length - lengths[index - 1]) / span)); + return [points[index - 1][0] + (points[index][0] - points[index - 1][0]) * t, points[index - 1][1] + (points[index][1] - points[index - 1][1]) * t]; + }; + const pieceOf = (from, to) => [at(from), ...points.filter((_, index) => lengths[index] > from + 1e-6 && lengths[index] < to - 1e-6), at(to)]; + const total = lengths[lengths.length - 1]; + if (total < 1e-6) + return ""; + let [start, end] = Array.isArray(fade) ? fade : [fade, fade]; + if (start + end > total) { + const share = total / (start + end); + start *= share; + end *= share; + } + const paint = `var(--anatomy-${tone})`; + const ramp = (piece, rising) => { + const d = pathOf(piece); + const [x1, y1] = piece[0]; + const [x2, y2] = piece[piece.length - 1]; + const id = `iso-fade-${hashOf(`${d}|${tone}|${rising}`)}`; + const marks = Array.from({ length: stops + 1 }, (_, index) => { + const t = index / stops; + const alpha = ease(rising ? t : 1 - t); + return ``; + }).join(""); + const gradient = `${marks}`; + return `${gradient}`; + }; + const parts = []; + if (start > 0.05) + parts.push(ramp(pieceOf(0, start), true)); + if (total - end - start > 0.05) + parts.push(lineSvg(pathOf(pieceOf(start, total - end)), { tone, className })); + if (end > 0.05) + parts.push(ramp(pieceOf(total - end, total), false)); + return `${parts.join("")}`; +} +function wireSvg(from, to, { via = [], tone = "lo", dotted = false, dashed = false, flow = false, ends = "both", size = 1.2 } = {}) { + const d = pathOf([from, ...via, to]); + const end = ([x, y]) => ``; + return `${lineSvg(d, { tone, dotted, dashed, flow })}${ends === "both" || ends === "from" ? end(from) : ""}${ends === "both" || ends === "to" ? end(to) : ""}`; +} +function faceTextSvg(transform, text, { size = 3, tone = "mid", anchor = "start" } = {}) { + return `${escape(text)}`; +} +function groupSvg(children, extra = {}) { + return `${Array.isArray(children) ? children.join("") : children}`; +} +function figureSvg({ width, height, label, body, standalone = false, className }) { + const style = standalone ? `` : ""; + return `${style}${Array.isArray(body) ? body.join("") : body}`; +} +function swatchSvg(mark) { + if (mark === "raised" || mark === "flat" || mark === "lit") { + const body = mark === "flat" ? "M2 6.5 8 3.5 14 6.5 8 9.5Z" : "M2 4.5 8 1.5 14 4.5 14 6 8 9 2 6Z"; + const crease = mark === "flat" ? "" : ``; + return ``; + } + return ``; +} +function plateHtml({ fig, title, hint, readout = "", keys = [], caption, body }) { + const corner = (name, text, extra = "") => `${escape(text)}`; + const legend = keys.length || caption + ? `
    ${keys.length ? `${keys.map((key) => `${swatchSvg(key.mark)}${escape(key.label)}`).join("")}` : ""}${caption ? `${escape(caption)}` : ""}
    ` + : ""; + return `
    ${body}
    ${legend}
    `; +} +function pageHtml({ title, theme = "dark", body, script = "", width = 650 }) { + return `${escape(title)}
    ${body}
    ${script ? `\x3cscript type="module">${script.replace(/<\/(script)/gi, "<\\/$1")}<\/script>` : ""}`; +} +const PAGE_CSS = ` +*{box-sizing:border-box} +html{color-scheme:dark} +body[data-theme="light"]{color-scheme:light} +body{margin:0;background:var(--anatomy-page);color:var(--anatomy-text);font:14px/1.6 ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;-webkit-font-smoothing:antialiased} +.iso-page{width:min(__WIDTH__px,100% - 32px);margin:64px auto 96px} +`; +const ISO_CSS = ` +.iso{ + --anatomy-page:#0b0b0b;--anatomy-card:#131313;--anatomy-text:#a3a3a3;--anatomy-strong:#e6e6e6;--anatomy-muted:#6e6e6e; + --anatomy-paper:#131313;--anatomy-face:#1b1b1b;--anatomy-top:#1c1c1c; + --anatomy-shade-0:#0f0f0f;--anatomy-shade-1:#121212;--anatomy-shade-2:#151515;--anatomy-shade-3:#181818; + --anatomy-lit-top:#262626;--anatomy-lit-shade:#1d1d1d; + --anatomy-hi:#d4d4d4;--anatomy-mid:#6a6a6a;--anatomy-lo:#3e3e3e;--anatomy-faint:#2a2a2a;--anatomy-lit:#ffffff; + --anatomy-halo:#050505;--anatomy-halo-opacity:.9;--anatomy-dot:#e6e6e6; + --anatomy-red:#ff6b6b;--anatomy-green:#6be0a0;--anatomy-blue:#6b9bff; + --anatomy-weight:.6px;--anatomy-ease:cubic-bezier(.32,.72,0,1); + --anatomy-mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace; + --anatomy-lift:inset 0 1px 0 0 #ffffff0a,inset 0 0 0 1px #ffffff06,0 0 0 1px #00000026,0 1px 1px -.5px #00000014,0 3px 3px -1.5px #00000014,0 6px 6px -3px #00000014,0 12px 12px -6px #00000014; +} +.iso[data-theme="light"]{ + --anatomy-page:#f2f2ef;--anatomy-card:#fbfbfa;--anatomy-text:#57575a;--anatomy-strong:#18181a;--anatomy-muted:#8c8c90; + --anatomy-paper:#fbfbfa;--anatomy-face:#ffffff;--anatomy-top:#ffffff; + --anatomy-shade-0:#dcdcd9;--anatomy-shade-1:#e5e5e2;--anatomy-shade-2:#ededea;--anatomy-shade-3:#f4f4f1; + --anatomy-lit-top:#ffffff;--anatomy-lit-shade:#eeeeeb; + --anatomy-hi:#26262a;--anatomy-mid:#8a8a8f;--anatomy-lo:#b4b4b8;--anatomy-faint:#d4d4d6;--anatomy-lit:#000000; + --anatomy-halo:#8a8a80;--anatomy-halo-opacity:.32;--anatomy-dot:#3a3a3e; + --anatomy-red:#e5484d;--anatomy-green:#30a46c;--anatomy-blue:#3e63dd; + --anatomy-lift:inset 0 1px 0 0 #ffffff,0 0 0 1px #0000000f,0 1px 1px -.5px #0000000a,0 3px 3px -1.5px #0000000a,0 6px 6px -3px #0000000a,0 12px 12px -6px #0000000a; +} +.iso-svg{display:block;width:100%;height:auto;overflow:visible} +.iso-svg path,.iso-svg circle{vector-effect:non-scaling-stroke} +.iso-fill{fill:var(--anatomy-paper);stroke:none} +.iso-solid[data-lit]>.iso-fill{fill:var(--anatomy-face)} +.iso-shade,.iso-top{stroke:none} +.iso-shade[data-shade="0"]{fill:var(--anatomy-shade-0)} +.iso-shade[data-shade="1"]{fill:var(--anatomy-shade-1)} +.iso-shade[data-shade="2"]{fill:var(--anatomy-shade-2)} +.iso-shade[data-shade="3"]{fill:var(--anatomy-shade-3)} +.iso-top{fill:var(--anatomy-top)} +.iso-solid[data-lit]>.iso-top{fill:var(--anatomy-lit-top)} +.iso-solid[data-lit]>.iso-shade{fill:var(--anatomy-lit-shade)} +.iso-solid[data-seamless]>.iso-shade,.iso-solid[data-seamless]>.iso-top{stroke-width:.4;stroke-linejoin:round} +.iso-solid[data-seamless]>.iso-shade[data-shade="0"]{stroke:var(--anatomy-shade-0)} +.iso-solid[data-seamless]>.iso-shade[data-shade="1"]{stroke:var(--anatomy-shade-1)} +.iso-solid[data-seamless]>.iso-shade[data-shade="2"]{stroke:var(--anatomy-shade-2)} +.iso-solid[data-seamless]>.iso-shade[data-shade="3"]{stroke:var(--anatomy-shade-3)} +.iso-solid[data-seamless]>.iso-top{stroke:var(--anatomy-top)} +.iso-solid[data-seamless][data-lit]>.iso-top{stroke:var(--anatomy-lit-top)} +.iso-solid[data-seamless][data-lit]>.iso-shade{stroke:var(--anatomy-lit-shade)} +.iso-solid[data-lit]>.iso-edge{stroke:var(--anatomy-lit)} +.iso-halo{fill:var(--anatomy-halo);stroke:none;filter:blur(9px);opacity:var(--anatomy-halo-opacity);transform:translateY(3px)} +.iso-line{fill:none;stroke:var(--anatomy-lo);stroke-width:var(--anatomy-weight);stroke-linecap:round;stroke-linejoin:round} +.iso-line[data-tone="hi"]{stroke:var(--anatomy-hi)} +.iso-line[data-tone="mid"]{stroke:var(--anatomy-mid)} +.iso-line[data-tone="lo"]{stroke:var(--anatomy-lo)} +.iso-line[data-tone="faint"]{stroke:var(--anatomy-faint)} +.iso-line[data-tone="lit"]{stroke:var(--anatomy-lit)} +.iso-line[data-tone="red"]{stroke:var(--anatomy-red)} +.iso-line[data-tone="green"]{stroke:var(--anatomy-green)} +.iso-line[data-tone="blue"]{stroke:var(--anatomy-blue)} +.iso-line[data-dotted]{stroke-dasharray:.01 3.5;stroke-width:1} +.iso-line[data-dashed]{stroke-dasharray:3 3} +.iso-line[data-flow]{stroke-dasharray:2 6} +.iso-wire-end{fill:var(--anatomy-paper);stroke:var(--anatomy-lo);stroke-width:var(--anatomy-weight)} +.iso-wire[data-tone="hi"] .iso-wire-end{stroke:var(--anatomy-hi)} +.iso-wire[data-tone="mid"] .iso-wire-end{stroke:var(--anatomy-mid)} +.iso-wire[data-tone="faint"] .iso-wire-end{stroke:var(--anatomy-faint)} +.iso-wire[data-tone="lit"] .iso-wire-end{stroke:var(--anatomy-lit)} +.iso-dots circle{fill:var(--anatomy-dot);stroke:none;opacity:.32} +.iso-dots[data-tone="hi"] circle{opacity:.85} +.iso-dots[data-tone="mid"] circle{opacity:.45} +.iso-dots[data-tone="lo"] circle{opacity:.22} +.iso-dots[data-tone="faint"] circle{opacity:.12} +.iso-face-text{fill:var(--anatomy-mid);font-family:var(--anatomy-mono);letter-spacing:.04em} +.iso-face-text[data-tone="hi"]{fill:var(--anatomy-hi)} +.iso-face-text[data-tone="lo"]{fill:var(--anatomy-lo)} +.iso-face-text[data-tone="faint"]{fill:var(--anatomy-faint)} +.tb-body{fill:var(--anatomy-shade-2);stroke:none} +.tb-shine{fill:var(--anatomy-top);stroke:none} +.tb-shade{fill:var(--anatomy-shade-0);stroke:none} +.tb-ring{fill:none;stroke:var(--anatomy-lo);stroke-width:.45;stroke-linecap:butt;stroke-linejoin:round} +.tb-ring[data-tone="mid"]{stroke:var(--anatomy-mid)} +.tb-ring[data-tone="faint"]{stroke:var(--anatomy-faint)} +.tb-ring[data-tone="hi"]{stroke:var(--anatomy-hi)} +@keyframes iso-flow{to{stroke-dashoffset:-32}} +@keyframes iso-pulse{0%,100%{opacity:.18}12%{opacity:1}40%{opacity:.18}} +.iso-line[data-flow]{animation:iso-flow 1600ms linear infinite} +.iso-dots circle[data-pulse]{animation:iso-pulse 1600ms steps(1,end) infinite;animation-delay:var(--pulse-at,0ms)} +.iso-figure{margin:32px 0 36px} +.iso-plate{position:relative;padding:52px 24px;border-radius:0;background:var(--anatomy-card);box-shadow:0 0 0 1px var(--anatomy-mid);--anatomy-paper:var(--anatomy-card)} +.iso-plate-corners{position:absolute;inset:18px 20px;pointer-events:none} +.iso-plate-corner{position:absolute;color:var(--anatomy-muted);font-family:var(--anatomy-mono);font-size:10.5px;line-height:1;letter-spacing:.04em;white-space:nowrap} +.iso-plate-corner[data-corner="fig"]{top:0;left:0;color:var(--anatomy-text)} +.iso-plate-corner[data-corner="title"]{top:0;right:0;text-transform:uppercase;letter-spacing:.1em} +.iso-plate-corner[data-corner="hint"]{bottom:0;left:0;text-transform:uppercase;letter-spacing:.1em} +.iso-plate-corner[data-corner="readout"]{right:0;bottom:0;color:var(--anatomy-text);font-variant-numeric:tabular-nums} +.iso-plate .iso-svg{margin:0 auto} +.iso-legend{display:grid;gap:8px;margin-top:12px;color:var(--anatomy-muted);font-size:13px;line-height:1.55} +.iso-keys{display:flex;flex-wrap:wrap;gap:6px 18px} +.iso-key{display:inline-flex;align-items:center;gap:7px} +.iso-swatch{width:16px;height:10px;flex:none;overflow:visible} +.iso-swatch path{fill:none;stroke:var(--anatomy-hi);stroke-width:1;vector-effect:non-scaling-stroke;stroke-linejoin:round;stroke-linecap:round} +.iso-swatch[data-mark="raised"] path,.iso-swatch[data-mark="flat"] path{stroke:var(--anatomy-mid)} +.iso-swatch[data-mark="lit"] path{stroke:var(--anatomy-lit)} +.iso-swatch[data-mark="dim"] path{stroke:var(--anatomy-lo)} +.iso-swatch[data-mark="dotted"] path{stroke-dasharray:.01 3;stroke-width:1.4} +.iso-swatch[data-mark="dashed"] path{stroke-dasharray:3 2.5} +.iso-swatch[data-mark="red"] path{stroke:var(--anatomy-red)} +.iso-swatch[data-mark="green"] path{stroke:var(--anatomy-green)} +.iso-swatch[data-mark="blue"] path{stroke:var(--anatomy-blue)} +.iso-swatch-crease{stroke:var(--anatomy-lo)!important} +@media (max-width:560px){.iso-plate{padding:48px 12px 70px}.iso-plate-corner[data-corner="hint"]{bottom:18px}} +@media (prefers-reduced-motion:reduce){.iso-line[data-flow],.iso-dots circle[data-pulse]{animation:none}} +`; + +return { AZIMUTH, ELEVATION, SHADES, cameraOf, iso, towardViewer, depthOf, pathOf, joinPaths, roundedPlan, extrude, ringOf, slabOf, circleRing, cylinder, insetPlan, onTop, planOutline, lineOnTop, segment, rise, ring, pointsAt, dotGrid, corners, boxCorners, knurl, sideArc, ringSeam, sideSeam, sideRing, coil, topTicks, sideTicks, radialTicks, haloOf, axisVector, translateAlong, holdStill, topMatrix, sideMatrix, screenRise, fitProjection, solidSvg, lineSvg, dotsSvg, fadedSvg, fadeLineSvg, wireSvg, faceTextSvg, groupSvg, figureSvg, plateHtml, pageHtml, ISO_CSS }; +})(); diff --git a/site/static/sample-report.html b/site/static/sample-report.html index 66c8e4b..e591c29 100644 --- a/site/static/sample-report.html +++ b/site/static/sample-report.html @@ -3,207 +3,298 @@ - CloudG Security Report - + + + cloudg security report · N/A @@ -232,130 +323,1755 @@ - - - -
    - -
    -
    -

    ☁️ CloudG Security Report

    -

    Scan: e9c53ac7-288b-41b2-8756-55f288a68a46 · Provider: Multi-Cloud · Account: N/A · 2026-10-21 02:48:54.246095

    -
    - -
    - -
    -
    Total Assets
    44
    -
    Total Findings
    12
    -
    Critical
    2
    -
    High
    5
    -
    Medium
    2
    -
    Low
    2
    -
    + + + + + + + - -
    -
    -
    - - - - +
    +
    + + cloudgSecurity report +
    +
    + + + + +
    + +
    +
    +
    +
    +
    Security report · Multi-Cloud
    +

    Estate posture

    +
    +

    12 findings across 44 assets.

    +
    + Scan a99507d7-b124-42a8-bb1d-73e3c8a7a83d + Account: N/A + Providers AWS, AZURE, GCP + Regions 4 + Started 2026-10-25 07:28:22.974673 +
    +
    +
    Assets
    44
    56 relations
    +
    Findings
    12
    5 sources
    +
    Critical
    2
    High 5
    +
    Exposed
    6
    reachable from the internet
    -
    -
    Legend
    -
    Account
    -
    Region
    -
    VPC
    -
    Subnet
    -
    Compute
    -
    Storage
    -
    Security
    -
    Other
    -
    Has Findings
    -
    Internet Exposed
    -
    Ontology Entities
    -
    Security Finding (▲)
    -
    Compliance Framework (◆)
    -
    ─ ─ FINDING_AFFECTS
    -
    ─ ─ COMPLIANCE_GOVERNS
    +
    + +
    -
    +
    +
    + + + + 0% + +
    +

    Assets with no open findings

    +

    34 of 44 assets have nothing open against them. The rest carry at least one finding.

    +
    +
    +
    +
    Top risks
    +
      +
      +
      +
      + + + + +
      +
      +
      +
      + + +
      + + + + + + + + + + + +
      +
      +
      +
      + Fig 1 + Estate map + Drag to pan · scroll to zoom · click an asset + rest + +
      +
      + +
      +
      +
      Each provider is a base plate. Accounts and regions stand on it as platforms, networks as slabs and subnets as tiles; every asset stands where it was found. Identity and governance assets sit on the control deck behind and above the plate. Trace lines follow each relation from its source port to its target port on a lane of their own, and a beacon above an asset shows its worst open finding.
      +
      +
      -
      -
      - + - + - + +
      - - - - - - - - - - - -
      SeverityTitleResourceSourceRisk Score
      -
      +
      + + + + + +
      SeverityFindingResourceSourceRisk
      + +
      + + + +
      +
      + + + +
      +
      + + + + + +
      AssetTypeProviderRegionFindings
      +
      +
      -
      +
      -
      -

      Severity Distribution

      - -
      -
      -

      Resource Types

      - -
      +

      Severity distribution 12 findings

      +

      Resource types top 12

      -
      -

      Findings by Source

      - -
      -
      -

      Compliance Overview

      - -
      +

      Findings by source 5 sources

      +

      Compliance overview pass and fail

      -
      + -
      +
      -
      -
      + + - -
      - +
      + Generated by cloudg 0.6.0 · scan a99507d7-b124-42a8-bb1d-73e3c8a7a83d + Map drawn with Anatomy and Hairline (MIT) · charts by Chart.js (MIT) and D3 (ISC) +
      + + +
      + @@ -364,29 +2080,32 @@

      Compliance Overview

      const graphData = {"nodes": [{"id": "org", "name": "sample-org", "type": "ORGANIZATION", "provider": "AWS", "region": "global", "arn": "arn:aws:organizations::111111111111:organization/o-sample", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "ou-workloads", "name": "Workloads", "type": "ORG_UNIT", "provider": "AWS", "region": "global", "arn": "arn:aws:organizations::111111111111:ou/o-sample/ou-work", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "acct-prod", "name": "prod", "type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::111111111111:root", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "acct-shared", "name": "shared-services", "type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::222222222222:root", "account_id": "222222222222", "is_internet_exposed": false, "is_external": false}, {"id": "acct-vendor", "name": "vendor-co", "type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::999999999999:root", "account_id": "999999999999", "is_internet_exposed": false, "is_external": false}, {"id": "scp-deny-regions", "name": "deny-unapproved-regions", "type": "ORG_POLICY", "provider": "AWS", "region": "global", "arn": "arn:aws:organizations::111111111111:policy/o-sample/service_control_policy/p-1", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "vpc-prod", "name": "prod-vpc", "type": "VPC", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:vpc/vpc-0prod", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "subnet-public", "name": "prod-public-a", "type": "SUBNET", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0pub", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "subnet-private", "name": "prod-private-a", "type": "SUBNET", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0priv", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "sg-web", "name": "sg-web", "type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "sg-admin", "name": "sg-admin", "type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "sg-app", "name": "sg-app", "type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0app", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "sg-db", "name": "sg-db", "type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0db", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "alb-web", "name": "web-alb", "type": "LOAD_BALANCER", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:loadbalancer/app/web-alb/abc", "account_id": "111111111111", "is_internet_exposed": true, "is_external": false}, {"id": "tg-web", "name": "web-tg", "type": "TARGET_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:targetgroup/web-tg/def", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "web-1", "name": "web-1", "type": "EC2", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web1", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "web-2", "name": "web-2", "type": "EC2", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web2", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "bastion", "name": "bastion", "type": "EC2", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0bast", "account_id": "111111111111", "is_internet_exposed": true, "is_external": false}, {"id": "orders-db", "name": "orders-db", "type": "RDS_INSTANCE", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:rds:us-east-1:111111111111:db:orders-db", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "data-bucket", "name": "prod-data", "type": "S3_BUCKET", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:s3:::prod-data", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "logs-bucket", "name": "prod-logs", "type": "S3_BUCKET", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:s3:::prod-logs", "account_id": "111111111111", "is_internet_exposed": true, "is_external": false}, {"id": "kms-main", "name": "prod-main-key", "type": "KMS_KEY", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:kms:us-east-1:111111111111:key/1111-2222", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "db-secret", "name": "orders-db-credentials", "type": "SECRET", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:secretsmanager:us-east-1:111111111111:secret:orders-db-credentials-AbC", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "app-role", "name": "app-role", "type": "IAM_ROLE", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::111111111111:role/app-role", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "admin-role", "name": "bastion-admin", "type": "IAM_ROLE", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::111111111111:role/bastion-admin", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "lambda-role", "name": "api-lambda-role", "type": "IAM_ROLE", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::111111111111:role/api-lambda-role", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "api-gw", "name": "public-api", "type": "API_GATEWAY", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:apigateway:us-east-1::/restapis/a1b2c3", "account_id": "111111111111", "is_internet_exposed": true, "is_external": false}, {"id": "api-handler", "name": "api-handler", "type": "LAMBDA_FUNCTION", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:lambda:us-east-1:111111111111:function:api-handler", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "log-group", "name": "/aws/lambda/api-handler", "type": "LOG_GROUP", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:logs:us-east-1:111111111111:log-group:/aws/lambda/api-handler", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "waf-web", "name": "web-acl", "type": "WAF_WEB_ACL", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:wafv2:us-east-1:111111111111:regional/webacl/web-acl/1", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "guardduty", "name": "guardduty-detector", "type": "THREAT_DETECTOR", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:guardduty:us-east-1:111111111111:detector/d1", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "securityhub", "name": "security-hub", "type": "SECURITY_HUB", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:securityhub:us-east-1:111111111111:hub/default", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "inspector", "name": "inspector", "type": "VULNERABILITY_SCANNER", "provider": "AWS", "region": "us-east-1", "arn": "arn:aws:inspector2:us-east-1:111111111111:inspector", "account_id": "111111111111", "is_internet_exposed": false, "is_external": false}, {"id": "deploy-role", "name": "deploy-role", "type": "IAM_ROLE", "provider": "AWS", "region": "global", "arn": "arn:aws:iam::222222222222:role/deploy-role", "account_id": "222222222222", "is_internet_exposed": false, "is_external": false}, {"id": "artifacts-bucket", "name": "shared-artifacts", "type": "S3_BUCKET", "provider": "AWS", "region": "eu-west-1", "arn": "arn:aws:s3:::shared-artifacts", "account_id": "222222222222", "is_internet_exposed": false, "is_external": false}, {"id": "ecr-repo", "name": "app-images", "type": "CONTAINER_REGISTRY", "provider": "AWS", "region": "eu-west-1", "arn": "arn:aws:ecr:eu-west-1:222222222222:repository/app-images", "account_id": "222222222222", "is_internet_exposed": false, "is_external": false}, {"id": "az-vnet", "name": "hub-vnet", "type": "VNET", "provider": "AZURE", "region": "westeurope", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/virtualNetworks/hub-vnet", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "is_internet_exposed": false, "is_external": false}, {"id": "az-nsg", "name": "jump-nsg", "type": "NSG", "provider": "AZURE", "region": "westeurope", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/networkSecurityGroups/jump-nsg", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "is_internet_exposed": false, "is_external": false}, {"id": "az-vm", "name": "jump-vm", "type": "VIRTUAL_MACHINE", "provider": "AZURE", "region": "westeurope", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Compute/virtualMachines/jump-vm", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "is_internet_exposed": true, "is_external": false}, {"id": "az-kv", "name": "hub-kv", "type": "KEY_VAULT", "provider": "AZURE", "region": "westeurope", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.KeyVault/vaults/hub-kv", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "is_internet_exposed": false, "is_external": false}, {"id": "az-blob", "name": "archive", "type": "BLOB_STORAGE", "provider": "AZURE", "region": "westeurope", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Storage/storageAccounts/archive", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "is_internet_exposed": false, "is_external": false}, {"id": "gcp-gcs", "name": "raw-uploads", "type": "GCS_BUCKET", "provider": "GCP", "region": "europe-west1", "arn": "//storage.googleapis.com/projects/_/buckets/raw-uploads", "account_id": "proj-analytics", "is_internet_exposed": true, "is_external": false}, {"id": "gcp-bq", "name": "analytics", "type": "DATA_WAREHOUSE", "provider": "GCP", "region": "europe-west1", "arn": "//bigquery.googleapis.com/projects/proj-analytics/datasets/analytics", "account_id": "proj-analytics", "is_internet_exposed": false, "is_external": false}, {"id": "gcp-vm", "name": "worker-1", "type": "GCE_INSTANCE", "provider": "GCP", "region": "europe-west1", "arn": "//compute.googleapis.com/projects/proj-analytics/zones/europe-west1-b/instances/worker-1", "account_id": "proj-analytics", "is_internet_exposed": false, "is_external": false}, {"id": "0.0.0.0/0", "name": "0.0.0.0/0", "type": "EXTERNAL", "provider": "EXTERNAL", "region": "", "arn": "", "account_id": "", "is_internet_exposed": false, "is_external": true}, {"id": "finding_f-ssh-open", "name": "Security group allows SSH (22) from 0.0.0.0/0", "type": "SECURITY_FINDING", "severity": "CRITICAL", "source_tool": "prowler", "risk_score": 9.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "compliance_CIS-AWS", "name": "CIS-AWS", "type": "COMPLIANCE_FRAMEWORK", "severity": "", "source_tool": "", "risk_score": 0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "compliance_PCI-DSS", "name": "PCI-DSS", "type": "COMPLIANCE_FRAMEWORK", "severity": "", "source_tool": "", "risk_score": 0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-bastion-imds", "name": "EC2 instance allows IMDSv1", "type": "SECURITY_FINDING", "severity": "HIGH", "source_tool": "prowler", "risk_score": 7.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-db-backup", "name": "RDS backup retention below 7 days", "type": "SECURITY_FINDING", "severity": "MEDIUM", "source_tool": "prowler", "risk_score": 5.0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "compliance_NIST-800-53", "name": "NIST-800-53", "type": "COMPLIANCE_FRAMEWORK", "severity": "", "source_tool": "", "risk_score": 0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-logs-public", "name": "S3 bucket allows public read", "type": "SECURITY_FINDING", "severity": "HIGH", "source_tool": "scoutsuite", "risk_score": 7.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-web-cve", "name": "CVE-2024-1234 in openssl 3.0.1", "type": "SECURITY_FINDING", "severity": "HIGH", "source_tool": "trivy", "risk_score": 7.8, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-lambda-runtime", "name": "Lambda runtime python3.8 is deprecated", "type": "SECURITY_FINDING", "severity": "LOW", "source_tool": "checkov", "risk_score": 2.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-deploy-trust", "name": "Role trusts an external account without ExternalId", "type": "SECURITY_FINDING", "severity": "HIGH", "source_tool": "iam", "risk_score": 7.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-rdp-open", "name": "NSG allows RDP (3389) from Internet", "type": "SECURITY_FINDING", "severity": "CRITICAL", "source_tool": "scoutsuite", "risk_score": 9.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "compliance_CIS-Azure", "name": "CIS-Azure", "type": "COMPLIANCE_FRAMEWORK", "severity": "", "source_tool": "", "risk_score": 0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-gcs-public", "name": "Bucket is publicly readable (allUsers)", "type": "SECURITY_FINDING", "severity": "HIGH", "source_tool": "prowler", "risk_score": 7.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "compliance_CIS-GCP", "name": "CIS-GCP", "type": "COMPLIANCE_FRAMEWORK", "severity": "", "source_tool": "", "risk_score": 0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-default-vpc", "name": "VPC flow logs not enabled", "type": "SECURITY_FINDING", "severity": "INFO", "source_tool": "prowler", "risk_score": 0.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-ghost", "name": "Bucket without lifecycle policy", "type": "SECURITY_FINDING", "severity": "MEDIUM", "source_tool": "scoutsuite", "risk_score": 5.0, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}, {"id": "finding_f-versioning", "name": "S3 versioning disabled", "type": "SECURITY_FINDING", "severity": "LOW", "source_tool": "prowler", "risk_score": 2.5, "is_external": false, "is_internet_exposed": false, "region": "", "arn": ""}], "links": [{"source": "org", "target": "ou-workloads", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "ou-workloads", "target": "acct-prod", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "ou-workloads", "target": "acct-shared", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "acct-prod", "target": "vpc-prod", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "acct-vendor", "target": "deploy-role", "type": "IAM_TRUST", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "CROSS_ACCOUNT_TRUST", "description": ""}, {"source": "scp-deny-regions", "target": "ou-workloads", "type": "GOVERNS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "vpc-prod", "target": "subnet-public", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "vpc-prod", "target": "subnet-private", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "subnet-public", "target": "alb-web", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "subnet-public", "target": "bastion", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "subnet-private", "target": "web-1", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "subnet-private", "target": "web-2", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "subnet-private", "target": "orders-db", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "sg-app", "target": "sg-db", "type": "SECURITY_GROUP_RULE", "port_range": "5432", "protocol": "TCP", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "alb-web", "target": "sg-web", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "alb-web", "target": "tg-web", "type": "LOAD_BALANCER_TARGET", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "tg-web", "target": "web-1", "type": "LOAD_BALANCER_TARGET", "port_range": "8080", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "tg-web", "target": "web-2", "type": "LOAD_BALANCER_TARGET", "port_range": "8080", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "web-1", "target": "sg-app", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "web-1", "target": "app-role", "type": "ASSUMES_ROLE", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "web-2", "target": "sg-app", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "web-2", "target": "app-role", "type": "ASSUMES_ROLE", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "bastion", "target": "sg-admin", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "bastion", "target": "admin-role", "type": "ASSUMES_ROLE", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "orders-db", "target": "sg-db", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "orders-db", "target": "kms-main", "type": "REFERENCES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "ENCRYPTED_BY_KMS", "description": ""}, {"source": "data-bucket", "target": "api-handler", "type": "INVOKES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "TRIGGERED_BY", "description": ""}, {"source": "data-bucket", "target": "kms-main", "type": "REFERENCES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "ENCRYPTED_BY_KMS", "description": ""}, {"source": "db-secret", "target": "kms-main", "type": "REFERENCES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "ENCRYPTED_BY_KMS", "description": ""}, {"source": "app-role", "target": "data-bucket", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "READS_FROM", "description": ""}, {"source": "app-role", "target": "orders-db", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "app-role", "target": "deploy-role", "type": "IAM_TRUST", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "CROSS_ACCOUNT_TRUST", "description": ""}, {"source": "admin-role", "target": "db-secret", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "lambda-role", "target": "db-secret", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "api-gw", "target": "api-handler", "type": "INVOKES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "api-handler", "target": "lambda-role", "type": "ASSUMES_ROLE", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "api-handler", "target": "log-group", "type": "LOGS_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "api-handler", "target": "kms-main", "type": "REFERENCES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "ENCRYPTED_BY_KMS", "description": ""}, {"source": "api-handler", "target": "ecr-repo", "type": "USES_IMAGE", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "waf-web", "target": "alb-web", "type": "PROTECTS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "inspector", "target": "web-1", "type": "MONITORS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "deploy-role", "target": "artifacts-bucket", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "deploy-role", "target": "ecr-repo", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "az-vnet", "target": "az-vm", "type": "CONTAINS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "az-vm", "target": "az-nsg", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "az-vm", "target": "az-kv", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "az-vm", "target": "az-blob", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "gcp-vm", "target": "gcp-gcs", "type": "GRANTS_ACCESS", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "gcp-vm", "target": "gcp-bq", "type": "REFERENCES", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "", "description": ""}, {"source": "0.0.0.0/0", "target": "sg-web", "type": "SECURITY_GROUP_RULE", "port_range": "80,443", "protocol": "TCP", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "", "description": ""}, {"source": "0.0.0.0/0", "target": "sg-admin", "type": "SECURITY_GROUP_RULE", "port_range": "22", "protocol": "TCP", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "", "description": ""}, {"source": "0.0.0.0/0", "target": "alb-web", "type": "INTERNET_EXPOSED", "port_range": "443", "protocol": "", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "INTERNET_REACHABLE", "description": ""}, {"source": "0.0.0.0/0", "target": "api-gw", "type": "INTERNET_EXPOSED", "port_range": "443", "protocol": "", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "INTERNET_REACHABLE", "description": ""}, {"source": "0.0.0.0/0", "target": "logs-bucket", "type": "INTERNET_EXPOSED", "port_range": "", "protocol": "", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "INTERNET_REACHABLE", "description": "bucket policy Principal *"}, {"source": "0.0.0.0/0", "target": "az-nsg", "type": "SECURITY_GROUP_RULE", "port_range": "3389", "protocol": "TCP", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "", "description": ""}, {"source": "0.0.0.0/0", "target": "gcp-gcs", "type": "INTERNET_EXPOSED", "port_range": "", "protocol": "", "cidr": "0.0.0.0/0", "direction": "ingress", "relationship": "INTERNET_REACHABLE", "description": ""}, {"source": "finding_f-ssh-open", "target": "sg-admin", "relation": "FINDING_AFFECTS", "severity": "CRITICAL"}, {"source": "compliance_CIS-AWS", "target": "finding_f-ssh-open", "relation": "COMPLIANCE_GOVERNS"}, {"source": "compliance_PCI-DSS", "target": "finding_f-ssh-open", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-bastion-imds", "target": "bastion", "relation": "FINDING_AFFECTS", "severity": "HIGH"}, {"source": "compliance_CIS-AWS", "target": "finding_f-bastion-imds", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-db-backup", "target": "orders-db", "relation": "FINDING_AFFECTS", "severity": "MEDIUM"}, {"source": "compliance_CIS-AWS", "target": "finding_f-db-backup", "relation": "COMPLIANCE_GOVERNS"}, {"source": "compliance_NIST-800-53", "target": "finding_f-db-backup", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-logs-public", "target": "logs-bucket", "relation": "FINDING_AFFECTS", "severity": "HIGH"}, {"source": "compliance_CIS-AWS", "target": "finding_f-logs-public", "relation": "COMPLIANCE_GOVERNS"}, {"source": "compliance_PCI-DSS", "target": "finding_f-logs-public", "relation": "COMPLIANCE_GOVERNS"}, {"source": "compliance_NIST-800-53", "target": "finding_f-logs-public", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-web-cve", "target": "web-1", "relation": "FINDING_AFFECTS", "severity": "HIGH"}, {"source": "compliance_PCI-DSS", "target": "finding_f-web-cve", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-lambda-runtime", "target": "api-handler", "relation": "FINDING_AFFECTS", "severity": "LOW"}, {"source": "finding_f-deploy-trust", "target": "deploy-role", "relation": "FINDING_AFFECTS", "severity": "HIGH"}, {"source": "compliance_CIS-AWS", "target": "finding_f-deploy-trust", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-rdp-open", "target": "az-nsg", "relation": "FINDING_AFFECTS", "severity": "CRITICAL"}, {"source": "compliance_CIS-Azure", "target": "finding_f-rdp-open", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-gcs-public", "target": "gcp-gcs", "relation": "FINDING_AFFECTS", "severity": "HIGH"}, {"source": "compliance_CIS-GCP", "target": "finding_f-gcs-public", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-default-vpc", "target": "vpc-prod", "relation": "FINDING_AFFECTS", "severity": "INFO"}, {"source": "compliance_CIS-AWS", "target": "finding_f-ghost", "relation": "COMPLIANCE_GOVERNS"}, {"source": "finding_f-versioning", "target": "data-bucket", "relation": "FINDING_AFFECTS", "severity": "LOW"}, {"source": "compliance_CIS-AWS", "target": "finding_f-versioning", "relation": "COMPLIANCE_GOVERNS"}]}; const hierarchyData = {"name": "Cloud Account", "type": "account", "children": [{"name": "us-east-1", "type": "region", "children": [{"name": "_no_vpc", "type": "vpc", "children": [{"name": "prod-private-a", "type": "subnet", "children": [{"id": "subnet-private", "name": "prod-private-a", "type": "SUBNET", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0priv", "internet_exposed": false, "findings": {}}]}, {"name": "prod-public-a", "type": "subnet", "children": [{"id": "subnet-public", "name": "prod-public-a", "type": "SUBNET", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0pub", "internet_exposed": false, "findings": {}}]}]}, {"name": "prod-vpc", "type": "vpc", "children": []}]}, {"name": "westeurope", "type": "region", "children": [{"name": "hub-vnet", "type": "vpc", "children": []}]}, {"name": "Global / Ungrouped", "type": "global", "children": [{"id": "org", "name": "sample-org", "type": "ORGANIZATION", "arn": "arn:aws:organizations::111111111111:organization/o-sample", "internet_exposed": false, "findings": {}}, {"id": "ou-workloads", "name": "Workloads", "type": "ORG_UNIT", "arn": "arn:aws:organizations::111111111111:ou/o-sample/ou-work", "internet_exposed": false, "findings": {}}, {"id": "acct-prod", "name": "prod", "type": "CLOUD_ACCOUNT", "arn": "arn:aws:iam::111111111111:root", "internet_exposed": false, "findings": {}}, {"id": "acct-shared", "name": "shared-services", "type": "CLOUD_ACCOUNT", "arn": "arn:aws:iam::222222222222:root", "internet_exposed": false, "findings": {}}, {"id": "acct-vendor", "name": "vendor-co", "type": "CLOUD_ACCOUNT", "arn": "arn:aws:iam::999999999999:root", "internet_exposed": false, "findings": {}}, {"id": "scp-deny-regions", "name": "deny-unapproved-regions", "type": "ORG_POLICY", "arn": "arn:aws:organizations::111111111111:policy/o-sample/service_control_policy/p-1", "internet_exposed": false, "findings": {}}, {"id": "sg-web", "name": "sg-web", "type": "SECURITY_GROUP", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web", "internet_exposed": false, "findings": {}}, {"id": "sg-admin", "name": "sg-admin", "type": "SECURITY_GROUP", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 1, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "sg-app", "name": "sg-app", "type": "SECURITY_GROUP", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0app", "internet_exposed": false, "findings": {}}, {"id": "sg-db", "name": "sg-db", "type": "SECURITY_GROUP", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0db", "internet_exposed": false, "findings": {}}, {"id": "alb-web", "name": "web-alb", "type": "LOAD_BALANCER", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:loadbalancer/app/web-alb/abc", "internet_exposed": true, "findings": {}}, {"id": "tg-web", "name": "web-tg", "type": "TARGET_GROUP", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:targetgroup/web-tg/def", "internet_exposed": false, "findings": {}}, {"id": "web-1", "name": "web-1", "type": "EC2", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web1", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "web-2", "name": "web-2", "type": "EC2", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web2", "internet_exposed": false, "findings": {}}, {"id": "bastion", "name": "bastion", "type": "EC2", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0bast", "internet_exposed": true, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "orders-db", "name": "orders-db", "type": "RDS_INSTANCE", "arn": "arn:aws:rds:us-east-1:111111111111:db:orders-db", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 1, "LOW": 0, "INFO": 0}}, {"id": "data-bucket", "name": "prod-data", "type": "S3_BUCKET", "arn": "arn:aws:s3:::prod-data", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 1, "INFO": 0}}, {"id": "logs-bucket", "name": "prod-logs", "type": "S3_BUCKET", "arn": "arn:aws:s3:::prod-logs", "internet_exposed": true, "findings": {}}, {"id": "kms-main", "name": "prod-main-key", "type": "KMS_KEY", "arn": "arn:aws:kms:us-east-1:111111111111:key/1111-2222", "internet_exposed": false, "findings": {}}, {"id": "db-secret", "name": "orders-db-credentials", "type": "SECRET", "arn": "arn:aws:secretsmanager:us-east-1:111111111111:secret:orders-db-credentials-AbC", "internet_exposed": false, "findings": {}}, {"id": "app-role", "name": "app-role", "type": "IAM_ROLE", "arn": "arn:aws:iam::111111111111:role/app-role", "internet_exposed": false, "findings": {}}, {"id": "admin-role", "name": "bastion-admin", "type": "IAM_ROLE", "arn": "arn:aws:iam::111111111111:role/bastion-admin", "internet_exposed": false, "findings": {}}, {"id": "lambda-role", "name": "api-lambda-role", "type": "IAM_ROLE", "arn": "arn:aws:iam::111111111111:role/api-lambda-role", "internet_exposed": false, "findings": {}}, {"id": "api-gw", "name": "public-api", "type": "API_GATEWAY", "arn": "arn:aws:apigateway:us-east-1::/restapis/a1b2c3", "internet_exposed": true, "findings": {}}, {"id": "api-handler", "name": "api-handler", "type": "LAMBDA_FUNCTION", "arn": "arn:aws:lambda:us-east-1:111111111111:function:api-handler", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 1, "INFO": 0}}, {"id": "log-group", "name": "/aws/lambda/api-handler", "type": "LOG_GROUP", "arn": "arn:aws:logs:us-east-1:111111111111:log-group:/aws/lambda/api-handler", "internet_exposed": false, "findings": {}}, {"id": "waf-web", "name": "web-acl", "type": "WAF_WEB_ACL", "arn": "arn:aws:wafv2:us-east-1:111111111111:regional/webacl/web-acl/1", "internet_exposed": false, "findings": {}}, {"id": "guardduty", "name": "guardduty-detector", "type": "THREAT_DETECTOR", "arn": "arn:aws:guardduty:us-east-1:111111111111:detector/d1", "internet_exposed": false, "findings": {}}, {"id": "securityhub", "name": "security-hub", "type": "SECURITY_HUB", "arn": "arn:aws:securityhub:us-east-1:111111111111:hub/default", "internet_exposed": false, "findings": {}}, {"id": "inspector", "name": "inspector", "type": "VULNERABILITY_SCANNER", "arn": "arn:aws:inspector2:us-east-1:111111111111:inspector", "internet_exposed": false, "findings": {}}, {"id": "deploy-role", "name": "deploy-role", "type": "IAM_ROLE", "arn": "arn:aws:iam::222222222222:role/deploy-role", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "artifacts-bucket", "name": "shared-artifacts", "type": "S3_BUCKET", "arn": "arn:aws:s3:::shared-artifacts", "internet_exposed": false, "findings": {}}, {"id": "ecr-repo", "name": "app-images", "type": "CONTAINER_REGISTRY", "arn": "arn:aws:ecr:eu-west-1:222222222222:repository/app-images", "internet_exposed": false, "findings": {}}, {"id": "az-nsg", "name": "jump-nsg", "type": "NSG", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/networkSecurityGroups/jump-nsg", "internet_exposed": false, "findings": {"total": 1, "CRITICAL": 1, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "az-vm", "name": "jump-vm", "type": "VIRTUAL_MACHINE", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Compute/virtualMachines/jump-vm", "internet_exposed": true, "findings": {}}, {"id": "az-kv", "name": "hub-kv", "type": "KEY_VAULT", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.KeyVault/vaults/hub-kv", "internet_exposed": false, "findings": {}}, {"id": "az-blob", "name": "archive", "type": "BLOB_STORAGE", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Storage/storageAccounts/archive", "internet_exposed": false, "findings": {}}, {"id": "gcp-gcs", "name": "raw-uploads", "type": "GCS_BUCKET", "arn": "//storage.googleapis.com/projects/_/buckets/raw-uploads", "internet_exposed": true, "findings": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}}, {"id": "gcp-bq", "name": "analytics", "type": "DATA_WAREHOUSE", "arn": "//bigquery.googleapis.com/projects/proj-analytics/datasets/analytics", "internet_exposed": false, "findings": {}}, {"id": "gcp-vm", "name": "worker-1", "type": "GCE_INSTANCE", "arn": "//compute.googleapis.com/projects/proj-analytics/zones/europe-west1-b/instances/worker-1", "internet_exposed": false, "findings": {}}]}]}; const findingsData = [{"id": "f-ssh-open", "resource_id": "sg-admin", "resource_arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin", "severity": "CRITICAL", "title": "Security group allows SSH (22) from 0.0.0.0/0", "description": "Security group allows SSH (22) from 0.0.0.0/0.", "evidence": "IpPermissions 0.0.0.0/0 tcp 22", "remediation": "Restrict 22 to the VPN.", "source_tool": "prowler", "source_finding_id": "prowler-aws-ec2_sg_open_22-1", "compliance_frameworks": ["CIS-AWS", "PCI-DSS"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 9.5}, {"id": "f-bastion-imds", "resource_id": "bastion", "resource_arn": null, "severity": "HIGH", "title": "EC2 instance allows IMDSv1", "description": "EC2 instance allows IMDSv1.", "evidence": null, "remediation": "Require IMDSv2.", "source_tool": "prowler", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 7.5}, {"id": "f-db-backup", "resource_id": "arn:aws:rds:us-east-1:111111111111:db:orders-db", "resource_arn": null, "severity": "MEDIUM", "title": "RDS backup retention below 7 days", "description": "RDS backup retention below 7 days.", "evidence": null, "remediation": null, "source_tool": "prowler", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS", "NIST-800-53"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 5.0}, {"id": "f-logs-public", "resource_id": "prod-logs", "resource_arn": "arn:aws:s3:::prod-logs", "severity": "HIGH", "title": "S3 bucket allows public read", "description": "S3 bucket allows public read.", "evidence": null, "remediation": null, "source_tool": "scoutsuite", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS", "PCI-DSS", "NIST-800-53"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 7.5}, {"id": "f-web-cve", "resource_id": "web-1", "resource_arn": null, "severity": "HIGH", "title": "CVE-2024-1234 in openssl 3.0.1", "description": "CVE-2024-1234 in openssl 3.0.1.", "evidence": null, "remediation": null, "source_tool": "trivy", "source_finding_id": "CVE-2024-1234", "compliance_frameworks": ["PCI-DSS"], "cvss_score": 8.1, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 7.8}, {"id": "f-lambda-runtime", "resource_id": "api-handler", "resource_arn": null, "severity": "LOW", "title": "Lambda runtime python3.8 is deprecated", "description": "Lambda runtime python3.8 is deprecated.", "evidence": null, "remediation": null, "source_tool": "checkov", "source_finding_id": "CKV_AWS_363", "compliance_frameworks": [], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 2.5}, {"id": "f-deploy-trust", "resource_id": "deploy-role", "resource_arn": "arn:aws:iam::222222222222:role/deploy-role", "severity": "HIGH", "title": "Role trusts an external account without ExternalId", "description": "Role trusts an external account without ExternalId.", "evidence": null, "remediation": null, "source_tool": "iam", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 7.5}, {"id": "f-rdp-open", "resource_id": "az-nsg", "resource_arn": null, "severity": "CRITICAL", "title": "NSG allows RDP (3389) from Internet", "description": "NSG allows RDP (3389) from Internet.", "evidence": null, "remediation": null, "source_tool": "scoutsuite", "source_finding_id": null, "compliance_frameworks": ["CIS-Azure"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 9.5}, {"id": "f-gcs-public", "resource_id": "gcp-gcs", "resource_arn": null, "severity": "HIGH", "title": "Bucket is publicly readable (allUsers)", "description": "Bucket is publicly readable (allUsers).", "evidence": null, "remediation": null, "source_tool": "prowler", "source_finding_id": null, "compliance_frameworks": ["CIS-GCP"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 7.5}, {"id": "f-default-vpc", "resource_id": "vpc-prod", "resource_arn": null, "severity": "INFO", "title": "VPC flow logs not enabled", "description": "VPC flow logs not enabled.", "evidence": null, "remediation": null, "source_tool": "prowler", "source_finding_id": null, "compliance_frameworks": [], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 0.5}, {"id": "f-ghost", "resource_id": "arn:aws:s3:::ghost-bucket", "resource_arn": "arn:aws:s3:::ghost-bucket", "severity": "MEDIUM", "title": "Bucket without lifecycle policy", "description": "Bucket without lifecycle policy.", "evidence": null, "remediation": null, "source_tool": "scoutsuite", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": false, "risk_score": 5.0}, {"id": "f-versioning", "resource_id": "data-bucket", "resource_arn": null, "severity": "LOW", "title": "S3 versioning disabled", "description": "S3 versioning disabled.", "evidence": null, "remediation": null, "source_tool": "prowler", "source_finding_id": null, "compliance_frameworks": ["CIS-AWS"], "cvss_score": null, "detected_at": "2026-01-15T12:00:00Z", "is_suppressed": true, "risk_score": 2.5}]; + const assetsData = [{"id": "org", "arn": "arn:aws:organizations::111111111111:organization/o-sample", "name": "sample-org", "asset_type": "ORGANIZATION", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:organizations::111111111111:organization/o-sample"}, {"id": "ou-workloads", "arn": "arn:aws:organizations::111111111111:ou/o-sample/ou-work", "name": "Workloads", "asset_type": "ORG_UNIT", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:organizations::111111111111:ou/o-sample/ou-work"}, {"id": "acct-prod", "arn": "arn:aws:iam::111111111111:root", "name": "prod", "asset_type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::111111111111:root"}, {"id": "acct-shared", "arn": "arn:aws:iam::222222222222:root", "name": "shared-services", "asset_type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "account_id": "222222222222", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::222222222222:root"}, {"id": "acct-vendor", "arn": "arn:aws:iam::999999999999:root", "name": "vendor-co", "asset_type": "CLOUD_ACCOUNT", "provider": "AWS", "region": "global", "account_id": "999999999999", "tags": {}, "metadata": {"external": true}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::999999999999:root"}, {"id": "scp-deny-regions", "arn": "arn:aws:organizations::111111111111:policy/o-sample/service_control_policy/p-1", "name": "deny-unapproved-regions", "asset_type": "ORG_POLICY", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:organizations::111111111111:policy/o-sample/service_control_policy/p-1"}, {"id": "vpc-prod", "arn": "arn:aws:ec2:us-east-1:111111111111:vpc/vpc-0prod", "name": "prod-vpc", "asset_type": "VPC", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "owner": "platform"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:vpc/vpc-0prod"}, {"id": "subnet-public", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0pub", "name": "prod-public-a", "asset_type": "SUBNET", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0pub"}, {"id": "subnet-private", "arn": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0priv", "name": "prod-private-a", "asset_type": "SUBNET", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:subnet/subnet-0priv"}, {"id": "sg-web", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web", "name": "sg-web", "asset_type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web"}, {"id": "sg-admin", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin", "name": "sg-admin", "asset_type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin"}, {"id": "sg-app", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0app", "name": "sg-app", "asset_type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0app"}, {"id": "sg-db", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0db", "name": "sg-db", "asset_type": "SECURITY_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0db"}, {"id": "alb-web", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:loadbalancer/app/web-alb/abc", "name": "web-alb", "asset_type": "LOAD_BALANCER", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "owner": "web-team"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "arn:aws:elasticloadbalancing:us-east-1:111111111111:loadbalancer/app/web-alb/abc"}, {"id": "tg-web", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:targetgroup/web-tg/def", "name": "web-tg", "asset_type": "TARGET_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:elasticloadbalancing:us-east-1:111111111111:targetgroup/web-tg/def"}, {"id": "web-1", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web1", "name": "web-1", "asset_type": "EC2", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "owner": "web-team"}, "metadata": {"instance_type": "t3.large", "imds_v2": false, "user_data": "export DB_PASSWORD=hunter2"}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web1"}, {"id": "web-2", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web2", "name": "web-2", "asset_type": "EC2", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "owner": "web-team"}, "metadata": {"instance_type": "t3.large"}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web2"}, {"id": "bastion", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0bast", "name": "bastion", "asset_type": "EC2", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "owner": "platform"}, "metadata": {"public_ip": "203.0.113.10"}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "arn:aws:ec2:us-east-1:111111111111:instance/i-0bast"}, {"id": "orders-db", "arn": "arn:aws:rds:us-east-1:111111111111:db:orders-db", "name": "orders-db", "asset_type": "RDS_INSTANCE", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"env": "prod", "data": "pii"}, "metadata": {"engine": "postgres", "backup_retention": 1}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:rds:us-east-1:111111111111:db:orders-db"}, {"id": "data-bucket", "arn": "arn:aws:s3:::prod-data", "name": "prod-data", "asset_type": "S3_BUCKET", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"data": "pii"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:s3:::prod-data"}, {"id": "logs-bucket", "arn": "arn:aws:s3:::prod-logs", "name": "prod-logs", "asset_type": "S3_BUCKET", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "arn:aws:s3:::prod-logs"}, {"id": "kms-main", "arn": "arn:aws:kms:us-east-1:111111111111:key/1111-2222", "name": "prod-main-key", "asset_type": "KMS_KEY", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:kms:us-east-1:111111111111:key/1111-2222"}, {"id": "db-secret", "arn": "arn:aws:secretsmanager:us-east-1:111111111111:secret:orders-db-credentials-AbC", "name": "orders-db-credentials", "asset_type": "SECRET", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:secretsmanager:us-east-1:111111111111:secret:orders-db-credentials-AbC"}, {"id": "app-role", "arn": "arn:aws:iam::111111111111:role/app-role", "name": "app-role", "asset_type": "IAM_ROLE", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {"policies": ["s3:GetObject", "rds-db:connect"]}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::111111111111:role/app-role"}, {"id": "admin-role", "arn": "arn:aws:iam::111111111111:role/bastion-admin", "name": "bastion-admin", "asset_type": "IAM_ROLE", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::111111111111:role/bastion-admin"}, {"id": "lambda-role", "arn": "arn:aws:iam::111111111111:role/api-lambda-role", "name": "api-lambda-role", "asset_type": "IAM_ROLE", "provider": "AWS", "region": "global", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::111111111111:role/api-lambda-role"}, {"id": "api-gw", "arn": "arn:aws:apigateway:us-east-1::/restapis/a1b2c3", "name": "public-api", "asset_type": "API_GATEWAY", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "arn:aws:apigateway:us-east-1::/restapis/a1b2c3"}, {"id": "api-handler", "arn": "arn:aws:lambda:us-east-1:111111111111:function:api-handler", "name": "api-handler", "asset_type": "LAMBDA_FUNCTION", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {"owner": "api-team"}, "metadata": {"runtime": "python3.8"}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:lambda:us-east-1:111111111111:function:api-handler"}, {"id": "log-group", "arn": "arn:aws:logs:us-east-1:111111111111:log-group:/aws/lambda/api-handler", "name": "/aws/lambda/api-handler", "asset_type": "LOG_GROUP", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:logs:us-east-1:111111111111:log-group:/aws/lambda/api-handler"}, {"id": "waf-web", "arn": "arn:aws:wafv2:us-east-1:111111111111:regional/webacl/web-acl/1", "name": "web-acl", "asset_type": "WAF_WEB_ACL", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:wafv2:us-east-1:111111111111:regional/webacl/web-acl/1"}, {"id": "guardduty", "arn": "arn:aws:guardduty:us-east-1:111111111111:detector/d1", "name": "guardduty-detector", "asset_type": "THREAT_DETECTOR", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {"security_service": "guardduty", "enabled": true}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:guardduty:us-east-1:111111111111:detector/d1"}, {"id": "securityhub", "arn": "arn:aws:securityhub:us-east-1:111111111111:hub/default", "name": "security-hub", "asset_type": "SECURITY_HUB", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {"security_service": "securityhub", "enabled": false}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:securityhub:us-east-1:111111111111:hub/default"}, {"id": "inspector", "arn": "arn:aws:inspector2:us-east-1:111111111111:inspector", "name": "inspector", "asset_type": "VULNERABILITY_SCANNER", "provider": "AWS", "region": "us-east-1", "account_id": "111111111111", "tags": {}, "metadata": {"security_service": "inspector", "enabled": true}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:inspector2:us-east-1:111111111111:inspector"}, {"id": "deploy-role", "arn": "arn:aws:iam::222222222222:role/deploy-role", "name": "deploy-role", "asset_type": "IAM_ROLE", "provider": "AWS", "region": "global", "account_id": "222222222222", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:iam::222222222222:role/deploy-role"}, {"id": "artifacts-bucket", "arn": "arn:aws:s3:::shared-artifacts", "name": "shared-artifacts", "asset_type": "S3_BUCKET", "provider": "AWS", "region": "eu-west-1", "account_id": "222222222222", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:s3:::shared-artifacts"}, {"id": "ecr-repo", "arn": "arn:aws:ecr:eu-west-1:222222222222:repository/app-images", "name": "app-images", "asset_type": "CONTAINER_REGISTRY", "provider": "AWS", "region": "eu-west-1", "account_id": "222222222222", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "arn:aws:ecr:eu-west-1:222222222222:repository/app-images"}, {"id": "az-vnet", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/virtualNetworks/hub-vnet", "name": "hub-vnet", "asset_type": "VNET", "provider": "AZURE", "region": "westeurope", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/virtualNetworks/hub-vnet"}, {"id": "az-nsg", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/networkSecurityGroups/jump-nsg", "name": "jump-nsg", "asset_type": "NSG", "provider": "AZURE", "region": "westeurope", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/networkSecurityGroups/jump-nsg"}, {"id": "az-vm", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Compute/virtualMachines/jump-vm", "name": "jump-vm", "asset_type": "VIRTUAL_MACHINE", "provider": "AZURE", "region": "westeurope", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "tags": {"owner": "it-ops"}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Compute/virtualMachines/jump-vm"}, {"id": "az-kv", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.KeyVault/vaults/hub-kv", "name": "hub-kv", "asset_type": "KEY_VAULT", "provider": "AZURE", "region": "westeurope", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.KeyVault/vaults/hub-kv"}, {"id": "az-blob", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Storage/storageAccounts/archive", "name": "archive", "asset_type": "BLOB_STORAGE", "provider": "AZURE", "region": "westeurope", "account_id": "00000000-aaaa-bbbb-cccc-000000000001", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Storage/storageAccounts/archive"}, {"id": "gcp-gcs", "arn": "//storage.googleapis.com/projects/_/buckets/raw-uploads", "name": "raw-uploads", "asset_type": "GCS_BUCKET", "provider": "GCP", "region": "europe-west1", "account_id": "proj-analytics", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": true, "display_id": "//storage.googleapis.com/projects/_/buckets/raw-uploads"}, {"id": "gcp-bq", "arn": "//bigquery.googleapis.com/projects/proj-analytics/datasets/analytics", "name": "analytics", "asset_type": "DATA_WAREHOUSE", "provider": "GCP", "region": "europe-west1", "account_id": "proj-analytics", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "//bigquery.googleapis.com/projects/proj-analytics/datasets/analytics"}, {"id": "gcp-vm", "arn": "//compute.googleapis.com/projects/proj-analytics/zones/europe-west1-b/instances/worker-1", "name": "worker-1", "asset_type": "GCE_INSTANCE", "provider": "GCP", "region": "europe-west1", "account_id": "proj-analytics", "tags": {}, "metadata": {}, "collected_at": "2026-01-15T12:00:00Z", "is_internet_exposed": false, "display_id": "//compute.googleapis.com/projects/proj-analytics/zones/europe-west1-b/instances/worker-1"}]; const findingsPerResource = {"sg-admin": {"total": 1, "CRITICAL": 1, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "bastion": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "arn:aws:rds:us-east-1:111111111111:db:orders-db": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 1, "LOW": 0, "INFO": 0}, "prod-logs": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "web-1": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "api-handler": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 1, "INFO": 0}, "deploy-role": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "az-nsg": {"total": 1, "CRITICAL": 1, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "gcp-gcs": {"total": 1, "CRITICAL": 0, "HIGH": 1, "MEDIUM": 0, "LOW": 0, "INFO": 0}, "vpc-prod": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 1}, "arn:aws:s3:::ghost-bucket": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 1, "LOW": 0, "INFO": 0}, "data-bucket": {"total": 1, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 1, "INFO": 0}}; const severityCounts = {"CRITICAL": 2, "HIGH": 5, "MEDIUM": 2, "LOW": 2, "INFO": 1}; const resourceTypes = {"ORGANIZATION": 1, "ORG_UNIT": 1, "CLOUD_ACCOUNT": 3, "ORG_POLICY": 1, "VPC": 1, "SUBNET": 2, "SECURITY_GROUP": 4, "LOAD_BALANCER": 1, "TARGET_GROUP": 1, "EC2": 3, "RDS_INSTANCE": 1, "S3_BUCKET": 3, "KMS_KEY": 1, "SECRET": 1, "IAM_ROLE": 4, "API_GATEWAY": 1, "LAMBDA_FUNCTION": 1, "LOG_GROUP": 1, "WAF_WEB_ACL": 1, "THREAT_DETECTOR": 1, "SECURITY_HUB": 1, "VULNERABILITY_SCANNER": 1, "CONTAINER_REGISTRY": 1, "VNET": 1, "NSG": 1, "VIRTUAL_MACHINE": 1, "KEY_VAULT": 1, "BLOB_STORAGE": 1, "GCS_BUCKET": 1, "DATA_WAREHOUSE": 1, "GCE_INSTANCE": 1}; const sourceTools = {"prowler": 6, "scoutsuite": 3, "trivy": 1, "checkov": 1, "iam": 1}; const complianceSummary = {"CIS-AWS": {"pass": 1, "fail": 3}, "PCI-DSS": {"pass": 0, "fail": 1}, "NIST-800-53": {"pass": 0, "fail": 1}, "CIS-Azure": {"pass": 0, "fail": 1}, "CIS-GCP": {"pass": 0, "fail": 1}}; + const reportSummary = {"providers": ["AWS", "AZURE", "GCP"], "accounts": ["00000000-aaaa-bbbb-cccc-000000000001", "111111111111", "222222222222", "999999999999", "proj-analytics"], "regions": ["eu-west-1", "europe-west1", "us-east-1", "westeurope"], "exposed": 6, "clean": 34, "edges": 56, "version": "0.6.0"}; + const topRisks = [0, 7, 4, 1, 3, 6]; + const isoScene = {"version": 1, "units": "world", "platforms": [{"id": "p:AWS", "kind": "provider", "label": "AWS", "provider": "AWS", "asset_id": null, "parent": null, "x": 0.0, "y": 0.0, "w": 184.0, "d": 172.0, "z": 0.0, "h": 6.0, "level": 0}, {"id": "r:AWS/111111111111/us-east-1", "kind": "region", "label": "111111111111 \u00b7 us-east-1", "provider": "AWS", "asset_id": null, "parent": "p:AWS", "x": 8.0, "y": 8.0, "w": 168.0, "d": 114.0, "z": 6.0, "h": 4.0, "level": 1}, {"id": "c:vpc-prod", "kind": "network", "label": "prod-vpc", "provider": "AWS", "asset_id": "vpc-prod", "parent": "r:AWS/111111111111/us-east-1", "x": 16.0, "y": 16.0, "w": 62.0, "d": 98.0, "z": 10.0, "h": 3.0, "level": 2}, {"id": "c:subnet-private", "kind": "subnet", "label": "prod-private-a", "provider": "AWS", "asset_id": "subnet-private", "parent": "c:vpc-prod", "x": 24.0, "y": 24.0, "w": 46.0, "d": 46.0, "z": 13.0, "h": 2.0, "level": 3}, {"id": "c:subnet-public", "kind": "subnet", "label": "prod-public-a", "provider": "AWS", "asset_id": "subnet-public", "parent": "c:vpc-prod", "x": 24.0, "y": 78.0, "w": 46.0, "d": 28.0, "z": 13.0, "h": 2.0, "level": 3}, {"id": "r:AWS/111111111111/us-east-1/zone", "kind": "zone", "label": "111111111111 \u00b7 us-east-1 services", "provider": "AWS", "asset_id": null, "parent": "r:AWS/111111111111/us-east-1", "x": 86.0, "y": 16.0, "w": 82.0, "d": 82.0, "z": 10.0, "h": 2.0, "level": 2}, {"id": "r:AWS/222222222222/eu-west-1", "kind": "region", "label": "222222222222 \u00b7 eu-west-1", "provider": "AWS", "asset_id": null, "parent": "p:AWS", "x": 8.0, "y": 130.0, "w": 52.0, "d": 34.0, "z": 6.0, "h": 4.0, "level": 1}, {"id": "d:AWS", "kind": "deck", "label": "AWS control plane", "provider": "AWS", "asset_id": null, "parent": null, "x": 0.0, "y": -86.0, "w": 82.0, "d": 64.0, "z": 54.0, "h": 3.0, "level": 1}, {"id": "p:AZURE", "kind": "provider", "label": "AZURE", "provider": "AZURE", "asset_id": null, "parent": null, "x": 224.0, "y": 0.0, "w": 78.0, "d": 120.0, "z": 0.0, "h": 6.0, "level": 0}, {"id": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope", "kind": "region", "label": "00000000-aaaa-bbbb-cccc-000000000001 \u00b7 westeurope", "provider": "AZURE", "asset_id": null, "parent": "p:AZURE", "x": 232.0, "y": 8.0, "w": 62.0, "d": 104.0, "z": 6.0, "h": 4.0, "level": 1}, {"id": "c:az-vnet", "kind": "network", "label": "hub-vnet", "provider": "AZURE", "asset_id": "az-vnet", "parent": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope", "x": 240.0, "y": 70.0, "w": 34.0, "d": 34.0, "z": 10.0, "h": 3.0, "level": 2}, {"id": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope/zone", "kind": "zone", "label": "00000000-aaaa-bbbb-cccc-000000000001 \u00b7 westeurope services", "provider": "AZURE", "asset_id": null, "parent": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope", "x": 240.0, "y": 16.0, "w": 46.0, "d": 46.0, "z": 10.0, "h": 2.0, "level": 2}, {"id": "p:GCP", "kind": "provider", "label": "GCP", "provider": "GCP", "asset_id": null, "parent": null, "x": 342.0, "y": 0.0, "w": 68.0, "d": 68.0, "z": 0.0, "h": 6.0, "level": 0}, {"id": "r:GCP/proj-analytics/europe-west1", "kind": "region", "label": "proj-analytics \u00b7 europe-west1", "provider": "GCP", "asset_id": null, "parent": "p:GCP", "x": 350.0, "y": 8.0, "w": 52.0, "d": 52.0, "z": 6.0, "h": 4.0, "level": 1}], "decks": [{"provider": "AWS", "id": "d:AWS", "z": 54.0}], "nodes": [{"id": "web-1", "name": "web-1", "type": "EC2", "category": "compute", "shape": "server", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web1", "exposed": false, "x": 38.0, "y": 38.0, "z": 15.0, "w": 11, "d": 11, "h": 10, "findings": {"HIGH": 1}, "worst": "HIGH", "parent": "c:subnet-private"}, {"id": "web-2", "name": "web-2", "type": "EC2", "category": "compute", "shape": "server", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0web2", "exposed": false, "x": 56.0, "y": 38.0, "z": 15.0, "w": 11, "d": 11, "h": 10, "findings": {}, "worst": null, "parent": "c:subnet-private"}, {"id": "orders-db", "name": "orders-db", "type": "RDS_INSTANCE", "category": "database", "shape": "database", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:rds:us-east-1:111111111111:db:orders-db", "exposed": false, "x": 38.0, "y": 56.0, "z": 15.0, "w": 11, "d": 11, "h": 11, "findings": {"MEDIUM": 1}, "worst": "MEDIUM", "parent": "c:subnet-private"}, {"id": "bastion", "name": "bastion", "type": "EC2", "category": "compute", "shape": "server", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:instance/i-0bast", "exposed": true, "x": 38.0, "y": 92.0, "z": 15.0, "w": 11, "d": 11, "h": 10, "findings": {"HIGH": 1}, "worst": "HIGH", "parent": "c:subnet-public"}, {"id": "alb-web", "name": "web-alb", "type": "LOAD_BALANCER", "category": "network", "shape": "gateway", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:loadbalancer/app/web-alb/abc", "exposed": true, "x": 56.0, "y": 92.0, "z": 15.0, "w": 13, "d": 9, "h": 4, "findings": {}, "worst": null, "parent": "c:subnet-public"}, {"id": "api-gw", "name": "public-api", "type": "API_GATEWAY", "category": "network", "shape": "gateway", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:apigateway:us-east-1::/restapis/a1b2c3", "exposed": true, "x": 100.0, "y": 30.0, "z": 12.0, "w": 13, "d": 9, "h": 4, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "kms-main", "name": "prod-main-key", "type": "KMS_KEY", "category": "secret", "shape": "key", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:kms:us-east-1:111111111111:key/1111-2222", "exposed": false, "x": 118.0, "y": 30.0, "z": 12.0, "w": 8, "d": 8, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "api-handler", "name": "api-handler", "type": "LAMBDA_FUNCTION", "category": "function", "shape": "function", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:lambda:us-east-1:111111111111:function:api-handler", "exposed": false, "x": 136.0, "y": 30.0, "z": 12.0, "w": 7, "d": 7, "h": 13, "findings": {"LOW": 1}, "worst": "LOW", "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "log-group", "name": "/aws/lambda/api-handler", "type": "LOG_GROUP", "category": "observability", "shape": "log", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:logs:us-east-1:111111111111:log-group:/aws/lambda/api-handler", "exposed": false, "x": 154.0, "y": 30.0, "z": 12.0, "w": 11, "d": 9, "h": 7, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "data-bucket", "name": "prod-data", "type": "S3_BUCKET", "category": "storage", "shape": "bucket", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:s3:::prod-data", "exposed": false, "x": 100.0, "y": 48.0, "z": 12.0, "w": 11, "d": 11, "h": 8, "findings": {"LOW": 1}, "worst": "LOW", "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "logs-bucket", "name": "prod-logs", "type": "S3_BUCKET", "category": "storage", "shape": "bucket", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:s3:::prod-logs", "exposed": true, "x": 118.0, "y": 48.0, "z": 12.0, "w": 11, "d": 11, "h": 8, "findings": {"HIGH": 1}, "worst": "HIGH", "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "db-secret", "name": "orders-db-credentials", "type": "SECRET", "category": "secret", "shape": "key", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:secretsmanager:us-east-1:111111111111:secret:orders-db-credentials-AbC", "exposed": false, "x": 136.0, "y": 48.0, "z": 12.0, "w": 8, "d": 8, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "sg-admin", "name": "sg-admin", "type": "SECURITY_GROUP", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0admin", "exposed": false, "x": 154.0, "y": 48.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {"CRITICAL": 1}, "worst": "CRITICAL", "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "sg-app", "name": "sg-app", "type": "SECURITY_GROUP", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0app", "exposed": false, "x": 100.0, "y": 66.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "sg-db", "name": "sg-db", "type": "SECURITY_GROUP", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0db", "exposed": false, "x": 118.0, "y": 66.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "sg-web", "name": "sg-web", "type": "SECURITY_GROUP", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web", "exposed": false, "x": 136.0, "y": 66.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "securityhub", "name": "security-hub", "type": "SECURITY_HUB", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:securityhub:us-east-1:111111111111:hub/default", "exposed": false, "x": 154.0, "y": 66.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "tg-web", "name": "web-tg", "type": "TARGET_GROUP", "category": "network", "shape": "gateway", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:elasticloadbalancing:us-east-1:111111111111:targetgroup/web-tg/def", "exposed": false, "x": 100.0, "y": 84.0, "z": 12.0, "w": 13, "d": 9, "h": 4, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "guardduty", "name": "guardduty-detector", "type": "THREAT_DETECTOR", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:guardduty:us-east-1:111111111111:detector/d1", "exposed": false, "x": 118.0, "y": 84.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "inspector", "name": "inspector", "type": "VULNERABILITY_SCANNER", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:inspector2:us-east-1:111111111111:inspector", "exposed": false, "x": 136.0, "y": 84.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "waf-web", "name": "web-acl", "type": "WAF_WEB_ACL", "category": "security", "shape": "shield", "provider": "AWS", "region": "us-east-1", "account": "111111111111", "arn": "arn:aws:wafv2:us-east-1:111111111111:regional/webacl/web-acl/1", "exposed": false, "x": 154.0, "y": 84.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {}, "worst": null, "parent": "r:AWS/111111111111/us-east-1/zone"}, {"id": "ecr-repo", "name": "app-images", "type": "CONTAINER_REGISTRY", "category": "storage", "shape": "bucket", "provider": "AWS", "region": "eu-west-1", "account": "222222222222", "arn": "arn:aws:ecr:eu-west-1:222222222222:repository/app-images", "exposed": false, "x": 25.0, "y": 147.0, "z": 10.0, "w": 11, "d": 11, "h": 8, "findings": {}, "worst": null, "parent": "r:AWS/222222222222/eu-west-1"}, {"id": "artifacts-bucket", "name": "shared-artifacts", "type": "S3_BUCKET", "category": "storage", "shape": "bucket", "provider": "AWS", "region": "eu-west-1", "account": "222222222222", "arn": "arn:aws:s3:::shared-artifacts", "exposed": false, "x": 43.0, "y": 147.0, "z": 10.0, "w": 11, "d": 11, "h": 8, "findings": {}, "worst": null, "parent": "r:AWS/222222222222/eu-west-1"}, {"id": "acct-prod", "name": "prod", "type": "CLOUD_ACCOUNT", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:iam::111111111111:root", "exposed": false, "x": 14.0, "y": -72.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "acct-shared", "name": "shared-services", "type": "CLOUD_ACCOUNT", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "222222222222", "arn": "arn:aws:iam::222222222222:root", "exposed": false, "x": 32.0, "y": -72.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "acct-vendor", "name": "vendor-co", "type": "CLOUD_ACCOUNT", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "999999999999", "arn": "arn:aws:iam::999999999999:root", "exposed": false, "x": 50.0, "y": -72.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "lambda-role", "name": "api-lambda-role", "type": "IAM_ROLE", "category": "identity", "shape": "principal", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:iam::111111111111:role/api-lambda-role", "exposed": false, "x": 68.0, "y": -72.0, "z": 57.0, "w": 7, "d": 7, "h": 10, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "app-role", "name": "app-role", "type": "IAM_ROLE", "category": "identity", "shape": "principal", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:iam::111111111111:role/app-role", "exposed": false, "x": 14.0, "y": -54.0, "z": 57.0, "w": 7, "d": 7, "h": 10, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "admin-role", "name": "bastion-admin", "type": "IAM_ROLE", "category": "identity", "shape": "principal", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:iam::111111111111:role/bastion-admin", "exposed": false, "x": 32.0, "y": -54.0, "z": 57.0, "w": 7, "d": 7, "h": 10, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "deploy-role", "name": "deploy-role", "type": "IAM_ROLE", "category": "identity", "shape": "principal", "provider": "AWS", "region": "global", "account": "222222222222", "arn": "arn:aws:iam::222222222222:role/deploy-role", "exposed": false, "x": 50.0, "y": -54.0, "z": 57.0, "w": 7, "d": 7, "h": 10, "findings": {"HIGH": 1}, "worst": "HIGH", "parent": "d:AWS"}, {"id": "org", "name": "sample-org", "type": "ORGANIZATION", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:organizations::111111111111:organization/o-sample", "exposed": false, "x": 68.0, "y": -54.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "scp-deny-regions", "name": "deny-unapproved-regions", "type": "ORG_POLICY", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:organizations::111111111111:policy/o-sample/service_control_policy/p-1", "exposed": false, "x": 14.0, "y": -36.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "ou-workloads", "name": "Workloads", "type": "ORG_UNIT", "category": "governance", "shape": "plate", "provider": "AWS", "region": "global", "account": "111111111111", "arn": "arn:aws:organizations::111111111111:ou/o-sample/ou-work", "exposed": false, "x": 32.0, "y": -36.0, "z": 57.0, "w": 12, "d": 12, "h": 3, "findings": {}, "worst": null, "parent": "d:AWS"}, {"id": "az-vm", "name": "jump-vm", "type": "VIRTUAL_MACHINE", "category": "compute", "shape": "server", "provider": "AZURE", "region": "westeurope", "account": "00000000-aaaa-bbbb-cccc-000000000001", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Compute/virtualMachines/jump-vm", "exposed": true, "x": 257.0, "y": 87.0, "z": 13.0, "w": 11, "d": 11, "h": 10, "findings": {}, "worst": null, "parent": "c:az-vnet"}, {"id": "az-blob", "name": "archive", "type": "BLOB_STORAGE", "category": "storage", "shape": "bucket", "provider": "AZURE", "region": "westeurope", "account": "00000000-aaaa-bbbb-cccc-000000000001", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Storage/storageAccounts/archive", "exposed": false, "x": 254.0, "y": 30.0, "z": 12.0, "w": 11, "d": 11, "h": 8, "findings": {}, "worst": null, "parent": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope/zone"}, {"id": "az-kv", "name": "hub-kv", "type": "KEY_VAULT", "category": "secret", "shape": "key", "provider": "AZURE", "region": "westeurope", "account": "00000000-aaaa-bbbb-cccc-000000000001", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.KeyVault/vaults/hub-kv", "exposed": false, "x": 272.0, "y": 30.0, "z": 12.0, "w": 8, "d": 8, "h": 6, "findings": {}, "worst": null, "parent": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope/zone"}, {"id": "az-nsg", "name": "jump-nsg", "type": "NSG", "category": "security", "shape": "shield", "provider": "AZURE", "region": "westeurope", "account": "00000000-aaaa-bbbb-cccc-000000000001", "arn": "/subscriptions/00000000-aaaa-bbbb-cccc-000000000001/resourceGroups/rg-hub/providers/Microsoft.Network/networkSecurityGroups/jump-nsg", "exposed": false, "x": 254.0, "y": 48.0, "z": 12.0, "w": 11, "d": 11, "h": 6, "findings": {"CRITICAL": 1}, "worst": "CRITICAL", "parent": "r:AZURE/00000000-aaaa-bbbb-cccc-000000000001/westeurope/zone"}, {"id": "gcp-bq", "name": "analytics", "type": "DATA_WAREHOUSE", "category": "database", "shape": "database", "provider": "GCP", "region": "europe-west1", "account": "proj-analytics", "arn": "//bigquery.googleapis.com/projects/proj-analytics/datasets/analytics", "exposed": false, "x": 367.0, "y": 25.0, "z": 10.0, "w": 11, "d": 11, "h": 11, "findings": {}, "worst": null, "parent": "r:GCP/proj-analytics/europe-west1"}, {"id": "gcp-vm", "name": "worker-1", "type": "GCE_INSTANCE", "category": "compute", "shape": "server", "provider": "GCP", "region": "europe-west1", "account": "proj-analytics", "arn": "//compute.googleapis.com/projects/proj-analytics/zones/europe-west1-b/instances/worker-1", "exposed": false, "x": 385.0, "y": 25.0, "z": 10.0, "w": 11, "d": 11, "h": 10, "findings": {}, "worst": null, "parent": "r:GCP/proj-analytics/europe-west1"}, {"id": "gcp-gcs", "name": "raw-uploads", "type": "GCS_BUCKET", "category": "storage", "shape": "bucket", "provider": "GCP", "region": "europe-west1", "account": "proj-analytics", "arn": "//storage.googleapis.com/projects/_/buckets/raw-uploads", "exposed": true, "x": 367.0, "y": 43.0, "z": 10.0, "w": 11, "d": 11, "h": 8, "findings": {"HIGH": 1}, "worst": "HIGH", "parent": "r:GCP/proj-analytics/europe-west1"}], "externals": [{"id": "0.0.0.0/0", "x": 205.0, "y": 218.0, "z": 0.0, "w": 14, "d": 14, "h": 34, "name": "0.0.0.0/0", "kind": "internet"}], "traces": [{"id": "t0", "source": "sg-app", "target": "sg-db", "type": "SECURITY_GROUP_RULE", "relationship": null, "family": "network", "label": "SECURITY_GROUP_RULE", "ports": "5432", "protocol": "TCP", "description": "", "count": 1, "points": [[100.0, 66.0, 18.0], [100.0, 66.0, 26.0], [118.0, 66.0, 26.0], [118.0, 66.0, 26.0], [118.0, 66.0, 18.0]]}, {"id": "t1", "source": "alb-web", "target": "sg-web", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[56.0, 92.0, 19.0], [56.0, 92.0, 28.0], [136.0, 92.0, 28.0], [136.0, 66.0, 28.0], [136.0, 66.0, 18.0]]}, {"id": "t2", "source": "alb-web", "target": "tg-web", "type": "LOAD_BALANCER_TARGET", "relationship": null, "family": "network", "label": "LOAD_BALANCER_TARGET", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[54.38, 93.49, 19.0], [54.38, 93.49, 28.0], [100.0, 93.49, 28.0], [100.0, 84.0, 28.0], [100.0, 84.0, 16.0]]}, {"id": "t3", "source": "tg-web", "target": "web-1", "type": "LOAD_BALANCER_TARGET", "relationship": null, "family": "network", "label": "LOAD_BALANCER_TARGET", "ports": "8080", "protocol": "", "description": "", "count": 1, "points": [[98.38, 85.49, 16.0], [98.38, 85.49, 34.0], [38.0, 85.49, 34.0], [38.0, 38.0, 34.0], [38.0, 38.0, 25.0]]}, {"id": "t4", "source": "tg-web", "target": "web-2", "type": "LOAD_BALANCER_TARGET", "relationship": null, "family": "network", "label": "LOAD_BALANCER_TARGET", "ports": "8080", "protocol": "", "description": "", "count": 1, "points": [[100.27, 80.9, 16.0], [100.27, 80.9, 34.0], [56.0, 80.9, 34.0], [56.0, 38.0, 34.0], [56.0, 38.0, 25.0]]}, {"id": "t5", "source": "web-1", "target": "sg-app", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[36.38, 39.49, 25.0], [36.38, 39.49, 34.0], [98.38, 39.49, 34.0], [98.38, 67.49, 34.0], [98.38, 67.49, 18.0]]}, {"id": "t6", "source": "web-2", "target": "sg-app", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[54.38, 39.49, 25.0], [54.38, 39.49, 34.0], [54.38, 62.9, 34.0], [100.27, 62.9, 34.0], [100.27, 62.9, 18.0]]}, {"id": "t7", "source": "bastion", "target": "sg-admin", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[38.0, 92.0, 25.0], [38.0, 92.0, 34.0], [154.0, 92.0, 34.0], [154.0, 48.0, 34.0], [154.0, 48.0, 18.0]]}, {"id": "t8", "source": "orders-db", "target": "sg-db", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[38.0, 56.0, 26.0], [38.0, 56.0, 34.0], [116.38, 56.0, 34.0], [116.38, 67.49, 34.0], [116.38, 67.49, 18.0]]}, {"id": "t9", "source": "orders-db", "target": "kms-main", "type": "REFERENCES", "relationship": "ENCRYPTED_BY_KMS", "family": "data", "label": "ENCRYPTED_BY_KMS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[36.38, 57.49, 26.0], [36.38, 57.49, 38.0], [118.0, 57.49, 38.0], [118.0, 30.0, 38.0], [118.0, 30.0, 18.0]]}, {"id": "t10", "source": "data-bucket", "target": "api-handler", "type": "INVOKES", "relationship": "TRIGGERED_BY", "family": "data", "label": "TRIGGERED_BY", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[100.0, 48.0, 20.0], [100.0, 48.0, 36.0], [136.0, 48.0, 36.0], [136.0, 30.0, 36.0], [136.0, 30.0, 25.0]]}, {"id": "t11", "source": "data-bucket", "target": "kms-main", "type": "REFERENCES", "relationship": "ENCRYPTED_BY_KMS", "family": "data", "label": "ENCRYPTED_BY_KMS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[98.38, 49.49, 20.0], [98.38, 49.49, 32.0], [116.38, 49.49, 32.0], [116.38, 31.49, 32.0], [116.38, 31.49, 18.0]]}, {"id": "t12", "source": "db-secret", "target": "kms-main", "type": "REFERENCES", "relationship": "ENCRYPTED_BY_KMS", "family": "data", "label": "ENCRYPTED_BY_KMS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[136.0, 48.0, 18.0], [136.0, 48.0, 30.0], [118.27, 48.0, 30.0], [118.27, 26.9, 30.0], [118.27, 26.9, 18.0]]}, {"id": "t13", "source": "api-gw", "target": "api-handler", "type": "INVOKES", "relationship": null, "family": "data", "label": "INVOKES", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[100.0, 30.0, 16.0], [100.0, 30.0, 36.0], [134.38, 30.0, 36.0], [134.38, 31.49, 36.0], [134.38, 31.49, 25.0]]}, {"id": "t14", "source": "api-handler", "target": "log-group", "type": "LOGS_TO", "relationship": null, "family": "data", "label": "LOGS_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[136.26, 27.07, 25.0], [136.26, 27.07, 36.0], [154.0, 27.07, 36.0], [154.0, 30.0, 36.0], [154.0, 30.0, 19.0]]}, {"id": "t15", "source": "api-handler", "target": "kms-main", "type": "REFERENCES", "relationship": "ENCRYPTED_BY_KMS", "family": "data", "label": "ENCRYPTED_BY_KMS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[137.79, 32.33, 25.0], [137.79, 32.33, 36.0], [120.04, 32.33, 36.0], [120.04, 32.67, 36.0], [120.04, 32.67, 18.0]]}, {"id": "t16", "source": "api-handler", "target": "ecr-repo", "type": "USES_IMAGE", "relationship": null, "family": "data", "label": "USES_IMAGE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[133.1, 29.49, 25.0], [133.1, 29.49, 36.0], [133.1, 147.0, 36.0], [25.0, 147.0, 36.0], [25.0, 147.0, 18.0]]}, {"id": "t17", "source": "waf-web", "target": "alb-web", "type": "PROTECTS", "relationship": null, "family": "governance", "label": "PROTECTS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[154.0, 84.0, 18.0], [154.0, 84.0, 36.0], [56.27, 84.0, 36.0], [56.27, 88.9, 36.0], [56.27, 88.9, 19.0]]}, {"id": "t18", "source": "inspector", "target": "web-1", "type": "MONITORS", "relationship": null, "family": "governance", "label": "MONITORS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[136.0, 84.0, 18.0], [136.0, 84.0, 42.0], [38.27, 84.0, 42.0], [38.27, 34.9, 42.0], [38.27, 34.9, 25.0]]}, {"id": "t19", "source": "az-vm", "target": "az-nsg", "type": "ATTACHED_TO", "relationship": null, "family": "network", "label": "ATTACHED_TO", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[257.0, 87.0, 23.0], [257.0, 87.0, 32.0], [254.0, 87.0, 32.0], [254.0, 48.0, 32.0], [254.0, 48.0, 18.0]]}, {"id": "t20", "source": "az-vm", "target": "az-kv", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[255.38, 88.49, 23.0], [255.38, 88.49, 38.0], [272.0, 88.49, 38.0], [272.0, 30.0, 38.0], [272.0, 30.0, 18.0]]}, {"id": "t21", "source": "az-vm", "target": "az-blob", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[257.27, 83.9, 23.0], [257.27, 83.9, 38.0], [254.0, 83.9, 38.0], [254.0, 30.0, 38.0], [254.0, 30.0, 20.0]]}, {"id": "t22", "source": "gcp-vm", "target": "gcp-gcs", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[385.0, 25.0, 20.0], [385.0, 25.0, 34.0], [367.0, 25.0, 34.0], [367.0, 43.0, 34.0], [367.0, 43.0, 18.0]]}, {"id": "t23", "source": "gcp-vm", "target": "gcp-bq", "type": "REFERENCES", "relationship": null, "family": "data", "label": "REFERENCES", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[383.38, 26.49, 20.0], [383.38, 26.49, 32.0], [367.0, 26.49, 32.0], [367.0, 25.0, 32.0], [367.0, 25.0, 21.0]]}, {"id": "t24", "source": "0.0.0.0/0", "target": "sg-web", "type": "SECURITY_GROUP_RULE", "relationship": null, "family": "network", "label": "SECURITY_GROUP_RULE", "ports": "80,443", "protocol": "TCP", "description": "", "count": 1, "points": [[205.0, 218.0, 34.0], [205.0, 218.0, 42.0], [134.38, 218.0, 42.0], [134.38, 67.49, 42.0], [134.38, 67.49, 18.0]]}, {"id": "t25", "source": "0.0.0.0/0", "target": "sg-admin", "type": "SECURITY_GROUP_RULE", "relationship": null, "family": "network", "label": "SECURITY_GROUP_RULE", "ports": "22", "protocol": "TCP", "description": "", "count": 1, "points": [[203.38, 219.49, 34.0], [203.38, 219.49, 42.0], [152.38, 219.49, 42.0], [152.38, 49.49, 42.0], [152.38, 49.49, 18.0]]}, {"id": "t26", "source": "0.0.0.0/0", "target": "alb-web", "type": "INTERNET_EXPOSED", "relationship": "INTERNET_REACHABLE", "family": "exposure", "label": "INTERNET_REACHABLE", "ports": "443", "protocol": "", "description": "", "count": 1, "points": [[205.27, 214.9, 34.0], [205.27, 214.9, 44.0], [58.32, 214.9, 44.0], [58.32, 95.02, 44.0], [58.32, 95.02, 19.0]]}, {"id": "t27", "source": "0.0.0.0/0", "target": "api-gw", "type": "INTERNET_EXPOSED", "relationship": "INTERNET_REACHABLE", "family": "exposure", "label": "INTERNET_REACHABLE", "ports": "443", "protocol": "", "description": "", "count": 1, "points": [[207.32, 221.02, 34.0], [207.32, 221.02, 44.0], [98.38, 221.02, 44.0], [98.38, 31.49, 44.0], [98.38, 31.49, 16.0]]}, {"id": "t28", "source": "0.0.0.0/0", "target": "logs-bucket", "type": "INTERNET_EXPOSED", "relationship": "INTERNET_REACHABLE", "family": "exposure", "label": "INTERNET_REACHABLE", "ports": "", "protocol": "", "description": "bucket policy Principal *", "count": 1, "points": [[200.67, 217.23, 34.0], [200.67, 217.23, 44.0], [118.0, 217.23, 44.0], [118.0, 48.0, 44.0], [118.0, 48.0, 20.0]]}, {"id": "t29", "source": "0.0.0.0/0", "target": "az-nsg", "type": "SECURITY_GROUP_RULE", "relationship": null, "family": "network", "label": "SECURITY_GROUP_RULE", "ports": "3389", "protocol": "TCP", "description": "", "count": 1, "points": [[209.15, 215.36, 34.0], [209.15, 215.36, 42.0], [252.38, 215.36, 42.0], [252.38, 49.49, 42.0], [252.38, 49.49, 18.0]]}, {"id": "t30", "source": "0.0.0.0/0", "target": "gcp-gcs", "type": "INTERNET_EXPOSED", "relationship": "INTERNET_REACHABLE", "family": "exposure", "label": "INTERNET_REACHABLE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[203.6, 223.2, 34.0], [203.6, 223.2, 44.0], [365.38, 223.2, 44.0], [365.38, 44.49, 44.0], [365.38, 44.49, 18.0]]}, {"id": "t31", "source": "acct-vendor", "target": "deploy-role", "type": "IAM_TRUST", "relationship": "CROSS_ACCOUNT_TRUST", "family": "identity", "label": "CROSS_ACCOUNT_TRUST", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[50.0, -72.0, 60.0], [50.0, -72.0, 82.0], [50.0, -72.0, 82.0], [50.0, -54.0, 82.0], [50.0, -54.0, 67.0]]}, {"id": "t32", "source": "scp-deny-regions", "target": "ou-workloads", "type": "GOVERNS", "relationship": null, "family": "governance", "label": "GOVERNS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[14.0, -36.0, 60.0], [14.0, -36.0, 84.0], [32.0, -36.0, 84.0], [32.0, -36.0, 84.0], [32.0, -36.0, 60.0]]}, {"id": "t33", "source": "web-1", "target": "app-role", "type": "ASSUMES_ROLE", "relationship": null, "family": "identity", "label": "ASSUMES_ROLE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[40.32, 41.02, 25.0], [40.32, 41.02, 82.0], [14.0, 41.02, 82.0], [14.0, -54.0, 82.0], [14.0, -54.0, 67.0]]}, {"id": "t34", "source": "web-2", "target": "app-role", "type": "ASSUMES_ROLE", "relationship": null, "family": "identity", "label": "ASSUMES_ROLE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[56.27, 34.9, 25.0], [56.27, 34.9, 82.0], [12.38, 34.9, 82.0], [12.38, -52.51, 82.0], [12.38, -52.51, 67.0]]}, {"id": "t35", "source": "bastion", "target": "admin-role", "type": "ASSUMES_ROLE", "relationship": null, "family": "identity", "label": "ASSUMES_ROLE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[36.38, 93.49, 25.0], [36.38, 93.49, 82.0], [32.0, 93.49, 82.0], [32.0, -54.0, 82.0], [32.0, -54.0, 67.0]]}, {"id": "t36", "source": "app-role", "target": "data-bucket", "type": "GRANTS_ACCESS", "relationship": "READS_FROM", "family": "identity", "label": "READS_FROM", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[14.26, -56.93, 67.0], [14.26, -56.93, 82.0], [100.27, -56.93, 82.0], [100.27, 44.9, 82.0], [100.27, 44.9, 20.0]]}, {"id": "t37", "source": "app-role", "target": "orders-db", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[15.79, -51.67, 67.0], [15.79, -51.67, 82.0], [38.27, -51.67, 82.0], [38.27, 52.9, 82.0], [38.27, 52.9, 26.0]]}, {"id": "t38", "source": "app-role", "target": "deploy-role", "type": "IAM_TRUST", "relationship": "CROSS_ACCOUNT_TRUST", "family": "identity", "label": "CROSS_ACCOUNT_TRUST", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[11.1, -54.51, 67.0], [11.1, -54.51, 82.0], [48.38, -54.51, 82.0], [48.38, -52.51, 82.0], [48.38, -52.51, 67.0]]}, {"id": "t39", "source": "admin-role", "target": "db-secret", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[30.38, -52.51, 67.0], [30.38, -52.51, 82.0], [30.38, 49.49, 82.0], [134.38, 49.49, 82.0], [134.38, 49.49, 18.0]]}, {"id": "t40", "source": "lambda-role", "target": "db-secret", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[68.0, -72.0, 67.0], [68.0, -72.0, 82.0], [136.27, -72.0, 82.0], [136.27, 44.9, 82.0], [136.27, 44.9, 18.0]]}, {"id": "t41", "source": "api-handler", "target": "lambda-role", "type": "ASSUMES_ROLE", "relationship": null, "family": "identity", "label": "ASSUMES_ROLE", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[138.48, 28.42, 25.0], [138.48, 28.42, 82.0], [66.38, 28.42, 82.0], [66.38, -70.51, 82.0], [66.38, -70.51, 67.0]]}, {"id": "t42", "source": "deploy-role", "target": "artifacts-bucket", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[50.26, -56.93, 67.0], [50.26, -56.93, 84.0], [43.0, -56.93, 84.0], [43.0, 147.0, 84.0], [43.0, 147.0, 18.0]]}, {"id": "t43", "source": "deploy-role", "target": "ecr-repo", "type": "GRANTS_ACCESS", "relationship": null, "family": "identity", "label": "GRANTS_ACCESS", "ports": "", "protocol": "", "description": "", "count": 1, "points": [[51.79, -51.67, 67.0], [51.79, -51.67, 82.0], [51.79, 148.49, 82.0], [23.38, 148.49, 82.0], [23.38, 148.49, 18.0]]}], "bounds": {"x": [0.0, 410.0], "y": [-86.0, 232.0], "z": [0.0, 84.0]}, "families": ["data", "exposure", "governance", "identity", "network"]}; let currentLayout = 'tree'; // 'tree' or 'force' // ── Tab Switching ── function switchTab(tabId) { document.querySelectorAll('.tab-content').forEach(el => el.classList.remove('active')); - document.querySelectorAll('.tab').forEach(el => el.classList.remove('active')); + document.querySelectorAll('.tab').forEach(el => el.classList.toggle('active', el.dataset.tab === tabId)); document.getElementById('tab-' + tabId).classList.add('active'); - event.target.classList.add('active'); - if (tabId === 'topology') { - setTimeout(() => renderTopology(), 50); - } + if (tabId === 'topology') setTimeout(() => renderTopology(), 30); + if (tabId === 'charts') setTimeout(() => renderCharts(), 30); } // ── HTML escaping ── // Finding, asset and compliance values come from scanner output and // cloud metadata. Markup is only ever built with the html`` tag below, - // which escapes every interpolated value. + // which escapes every interpolated value. The map builds its SVG with + // createElementNS and textContent, so it never parses markup at all. function escapeHtml(value) { if (value === null || value === undefined) return ''; return String(value) @@ -407,6 +2126,16 @@

      Compliance Overview

      function severityClass(severity) { return 'severity-' + (SEVERITIES.includes(severity) ? severity : 'INFO'); } + function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + } + + // Asset names for resource columns, by id and by ARN + const assetById = new Map(); + assetsData.forEach(a => { assetById.set(a.id, a); if (a.arn) assetById.set(a.arn, a); }); + function assetOf(f) { + return assetById.get(f.resource_id) || assetById.get(f.resource_arn) || null; + } // ── Findings Table ── // Rows keep the finding's position in findingsData, so the detail panel @@ -415,22 +2144,24 @@

      Compliance Overview

      function renderFindings(data) { const tbody = document.getElementById('findings-body'); - tbody.innerHTML = data.map(f => html` + tbody.innerHTML = data.map(f => { + const asset = assetOf(f); + return html` ${f.severity} - ${f.title} - ${f.resource_arn || f.resource_id || ''} - ${f.source_tool} - ${f.risk_score || '-'} +
      ${f.title}
      + ${asset ? asset.name : ''}
      ${f.resource_arn || f.resource_id || ''}
      + ${f.source_tool} + ${f.risk_score || '-'} - `).join(''); + `; + }).join(''); + const count = document.getElementById('findings-count'); + count.textContent = data.length + ' of ' + findingsData.length; + const empty = document.getElementById('findings-empty'); + empty.hidden = data.length > 0; } - document.getElementById('findings-body').addEventListener('click', (e) => { - const row = e.target.closest('tr[data-index]'); - if (row) showDetail(Number(row.dataset.index)); - }); - function filterFindings() { const severity = document.getElementById('filter-severity').value; const source = document.getElementById('filter-source').value; @@ -441,219 +2172,840 @@

      Compliance Overview

      if (search) filtered = filtered.filter(f => String(f.title || '').toLowerCase().includes(search) || String(f.resource_arn || '').toLowerCase().includes(search) || + String(f.resource_id || '').toLowerCase().includes(search) || String(f.description || '').toLowerCase().includes(search) ); renderFindings(filtered); } - // Populate source filter - const sourceSelect = document.getElementById('filter-source'); - Object.keys(sourceTools).forEach(tool => { - const opt = document.createElement('option'); - opt.value = tool; opt.textContent = tool; - sourceSelect.appendChild(opt); - }); - // ── Detail Panel ── function showDetail(index) { const f = findingsData[index]; if (!f) return; const panel = document.getElementById('detail-panel'); const content = document.getElementById('detail-content'); + const asset = assetOf(f); content.innerHTML = html` + ${f.severity}

      ${f.title}

      -
      Severity
      ${f.severity}
      -
      Resource
      ${f.resource_arn || f.resource_id || 'N/A'}
      +
      Resource
      ${asset ? asset.name : ''} ${f.resource_arn || f.resource_id || 'N/A'}
      Description
      ${f.description || 'N/A'}
      Evidence
      ${f.evidence || 'N/A'}
      Remediation
      ${f.remediation || 'N/A'}
      -
      Source Tool
      ${f.source_tool}
      -
      CVSS Score
      ${f.cvss_score || 'N/A'}
      -
      Compliance Frameworks
      ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
      -
      Risk Score
      ${f.risk_score || 'N/A'}
      +
      Source tool
      ${f.source_tool}
      +
      CVSS score
      ${f.cvss_score || 'N/A'}
      +
      Compliance frameworks
      ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
      +
      Risk score
      ${f.risk_score || 'N/A'}
      `; panel.classList.add('open'); + document.getElementById('scrim').classList.add('open'); + } + function closeDetail() { + document.getElementById('detail-panel').classList.remove('open'); + document.getElementById('scrim').classList.remove('open'); } - function closeDetail() { document.getElementById('detail-panel').classList.remove('open'); } - - // ── Hierarchical Topology ── - const typeColors = { - account: '#6366F1', region: '#22C55E', vpc: '#4A90D9', vnet: '#4A90D9', - subnet: '#7EC8E3', global: '#8B5CF6', - EC2: '#FF9900', VIRTUAL_MACHINE: '#FF9900', GCE_INSTANCE: '#FF9900', - VPC: '#4A90D9', VNET: '#4A90D9', SUBNET: '#7EC8E3', - RDS_INSTANCE: '#3B48CC', AURORA_CLUSTER: '#3B48CC', AZURE_SQL: '#3B48CC', CLOUD_SQL: '#3B48CC', - S3_BUCKET: '#569A31', BLOB_STORAGE: '#569A31', GCS_BUCKET: '#569A31', - SECURITY_GROUP: '#DD4B39', NSG: '#DD4B39', NACL: '#DD4B39', - LOAD_BALANCER: '#8C4FFF', CLOUDFRONT: '#8C4FFF', - IAM_USER: '#F472B6', IAM_ROLE: '#F472B6', IAM_POLICY: '#F472B6', - LAMBDA_FUNCTION: '#FF9900', CLOUD_FUNCTION: '#FF9900', - KMS_KEY: '#14B8A6', SECRET: '#14B8A6', - EBS_VOLUME: '#569A31', DYNAMODB_TABLE: '#3B48CC', - ECS_CLUSTER: '#FF9900', EKS_CLUSTER: '#FF9900', - SECURITY_FINDING: '#EF4444', COMPLIANCE_FRAMEWORK: '#14B8A6', - }; - const typeRadius = { - account: 22, region: 18, vpc: 16, vnet: 16, subnet: 14, global: 18, + // ── Compliance Grid ── + function renderCompliance() { + const grid = document.getElementById('compliance-grid'); + const entries = Object.entries(complianceSummary); + grid.innerHTML = entries.map(([fw, data]) => { + const pass = Number(data.pass) || 0; + const fail = Number(data.fail) || 0; + const total = pass + fail; + const pct = total > 0 ? Math.round((pass / total) * 100) : 0; + return html` +
      +

      ${fw}

      +
      ${pct}%
      +
      +
      ${pass} pass${fail} fail${total} controls
      +
      + `; + }).join(''); + document.getElementById('compliance-empty').hidden = entries.length > 0; + } + + // ── Overview ── + function renderOverview() { + const total = SEVERITIES.reduce((n, s) => n + (severityCounts[s] || 0), 0); + const bar = document.getElementById('sevbar'); + bar.innerHTML = SEVERITIES.filter(s => severityCounts[s]).map(s => + html``).join(''); + document.getElementById('sevlegend').innerHTML = SEVERITIES.map(s => + html`${s.charAt(0) + s.slice(1).toLowerCase()} ${severityCounts[s] || 0}`).join(''); + const assets = assetsData.length; + const pct = assets ? Math.round((reportSummary.clean / assets) * 100) : 100; + const ring = document.getElementById('ring-val'); + ring.style.strokeDashoffset = String(251.3 * (1 - pct / 100)); + if (pct < 50) ring.style.stroke = cssVar('--high'); + document.getElementById('ring-text').textContent = pct + '%'; + const worst = SEVERITIES.find(s => severityCounts[s]); + const providers = reportSummary.providers.length; + document.getElementById('lede').innerHTML = html`${total} findings across ${assets} assets in ${providers} ${providers === 1 ? 'provider' : 'providers'}. ${worst ? 'The worst open finding is ' + worst.toLowerCase() + '.' : 'Nothing is open.'} ${reportSummary.exposed} ${reportSummary.exposed === 1 ? 'asset is' : 'assets are'} reachable from the internet.`; + const list = document.getElementById('top-risks'); + list.innerHTML = topRisks.length ? topRisks.map(i => { + const f = findingsData[i]; + const asset = assetOf(f); + return html`
      • `; + }).join('') : html`
    • No open findings.
      • `; + } + + // ── Assets Table ── + function renderAssets() { + const provider = document.getElementById('asset-provider').value; + const search = document.getElementById('asset-search').value.toLowerCase(); + let rows = assetsData; + if (provider) rows = rows.filter(a => a.provider === provider); + if (search) rows = rows.filter(a => [a.name, a.asset_type, a.region, a.arn, a.id, a.account_id] + .some(v => String(v || '').toLowerCase().includes(search))); + const shown = rows.slice(0, 2000); + document.getElementById('assets-body').innerHTML = shown.map(a => { + const counts = findingsPerResource[a.id] || findingsPerResource[a.arn || ''] || null; + const worst = counts ? SEVERITIES.find(s => counts[s]) : null; + // Escaped fragments are joined, never nested: html`` would escape them again + const exposed = a.is_internet_exposed ? html` exposed` : ''; + const badge = worst ? html`${counts.total}` : html`none`; + return html` +
      ${a.name}
      ${a.arn || a.id}
      + ${a.asset_type}` + exposed + html` + ${a.provider} + ${a.region} + ` + badge + html` + `; + }).join(''); + document.getElementById('assets-count').textContent = rows.length + ' of ' + assetsData.length + (rows.length > shown.length ? ' (first 2000 shown)' : ''); + } + + // ── Isometric map ── + // The scene (cloudg/renderers/iso_scene.py) holds every position in world + // units. This code projects it with the Anatomy iso-kit camera, paints it + // back to front, and animates it. Nothing here parses markup: elements are + // made with createElementNS and names go in with textContent. + const SVGNS = 'http://www.w3.org/2000/svg'; + const FAMILIES = [ + ['exposure', 'Internet exposure'], ['network', 'Network path'], ['identity', 'Identity and access'], + ['data', 'Data and invocation'], ['governance', 'Governance and protection'], + ]; + const MAP = { + az: 45, el: 30, P: null, view: null, base: null, labels: false, + families: new Set(FAMILIES.map(f => f[0])), selected: null, focusTrace: null, + traceEls: new Map(), nodeEls: new Map(), byId: new Map(), pulses: [], raf: 0, + built: false, flat: false, turning: false, }; + // The drawing box: 1200 wide, as tall as the stage's aspect asks (set in buildMap) + let W = 1200, H = 760; + + function svgEl(tag, attrs, parent) { + const el = document.createElementNS(SVGNS, tag); + if (attrs) for (const k in attrs) if (attrs[k] !== undefined && attrs[k] !== null && attrs[k] !== '') el.setAttribute(k, attrs[k]); + if (parent) parent.appendChild(el); + return el; + } + function reducedMotion() { + return window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + } - let zoomBehavior, svgG; + // A kit solid as DOM: halo, fill, four side tones, top, crease, bevel, edge + function solid(parent, paths, opts) { + const o = opts || {}; + const g = svgEl('g', { class: 'iso-solid' + (o.cls ? ' ' + o.cls : '') }, parent); + if (o.lit) g.setAttribute('data-lit', ''); + if (o.halo) svgEl('path', { class: 'iso-halo', d: paths.fill }, g); + svgEl('path', { class: 'iso-fill', d: paths.fill }, g); + if (!o.flat) { + paths.shades.forEach((d, i) => { if (d) svgEl('path', { class: 'iso-shade', 'data-shade': i, d }, g); }); + svgEl('path', { class: 'iso-top', d: paths.top }, g); + } + if (paths.crease) svgEl('path', { class: 'iso-line iso-crease', 'data-tone': o.crease || 'faint', d: paths.crease }, g); + if (paths.bevel) svgEl('path', { class: 'iso-line iso-bevel', 'data-tone': o.crease || 'faint', d: paths.bevel }, g); + if (o.inner) o.inner(g); + svgEl('path', { class: 'iso-line iso-edge', 'data-tone': o.tone || 'hi', d: paths.outline }, g); + return g; + } + function line(parent, d, tone, cls) { + if (!d) return null; + return svgEl('path', { class: 'iso-line' + (cls ? ' ' + cls : ''), 'data-tone': tone || 'lo', d }, parent); + } + + // A matrix that lays text flat on a top face and keeps it reading left to right + function faceMatrix(at, z, P) { + const o = IsoKit.iso([at[0], at[1], z], P); + const vx = IsoKit.iso([at[0] + 1, at[1], z], P), vy = IsoKit.iso([at[0], at[1] + 1, z], P); + const ex = [vx[0] - o[0], vx[1] - o[1]], ey = [vy[0] - o[0], vy[1] - o[1]]; + const cands = [[ex, ey], [[-ex[0], -ex[1]], ey], [ey, ex], [[-ey[0], -ey[1]], ex]]; + let best = cands[0]; + for (const c of cands) if (c[0][0] > best[0][0]) best = c; + let [u, v] = best; + if (v[1] < 0) v = [-v[0], -v[1]]; + const f = n => Math.round(n * 1000) / 1000; + return 'matrix(' + [u[0], u[1], v[0], v[1], o[0], o[1]].map(f).join(' ') + ')'; + } + + // Draw order for siblings with disjoint footprints: a before b when b is + // nearer along the one axis that separates them (a topological sort, with + // centre depth breaking ties), so each platform's contents paint in order. + function orderSiblings(items, P) { + const [vx, vy] = IsoKit.towardViewer(P); + const n = items.length; + if (n < 2) return items.slice(); + const depth = it => (it.cx * vx + it.cy * vy); + if (n > 60) return items.slice().sort((a, b) => depth(a) - depth(b)); + const after = items.map(() => []), deg = new Array(n).fill(0); + for (let i = 0; i < n; i++) for (let j = i + 1; j < n; j++) { + const a = items[i], b = items[j]; + const sx = a.x1 <= b.x0 + 1e-6 ? 1 : (b.x1 <= a.x0 + 1e-6 ? -1 : 0); + const sy = a.y1 <= b.y0 + 1e-6 ? 1 : (b.y1 <= a.y0 + 1e-6 ? -1 : 0); + let aFirst = null; + if (sx && !sy) aFirst = sx * vx > 0; + else if (sy && !sx) aFirst = sy * vy > 0; + if (aFirst === null) continue; + if (aFirst) { after[i].push(j); deg[j]++; } else { after[j].push(i); deg[i]++; } + } + const out = [], ready = []; + for (let i = 0; i < n; i++) if (!deg[i]) ready.push(i); + while (ready.length) { + ready.sort((p, q) => depth(items[q]) - depth(items[p])); + const i = ready.pop(); + out.push(items[i]); + for (const j of after[i]) if (!--deg[j]) ready.push(j); + } + if (out.length < n) return items.slice().sort((a, b) => depth(a) - depth(b)); + return out; + } + function rectOf(x, y, w, d) { return { x0: x, y0: y, x1: x + w, y1: y + d, cx: x + w / 2, cy: y + d / 2 }; } + + // ── Parts ── + function drawPlatform(parent, p, P) { + const K = IsoKit; + const radius = { provider: 3.2, region: 2.4, network: 1.8, subnet: 1.2, zone: 1.2, deck: 2.4 }[p.kind] || 1.2; + const plan = { x: p.x, y: p.y, w: p.w, d: p.d, r: radius }; + const g = svgEl('g', { class: 'platform pl-' + p.kind }, parent); + if (p.kind === 'provider') { + // Feet under the plate, and a soft halo on the bench + for (const [fx, fy] of K.corners(plan, 7)) solid(g, K.cylinder(fx, fy, 2.6, -3, 3, P, 18), { tone: 'mid' }); + } + const top = p.z + p.h; + const tones = { provider: 'mid', region: 'mid', network: 'mid', subnet: 'lo', zone: 'lo', deck: 'mid' }; + solid(g, K.slabOf(plan, p.z, p.h, P, 8, p.kind === 'provider' ? 1.4 : 0.8), { + tone: tones[p.kind] || 'lo', halo: p.kind === 'provider' || p.kind === 'deck', + inner: s => { + const inset = K.insetPlan(plan, Math.min(3.5, Math.min(p.w, p.d) / 6)); + line(s, K.planOutline(inset, top, P), 'faint'); + if (p.kind === 'provider' || p.kind === 'deck') { + // Ruler ticks along the front edge and a screw at each corner + const t = K.topTicks(p.x + 6, p.x + p.w - 6, 4, 5, p.y + p.d - 2.2, top, [1.0, 1.9], P, 'x'); + line(s, t.minor, 'faint'); line(s, t.major, 'lo'); + for (const [sx, sy] of K.corners(plan, 3.2)) { + const c = K.iso([sx, sy, top], P); + svgEl('circle', { class: 'screw', cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 1.1 }, s); + } + } + if (p.kind === 'network') line(s, K.sideSeam(plan, p.z + p.h * 0.5, P), 'faint'); + }, + }); + if (p.kind !== 'zone' && p.w > 30) { + const label = svgEl('text', { + class: 'pl-label' + (p.kind === 'provider' || p.kind === 'deck' ? ' major' : ''), + transform: faceMatrix([p.x + 4.5, p.y + 3.2], top, P), 'font-size': p.kind === 'provider' ? 4.2 : 3.2, + 'dominant-baseline': 'hanging', + }, g); + const max = Math.max(6, Math.floor((p.w - 10) / (p.kind === 'provider' ? 2.6 : 2.0))); + const text = p.label.length > max ? p.label.slice(0, max - 1) + '…' : p.label; + label.textContent = text; + } + return g; + } + + function drawNode(parent, n, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node c-' + n.category, 'data-node': n.id, tabindex: '-1' }, parent); + const title = svgEl('title', null, g); + title.textContent = n.name + ' · ' + n.type + (n.region ? ' · ' + n.region : ''); + const { x, y, z, w, d, h } = n; + const plan = { x: x - w / 2, y: y - d / 2, w, d, r: 1.2 }; + const top = z + h; + if (n.exposed) line(g, K.ring(x, y, Math.max(w, d) / 2 + 2.4, z + 0.05, P, 36), null, 'exposed-ring'); + const box = (pl, z0, hh, opt) => solid(g, K.slabOf(pl, z0, hh, P, 6, 0.5), opt || {}); + const cyl = (r, z0, hh, steps, opt) => solid(g, K.cylinder(x, y, r, z0, hh, P, steps || 28, 0.4), opt || {}); + switch (n.shape) { + case 'server': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.sideSeam(plan, z + (h * i) / 4, P), 'faint'); + for (const dot of K.dotGrid(x - 3, y - 3, 3, 3, 3, top, P)) svgEl('circle', { class: 'iso-dot-vent', cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .45, fill: 'var(--anatomy-lo)' }, s); + } }); + break; + case 'cluster': { + box({ x: plan.x, y: plan.y, w, d, r: 1.4 }, z, 3); + const q = w / 2 - 0.8; + for (const [ox, oy] of [[-1, -1], [1, -1], [-1, 1], [1, 1]].sort((a, b) => (a[0] + a[1]) - (b[0] + b[1]))) + box({ x: x + (ox < 0 ? -q - 0.4 : 0.4), y: y + (oy < 0 ? -q - 0.4 : 0.4), w: q, d: q, r: 0.8 }, z + 3, h - 3); + break; + } + case 'function': + box(plan, z, h, { inner: s => line(s, K.sideSeam(plan, z + h * 0.7, P), 'faint') }); + box({ x: x - w / 2 - 0.6, y: y - d / 2 - 0.6, w: w + 1.2, d: d + 1.2, r: 1 }, top, 1.2); + break; + case 'database': + cyl(w / 2, z, h, 30, { inner: s => { + line(s, K.sideArc(x, y, w / 2, z + h / 3, P), 'faint'); + line(s, K.sideArc(x, y, w / 2, z + (2 * h) / 3, P), 'faint'); + } }); + break; + case 'bucket': + cyl(w / 2 - 0.6, z, h, 30); + line(g, K.ring(x, y, w / 2 - 2, top, P, 30), 'lo'); + break; + case 'queue': { + box(plan, z, h, { inner: s => { + for (let i = 1; i < 4; i++) line(s, K.lineOnTop([plan.x + (w * i) / 4, plan.y + 1], [plan.x + (w * i) / 4, plan.y + d - 1], top, P), 'faint'); + } }); + break; + } + case 'gateway': + box(plan, z, h, { inner: s => { + for (const dot of K.dotGrid(x - 4, y, 3, 1, 4, top, P)) svgEl('circle', { class: 'led' + (n.exposed ? ' exposed' : ''), cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .9 }, s); + } }); + break; + case 'shield': + cyl(w / 2, z, h, 8, { inner: s => line(s, K.ring(x, y, w / 2 - 2, top, P, 8), 'faint') }); + break; + case 'key': + cyl(w / 2, z, h * 0.6, 24); + cyl(w / 4, z + h * 0.6, h * 0.4, 18); + break; + case 'log': + for (let i = 0; i < 3; i++) box({ x: plan.x + i * 0.4, y: plan.y + i * 0.4, w: w - i * 0.8, d: d - i * 0.8, r: 1 }, z + i * (h / 3), h / 3 - 0.3); + break; + case 'core': + cyl(w / 2, z, h * 0.7, 6); + cyl(w / 4, z + h * 0.7, h * 0.3, 20, { lit: true }); + break; + case 'principal': + cyl(w / 2 - 1, z, h * 0.65, 22); + cyl(w / 2, z + h * 0.65, h * 0.35, 22); + break; + case 'plate': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.lineOnTop([plan.x + 2.5, plan.y + (d * i) / 4.4], [plan.x + w - 2.5, plan.y + (d * i) / 4.4], top, P), 'faint'); + } }); + break; + default: + box(plan, z, h); + } + return g; + } + + function drawBeacon(parent, n, P) { + if (!n.worst) return null; + const K = IsoKit; + const rank = SEVERITIES.indexOf(n.worst); + const mastTop = n.z + n.h + 7 + (4 - rank) * 2.6; + const g = svgEl('g', { class: 'beacon-group sev-' + n.worst, 'data-node': n.id }, parent); + svgEl('path', { class: 'mast', d: K.segment([n.x, n.y, n.z + n.h], [n.x, n.y, mastTop], P) }, g); + solid(g, K.cylinder(n.x, n.y, 1.9, mastTop, 2.4, P, 18), { cls: 'beacon', tone: 'hi' }); + const c = K.iso([n.x, n.y, mastTop + 2.4], P); + const halo = svgEl('circle', { class: 'beacon-halo' + (rank <= 1 ? ' pulse' : ''), cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 3.2 }, g); + halo.style.animationDelay = ((n.x * 7 + n.y * 13) % 2400) + 'ms'; + return g; + } + + function drawExternal(parent, e, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node external ' + (e.kind === 'internet' ? 'internet' : ''), 'data-node': e.id }, parent); + const t = svgEl('title', null, g); + t.textContent = e.kind === 'internet' ? 'The internet (' + e.name + ')' : 'Outside the estate: ' + e.name; + if (e.kind === 'internet') { + // An uplink mast: a weighted foot, a tapering column, three dish rings + solid(g, K.cylinder(e.x, e.y, 9, 0, 3, P, 36, 0.6), { tone: 'mid', halo: true }); + solid(g, K.cylinder(e.x, e.y, 3.4, 3, e.h - 9, P, 24), { tone: 'hi' }); + for (let i = 0; i < 3; i++) line(g, K.sideArc(e.x, e.y, 3.4, 8 + i * 6, P), 'faint'); + solid(g, K.cylinder(e.x, e.y, 6.5, e.h - 6, 2, P, 32, 0.4), { tone: 'hi' }); + solid(g, K.cylinder(e.x, e.y, 2.2, e.h - 4, 4, P, 20), { tone: 'hi', lit: true }); + } else { + solid(g, K.slabOf({ x: e.x - e.w / 2, y: e.y - e.d / 2, w: e.w, d: e.d, r: 1.4 }, 0, e.h, P, 6, 0.5), { tone: 'mid' }); + } + return g; + } + + function roundedPath(pts, r) { + if (pts.length < 3) return 'M' + pts.map(p => p[0].toFixed(1) + ' ' + p[1].toFixed(1)).join('L'); + let d = 'M' + pts[0][0].toFixed(1) + ' ' + pts[0][1].toFixed(1); + for (let i = 1; i < pts.length - 1; i++) { + const [ax, ay] = pts[i - 1], [bx, by] = pts[i], [cx, cy] = pts[i + 1]; + const l1 = Math.hypot(bx - ax, by - ay), l2 = Math.hypot(cx - bx, cy - by); + const rr = Math.min(r, l1 / 2, l2 / 2); + if (rr < 0.3) { d += 'L' + bx.toFixed(1) + ' ' + by.toFixed(1); continue; } + const p1 = [bx - ((bx - ax) / l1) * rr, by - ((by - ay) / l1) * rr]; + const p2 = [bx + ((cx - bx) / l2) * rr, by + ((cy - by) / l2) * rr]; + d += 'L' + p1[0].toFixed(1) + ' ' + p1[1].toFixed(1) + 'Q' + bx.toFixed(1) + ' ' + by.toFixed(1) + ' ' + p2[0].toFixed(1) + ' ' + p2[1].toFixed(1); + } + const last = pts[pts.length - 1]; + return d + 'L' + last[0].toFixed(1) + ' ' + last[1].toFixed(1); + } + + function drawTrace(parent, t, P) { + const g = svgEl('g', { class: 'trace f-' + t.family + (MAP.families.has(t.family) ? '' : ' hidden-family'), 'data-trace': t.id }, parent); + const pts = t.points.map(p => IsoKit.iso(p, P)).filter((p, i, a) => i === 0 || Math.hypot(p[0] - a[i - 1][0], p[1] - a[i - 1][1]) > 0.05); + const d = roundedPath(pts, 4); + const title = svgEl('title', null, g); + title.textContent = nameOf(t.source) + ' → ' + nameOf(t.target) + ' · ' + t.label + (t.ports ? ' · ' + t.ports : ''); + svgEl('path', { class: 'trace-hit', d }, g); + const lineEl = svgEl('path', { class: 'trace-line', d }, g); + svgEl('path', { class: 'trace-flow', d }, g); + const a = pts[0], b = pts[pts.length - 1]; + svgEl('circle', { class: 'trace-end', cx: a[0].toFixed(1), cy: a[1].toFixed(1), r: 1.5 }, g); + svgEl('circle', { class: 'trace-end to', cx: b[0].toFixed(1), cy: b[1].toFixed(1), r: 1.7 }, g); + MAP.traceEls.set(t.id, { g, line: lineEl, t }); + return g; + } + + function nameOf(id) { + const n = MAP.byId.get(id); + return n ? n.name : id; + } + + // ── Build ── + function buildMap() { + const svg = document.getElementById('iso-svg'); + while (svg.firstChild) svg.removeChild(svg.firstChild); + MAP.traceEls.clear(); MAP.nodeEls.clear(); + const s = isoScene; + const b = s.bounds; + const stageEl = document.getElementById('stage'); + const sw = stageEl.clientWidth || 1200, sh = stageEl.clientHeight || 760; + const nextH = Math.round(W * Math.min(1.6, Math.max(0.45, sh / sw))); + if (nextH !== H) { H = nextH; MAP.view = null; } + const corners = []; + for (const x of b.x) for (const y of b.y) for (const z of b.z) corners.push([x, y, z]); + corners.push([b.x[0], b.y[0], -3]); + const P = IsoKit.fitProjection(corners, W, H, { pad: 46, azimuth: MAP.az, elevation: MAP.el }); + MAP.P = P; + if (!MAP.view) MAP.view = [0, 0, W, H]; + svg.setAttribute('viewBox', MAP.view.join(' ')); + svg.setAttribute('preserveAspectRatio', 'xMidYMid meet'); + svg.classList.add('iso-svg'); + + const kids = new Map(); + const push = (key, item) => { if (!kids.has(key)) kids.set(key, []); kids.get(key).push(item); }; + s.platforms.forEach(p => push(p.parent || '', { kind: 'platform', p, ...rectOf(p.x, p.y, p.w, p.d) })); + s.nodes.forEach(n => push(n.parent, { kind: 'node', n, ...rectOf(n.x - 9, n.y - 9, 18, 18) })); + + const layers = { decks: svgEl('g', { class: 'layer-decks' }), ground: svgEl('g', { class: 'layer-ground' }), + groundTraces: svgEl('g', { class: 'layer-traces' }), deckTraces: svgEl('g', { class: 'layer-traces deck' }), + labels: svgEl('g', { class: 'layer-labels' }) }; + const rising = []; + const paint = (parent, item, depthIndex) => { + if (item.kind === 'platform') { + const p = item.p; + const g = svgEl('g', { class: 'stack' }, parent); + if (p.kind === 'deck') { + // Four posts carry the control deck above the plate + const posts = IsoKit.corners({ x: p.x, y: p.y, w: p.w, d: p.d }, 4).map(([px, py]) => ({ px, py, ...rectOf(px - 1, py - 1, 2, 2) })); + for (const post of orderSiblings(posts, P)) solid(g, IsoKit.cylinder(post.px, post.py, 1.5, 0, p.z, P, 16), { cls: 'post', tone: 'lo' }); + } + drawPlatform(g, p, P); + rising.push(g); + for (const child of orderSiblings(kids.get(p.id) || [], P)) paint(g, child, depthIndex + 1); + } else { + const n = item.n; + const g = drawNode(parent, n, P); + MAP.nodeEls.set(n.id, g); + drawBeacon(parent, n, P); + } + }; + const roots = orderSiblings(kids.get('') || [], P); + for (const root of roots) paint(root.p.kind === 'deck' ? layers.decks : layers.ground, root, 0); + const externals = s.externals.map(e => ({ e, ...rectOf(e.x - e.w, e.y - e.d, e.w * 2, e.d * 2) })); + for (const ext of orderSiblings(externals, P)) { const g = drawExternal(layers.ground, ext.e, P); MAP.nodeEls.set(ext.e.id, g); rising.push(g); } + + const deckIds = new Set(s.nodes.filter(n => n.parent && n.parent.startsWith('d:')).map(n => n.id)); + for (const t of s.traces) drawTrace(deckIds.has(t.source) || deckIds.has(t.target) ? layers.deckTraces : layers.groundTraces, t, P); + + // The deck is behind the plate (toward -y): paint it first unless the camera looks from behind + const deckInFront = IsoKit.towardViewer(P)[1] < 0; + const order = deckInFront + ? [layers.ground, layers.groundTraces, layers.decks, layers.deckTraces, layers.labels] + : [layers.decks, layers.ground, layers.groundTraces, layers.deckTraces, layers.labels]; + order.forEach(l => svg.appendChild(l)); + MAP.layers = layers; + MAP.rising = rising; + renderLabels(); + applyFocus(); + MAP.built = true; + } + + function renderLabels() { + const layer = MAP.layers && MAP.layers.labels; + if (!layer) return; + while (layer.firstChild) layer.removeChild(layer.firstChild); + const ids = new Set(); + if (MAP.labels) isoScene.nodes.forEach(n => ids.add(n.id)); + if (MAP.selected) { + ids.add(MAP.selected); + for (const t of isoScene.traces) if (t.source === MAP.selected || t.target === MAP.selected) { ids.add(t.source); ids.add(t.target); } + } + if (MAP.hover) ids.add(MAP.hover); + // Labels are placed top to bottom; one that would overlap a placed label + // moves up in steps, and a leader then ties it to its asset's top + const anchors = [...ids].map(id => MAP.byId.get(id)).filter(Boolean).map(n => { + const top = (n.z || 0) + (n.h || 0) + (n.worst ? 18 : 5); + return { n, a: IsoKit.iso([n.x, n.y, top], MAP.P) }; + }).sort((p, q) => p.a[1] - q.a[1]); + const placed = []; + const hits = (r) => placed.some(q => r.x0 < q.x1 && q.x0 < r.x1 && r.y0 < q.y1 && q.y0 < r.y1); + for (const { n, a } of anchors) { + const name = n.name.length > 34 ? n.name.slice(0, 33) + '…' : n.name; + const half = Math.max(name.length * 3.1, String(n.type || '').length * 2.7) + 4; + let dy = 0, rect; + for (let k = 0; k < 14; k++) { + rect = { x0: a[0] - half, x1: a[0] + half, y0: a[1] - 16 - dy, y1: a[1] + 9 - dy }; + if (!hits(rect)) break; + dy += 13; + } + placed.push(rect); + if (dy) svgEl('path', { class: 'iso-line', 'data-tone': 'mid', d: 'M' + a[0].toFixed(1) + ' ' + (a[1] + 2).toFixed(1) + 'V' + (a[1] - dy + 10).toFixed(1) }, layer); + if (dy) svgEl('circle', { cx: a[0].toFixed(1), cy: (a[1] + 2).toFixed(1), r: 1.2, fill: 'var(--anatomy-mid)' }, layer); + const text = svgEl('text', { class: 'node-label', x: a[0].toFixed(1), y: (a[1] - 4 - dy).toFixed(1), 'text-anchor': 'middle' }, layer); + text.appendChild(document.createTextNode(name)); + const sub = svgEl('tspan', { x: a[0].toFixed(1), dy: '11' }, text); + sub.textContent = n.type || n.kind || ''; + } + } + + // ── Intro: plates rise, assets settle, traces draw in, then flow ── + function playIntro() { + const svg = document.getElementById('iso-svg'); + svg.classList.remove('flowing'); + if (reducedMotion()) { svg.classList.add('flowing'); return; } + const groups = MAP.rising || []; + groups.forEach(g => { g.classList.remove('risen'); g.classList.add('rise-in'); }); + const traces = [...MAP.traceEls.values()]; + traces.forEach(({ line: l }) => { + const len = l.getTotalLength ? l.getTotalLength() : 0; + l.style.transition = 'none'; + l.style.strokeDasharray = len + ' ' + len; + l.style.strokeDashoffset = String(len); + }); + svg.getBoundingClientRect(); + const step = Math.min(70, 900 / Math.max(1, groups.length)); + groups.forEach((g, i) => setTimeout(() => { g.classList.remove('rise-in'); g.classList.add('risen'); }, 80 + i * step)); + const start = 260 + groups.length * step; + const famDelay = { network: 0, data: 220, identity: 440, governance: 660, exposure: 880 }; + traces.forEach(({ line: l, t }, i) => { + const delay = start + (famDelay[t.family] || 0) + (i % 12) * 30; + setTimeout(() => { + l.style.transition = 'stroke-dashoffset 1.1s cubic-bezier(.32,.72,0,1)'; + l.style.strokeDashoffset = '0'; + }, delay); + }); + clearTimeout(MAP.flowTimer); + MAP.flowTimer = setTimeout(() => { + traces.forEach(({ line: l }) => { l.style.strokeDasharray = ''; l.style.strokeDashoffset = ''; l.style.transition = ''; }); + svg.classList.add('flowing'); + }, start + 2200); + } + + // ── Focus: a selected asset, a traced path ── + function connectedTraces(id) { + return isoScene.traces.filter(t => t.source === id || t.target === id); + } + function applyFocus(traceIds) { + const svg = document.getElementById('iso-svg'); + const on = new Set(), onNodes = new Set(); + if (traceIds) traceIds.forEach(id => on.add(id)); + else if (MAP.selected) connectedTraces(MAP.selected).forEach(t => on.add(t.id)); + for (const id of on) { const t = MAP.traceEls.get(id); if (t) { onNodes.add(t.t.source); onNodes.add(t.t.target); } } + if (MAP.selected) onNodes.add(MAP.selected); + svg.classList.toggle('focus', on.size > 0 || !!MAP.selected); + MAP.traceEls.forEach(({ g }, id) => g.classList.toggle('on', on.has(id))); + svg.querySelectorAll('[data-node]').forEach(el => { + const id = el.getAttribute('data-node'); + el.classList.toggle('on', onNodes.has(id)); + el.classList.toggle('sel', id === MAP.selected); + }); + startPulses([...on]); + } + + // A dot runs along each lit trace from source to target, so the eye can follow it + function startPulses(ids, hops) { + cancelAnimationFrame(MAP.raf); + MAP.pulses.forEach(p => p.el.remove()); + MAP.pulses = []; + if (!ids.length || reducedMotion()) return; + const svg = document.getElementById('iso-svg'); + ids.slice(0, 40).forEach((id, i) => { + const t = MAP.traceEls.get(id); + if (!t || t.g.classList.contains('hidden-family')) return; + const el = svgEl('circle', { class: 'pulse-dot', r: 2.3 }, t.g); + MAP.pulses.push({ el, path: t.line, len: t.line.getTotalLength(), delay: hops ? (hops.get(id) || 0) * 420 : i * 90 }); + }); + const t0 = performance.now(); + const tick = now => { + for (const p of MAP.pulses) { + const e = now - t0 - p.delay; + if (e < 0) { p.el.setAttribute('opacity', '0'); continue; } + const dur = Math.max(900, p.len * 9); + const f = (e % (dur + 500)) / dur; + if (f > 1) { p.el.setAttribute('opacity', '0'); continue; } + const eased = f < .5 ? 2 * f * f : 1 - Math.pow(-2 * f + 2, 2) / 2; + const pt = p.path.getPointAtLength(eased * p.len); + p.el.setAttribute('cx', pt.x.toFixed(1)); + p.el.setAttribute('cy', pt.y.toFixed(1)); + p.el.setAttribute('opacity', String(Math.min(1, Math.sin(f * Math.PI) * 1.6))); + } + MAP.raf = requestAnimationFrame(tick); + }; + MAP.raf = requestAnimationFrame(tick); + } + + function selectNode(id, opts) { + MAP.selected = id && MAP.byId.has(id) ? id : null; + id = MAP.selected; + MAP.focusTrace = null; + applyFocus(); + renderLabels(); + renderInspector(); + if (id && opts && opts.center) centerOn(id); + const n = MAP.byId.get(id); + setReadout(n ? n.name + ' · ' + (n.type || n.kind) + ' · ' + connectedTraces(id).length + ' relations' : null); + } + + function focusTrace(traceId) { + const t = MAP.traceEls.get(traceId); + if (!t) return; + MAP.focusTrace = traceId; + applyFocus([traceId]); + document.querySelectorAll('.conn-item').forEach(el => el.classList.toggle('on', el.dataset.trace === traceId)); + setReadout(nameOf(t.t.source) + ' → ' + nameOf(t.t.target) + ' · ' + t.t.label); + } + + function traceFromInternet() { + const starts = isoScene.externals.filter(e => e.kind === 'internet').map(e => e.id); + if (!starts.length) { setReadout('nothing in this scan is reachable from the internet'); return; } + const out = new Map(); + isoScene.traces.forEach(t => { if (!out.has(t.source)) out.set(t.source, []); out.get(t.source).push(t); }); + const hops = new Map(), seen = new Set(starts); + let frontier = starts.slice(), depth = 0; + while (frontier.length && depth < 5) { + const next = []; + for (const id of frontier) for (const t of out.get(id) || []) { + if (!MAP.families.has(t.family)) continue; + if (!hops.has(t.id)) hops.set(t.id, depth); + if (!seen.has(t.target)) { seen.add(t.target); next.push(t.target); } + } + frontier = next; depth++; + } + const maxHop = hops.size ? Math.max(...hops.values()) + 1 : 0; + MAP.selected = null; + applyFocus([...hops.keys()]); + startPulses([...hops.keys()], hops); + renderLabels(); + renderInspector({ internet: { reached: seen.size - starts.length, hops: maxHop, traces: hops.size } }); + setReadout('the internet reaches ' + (seen.size - starts.length) + ' assets in up to ' + maxHop + (maxHop === 1 ? ' hop' : ' hops')); + } + + function setReadout(text) { + document.getElementById('readout').textContent = text || (isoScene.nodes ? isoScene.nodes.length + ' assets · ' + isoScene.traces.length + ' traces' : 'rest'); + } + + function renderInspector(extra) { + const box = document.getElementById('inspector'); + const n = MAP.selected ? MAP.byId.get(MAP.selected) : null; + if (extra && extra.internet) { + const x = extra.internet; + box.innerHTML = html`
      Attack surface

      Reachable from the internet

      +
      Assets${x.reached}Hops${x.hops}Traces${x.traces}
      +

      The lit traces start at the internet mast and follow each relation's direction, one hop after another. Click an asset to see its own connections.

      `; + return; + } + if (!n) { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + box.innerHTML = html`
      Inspector

      Pick an asset

      +

      Click any asset to light its relations; each lit trace carries a dot from its source to its target. Click a relation in this panel to follow it alone.

      +
      Assets${(isoScene.nodes || []).length}Platforms${(isoScene.platforms || []).length}Traces${(isoScene.traces || []).length}
      +

      [ ] turn · + - zoom · 0 fit · Esc clear

      `; + return; + } + const conns = connectedTraces(n.id); + const connItems = conns.map(t => html` + `).join(''); + const f = n.findings || {}; + const fsum = SEVERITIES.filter(s => f[s]).map(s => s.charAt(0) + s.slice(1).toLowerCase() + ' ' + f[s]).join(', '); + box.innerHTML = html`
      ${n.type || n.kind}

      ${n.name}

      +
      + Provider${n.provider || ''} + Region${n.region || ''} + Account${n.account || ''} + Identifier${n.arn || n.id} + Exposure${n.exposed ? 'Reachable from the internet' : 'Not exposed'} + Findings${fsum || 'None open'} +
      +
      Relations ${conns.length}
      ` + connItems + html`
      `; + } + + // ── Pan, zoom, turn ── + function setView(v) { + MAP.view = v; + document.getElementById('iso-svg').setAttribute('viewBox', v.map(n => n.toFixed(1)).join(' ')); + } + function zoomBy(factor, cx, cy) { + const [x, y, w, h] = MAP.view; + const nw = Math.min(W * 4, Math.max(W / 14, w / factor)), nh = nw * (h / w); + const px = cx === undefined ? x + w / 2 : cx, py = cy === undefined ? y + h / 2 : cy; + setView([px - (px - x) * (nw / w), py - (py - y) * (nh / h), nw, nh]); + } + function fitView() { setView([0, 0, W, H]); } + function centerOn(id) { + const n = MAP.byId.get(id); + if (!n || !MAP.P) return; + const p = IsoKit.iso([n.x, n.y, (n.z || 0) + (n.h || 0)], MAP.P); + const [, , w, h] = MAP.view; + const nw = Math.min(w, W / 2.2), nh = nw * (h / w); + animateView([p[0] - nw / 2, p[1] - nh / 2, nw, nh]); + } + function animateView(target) { + if (reducedMotion()) { setView(target); return; } + const from = MAP.view.slice(), t0 = performance.now(), dur = 520; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = 1 - Math.pow(1 - k, 3); + setView(from.map((v, i) => v + (target[i] - v) * e)); + if (k < 1) requestAnimationFrame(step); + }; + requestAnimationFrame(step); + } + function turn(by) { + if (MAP.turning) return; + const from = MAP.az, to = MAP.az + by; + const heavy = (isoScene.nodes || []).length > 260 || reducedMotion(); + const finish = () => { MAP.az = ((to % 360) + 360) % 360; MAP.turning = false; buildMap(); document.getElementById('iso-svg').classList.add('flowing'); }; + if (heavy) { finish(); return; } + MAP.turning = true; + const t0 = performance.now(), dur = 640; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = k < .5 ? 4 * k * k * k : 1 - Math.pow(-2 * k + 2, 3) / 2; + MAP.az = from + (to - from) * e; + buildMap(); + if (k < 1) requestAnimationFrame(step); else finish(); + }; + requestAnimationFrame(step); + } + function setFlat(flat) { + MAP.flat = flat; + document.getElementById('stage').classList.toggle('flat', flat); + document.getElementById('view-3d').setAttribute('aria-pressed', String(!flat)); + document.getElementById('view-2d').setAttribute('aria-pressed', String(flat)); + document.getElementById('layout-2d').hidden = !flat; + ['rot-left', 'rot-right', 'labels-btn', 'trace-internet', 'replay-btn'].forEach(id => { document.getElementById(id).disabled = flat; }); + renderTopology(); + } + + function renderFamilies() { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + document.getElementById('families').innerHTML = FAMILIES.map(([key, label]) => + html``).join(''); + } + + function renderLegend() { + const legend = document.getElementById('legend'); + legend.innerHTML = FAMILIES.map(([key, label]) => html`${label}`).join('') + + SEVERITIES.slice(0, 4).map(s => html`${s.charAt(0) + s.slice(1).toLowerCase()} beacon`).join('') + + html`Internet exposed`; + } + + // ── Topology entry point: the 3D map, or the 2D tree and force views ── function renderTopology() { const container = document.getElementById('topology-container'); - // Remove existing SVG only - container.querySelectorAll('svg').forEach(s => s.remove()); + const stage = document.getElementById('stage'); + const can3d = typeof IsoKit !== 'undefined' && isoScene && isoScene.nodes && !isoScene.too_large && !isoScene.error && isoScene.nodes.length > 0; + if (!can3d && !MAP.flat) { + MAP.flat = true; + stage.classList.add('flat'); + document.getElementById('view-3d').disabled = true; + document.getElementById('view-3d').setAttribute('aria-pressed', 'false'); + document.getElementById('view-2d').setAttribute('aria-pressed', 'true'); + document.getElementById('layout-2d').hidden = false; + const hint = document.getElementById('map-hint'); + if (isoScene && isoScene.too_large) hint.textContent = 'The 3D map is drawn for up to ' + isoScene.limit + ' assets; this scan has ' + isoScene.asset_count; + else hint.textContent = 'Scroll to zoom · drag to pan'; + } + if (!MAP.flat) { + if (!MAP.built) { buildMap(); setReadout(); renderInspector(); playIntro(); } + return; + } + container.querySelectorAll('svg, p').forEach(s => s.remove()); const width = container.clientWidth || 1200; - const height = 600; + const height = container.clientHeight || 640; if (!hierarchyData || !hierarchyData.children || hierarchyData.children.length === 0) { - // Fallback to force-directed if no hierarchy if (graphData.nodes && graphData.nodes.length > 0) { renderForceTopology(container, width, height); } else { const msg = document.createElement('p'); - msg.style.cssText = 'padding:40px;text-align:center;color:var(--text-muted)'; - msg.textContent = 'No topology data available. Run a collection first.'; + msg.className = 'map-note'; + msg.textContent = 'No topology data in this scan. Run cloudg collect or cloudg map first.'; container.appendChild(msg); } return; } + if (currentLayout === 'force') renderForceTopology(container, width, height); + else renderTreeTopology(container, width, height); + } - if (currentLayout === 'force') { - renderForceTopology(container, width, height); - } else { - renderTreeTopology(container, width, height); - } + // ── 2D views (D3) ── + const CATEGORY_COLORS = { + account: '--accent', region: '--f-governance', vpc: '--f-network', vnet: '--f-network', subnet: '--f-network', global: '--f-data', + }; + function typeColor(type) { + if (CATEGORY_COLORS[type]) return cssVar(CATEGORY_COLORS[type]); + if (type === 'SECURITY_FINDING') return cssVar('--critical'); + if (type === 'COMPLIANCE_FRAMEWORK') return cssVar('--f-governance'); + return cssVar('--muted'); } + let zoomBehavior, svgG; function renderTreeTopology(container, width, height) { const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); - const g = svg.append('g').attr('transform', 'translate(80, 20)'); + const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.1, 4]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - const root = d3.hierarchy(hierarchyData); - root.descendants().forEach(d => { d._children = d.children; }); - - const treeLayout = d3.tree().nodeSize([28, 220]); - treeLayout(root); - - // Center vertically + d3.tree().nodeSize([26, 230])(root); const yExtent = d3.extent(root.descendants(), d => d.x); - const yOffset = height / 2 - (yExtent[0] + yExtent[1]) / 2; - g.attr('transform', `translate(80, ${yOffset})`); - - // Links + svg.call(zoomBehavior.transform, d3.zoomIdentity.translate(80, height / 2 - (yExtent[0] + yExtent[1]) / 2)); g.selectAll('.link').data(root.links()).join('path') - .attr('class', 'link') .attr('d', d3.linkHorizontal().x(d => d.y).y(d => d.x)) - .attr('fill', 'none') - .attr('stroke', '#475569') - .attr('stroke-width', 1.5) - .attr('stroke-opacity', 0.6); - - // Nodes + .attr('fill', 'none').attr('stroke', cssVar('--line-strong')).attr('stroke-width', 1.2); const node = g.selectAll('.node').data(root.descendants()).join('g') - .attr('class', 'node') - .attr('transform', d => `translate(${d.y},${d.x})`) - .style('cursor', d => d.children || d._children ? 'pointer' : 'default') - .on('click', (e, d) => { - if (d.children) { d._children = d.children; d.children = null; } - else if (d._children) { d.children = d._children; d._children = null; } - renderTopology(); - }); - - // Node circles + .attr('transform', d => `translate(${d.y},${d.x})`); node.append('circle') - .attr('r', d => { - const nodeData = d.data; - return typeRadius[nodeData.type] || 10; - }) - .attr('fill', d => typeColors[d.data.type] || '#95A5A6') + .attr('r', d => d.children ? 7 : 5) + .attr('fill', d => d.children ? cssVar('--surface') : typeColor(d.data.type)) .attr('stroke', d => { - const nodeData = d.data; - if (nodeData.findings && nodeData.findings.total > 0) { - if (nodeData.findings.CRITICAL > 0) return '#EF4444'; - if (nodeData.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - } - if (nodeData.internet_exposed) return '#FF9900'; - return 'rgba(255,255,255,0.3)'; + const f = d.data.findings; + if (f && f.total > 0) return cssVar(f.CRITICAL ? '--critical' : f.HIGH ? '--high' : '--medium'); + if (d.data.internet_exposed) return cssVar('--f-exposure'); + return typeColor(d.data.type); }) - .attr('stroke-width', d => { - const nodeData = d.data; - if ((nodeData.findings && nodeData.findings.total > 0) || nodeData.internet_exposed) return 3; - return 1.5; - }); - - // Findings badge - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('circle') - .attr('cx', d => (typeRadius[d.data.type] || 10) - 2) - .attr('cy', d => -(typeRadius[d.data.type] || 10) + 2) - .attr('r', 8) - .attr('fill', d => { - if (d.data.findings.CRITICAL > 0) return '#EF4444'; - if (d.data.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - }); - - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) - 2) - .attr('y', d => -(typeRadius[d.data.type] || 10) + 6) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '9px') - .attr('font-weight', '700') - .text(d => d.data.findings.total); - - // Collapse indicator - node.filter(d => d._children && !d.children) - .append('text') - .attr('x', 0).attr('y', 4) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '14px') - .attr('font-weight', '700') - .text('+'); - - // Labels + .attr('stroke-width', d => (d.data.findings && d.data.findings.total) || d.data.internet_exposed ? 2.5 : 1.4); node.append('text') - .attr('x', d => (d.children || d._children ? -(typeRadius[d.data.type] || 10) - 6 : (typeRadius[d.data.type] || 10) + 6)) - .attr('dy', '0.35em') - .attr('text-anchor', d => d.children || d._children ? 'end' : 'start') - .attr('fill', '#CBD5E1') - .attr('font-size', d => ['account','region','vpc'].includes(d.data.type) ? '13px' : '11px') - .attr('font-weight', d => ['account','region','vpc'].includes(d.data.type) ? '600' : '400') - .text(d => { - const name = d.data.name || d.data.id || ''; - return name.length > 35 ? name.substring(0, 32) + '…' : name; - }); - - // Type badge for leaf nodes - node.filter(d => d.data.type && !['account','region','vpc','subnet','global','vnet'].includes(d.data.type)) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) + 6) - .attr('dy', '1.8em') - .attr('fill', '#64748B') - .attr('font-size', '9px') - .text(d => d.data.type); - - // Tooltips + .attr('x', d => d.children ? -12 : 10).attr('dy', '0.32em') + .attr('text-anchor', d => d.children ? 'end' : 'start') + .attr('fill', cssVar('--text')).attr('font-size', d => d.children ? '12px' : '11px') + .attr('font-weight', d => d.children ? 600 : 400) + .text(d => { const n = d.data.name || d.data.id || ''; return n.length > 36 ? n.slice(0, 33) + '…' : n; }); + node.filter(d => !d.children).append('text') + .attr('x', 10).attr('dy', '1.7em').attr('fill', cssVar('--muted')).attr('font-size', '9px') + .attr('font-family', 'ui-monospace, monospace').text(d => d.data.type); node.append('title').text(d => { let tip = `${d.data.type}: ${d.data.name || d.data.id || ''}`; if (d.data.arn) tip += `\nARN: ${d.data.arn}`; - if (d.data.findings && d.data.findings.total > 0) { - tip += `\n\nFindings: ${d.data.findings.total}`; - if (d.data.findings.CRITICAL) tip += `\n Critical: ${d.data.findings.CRITICAL}`; - if (d.data.findings.HIGH) tip += `\n High: ${d.data.findings.HIGH}`; - if (d.data.findings.MEDIUM) tip += `\n Medium: ${d.data.findings.MEDIUM}`; - } - if (d.data.internet_exposed) tip += '\n\n⚠ INTERNET EXPOSED'; + if (d.data.findings && d.data.findings.total > 0) tip += `\nFindings: ${d.data.findings.total}`; + if (d.data.internet_exposed) tip += '\nReachable from the internet'; return tip; }); } @@ -662,225 +3014,296 @@

      ${f.title}

      const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.2, 5]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - if (!graphData.nodes || graphData.nodes.length === 0) return; - - const sim = d3.forceSimulation(graphData.nodes) - .force('link', d3.forceLink(graphData.links).id(d => d.id).distance(100)) - .force('charge', d3.forceManyBody().strength(-200)) + const nodes = graphData.nodes.map(n => Object.assign({}, n)); + const links = graphData.links.map(l => Object.assign({}, l)); + const sim = d3.forceSimulation(nodes) + .force('link', d3.forceLink(links).id(d => d.id).distance(80)) + .force('charge', d3.forceManyBody().strength(-180)) .force('center', d3.forceCenter(width / 2, height / 2)) - .force('collision', d3.forceCollide().radius(30)); - - const link = g.append('g').selectAll('line').data(graphData.links).join('line') - .attr('stroke', d => { - if (d.relation === 'FINDING_AFFECTS') { - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB'}; - return sevColors[d.severity] || '#EF4444'; - } - if (d.relation === 'COMPLIANCE_GOVERNS') return '#14B8A6'; - return d.cidr === '0.0.0.0/0' ? '#E74C3C' : '#555'; - }) - .attr('stroke-width', d => { - if (d.relation === 'FINDING_AFFECTS') return 2; - if (d.relation === 'COMPLIANCE_GOVERNS') return 1.5; - return d.cidr === '0.0.0.0/0' ? 2 : 1; - }) - .attr('stroke-dasharray', d => { - if (d.relation === 'FINDING_AFFECTS' || d.relation === 'COMPLIANCE_GOVERNS') return '6,3'; - return null; - }) - .attr('stroke-opacity', d => d.relation ? 0.7 : 0.5); - - const node = g.append('g').selectAll('g').data(graphData.nodes).join('g') + .force('collision', d3.forceCollide().radius(22)); + const sevVar = { CRITICAL: '--critical', HIGH: '--high', MEDIUM: '--medium', LOW: '--low' }; + const link = g.append('g').selectAll('line').data(links).join('line') + .attr('stroke', d => d.relation === 'FINDING_AFFECTS' ? cssVar(sevVar[d.severity] || '--critical') + : d.relation === 'COMPLIANCE_GOVERNS' ? cssVar('--f-governance') : cssVar('--line-strong')) + .attr('stroke-width', d => d.relation ? 1.4 : 1) + .attr('stroke-dasharray', d => d.relation ? '5,3' : null); + const node = g.append('g').selectAll('g').data(nodes).join('g') .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended)); - - // Node shape: findings get triangles, compliance gets diamonds, rest get circles - node.each(function(d) { - const el = d3.select(this); - const r = d.is_external ? 8 : 16; - if (d.type === 'SECURITY_FINDING') { - // Triangle for findings - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB',INFO:'#64748B'}; - el.append('polygon') - .attr('points', `0,${-r} ${r},${r} ${-r},${r}`) - .attr('fill', sevColors[d.severity] || '#EF4444') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else if (d.type === 'COMPLIANCE_FRAMEWORK') { - // Diamond for compliance - el.append('polygon') - .attr('points', `0,${-r} ${r},0 0,${r} ${-r},0`) - .attr('fill', '#14B8A6') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else { - el.append('circle').attr('r', r) - .attr('fill', typeColors[d.type] || '#95A5A6') - .attr('stroke', d.is_internet_exposed ? '#FF0000' : '#fff') - .attr('stroke-width', d.is_internet_exposed ? 3 : 1.5); - } - }); - - node.append('text').text(d => d.name ? d.name.substring(0, 15) : d.id.substring(0, 10)) - .attr('dy', 28).attr('text-anchor', 'middle') - .attr('fill', '#CBD5E1').attr('font-size', '10px'); - - node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region}\nARN: ${d.arn || 'N/A'}`); - + node.append('circle').attr('r', d => d.is_external ? 6 : d.type === 'SECURITY_FINDING' ? 5 : 9) + .attr('fill', d => d.type === 'SECURITY_FINDING' ? cssVar(sevVar[d.severity] || '--critical') : typeColor(d.type)) + .attr('stroke', d => d.is_internet_exposed ? cssVar('--f-exposure') : cssVar('--surface')) + .attr('stroke-width', d => d.is_internet_exposed ? 2.5 : 1.5); + node.append('text').text(d => (d.name || d.id || '').slice(0, 18)) + .attr('dy', 20).attr('text-anchor', 'middle').attr('fill', cssVar('--muted')).attr('font-size', '10px'); + node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region || ''}\nARN: ${d.arn || 'N/A'}`); sim.on('tick', () => { - link.attr('x1', d => d.source.x).attr('y1', d => d.source.y) - .attr('x2', d => d.target.x).attr('y2', d => d.target.y); + link.attr('x1', d => d.source.x).attr('y1', d => d.source.y).attr('x2', d => d.target.x).attr('y2', d => d.target.y); node.attr('transform', d => `translate(${d.x},${d.y})`); }); - function dragstarted(e, d) { if (!e.active) sim.alphaTarget(0.3).restart(); d.fx = d.x; d.fy = d.y; } function dragged(e, d) { d.fx = e.x; d.fy = e.y; } function dragended(e, d) { if (!e.active) sim.alphaTarget(0); d.fx = null; d.fy = null; } } - function toggleLayout() { - currentLayout = currentLayout === 'tree' ? 'force' : 'tree'; - renderTopology(); - } - function resetZoom() { + if (!MAP.flat) { animateView([0, 0, W, H]); return; } const svg = d3.select('#topology-container svg'); - if (svg.node() && zoomBehavior) { - svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity); - } - } - - function expandAll() { - function expand(d) { if (d._children) { d.children = d._children; d._children = null; } if (d.children) d.children.forEach(expand); } - if (hierarchyData) { const root = d3.hierarchy(hierarchyData); expand(root); } - renderTopology(); - } - - function collapseAll() { - // Re-render with collapsed tree (only top 2 levels open) - renderTopology(); + if (svg.node() && zoomBehavior) svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity.translate(80, 20)); } // ── Chart.js Charts ── + let chartsBuilt = false; + const charts = []; function renderCharts() { - // Severity Donut - new Chart(document.getElementById('chart-severity'), { + if (chartsBuilt || typeof Chart === 'undefined') return; + chartsBuilt = true; + charts.splice(0).forEach(c => c.destroy()); + const text = cssVar('--text'), muted = cssVar('--muted'), grid = cssVar('--line'); + const sev = SEVERITIES.map(s => cssVar('--' + s.toLowerCase())); + Chart.defaults.font.family = getComputedStyle(document.body).fontFamily; + Chart.defaults.color = muted; + const legend = { position: 'bottom', labels: { color: text, usePointStyle: true, pointStyle: 'circle', boxWidth: 8, padding: 14 } }; + charts.push(new Chart(document.getElementById('chart-severity'), { type: 'doughnut', - data: { - labels: Object.keys(severityCounts), - datasets: [{ - data: Object.values(severityCounts), - backgroundColor: ['#EF4444', '#F97316', '#EAB308', '#3498DB', '#64748B'], - borderWidth: 0, hoverOffset: 8, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Resource Types Bar - new Chart(document.getElementById('chart-resources'), { + data: { labels: SEVERITIES, datasets: [{ data: SEVERITIES.map(s => severityCounts[s] || 0), backgroundColor: sev, borderColor: cssVar('--surface'), borderWidth: 3, hoverOffset: 6 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); + const types = Object.entries(resourceTypes).sort((a, b) => b[1] - a[1]).slice(0, 12); + charts.push(new Chart(document.getElementById('chart-resources'), { type: 'bar', - data: { - labels: Object.keys(resourceTypes), - datasets: [{ - data: Object.values(resourceTypes), - backgroundColor: '#6366F1', borderRadius: 6, - }] - }, - options: { - responsive: true, indexAxis: 'y', - plugins: { legend: { display: false } }, - scales: { - x: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' } }, - y: { ticks: { color: '#CBD5E1' }, grid: { display: false } }, - }, - } - }); - - // Sources Donut - new Chart(document.getElementById('chart-sources'), { + data: { labels: types.map(t => t[0]), datasets: [{ data: types.map(t => t[1]), backgroundColor: cssVar('--accent'), borderRadius: 4, barThickness: 12 }] }, + options: { responsive: true, maintainAspectRatio: false, indexAxis: 'y', plugins: { legend: { display: false } }, + scales: { x: { ticks: { color: muted }, grid: { color: grid } }, y: { ticks: { color: text, font: { family: 'ui-monospace, monospace', size: 10 } }, grid: { display: false } } } }, + })); + const fam = ['--f-network', '--f-data', '--f-identity', '--f-governance', '--accent', '--high'].map(cssVar); + charts.push(new Chart(document.getElementById('chart-sources'), { type: 'doughnut', - data: { - labels: Object.keys(sourceTools), - datasets: [{ - data: Object.values(sourceTools), - backgroundColor: ['#6366F1', '#8B5CF6', '#EC4899', '#14B8A6', '#F97316', '#EAB308'], - borderWidth: 0, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Compliance + data: { labels: Object.keys(sourceTools), datasets: [{ data: Object.values(sourceTools), backgroundColor: fam, borderColor: cssVar('--surface'), borderWidth: 3 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); const fws = Object.keys(complianceSummary); - const passData = fws.map(fw => complianceSummary[fw]?.pass || 0); - const failData = fws.map(fw => complianceSummary[fw]?.fail || 0); - new Chart(document.getElementById('chart-compliance'), { + charts.push(new Chart(document.getElementById('chart-compliance'), { type: 'bar', - data: { - labels: fws, - datasets: [ - { label: 'Pass', data: passData, backgroundColor: '#22C55E', borderRadius: 4 }, - { label: 'Fail', data: failData, backgroundColor: '#EF4444', borderRadius: 4 }, - ] - }, - options: { - responsive: true, - plugins: { legend: { labels: { color: '#CBD5E1' } } }, - scales: { - x: { ticks: { color: '#CBD5E1' }, grid: { display: false }, stacked: true }, - y: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' }, stacked: true }, - }, - } - }); + data: { labels: fws, datasets: [ + { label: 'Pass', data: fws.map(fw => complianceSummary[fw]?.pass || 0), backgroundColor: cssVar('--pass'), borderRadius: 3 }, + { label: 'Fail', data: fws.map(fw => complianceSummary[fw]?.fail || 0), backgroundColor: cssVar('--fail'), borderRadius: 3 }, + ] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, + scales: { x: { ticks: { color: text }, grid: { display: false }, stacked: true }, y: { ticks: { color: muted }, grid: { color: grid }, stacked: true } } }, + })); } - // ── Compliance Grid ── - function renderCompliance() { - const grid = document.getElementById('compliance-grid'); - grid.innerHTML = Object.entries(complianceSummary).map(([fw, data]) => { - const pass = Number(data.pass) || 0; - const fail = Number(data.fail) || 0; - const total = pass + fail; - const pct = total > 0 ? Math.round((pass / total) * 100) : 0; - const barColor = pct >= 80 ? 'var(--success)' : pct >= 50 ? 'var(--medium)' : 'var(--critical)'; - return html` -
      -

      ${fw}

      -
      - Pass: ${pass}Fail: ${fail}${pct}% -
      -
      -
      -
      -
      - `; - }).join(''); + // ── Empty states: Hairline figures ── + // cloudg/templates/figures/empty-tray.js is a figure in Hairline's own + // format; this is the small host that mounts it (the stage, its svg and a + // read-out the figure writes to) in place of Hairline's bench page. + function mountEmptyTray(host) { + const spec = window.HAIRLINE_FIGURES && window.HAIRLINE_FIGURES.tray; + if (typeof HL === 'undefined' || !spec || !host || host.dataset.mounted) return; + host.dataset.mounted = '1'; + HL.inject(document); + const svg = HL.mk('svg', { viewBox: '0 0 400 320', 'aria-hidden': 'true' }, host); + const read = document.createElement('span'); + read.setAttribute('data-hairline-live', ''); + host.appendChild(read); + spec.mount({ stage: host, svg, read }, spec.range[1]); } // ── Export ── function exportJSON() { - const blob = new Blob([JSON.stringify({findings: findingsData, graph: graphData, hierarchy: hierarchyData}, null, 2)], {type: 'application/json'}); + const blob = new Blob([JSON.stringify({ findings: findingsData, assets: assetsData, graph: graphData, hierarchy: hierarchyData, scene: isoScene }, null, 2)], { type: 'application/json' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = 'cloudg-report.json'; + a.click(); + } + function exportCSV() { + const cols = ['severity', 'title', 'resource_id', 'resource_arn', 'source_tool', 'risk_score', 'cvss_score', 'compliance_frameworks', 'remediation']; + const cell = v => { + let s = Array.isArray(v) ? v.join('; ') : (v === null || v === undefined ? '' : String(v)); + if (/^[=+\-@\t\r]/.test(s)) s = "'" + s; // keep spreadsheets from evaluating it + return '"' + s.replace(/"/g, '""') + '"'; + }; + const rows = [cols.join(',')].concat(findingsData.map(f => cols.map(c => cell(f[c])).join(','))); + const blob = new Blob([rows.join('\r\n')], { type: 'text/csv' }); const a = document.createElement('a'); a.href = URL.createObjectURL(blob); - a.download = 'findings.json'; + a.download = 'cloudg-findings.csv'; a.click(); } // ── Init ── (tests/test_html_report.py runs the script up to this line) - renderFindings(findingsData); - renderTopology(); - renderCharts(); - renderCompliance(); + (function init() { + const root = document.documentElement; + let theme = null; + try { theme = localStorage.getItem('cloudg-report-theme'); } catch (e) { theme = null; } + if (!theme) theme = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; + root.dataset.theme = theme; + document.body.dataset.theme = theme; + + // The kit's tones, scoped to the report's body.iso + if (typeof IsoKit !== 'undefined') { + const style = document.createElement('style'); + style.textContent = IsoKit.ISO_CSS; + document.head.insertBefore(style, document.head.firstChild); + } + (isoScene.nodes || []).forEach(n => MAP.byId.set(n.id, n)); + (isoScene.externals || []).forEach(e => MAP.byId.set(e.id, e)); + + document.getElementById('main-tabs').addEventListener('click', e => { + const tab = e.target.closest('[data-tab]'); + if (tab) switchTab(tab.dataset.tab); + }); + document.getElementById('theme-toggle').addEventListener('click', () => { + const next = root.dataset.theme === 'light' ? 'dark' : 'light'; + root.dataset.theme = next; document.body.dataset.theme = next; + try { localStorage.setItem('cloudg-report-theme', next); } catch (e) { /* private mode */ } + if (chartsBuilt) { chartsBuilt = false; if (document.getElementById('tab-charts').classList.contains('active')) renderCharts(); } + if (MAP.flat) renderTopology(); + }); + document.getElementById('print-btn').addEventListener('click', () => window.print()); + document.getElementById('csv-btn').addEventListener('click', exportCSV); + document.getElementById('scrim').addEventListener('click', closeDetail); + document.addEventListener('keydown', e => { if (e.key === 'Escape') closeDetail(); }); + + document.getElementById('findings-body').addEventListener('click', (e) => { + const row = e.target.closest('tr[data-index]'); + if (row) showDetail(Number(row.dataset.index)); + }); + document.getElementById('top-risks').addEventListener('click', e => { + const b = e.target.closest('[data-index]'); + if (b) showDetail(Number(b.dataset.index)); + }); + ['filter-severity', 'filter-source'].forEach(id => document.getElementById(id).addEventListener('change', filterFindings)); + document.getElementById('filter-search').addEventListener('input', filterFindings); + const sourceSelect = document.getElementById('filter-source'); + Object.keys(sourceTools).forEach(tool => { + const opt = document.createElement('option'); + opt.value = tool; opt.textContent = tool; + sourceSelect.appendChild(opt); + }); + const providerSelect = document.getElementById('asset-provider'); + [...new Set(assetsData.map(a => a.provider))].sort().forEach(p => { + const opt = document.createElement('option'); + opt.value = p; opt.textContent = p; + providerSelect.appendChild(opt); + }); + providerSelect.addEventListener('change', renderAssets); + document.getElementById('asset-search').addEventListener('input', renderAssets); + document.getElementById('assets-body').addEventListener('click', e => { + const row = e.target.closest('tr[data-asset]'); + if (!row || !MAP.byId.has(row.dataset.asset)) return; + switchTab('topology'); + if (MAP.flat && !document.getElementById('view-3d').disabled) setFlat(false); + setTimeout(() => selectNode(row.dataset.asset, { center: true }), 80); + }); + + // Map controls + const stage = document.getElementById('stage'); + const svg = document.getElementById('iso-svg'); + document.getElementById('view-3d').addEventListener('click', () => setFlat(false)); + document.getElementById('view-2d').addEventListener('click', () => setFlat(true)); + document.getElementById('layout-tree').addEventListener('click', () => { currentLayout = 'tree'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'true'); document.getElementById('layout-force').setAttribute('aria-pressed', 'false'); renderTopology(); }); + document.getElementById('layout-force').addEventListener('click', () => { currentLayout = 'force'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'false'); document.getElementById('layout-force').setAttribute('aria-pressed', 'true'); renderTopology(); }); + document.getElementById('rot-left').addEventListener('click', () => turn(-90)); + document.getElementById('rot-right').addEventListener('click', () => turn(90)); + document.getElementById('zoom-in').addEventListener('click', () => MAP.flat ? null : zoomBy(1.35)); + document.getElementById('zoom-out').addEventListener('click', () => MAP.flat ? null : zoomBy(1 / 1.35)); + document.getElementById('zoom-fit').addEventListener('click', resetZoom); + document.getElementById('labels-btn').addEventListener('click', e => { MAP.labels = !MAP.labels; e.currentTarget.setAttribute('aria-pressed', String(MAP.labels)); renderLabels(); }); + document.getElementById('trace-internet').addEventListener('click', traceFromInternet); + document.getElementById('replay-btn').addEventListener('click', () => { MAP.selected = null; applyFocus(); renderInspector(); playIntro(); }); + document.getElementById('families').addEventListener('click', e => { + const b = e.target.closest('[data-family]'); + if (!b) return; + const f = b.dataset.family; + if (MAP.families.has(f)) MAP.families.delete(f); else MAP.families.add(f); + b.setAttribute('aria-pressed', String(MAP.families.has(f))); + MAP.traceEls.forEach(({ g, t }) => g.classList.toggle('hidden-family', !MAP.families.has(t.family))); + applyFocus(MAP.focusTrace ? [MAP.focusTrace] : undefined); + }); + document.getElementById('inspector').addEventListener('click', e => { + const b = e.target.closest('[data-trace]'); + if (b) focusTrace(b.dataset.trace); + }); + svg.addEventListener('click', e => { + if (MAP.dragMoved) return; + const tr = e.target.closest('[data-trace]'); + if (tr && !e.target.closest('[data-node]')) { const t = MAP.traceEls.get(tr.dataset.trace); if (t) { MAP.selected = t.t.source; renderInspector(); focusTrace(tr.dataset.trace); renderLabels(); } return; } + const node = e.target.closest('[data-node]'); + selectNode(node ? node.getAttribute('data-node') : null); + }); + svg.addEventListener('pointerover', e => { + const node = e.target.closest('[data-node]'); + const id = node ? node.getAttribute('data-node') : null; + if (id === MAP.hover) return; + MAP.hover = id; renderLabels(); + const n = id && MAP.byId.get(id); + if (n) setReadout(n.name + ' · ' + (n.type || n.kind)); else if (!MAP.selected) setReadout(); + }); + // Pan with a drag, zoom with the wheel about the pointer + const toView = (cx, cy) => { + const r = svg.getBoundingClientRect(), [x, y, w, h] = MAP.view; + const s = Math.min(r.width / w, r.height / h); + const ox = (r.width - w * s) / 2, oy = (r.height - h * s) / 2; + return [x + (cx - r.left - ox) / s, y + (cy - r.top - oy) / s, s]; + }; + stage.addEventListener('wheel', e => { + if (MAP.flat) return; + e.preventDefault(); + const [px, py] = toView(e.clientX, e.clientY); + zoomBy(Math.exp(-e.deltaY * 0.0015), px, py); + }, { passive: false }); + let drag = null; + stage.addEventListener('pointerdown', e => { + if (MAP.flat || e.button !== 0) return; + drag = { x: e.clientX, y: e.clientY, view: MAP.view.slice(), s: toView(e.clientX, e.clientY)[2] }; + MAP.dragMoved = false; + }); + window.addEventListener('pointermove', e => { + if (!drag) return; + const dx = e.clientX - drag.x, dy = e.clientY - drag.y; + if (!MAP.dragMoved && Math.hypot(dx, dy) < 4) return; + MAP.dragMoved = true; stage.classList.add('dragging'); + setView([drag.view[0] - dx / drag.s, drag.view[1] - dy / drag.s, drag.view[2], drag.view[3]]); + }); + window.addEventListener('pointerup', () => { drag = null; stage.classList.remove('dragging'); setTimeout(() => { MAP.dragMoved = false; }, 0); }); + stage.addEventListener('keydown', e => { + if (MAP.flat) return; + const ids = (isoScene.nodes || []).map(n => n.id); + if (e.key === '[') turn(-90); + else if (e.key === ']') turn(90); + else if (e.key === '+' || e.key === '=') zoomBy(1.35); + else if (e.key === '-') zoomBy(1 / 1.35); + else if (e.key === '0') resetZoom(); + else if (e.key === 'Escape') selectNode(null); + else if (e.key === 'ArrowRight' || e.key === 'ArrowLeft' || e.key === 'ArrowDown' || e.key === 'ArrowUp') { + const i = ids.indexOf(MAP.selected), d = e.key === 'ArrowRight' || e.key === 'ArrowDown' ? 1 : -1; + selectNode(ids[(i + d + ids.length) % ids.length], { center: true }); + } else return; + e.preventDefault(); + }); + const list = document.getElementById('map-assets'); + (isoScene.nodes || []).slice(0, 3000).forEach(n => { const o = document.createElement('option'); o.value = n.name; list.appendChild(o); }); + document.getElementById('map-search').addEventListener('change', e => { + const q = e.target.value.trim().toLowerCase(); + const n = (isoScene.nodes || []).find(x => x.name.toLowerCase() === q) || (isoScene.nodes || []).find(x => x.name.toLowerCase().includes(q) || (x.arn || '').toLowerCase().includes(q)); + if (n) { if (MAP.flat) setFlat(false); selectNode(n.id, { center: true }); } + }); + + renderOverview(); + renderFindings(findingsData); + renderAssets(); + renderCompliance(); + renderFamilies(); + renderLegend(); + renderTopology(); + mountEmptyTray(document.getElementById('findings-empty-figure')); + mountEmptyTray(document.getElementById('compliance-empty-figure')); + })(); diff --git a/tests/test_html_report.py b/tests/test_html_report.py index fa3f6bc..bd39733 100644 --- a/tests/test_html_report.py +++ b/tests/test_html_report.py @@ -114,8 +114,10 @@ def test_hostile_values_never_appear_unescaped(tmp_path): assert "onerror=alert(1)>" not in page # Exactly the two library scripts plus the report's own script: no # value closed a script element or opened a new one - assert len(re.findall(r" ground_top + # The deck stands behind the plate, never over it + base = next( + p for p in scene["platforms"] if p["kind"] == "provider" and p["provider"] == "AWS" + ) + assert deck["y"] + deck["d"] <= base["y"] + + def test_findings_and_exposure_are_carried_on_nodes(self, estate): + _, _, scene = estate + nodes = {n["name"]: n for n in scene["nodes"]} + assert nodes["sg-admin"]["worst"] == "CRITICAL" + assert nodes["web-alb"]["exposed"] is True + assert nodes["web-tg"]["worst"] is None + + def test_internet_is_an_external_mast(self, estate): + _, _, scene = estate + assert [e["kind"] for e in scene["externals"]] == ["internet"] + mast = scene["externals"][0] + assert mast["y"] > max(p["y"] + p["d"] for p in scene["platforms"]) + + def test_scene_is_json_and_compact(self, estate): + _, _, scene = estate + text = json.dumps(scene) + assert len(text) < 60_000 + + +class TestTraces: + def test_containment_is_not_drawn_as_traces(self, estate): + _, _, scene = estate + assert all(t["type"] != "CONTAINS" for t in scene["traces"]) + assert len(scene["traces"]) == 44 + + def test_routes_are_orthogonal_and_end_on_their_assets(self, estate): + _, _, scene = estate + ends = {n["id"]: n for n in scene["nodes"] + scene["externals"]} + for trace in scene["traces"]: + pts = trace["points"] + for a, b in itertools.pairwise(pts): + assert sum(1 for i in range(3) if a[i] != b[i]) <= 1, trace["id"] + for end, node in ((pts[0], ends[trace["source"]]), (pts[-1], ends[trace["target"]])): + assert end[2] == pytest.approx(node["z"] + node["h"]) + assert abs(end[0] - node["x"]) <= max(node["w"], node["d"]) / 2 + assert abs(end[1] - node["y"]) <= max(node["w"], node["d"]) / 2 + # The lane is above both ends + assert pts[1][2] > max(pts[0][2], pts[-1][2]) + + def test_no_two_traces_share_a_horizontal_run(self, estate): + _, _, scene = estate + runs = [(t["id"], r) for t in scene["traces"] for r in _runs(t)] + for (ta, a), (tb, b) in itertools.combinations(runs, 2): + if ta == tb or a[0] != b[0] or abs(a[2] - b[2]) > 1e-6 or abs(a[1] - b[1]) >= 1.2: + continue + overlap = min(a[4], b[4]) - max(a[3], b[3]) + assert overlap <= 0.5, f"{ta} and {tb} share a run" + + def test_no_two_traces_share_a_port(self, estate): + _, _, scene = estate + ports = [tuple(t["points"][0]) for t in scene["traces"]] + ports += [tuple(t["points"][-1]) for t in scene["traces"]] + assert len(ports) == len(set(ports)) + + def test_families(self, estate): + _, _, scene = estate + fam = {(t["type"], t["relationship"]): t["family"] for t in scene["traces"]} + assert fam[("INTERNET_EXPOSED", "INTERNET_REACHABLE")] == "exposure" + assert fam[("ASSUMES_ROLE", None)] == "identity" + assert fam[("LOAD_BALANCER_TARGET", None)] == "network" + assert fam[("PROTECTS", None)] == "governance" + + def test_parallel_edges_collapse_into_one_counted_trace(self): + a = CloudAsset( + id="a", name="a", asset_type=AssetType.EC2, provider=CloudProvider.AWS, region="r" + ) + b = CloudAsset( + id="b", + name="b", + asset_type=AssetType.RDS_INSTANCE, + provider=CloudProvider.AWS, + region="r", + ) + edges = [ + NetworkEdge( + source_id="a", target_id="b", edge_type=EdgeType.SECURITY_GROUP_RULE, port_range=p + ) + for p in ("5432", "5433") + ] + scene = build_scene(ScanResult(assets=[a, b], edges=edges)) + assert len(scene["traces"]) == 1 + assert scene["traces"][0]["count"] == 2 + + +class TestEdgeCases: + def test_empty_scan(self): + scene = build_scene(ScanResult()) + assert scene["nodes"] == [] and scene["traces"] == [] and scene["platforms"] == [] + + def test_metadata_places_assets_without_contains_edges(self): + vpc = CloudAsset( + id="v", + name="vpc-1", + asset_type=AssetType.VPC, + provider=CloudProvider.AWS, + region="r", + metadata={"vpc_id": "vpc-1"}, + ) + vm = CloudAsset( + id="i", + name="vm", + asset_type=AssetType.EC2, + provider=CloudProvider.AWS, + region="r", + metadata={"vpc_id": "vpc-1"}, + ) + scene = build_scene(ScanResult(assets=[vpc, vm])) + node = scene["nodes"][0] + assert node["parent"] == "c:v" + + def test_too_large(self, monkeypatch): + monkeypatch.setattr("cloudg.renderers.iso_scene.MAX_SCENE_ASSETS", 1) + assets = [ + CloudAsset(id=str(i), name=str(i), asset_type=AssetType.EC2, provider=CloudProvider.AWS) + for i in range(2) + ] + scene = build_scene(ScanResult(assets=assets)) + assert scene == {"too_large": True, "asset_count": 2, "limit": 1} + assert MAX_SCENE_ASSETS >= 1000 + + @pytest.mark.parametrize( + "value, category", + [ + ("EC2", "compute"), + ("LAMBDA_FUNCTION", "function"), + ("EKS_CLUSTER", "cluster"), + ("RDS_INSTANCE", "database"), + ("S3_BUCKET", "storage"), + ("IAM_ROLE", "identity"), + ("KMS_KEY", "secret"), + ("SECURITY_GROUP", "security"), + ("ORG_POLICY", "governance"), + ("LOAD_BALANCER", "network"), + ("MESSAGE_QUEUE", "messaging"), + ("LOG_GROUP", "observability"), + ("ML_ENDPOINT", "ml"), + ("OTHER", "other"), + ], + ) + def test_categories(self, value, category): + assert category_of(value) == category + + def test_every_asset_type_has_a_category(self): + for value in AssetType: + assert category_of(value.value) + + def test_unknown_edge_types_are_data(self): + assert edge_family("SOMETHING_NEW") == "data" + + +class TestReport: + def test_report_embeds_the_scene_and_the_engines(self, estate, tmp_path): + scan, graph, _ = estate + page = HTMLReportGenerator(output_dir=str(tmp_path)).generate(scan, graph_json=graph) + text = page.read_text(encoding="utf-8") + assert "var IsoKit = (function" in text + assert "hairline kernel sha256" in text + assert '"traces":' in text and '"platforms":' in text + + def test_missing_engines_leave_the_2d_map(self, estate, tmp_path, monkeypatch): + scan, graph, _ = estate + monkeypatch.setattr(html_report, "MAP_ENGINES", {"isokit": "missing.js"}) + page = HTMLReportGenerator(output_dir=str(tmp_path)).generate(scan, graph_json=graph) + text = page.read_text(encoding="utf-8") + assert "var IsoKit = (function" not in text + assert "renderTreeTopology" in text + + def test_layout_failure_never_costs_the_report(self, estate, tmp_path, monkeypatch): + scan, graph, _ = estate + + def boom(*_a, **_k): + raise ValueError("bad layout") + + monkeypatch.setattr(html_report, "build_scene", boom) + page = HTMLReportGenerator(output_dir=str(tmp_path)).generate(scan, graph_json=graph) + assert '"error": "bad layout"' in page.read_text(encoding="utf-8") + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_inline_scripts_share_one_global_scope(estate, tmp_path): + """Every inline script runs in the same global scope, so a top-level name + declared twice (a figure's const against the report's function) stops the + report's own script from loading at all.""" + scan, graph, _ = estate + page = HTMLReportGenerator(output_dir=str(tmp_path)).generate(scan, graph_json=graph) + text = page.read_text(encoding="utf-8") + scripts = re.findall(r"", text, re.S) + assert len(scripts) == len(html_report.VENDORED_LIBRARIES) + len(html_report.MAP_ENGINES) + 1 + main = scripts[-1] + main = main[: main.index("// ── Init ──")] + stub = "globalThis.window = globalThis; globalThis.document = {};\n" + js = tmp_path / "all.js" + # Chart.js and D3 are skipped: they need a DOM to load and declare nothing that clashes + js.write_text( + stub + + "\n;\n".join(scripts[2:-1] + [main]) + + "\nconsole.log(typeof IsoKit, typeof HL, Object.keys(window.HAIRLINE_FIGURES), typeof buildMap)\n", + encoding="utf-8", + ) + proc = subprocess.run( + ["node", str(js)], capture_output=True, text=True, timeout=60, check=False + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.split() == ["object", "object", "[", "'tray'", "]", "function"] From 0a8ed715482031a1f645f42f4b6cfc54db8ea0fd Mon Sep 17 00:00:00 2001 From: morpheuslord Date: Sun, 25 Oct 2026 13:23:47 +0530 Subject: [PATCH 04/21] Build the docs site with MkDocs Material The custom generator (site/build.py, site/docsgen, site/templates and the old static CSS and JS) is replaced by MkDocs Material, set up like the Startup-SBOM docs: slate by default with a light toggle, navigation tabs, a release banner, code copy and annotations, linked tabs, mermaid and snippets. site/hooks.py and site/gen/ generate the reference pages at build time and never commit them: the CLI usage and options tables from the click commands, the config.yaml reference, the changelog, and the docs/*.md references split at their "##" headings. Split pages keep the anchors GitHub gives each heading, and every URL of the old site is still served. The API pages render their objects with mkdocstrings below the hand-written prose, with source links to GitHub. New pages: a landing page, an API overview, an Exceptions page for cloudg.exceptions, an Examples section whose scripts live in site/examples/ and are included with snippets, Credits, and a Sample report page that embeds the report published at /cloudg/sample-report.html. The docs tools are a "docs" dependency group, and the Pages workflow runs mkdocs build --strict. --- .github/workflows/pages.yml | 27 +- mkdocs.yml | 272 +++++++ pyproject.toml | 12 + site/AUTHORING.md | 257 +++--- site/build.py | 88 --- site/content/api/cloudgconfig.md | 57 +- site/content/api/cloudgengine.md | 118 +-- site/content/api/cloudontology.md | 167 ++-- site/content/api/dependencygraph.md | 118 ++- site/content/api/entry-points.md | 73 +- site/content/api/event-hooks.md | 81 +- site/content/api/exceptions.md | 205 +++++ site/content/api/findingsnormaliser.md | 77 +- site/content/api/graphbuilder.md | 174 ++--- site/content/api/index.md | 202 +++++ site/content/api/inventorymapper.md | 82 +- site/content/api/inventoryresult.md | 118 +-- site/content/api/mcp-layer.md | 90 ++- site/content/api/models.md | 94 ++- site/content/api/parse-report.md | 88 +-- site/content/api/ragexporter.md | 172 ++-- site/content/api/reachabilityanalyzer.md | 167 ++-- site/content/api/relationshiplinker.md | 58 +- site/content/api/resilience.md | 98 ++- site/content/api/results.md | 97 ++- .../img => content/assets/images}/favicon.svg | 0 site/content/assets/images/logo-mark.svg | 1 + site/content/assets/stylesheets/extra.css | 264 +++++++ site/content/cli/auth-flags.md | 42 +- site/content/cli/collect.md | 38 +- site/content/cli/deps.md | 81 +- site/content/cli/index.md | 11 + site/content/cli/ingest.md | 77 +- site/content/cli/map.md | 86 +- site/content/cli/mcp-call.md | 45 +- site/content/cli/mcp-config.md | 36 +- site/content/cli/mcp-prompts.md | 38 +- site/content/cli/mcp-read.md | 42 +- site/content/cli/mcp-resources.md | 40 +- site/content/cli/mcp-serve.md | 60 +- site/content/cli/mcp-tools.md | 38 +- site/content/cli/output-flags.md | 34 +- site/content/cli/provider-flags.md | 36 +- site/content/cli/report.md | 36 +- site/content/cli/run.md | 90 ++- site/content/cli/scan.md | 40 +- site/content/credits.md | 53 ++ site/content/examples/errors.md | 126 +++ site/content/examples/graph-and-exports.md | 188 +++++ site/content/examples/index.md | 55 ++ site/content/examples/ingest-and-report.md | 209 +++++ site/content/examples/map-and-query.md | 162 ++++ site/content/guides/authentication.md | 367 +++++---- site/content/guides/aws-organizations.md | 131 ++-- site/content/guides/ci-automation.md | 89 ++- site/content/guides/compliance.md | 93 ++- site/content/guides/dependencies.md | 49 +- site/content/guides/docker.md | 155 ++-- site/content/guides/graph-ontology-rag.md | 48 +- site/content/guides/ingesting.md | 220 ++---- site/content/guides/installation.md | 133 ++-- site/content/guides/introduction.md | 86 +- site/content/guides/inventory-mapping.md | 148 ++-- site/content/guides/pipeline.md | 85 +- site/content/guides/python-recipes.md | 137 ++-- site/content/guides/quickstart.md | 67 +- site/content/guides/rate-limits.md | 52 +- site/content/guides/reports.md | 75 +- site/content/guides/running-scanners.md | 133 ++-- site/content/index.md | 244 ++++++ site/content/mcp/connect.md | 305 ++++---- site/content/mcp/index.md | 100 ++- site/content/reference/output-files.md | 64 +- site/content/sample-report.md | 53 ++ site/data/nav.yaml | 195 ----- site/data/stats.yaml | 19 - site/docsgen/__init__.py | 1 - site/docsgen/discover.py | 107 --- site/docsgen/fences.py | 32 - site/docsgen/gen_api.py | 387 --------- site/docsgen/gen_changelog.py | 54 -- site/docsgen/highlight.py | 274 ------- site/docsgen/icons.py | 63 -- site/docsgen/links.py | 119 --- site/docsgen/md.py | 351 --------- site/docsgen/output.py | 196 ----- site/docsgen/page.py | 64 -- site/docsgen/render.py | 327 -------- site/docsgen/site.py | 78 -- site/docsgen/split.py | 217 ------ site/examples/errors/engine_errors.py | 42 + site/examples/errors/fail_fast.py | 41 + site/examples/errors/raise_for_errors.py | 19 + site/examples/errors/strict_parse_report.py | 15 + site/examples/errors/strict_scanner.py | 17 + site/examples/graph/build_graph.py | 99 +++ site/examples/graph/exposure.py | 96 +++ site/examples/graph/ontology_demo.py | 95 +++ site/examples/graph/rag_demo.py | 109 +++ site/examples/ingest/export_findings.py | 16 + site/examples/ingest/ingest_only.py | 18 + site/examples/ingest/normalise_demo.py | 29 + site/examples/ingest/parse_first.py | 23 + site/examples/ingest/save_result.py | 31 + site/examples/make_sample_reports.py | 80 ++ site/examples/map/blast_radius.py | 49 ++ site/examples/map/inventory_roundtrip.py | 48 ++ site/examples/map/map_only.py | 13 + site/examples/map/overlay_findings.py | 21 + site/examples/map/whats_exposed.py | 13 + site/gen/__init__.py | 1 + site/gen/changelog.py | 70 ++ site/{docsgen/gen_cli.py => gen/cli.py} | 87 ++- site/gen/common.py | 162 ++++ site/{data => gen}/config-notes.yaml | 0 .../gen_config.py => gen/config_ref.py} | 89 ++- site/gen/griffe_rst.py | 124 +++ site/gen/links.py | 104 +++ site/gen/split.py | 207 +++++ site/hooks.py | 399 ++++++++++ site/overrides/main.html | 10 + site/static/css/docs.css | 736 ------------------ site/static/img/logo.svg | 1 - site/static/js/code.js | 144 ---- site/static/js/config.js | 50 -- site/static/js/diagrams.js | 138 ---- site/static/js/docs.js | 20 - site/static/js/nav.js | 123 --- site/static/js/search-rank.js | 105 --- site/static/js/search-view.js | 101 --- site/static/js/search.js | 169 ---- site/static/js/theme.js | 23 - site/static/js/util.js | 50 -- site/templates/404.html | 22 - site/templates/_api_index.html | 11 - site/templates/_api_object.html | 68 -- site/templates/_cli_index.html | 23 - site/templates/_config.html | 20 - site/templates/_hero_graph.html | 51 -- site/templates/_logo_mark.html | 1 - site/templates/_options.html | 17 - site/templates/base.html | 83 -- site/templates/changelog.html | 36 - site/templates/home.html | 88 --- site/templates/page.html | 87 --- 145 files changed, 7335 insertions(+), 7197 deletions(-) create mode 100644 mkdocs.yml delete mode 100644 site/build.py create mode 100644 site/content/api/exceptions.md create mode 100644 site/content/api/index.md rename site/{static/img => content/assets/images}/favicon.svg (100%) create mode 100644 site/content/assets/images/logo-mark.svg create mode 100644 site/content/assets/stylesheets/extra.css create mode 100644 site/content/cli/index.md create mode 100644 site/content/credits.md create mode 100644 site/content/examples/errors.md create mode 100644 site/content/examples/graph-and-exports.md create mode 100644 site/content/examples/index.md create mode 100644 site/content/examples/ingest-and-report.md create mode 100644 site/content/examples/map-and-query.md create mode 100644 site/content/index.md create mode 100644 site/content/sample-report.md delete mode 100644 site/data/nav.yaml delete mode 100644 site/data/stats.yaml delete mode 100644 site/docsgen/__init__.py delete mode 100644 site/docsgen/discover.py delete mode 100644 site/docsgen/fences.py delete mode 100644 site/docsgen/gen_api.py delete mode 100644 site/docsgen/gen_changelog.py delete mode 100644 site/docsgen/highlight.py delete mode 100644 site/docsgen/icons.py delete mode 100644 site/docsgen/links.py delete mode 100644 site/docsgen/md.py delete mode 100644 site/docsgen/output.py delete mode 100644 site/docsgen/page.py delete mode 100644 site/docsgen/render.py delete mode 100644 site/docsgen/site.py delete mode 100644 site/docsgen/split.py create mode 100644 site/examples/errors/engine_errors.py create mode 100644 site/examples/errors/fail_fast.py create mode 100644 site/examples/errors/raise_for_errors.py create mode 100644 site/examples/errors/strict_parse_report.py create mode 100644 site/examples/errors/strict_scanner.py create mode 100644 site/examples/graph/build_graph.py create mode 100644 site/examples/graph/exposure.py create mode 100644 site/examples/graph/ontology_demo.py create mode 100644 site/examples/graph/rag_demo.py create mode 100644 site/examples/ingest/export_findings.py create mode 100644 site/examples/ingest/ingest_only.py create mode 100644 site/examples/ingest/normalise_demo.py create mode 100644 site/examples/ingest/parse_first.py create mode 100644 site/examples/ingest/save_result.py create mode 100644 site/examples/make_sample_reports.py create mode 100644 site/examples/map/blast_radius.py create mode 100644 site/examples/map/inventory_roundtrip.py create mode 100644 site/examples/map/map_only.py create mode 100644 site/examples/map/overlay_findings.py create mode 100644 site/examples/map/whats_exposed.py create mode 100644 site/gen/__init__.py create mode 100644 site/gen/changelog.py rename site/{docsgen/gen_cli.py => gen/cli.py} (66%) create mode 100644 site/gen/common.py rename site/{data => gen}/config-notes.yaml (100%) rename site/{docsgen/gen_config.py => gen/config_ref.py} (77%) create mode 100644 site/gen/griffe_rst.py create mode 100644 site/gen/links.py create mode 100644 site/gen/split.py create mode 100644 site/hooks.py create mode 100644 site/overrides/main.html delete mode 100644 site/static/css/docs.css delete mode 100644 site/static/img/logo.svg delete mode 100644 site/static/js/code.js delete mode 100644 site/static/js/config.js delete mode 100644 site/static/js/diagrams.js delete mode 100644 site/static/js/docs.js delete mode 100644 site/static/js/nav.js delete mode 100644 site/static/js/search-rank.js delete mode 100644 site/static/js/search-view.js delete mode 100644 site/static/js/search.js delete mode 100644 site/static/js/theme.js delete mode 100644 site/static/js/util.js delete mode 100644 site/templates/404.html delete mode 100644 site/templates/_api_index.html delete mode 100644 site/templates/_api_object.html delete mode 100644 site/templates/_cli_index.html delete mode 100644 site/templates/_config.html delete mode 100644 site/templates/_hero_graph.html delete mode 100644 site/templates/_logo_mark.html delete mode 100644 site/templates/_options.html delete mode 100644 site/templates/base.html delete mode 100644 site/templates/changelog.html delete mode 100644 site/templates/home.html delete mode 100644 site/templates/page.html diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d796f4d..3221d8f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,8 +1,9 @@ name: Docs -# Builds the documentation site (site/build.py) and deploys it to GitHub Pages -# on every push to main that touches the docs, the content they are generated -# from (click commands, docstrings, config.yaml, CHANGELOG.md) or the site itself. +# Builds the documentation site (MkDocs Material, mkdocs.yml) and deploys it to +# GitHub Pages on every push to main that touches the docs, the content they are +# generated from (click commands, docstrings, config.yaml, CHANGELOG.md) or the +# site itself. The site is served at https://morpheuslord.github.io/cloudg/. # # One-time setup: under Settings -> Pages, set "Build and deployment" # source to "GitHub Actions". The GITHUB_TOKEN cannot create the Pages @@ -18,12 +19,15 @@ on: - "cloudg/**" - "config.yaml" - "CHANGELOG.md" + - "README.md" - "pyproject.toml" + - "mkdocs.yml" - ".github/workflows/pages.yml" pull_request: paths: - "docs/**" - "site/**" + - "mkdocs.yml" - ".github/workflows/pages.yml" workflow_dispatch: @@ -46,14 +50,19 @@ jobs: with: python-version: "3.12" - # The CLI and API reference pages are generated by importing cloudg, - # so the package is installed; its core dependencies already include - # everything the site generator needs (markdown-it-py, Pygments, Jinja2). - - name: Install cloudg - run: pip install . + # The CLI, config and API reference pages are generated from the code, + # so cloudg is installed next to the docs dependency group. pip reads + # dependency groups from 25.1 on. + - name: Install cloudg and the docs tools + run: | + python -m pip install --upgrade "pip>=25.1" + pip install . --group docs - name: Build the site - run: python site/build.py --strict + env: + NO_MKDOCS_2_WARNING: "1" + DISABLE_MKDOCS_2_WARNING: "true" + run: mkdocs build --strict - uses: actions/upload-pages-artifact@v3 if: github.event_name != 'pull_request' diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..599e1e2 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,272 @@ +# MkDocs Material configuration for https://morpheuslord.github.io/cloudg/ +# +# Hand-written pages live in site/content/. The reference pages (CLI options, +# config.yaml, the changelog and the long docs/*.md references) are generated +# at build time by site/hooks.py and are never committed. site/AUTHORING.md +# explains how to write a page. +# +# .venv/bin/mkdocs serve -a 127.0.0.1:8807 # live preview +# .venv/bin/mkdocs build --strict # what CI runs + +site_name: cloudg +site_url: https://morpheuslord.github.io/cloudg/ +site_description: >- + cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, + runs security scanners over the same inventory and merges every finding + into one deduplicated, compliance-mapped result. +site_author: morpheuslord + +repo_url: https://github.com/morpheuslord/cloudg +repo_name: morpheuslord/cloudg +edit_uri: edit/main/site/content/ + +docs_dir: site/content +site_dir: site/dist + +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + anchors: warn + +watch: + - site/hooks.py + - site/gen + - site/overrides + - site/static + - docs + - cloudg + - CHANGELOG.md + - config.yaml + +hooks: + - site/hooks.py + +nav: + - Home: index.md + - Guides: + - Start: + - Introduction: guides/introduction.md + - Installation: guides/installation.md + - Quickstart: guides/quickstart.md + - The pipeline: guides/pipeline.md + - Mapping: + - Inventory mapping: guides/inventory-mapping.md + - Dependencies and blast radius: guides/dependencies.md + - AWS Organizations: guides/aws-organizations.md + - Scanning: + - Running scanners: guides/running-scanners.md + - Ingesting existing output: guides/ingesting.md + - Compliance frameworks: guides/compliance.md + - Analyse: + - Graph, ontology and RAG: guides/graph-ontology-rag.md + - Reports: guides/reports.md + - Sample report: sample-report.md + - Operate: + - Authentication: guides/authentication.md + - Rate limits and throttling: guides/rate-limits.md + - Docker: guides/docker.md + - CI and automation: guides/ci-automation.md + - Python recipes: guides/python-recipes.md + - Examples: + - examples/index.md + - Map and query: examples/map-and-query.md + - Ingest and report: examples/ingest-and-report.md + - Graph, ontology and RAG: examples/graph-and-exports.md + - Errors and strict mode: examples/errors.md + - CLI: + - cli/index.md + - Pipeline: + - cloudg run: cli/run.md + - cloudg collect: cli/collect.md + - cloudg scan: cli/scan.md + - cloudg report: cli/report.md + - Inventory: + - cloudg map: cli/map.md + - cloudg deps: cli/deps.md + - cloudg ingest: cli/ingest.md + - MCP: + - cloudg mcp serve: cli/mcp-serve.md + - cloudg mcp config: cli/mcp-config.md + - cloudg mcp tools: cli/mcp-tools.md + - cloudg mcp resources: cli/mcp-resources.md + - cloudg mcp prompts: cli/mcp-prompts.md + - cloudg mcp call: cli/mcp-call.md + - cloudg mcp read: cli/mcp-read.md + - Shared: + - Provider and region flags: cli/provider-flags.md + - Auth flags: cli/auth-flags.md + - Output flags: cli/output-flags.md + - Python API: + - api/index.md + - Engine: + - CloudGEngine: api/cloudgengine.md + - CloudGConfig: api/cloudgconfig.md + - Result objects: api/results.md + - Event hooks: api/event-hooks.md + - Exceptions: api/exceptions.md + - Inventory: + - InventoryMapper: api/inventorymapper.md + - InventoryResult: api/inventoryresult.md + - RelationshipLinker: api/relationshiplinker.md + - DependencyGraph: api/dependencygraph.md + - Graph: + - GraphBuilder: api/graphbuilder.md + - ReachabilityAnalyzer: api/reachabilityanalyzer.md + - CloudOntology: api/cloudontology.md + - RAGExporter: api/ragexporter.md + - Findings: + - FindingsNormaliser: api/findingsnormaliser.md + - parse_report: api/parse-report.md + - Data models: api/models.md + - Extend: + - Entry points: api/entry-points.md + - cloudg.resilience: api/resilience.md + - CloudGMCPLayer: api/mcp-layer.md + - MCP server: + - mcp/index.md + - Connect a client: mcp/connect.md + # docs/*.md entries are split into one page per section by site/hooks.py + - Server: docs/MCP.md + - Tools: docs/MCP_TOOLS.md + - Privacy and policy: docs/MCP_PRIVACY.md + - Internals: docs/MCP_INTERNALS.md + - Reference: + # one page per top-level section of config.yaml, generated by site/hooks.py + - config.yaml: config.yaml + - Outputs: + - Output files: reference/output-files.md + - Inventory structures: docs/INVENTORY_REFERENCE.md + - Catalogs: docs/INVENTORY_CATALOG.md + - Inventory internals: docs/INVENTORY_INTERNALS.md + - Resilience internals: docs/RESILIENCE.md + - About: + - Changelog: changelog.md + - Credits: credits.md + +theme: + name: material + # The release banner on every page (site/overrides/main.html; bump it with the version) + custom_dir: site/overrides + logo: assets/images/logo-mark.svg + favicon: assets/images/favicon.svg + icon: + repo: fontawesome/brands/github + palette: + # Dark by default: it is a security tool. + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: cyan + toggle: + icon: material/weather-sunny + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: blue grey + accent: cyan + toggle: + icon: material/weather-night + name: Switch to dark mode + features: + - navigation.tabs + - navigation.sections + - navigation.indexes + - navigation.top + - navigation.footer + - search.suggest + - search.highlight + - content.code.copy + - content.code.annotate + - content.code.select + - content.tabs.link + - content.action.edit + - toc.follow + +plugins: + - search + - redirects: + redirect_maps: + # folder addresses that never had a page of their own + guides/index.md: guides/introduction.md + reference/index.md: reference/config/providers.md + reference/config/index.md: reference/config/providers.md + about/index.md: changelog.md + - mkdocstrings: + default_handler: python + handlers: + python: + paths: [.] + options: + extensions: + - site/gen/griffe_rst.py:RstToMarkdown + docstring_style: google + docstring_options: + # cloudg's "Returns:" sections are prose that wraps at the same + # indent, not one "type: description" item per line + returns_multiple_items: false + docstring_section_style: table + merge_init_into_class: true + show_root_heading: true + show_root_full_path: false + show_root_members_full_path: false + show_object_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + show_signature_annotations: true + separate_signature: true + signature_crossrefs: true + show_source: true + show_bases: true + show_labels: true + members_order: source + filters: ["!^_"] + heading_level: 3 + line_length: 88 + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: [site/examples, .] + check_paths: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +extra_css: + - assets/stylesheets/extra.css + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/morpheuslord/cloudg + - icon: fontawesome/brands/python + link: https://pypi.org/project/cloudg/ + - icon: fontawesome/brands/docker + link: https://github.com/morpheuslord/cloudg/pkgs/container/cloudg + +# copyright and extra.release are set by site/hooks.py from CHANGELOG.md diff --git a/pyproject.toml b/pyproject.toml index 3a47f38..204ce0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,18 @@ full = [ "policy-sentry>=0.12", ] +# Builds the documentation site (mkdocs.yml): pip install --group docs +[dependency-groups] +docs = [ + "mkdocs>=1.6.1,<2", + "mkdocs-material>=9.8.0,<10", + "mkdocstrings[python]>=1.0.8,<2", + "mkdocstrings-python>=2.0.10,<3", + "mkdocs-redirects>=1.2.3,<2", + # formats the signatures mkdocstrings renders + "ruff>=0.17.2,<1", +] + [project.scripts] cloudg = "cloudg.cli:cli" cloudg-mcp = "cloudg.mcp.cli:main" diff --git a/site/AUTHORING.md b/site/AUTHORING.md index 4fdc01e..03fc8ae 100644 --- a/site/AUTHORING.md +++ b/site/AUTHORING.md @@ -1,190 +1,259 @@ # Writing pages for the cloudg docs site -The site is built by `site/build.py` from three kinds of input: +The site at https://morpheuslord.github.io/cloudg/ is built with [MkDocs](https://www.mkdocs.org/) and the [Material theme](https://squidfunk.github.io/mkdocs-material/). It has three kinds of page: -- hand-written pages in `site/content/` (guides, CLI examples, API prose) -- generated reference data (click commands, docstrings, `config.yaml`, `CHANGELOG.md`) -- the long reference documents in `docs/*.md`, which are split at their `##` headings +- hand-written pages in `site/content/` (guides, examples, CLI prose, API prose); +- reference pages generated at build time from the code and the repository files: the CLI options tables (click commands), the API signatures and parameter tables (docstrings, through mkdocstrings), the `config.yaml` reference, the changelog, and the long `docs/*.md` references split into one page per section; +- static files copied as they are: the sample report, `docs/viewer.html` and `docs/assets/`. -Build it with: +Generated pages are never committed. `site/hooks.py` creates them on every build. + +## Build and preview + +The docs tools are a dependency group in `pyproject.toml`. Install them next to cloudg in a virtual environment: ```bash -.venv/bin/python site/build.py # writes site/dist -.venv/bin/python site/build.py --serve # builds, then serves on http://127.0.0.1:8000/cloudg/ +python -m venv .venv +.venv/bin/pip install --upgrade "pip>=25.1" +.venv/bin/pip install -e . --group docs + +.venv/bin/mkdocs serve -a 127.0.0.1:8807 # live preview on http://127.0.0.1:8807/cloudg/ +.venv/bin/mkdocs build --strict # what CI runs; writes site/dist ``` +With uv: `uv pip install -e . --group docs`. `mkdocs build --strict` must finish with no warnings: a broken link, a missing anchor or a page left out of the nav fails the build in CI. Set `NO_MKDOCS_2_WARNING=1` and `DISABLE_MKDOCS_2_WARNING=true` to hide the banners some plugins print about the future of MkDocs. + +## Layout + +| Path | What it holds | +|---|---| +| `mkdocs.yml` | Site settings, theme, plugins, markdown extensions and the nav | +| `site/content/` | The docs directory: every hand-written page, plus `assets/` (logo, favicon, `extra.css`) | +| `site/overrides/main.html` | Theme override: the release banner at the top of every page | +| `site/hooks.py` | MkDocs hooks that generate the reference pages and copy the static files | +| `site/gen/` | The generators the hooks call: `cli.py`, `config_ref.py`, `changelog.py`, `split.py`, `links.py`, and `griffe_rst.py` for docstrings | +| `site/gen/config-notes.yaml` | Descriptions for `config.yaml` keys that have neither a comment in the file nor a `Field(description=...)` | +| `site/examples/` | Example scripts that pages include with snippets | +| `site/static/sample-report.html` | The sample report, built by `scripts/build_sample_report.py` | + ## Style rules - No em dashes anywhere, and no `--` or spaced en dash standing in for one. Use a comma, a colon, parentheses or two sentences. - Straight quotes only. No emoji. - Sentence-case headings. - Run every page you write through the `/human` skill's scanner before committing it. -- Every code example must work as written against the current release. Check signatures in the source; run the Python ones when you can (the sample scanner output inlined in `tests/test_ingest.py` is handy for ingest examples). +- Every code example must work as written against the current release. Check signatures in the source and run the Python ones when you can (the sample scanner output inlined in `tests/test_ingest.py` is handy for ingest examples). Prefer putting a longer example in `site/examples/` and including it, so the script on the page is a file you can run. + +## Adding a page -## Front matter +1. Create the Markdown file under `site/content/`. +2. Add it to `nav:` in `mkdocs.yml`. A page that is not in the nav fails the strict build. +3. Link to other pages with relative paths to their Markdown files, for example `[cloudg map](../cli/map.md#usage)`. MkDocs checks every link and anchor and turns them into the right URLs. Never write site-absolute links (`/cloudg/...`). -Every file starts with YAML front matter. +A page starts with optional front matter, then the body: -```yaml +```markdown --- -title: Inventory mapping -lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" -meta: # optional strip under the lede (guides only) - - [Command, "`cloudg map`"] - - [Providers, "AWS, Azure, GCP"] -source: docs/DOCUMENTATION.md # optional: what "Edit on GitHub" opens. Defaults to this file. -since: "0.5.1" # optional: shown as "Last changed in v0.5.1" +title: Inventory mapping # optional: the nav title is used otherwise +description: One sentence for search engines and link previews. --- -``` -`lede` and `meta` values are inline markdown. +`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together? +{ .lede } -## Links +**Command** `cloudg map` · **Providers** AWS, Azure, GCP +{ .page-meta } -Write site links as root-relative paths without the base: `/guides/quickstart/`, `/cli/map/#options`, `/reference/config/ratelimit/#aws.retry_budget`. The build adds the base path. The full URL map is in `site/data/nav.yaml`. +## First section +``` -## Markdown extensions +The `{ .lede }` paragraph is the larger introduction under the title. The `{ .page-meta }` line is the strip of facts under it; end it with `**Last changed** [v0.6.1](../changelog.md#0.6.1)` when a page documents behaviour that changed in a release. The body never starts with an `#` heading: the title comes from the nav or the front matter. -Standard CommonMark plus GFM tables. On top of that: +## Markdown -### Headings +The extensions are listed in `mkdocs.yml`. The ones pages use most: -`##` and `###` feed the "On this page" list. Anchors are generated from the text; force one with a trailing `{#my-anchor}`. Never skip a level. +### Headings and anchors -### Code blocks +`##` and `###` feed the table of contents. Anchors are generated from the text the way GitHub does it; set one yourself with a trailing `{ #my-anchor }`. Never skip a level. -The info string takes a language and optional attributes: +### Code blocks ````markdown -```python title="map_and_overlay.py" hl="4-6" +```python title="map_and_overlay.py" hl_lines="4-6" from cloudg import CloudGConfig, CloudGEngine ... ``` ```` -- `title` shows as the file name in the editor bar. Use a real file name for code people should save (`deploy.py`, `config.yaml`, `.github/workflows/cloudg.yml`). -- `hl` highlights lines: `hl="3"`, `hl="2,5-7"`. -- `bash` and `shell` blocks render as a terminal. Each command line gets a `$` prompt automatically; comment lines (`# ...`) are left out when the reader presses Copy. Put each command on its own line, or continue it with a trailing `\`. -- `console` is for a captured session with output. Write the prompt yourself (`$ cloudg deps ...`) and the output below it. -- Other languages (`python`, `yaml`, `json`, `toml`, `dockerfile`, `ini`, `text`) get line numbers. +- `title` shows as the file name above the block. Use a real file name for code people should save. +- `hl_lines` highlights lines: `hl_lines="3"`, `hl_lines="2 5-7"`. +- `bash` blocks hold commands only; `console` is for a captured session, with the `$` prompt written out and the output below it. +- Every block has a copy button. Annotations work too: put `# (1)!` at the end of a line and a numbered list after the block. -### Code tabs +### Code from a file -Consecutive fenced blocks that each carry a `tab` attribute become one tabbed block. Tabs with the same label are linked across the page, and the reader's choice is remembered. +`pymdownx.snippets` includes a file, so the code on the page is the script that was run. Paths are relative to `site/examples/` (then the repository root): ````markdown -```bash tab="CLI" -cloudg map -p aws --regions all +```python title="strict_scanner.py" +--8<-- "errors/strict_scanner.py" ``` +```` -```python tab="Python" title="map.py" -from cloudg import CloudGConfig, CloudGEngine -... -``` +A missing file fails the build. -```bash tab="Docker" -docker run --rm -v ~/.aws:/home/cloudg/.aws:ro -v "$PWD/reports:/app/reports" cloudg map -p aws --regions all -``` +### Tabs + +Tabs with the same label are linked across the whole site, so a reader who picks "Python" once sees Python everywhere. + +````markdown +=== "CLI" + + ```bash + cloudg map -p aws --regions all + ``` + +=== "Python" + + ```python title="map.py" + from cloudg import CloudGConfig, CloudGEngine + ... + ``` ```` -Use the labels `CLI`, `Python`, `Docker` for ways to do one thing, and `pip`, `uv`, `Docker` for installs. +Use the labels `CLI`, `Python`, `Docker` for ways to do one thing, and `pip`, `uv`, `Docker` for installs. The content of a tab is indented by four spaces. ### Diagrams -A `mermaid` block renders as a diagram. Add a caption with `caption="..."`. +A `mermaid` block renders as a diagram that follows the light and dark palette. A caption goes in a paragraph right after it: ````markdown -```mermaid caption="What cloudg run does, phase by phase" +```mermaid flowchart LR A[Collect] --> B[Graph] --> C[Scan] --> D[Normalise] --> E[Report] ``` +What cloudg run does, phase by phase. +{ .diagram-caption } ```` -Keep diagrams small enough to read at 680 px: up to about 12 nodes. Prefer `flowchart LR` or `TD`, `sequenceDiagram` for request flows and `stateDiagram-v2` for state machines. Don't set colours or `classDef` styles; the theme handles it. Quote labels that contain punctuation: `A["cloudg map --org"]`. +Keep diagrams small enough to read on a laptop: up to about 12 nodes. Prefer `flowchart LR` or `TD`, `sequenceDiagram` for request flows and `stateDiagram-v2` for state machines. Don't set colours or `classDef` styles. Quote labels that contain punctuation: `A["cloudg map --org"]`. -### Callouts +### Admonitions ```markdown -:::note -The map doesn't depend on the scanners. -::: +!!! note + The map doesn't depend on the scanners. -:::warning Admin rights -AWSControlTowerExecution has admin rights. -::: +!!! warning "AWSControlTowerExecution has admin rights" + Without `--org-role`, cloudg assumes that role in every member account. + +??? tip "Collapsed until clicked" + Use `???` for detail most readers can skip. ``` -Kinds: `note`, `tip`, `warning`, `danger`. Text after the kind replaces the default label. +Kinds: `note`, `tip`, `warning`, `danger`, `example`, `info`. -### Steps +### Steps, cells and link cards -Numbered steps, one `###` per step. Use four colons on the outer fence when a callout sits inside. +Numbered steps are `###` headings inside a `steps` block; the numbers come from the stylesheet: ```markdown -::::steps +
      + ### Install + ... -:::tip -... -::: + ### Authenticate + ... -:::: -``` -### Numbered cells +
      +``` -Two to four short items side by side, one `###` per cell. +Two to four short items side by side are cards in a grid: ```markdown -:::cells +
      +
      + ### Deep collectors -ECS, EKS, Lambda triggers... -### Breadth sweep -On AWS, a Cloud Control sweep... -::: -``` -### Link cards +131 dedicated AWS collector tasks... + +
      +
      +``` -A list of links rendered as a grid of cards. The text after the link is the description. +A list of links with a line of text each becomes a grid of cards: ```markdown -:::links -- [Inventory mapping](/guides/inventory-mapping/) What exists and how it is wired. -- [cloudg deps](/cli/deps/) Blast radius for one asset. -::: +
      + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + What exists and how it is wired. + +
      ``` ## Page kinds -### Guides: `site/content/guides/.md` +### Guides and examples -Front matter plus a body. The body starts with prose or an `##`, never with an `#` (the title comes from front matter). +Plain pages in `site/content/guides/` and `site/content/examples/`. ### CLI commands: `site/content/cli/.md` -The generator writes the usage line and the options table from the click command. Your file adds: +The front matter names the command, and a marker says where the generated "Usage" and "Options" sections go: -```yaml +```markdown --- command: map # or "mcp serve" -lede: Builds the scanner-independent inventory map. -intro: | # optional markdown shown above the usage line - ... +description: Builds the scanner-independent inventory map. --- + +One paragraph on what the command is for. + + + +## How it works +... ``` -The body follows the options table. Typical sections: `## Examples`, `## Output files`, `## Exit codes`, `## Notes`. Pages without a `command` key (the shared flag pages) are plain pages. +The tables come from the click definitions in `cloudg/cli*.py`, so they match `--help`. `cli/index.md` uses `` for the table of every command, in nav order. Typical sections after the options: `## Examples`, `## Output files`, `## Exit codes`, `## Notes`. ### Python API: `site/content/api/.md` -```yaml ---- -object: cloudg.api.CloudGEngine # import path; omit for a plain page -members: [run_pipeline, run_pipeline_sync, collect, map_inventory] # optional order/filter -lede: Runs the whole pipeline from Python, or any one phase of it. ---- +Prose and examples come first, then a `## Reference` section where mkdocstrings renders the objects from their docstrings: + +```markdown +## Reference + +::: cloudg.api.CloudGEngine + options: + members: + - run_pipeline + - map_inventory_sync ``` -The generator writes the kind tag, source link, signature and a ruled row per member (signature, docstring, parameter table). Your body follows the members. Typical sections: `## Examples`, `## Notes`. +`members` picks and orders the members; `members: false` shows the object without them. The signatures, parameter, return and raises tables, and the "source" link to GitHub come from the code. Docstrings follow the Google style; `site/gen/griffe_rst.py` turns the reST roles (`:class:`, double backticks, `::` blocks) they still use into Markdown while the site is built, without changing the source. + +### Generated reference pages + +Three kinds of nav entry are replaced by generated pages in `site/hooks.py`: + +- `docs/X.md`: the document is split into one page per `##` section under the prefix in `SPLITS`. A section over 40 KB with at least three `###` headings becomes a folder with one page per `###`. Every heading keeps the anchor GitHub gives it, so links into the single-file document keep working, and links between the documents are pointed at the page that now holds their target. +- `config.yaml`: one page per top-level section, from the comments in `config.yaml`, the `Field(description=...)` of the `CloudGConfig` models and `site/gen/config-notes.yaml`. +- `changelog.md`: `CHANGELOG.md`, with each release's anchor set to its version (`changelog/#0.6.1`). + +To change what those pages say, edit their source (the `docs/*.md` file, `config.yaml`, the models or `CHANGELOG.md`). The "Edit" button on a generated page opens its source on GitHub. + +### Sample report + +`sample-report.md` embeds `site/static/sample-report.html`, which the hooks publish at `/cloudg/sample-report.html` (and at the old address, `/cloudg/static/sample-report.html`). Rebuild the file with `python scripts/build_sample_report.py`; the page needs no change. + +## URLs + +Every page keeps the address it had on the earlier site generator: `guides/quickstart.md` is served at `/cloudg/guides/quickstart/`, `cli/index.md` at `/cloudg/cli/`. Folder addresses without a page of their own (`/cloudg/guides/`, `/cloudg/reference/`) redirect through the `redirects` plugin; add an entry under `redirect_maps` in `mkdocs.yml` when a page moves. diff --git a/site/build.py b/site/build.py deleted file mode 100644 index 6ed2ff3..0000000 --- a/site/build.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Build the cloudg documentation site into site/dist. - - python site/build.py # build - python site/build.py --serve # build, then serve on http://127.0.0.1:8000/cloudg/ - python site/build.py --strict # fail on broken internal links (used in CI) - python site/build.py --base / # build for serving at the domain root - -See site/AUTHORING.md for the page format. The generator itself lives in -site/docsgen/; this file is the command line around it. -""" - -from __future__ import annotations - -import argparse -import functools -import http.server -import os -import sys -from pathlib import Path -from urllib.parse import urlsplit - -SITE = Path(__file__).resolve().parent -REPO = SITE.parent -sys.path.insert(0, str(SITE)) -sys.path.insert(0, str(REPO)) - -from docsgen.site import Site # noqa: E402 - - -def serve(out: Path, base: str, port: int) -> None: - class Handler(http.server.SimpleHTTPRequestHandler): - def end_headers(self): - self.send_header("Cache-Control", "no-store") - super().end_headers() - - def translate_path(self, path): # noqa: D401 - parsed = urlsplit(path).path - if base != "/" and parsed.startswith(base): - parsed = "/" + parsed[len(base) :] - return str(out) + parsed - - def send_error(self, code, message=None, explain=None): - if code == 404 and (out / "404.html").exists(): - body = (out / "404.html").read_bytes() - self.send_response(404) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - return - super().send_error(code, message, explain) - - handler = functools.partial(Handler, directory=str(out)) - with http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) as httpd: - print(f"serving http://127.0.0.1:{port}{base}") - httpd.serve_forever() - - -def main() -> int: - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--out", default=str(SITE / "dist")) - ap.add_argument("--base", default=os.environ.get("DOCS_BASE", "/cloudg/")) - ap.add_argument("--strict", action="store_true", help="exit non-zero on broken internal links") - ap.add_argument("--serve", action="store_true") - ap.add_argument("--port", type=int, default=8000) - args = ap.parse_args() - - site = Site(args.base, args.strict) - site.discover() - site.render_all() - out = Path(args.out) - site.write(out) - problems = site.check_links() - for p in problems: - print(f"link: {p}", file=sys.stderr) - print(f"built {len(site.pages)} pages into {out} (base {site.base})") - if problems and args.strict: - return 1 - if args.serve: - serve(out, site.base, args.port) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/site/content/api/cloudgconfig.md b/site/content/api/cloudgconfig.md index 762177e..be990b0 100644 --- a/site/content/api/cloudgconfig.md +++ b/site/content/api/cloudgconfig.md @@ -1,10 +1,10 @@ --- -object: [cloudg.config.CloudGConfig, cloudg.config.load_config] -members: - cloudg.config.CloudGConfig: [] -lede: "`CloudGConfig` is the pydantic model behind `config.yaml`. Build it in code, load it from a file, or do both and override a few fields." +description: CloudGConfig is the pydantic model behind config.yaml. Build it in code, load it from a file, or do both and override a few fields. --- +`CloudGConfig` is the pydantic model behind `config.yaml`. Build it in code, load it from a file, or do both and override a few fields. +{ .lede } + ## Two ways to get a config `CloudGConfig()` with no arguments is a working configuration: AWS only, `us-east-1`, all five scanners enabled, ontology and RAG export on, Terraform export off. Every field has a default, so you only set what differs. @@ -19,9 +19,9 @@ config.aws.regions = ["eu-west-1", "eu-central-1"] config.scanners.enabled = ["prowler", "iam"] ``` -From a file, `load_config(path)` reads the YAML with `yaml.safe_load` and hands the dict to `CloudGConfig.model_validate()`. The file has the same shape as the model: top-level keys are the attributes below, nested keys are the fields of each section. The generated [config.yaml reference](/reference/config/providers/) documents every key with its default. +From a file, `load_config(path)` reads the YAML with `yaml.safe_load` and hands the dict to `CloudGConfig.model_validate()`. The file has the same shape as the model: top-level keys are the attributes below, nested keys are the fields of each section. The generated [config.yaml reference](../reference/config/providers.md) documents every key with its default. -```mermaid caption="What load_config does with the path it gets" +```mermaid flowchart TD P["load_config(path)"] --> N{"path is None?"} N -->|yes| D["CloudGConfig() defaults"] @@ -33,6 +33,8 @@ flowchart TD U --> C["CloudGConfig"] V -->|invalid value| X["pydantic ValidationError"] ``` +What load_config does with the path it gets +{ .diagram-caption } Two branches in that diagram return defaults instead of failing: a `None` path and a path that does not exist. A typo in the file name therefore gives you a run against `us-east-1` with every scanner enabled, and the only sign is a `WARNING cloudg.config: Config file ... not found, using defaults` line. If that matters, check `Path(path).is_file()` before calling, or use the strict loader in the examples. An empty file also gives defaults. The CLI calls the same function with the `-c` value and does not look for a `config.yaml` in the working directory on its own. @@ -77,7 +79,7 @@ Every section ignores keys it does not define, and says so. An unknown key at th | every Azure credential | `DefaultAzureCredential` | | `gcp.credentials_file` | application default credentials, which read `GOOGLE_APPLICATION_CREDENTIALS` | -Two variables change cloudg behaviour outside the config: `CLOUDG_CATALOG_DIR` overlays the inventory reference catalogs, and `CLOUDG_CACHE_DIR` moves the Cloud Control schema cache (default `~/.cache/cloudg`). [Authentication](/guides/authentication/) covers the full credential order for each provider. +Two variables change cloudg behaviour outside the config: `CLOUDG_CATALOG_DIR` overlays the inventory reference catalogs, and `CLOUDG_CACHE_DIR` moves the Cloud Control schema cache (default `~/.cache/cloudg`). [Authentication](../guides/authentication.md) covers the full credential order for each provider. ## Examples @@ -158,7 +160,6 @@ from pydantic import BaseModel from cloudg import CloudGConfig - def unknown_keys(data: object, model: type[BaseModel], path: str = "") -> list[str]: """Dotted paths of the keys in `data` that `model` does not define.""" if not isinstance(data, dict): @@ -172,7 +173,6 @@ def unknown_keys(data: object, model: type[BaseModel], path: str = "") -> list[s found += unknown_keys(value, field.annotation, f"{path}{key}.") return found - def load_strict(path: str) -> CloudGConfig: file = Path(path) if not file.is_file(): @@ -183,7 +183,6 @@ def load_strict(path: str) -> CloudGConfig: raise ValueError("unknown config keys: " + ", ".join(bad)) return CloudGConfig.model_validate(data) - if __name__ == "__main__": try: config = load_strict(sys.argv[1] if len(sys.argv) > 1 else "config.yaml") @@ -211,12 +210,36 @@ The patterns are enforced at construction: `aws.retry_mode` must be `legacy`, `s `InventoryMapper` deep-copies the config before a run, so organization discovery (which rewrites `aws.accounts`, `aws.role_name` and possibly `aws.regions`) never changes the object you passed in. +## Reference + +::: cloudg.config.CloudGConfig + options: + members: false + +::: cloudg.config.load_config + ## Related -:::links -- [config.yaml reference](/reference/config/providers/) Every key, its type and default. -- [Rate limit settings](/reference/config/ratelimit/) The `ratelimit` section in full. -- [Authentication](/guides/authentication/) Credential order for AWS, Azure and GCP. -- [CloudGEngine](/api/cloudgengine/) What runs on the config. -- [Rate limits and throttling](/guides/rate-limits/) When to change the `ratelimit` defaults. -::: +
      + +- [**config.yaml reference**](../reference/config/providers.md) + + Every key, its type and default. + +- [**Rate limit settings**](../reference/config/ratelimit.md) + + The `ratelimit` section in full. + +- [**Authentication**](../guides/authentication.md) + + Credential order for AWS, Azure and GCP. + +- [**CloudGEngine**](cloudgengine.md) + + What runs on the config. + +- [**Rate limits and throttling**](../guides/rate-limits.md) + + When to change the `ratelimit` defaults. + +
      diff --git a/site/content/api/cloudgengine.md b/site/content/api/cloudgengine.md index f9cc192..b2c7ec9 100644 --- a/site/content/api/cloudgengine.md +++ b/site/content/api/cloudgengine.md @@ -1,14 +1,15 @@ --- -object: cloudg.api.CloudGEngine -members: [run_pipeline, run_pipeline_sync, collect, scan, analyze, ingest_reports, normalise_findings, run_from_reports, run_from_reports_sync, map_inventory, map_inventory_sync] -lede: "The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: call the whole pipeline in one line, or drive each phase yourself." +description: 'The engine behind cloudg run, cloudg ingest and cloudg map: call the whole pipeline in one line, or drive each phase yourself.' --- +The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: call the whole pipeline in one line, or drive each phase yourself. +{ .lede } + ## Which method to call -`CloudGEngine` takes one argument, a [`CloudGConfig`](/api/cloudgconfig/), and keeps it on `engine.config`. Everything else is a method call. The methods fall into three groups, and they share very little state: the engine holds the config and the six [event hooks](/api/event-hooks/), nothing more. You can call `collect()` twice, or call `analyze()` on assets you built by hand, and nothing on the engine gets in the way. +`CloudGEngine` takes one argument, a [`CloudGConfig`](cloudgconfig.md), and keeps it on `engine.config`. Everything else is a method call. The methods fall into three groups, and they share very little state: the engine holds the config and the six [event hooks](event-hooks.md), nothing more. You can call `collect()` twice, or call `analyze()` on assets you built by hand, and nothing on the engine gets in the way. -```mermaid caption="What each entry point runs" +```mermaid flowchart LR RP["run_pipeline()"] --> C["collect()"] RP --> S["scan()"] @@ -21,6 +22,8 @@ flowchart LR RFR --> R MI["map_inventory()"] --> IM["InventoryMapper"] ``` +What each entry point runs +{ .diagram-caption } | You have | Call | Cloud access | Scanner binaries | |---|---|---|---| @@ -40,10 +43,10 @@ The `*_sync` variants wrap the async method in `asyncio.run()`. Use them from pl 1. `collect()` runs the collectors for every provider in `config.providers`, across `aws.regions`, `aws.accounts`, the Azure subscriptions and the GCP projects you configured. 2. `scan()` builds a graph of the collected assets and runs the reachability analysis on it, then starts every scanner in `scanners.enabled` in a thread pool with one worker per job: Prowler and ScoutSuite once per provider, with `aws.profile` and the rest of the configured AWS credentials and regions; Checkov once per IaC directory; Trivy once over `scanners.trivy_images`, or over the IaC directories when there are no images; scanner plugins by name. The IAM policy linter runs when `iam` is in the list and there are assets. Unknown names are logged and skipped. 3. `analyze()` rebuilds the graph and records node and edge counts, reachability findings and lateral movement paths. It then writes the ontology (`ontology.enabled`), the RAG chunks (`rag.enabled`) and the Terraform recreation (`terraform.enabled`). -4. The scanner findings and the reachability findings go through [`FindingsNormaliser`](/api/findingsnormaliser/): deduplication, cross-scanner merging, CVSS rescoring and compliance mapping. The reachability analysis ran in both step 2 and step 3. Both runs produce the same check on the same resource, so the deduplication pass folds the copies into one. +4. The scanner findings and the reachability findings go through [`FindingsNormaliser`](findingsnormaliser.md): deduplication, cross-scanner merging, CVSS rescoring and compliance mapping. The reachability analysis ran in both step 2 and step 3. Both runs produce the same check on the same resource, so the deduplication pass folds the copies into one. 5. The reports in `report.formats` are written: `findings.json` and `report.html` with the D3 graph embedded, and `topology.svg`. -Everything ends up in one [`PipelineResult`](/api/results/). +Everything ends up in one [`PipelineResult`](results.md). ### IaC targets for Checkov @@ -53,7 +56,7 @@ Everything ends up in one [`PipelineResult`](/api/results/). ### Full pipeline -This needs read access to the account (any method from [Authentication](/guides/authentication/) works) and the scanner binaries you enable. Missing binaries are skipped. +This needs read access to the account (any method from [Authentication](../guides/authentication.md) works) and the scanner binaries you enable. Missing binaries are skipped. ```python title="nightly_pipeline.py" import asyncio @@ -61,7 +64,6 @@ import json from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.profile = "security-audit" @@ -82,7 +84,6 @@ async def main() -> None: if cov.failed_services: print("incomplete:", cov.to_summary()) - asyncio.run(main()) ``` @@ -91,24 +92,7 @@ asyncio.run(main()) No credentials and no scanners: point the engine at reports produced elsewhere. Keys are tool names (`prowler`, `scoutsuite`, `checkov`, `trivy`); values are lists of files or directories. ```python title="ingest_only.py" -from cloudg import CloudGConfig, CloudGEngine - -engine = CloudGEngine(CloudGConfig()) -engine.on_phase_start = lambda phase: print("phase:", phase) - -result = engine.run_from_reports_sync( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy.json"], - }, - output_dir="./reports", -) - -print(result.total_findings, result.severity_breakdown) -print(result.report_paths) -for f in result.findings: - print(f.severity.value, f.risk_score, f.source_tool, f.title) +--8<-- "ingest/ingest_only.py" ``` With one failing Prowler check, one Checkov failure and one Trivy CVE, it prints: @@ -120,7 +104,7 @@ phase: normalisation phase: reporting 3 {'CRITICAL': 1, 'HIGH': 2} {'json': PosixPath('reports/findings.json'), 'html': PosixPath('reports/report.html')} -CRITICAL 9.5 trivy [Trivy] CVE-2024-0001: openssl (alpine) +CRITICAL 9.7 trivy [Trivy] CVE-2024-0001: openssl (alpine) HIGH 7.5 prowler S3 bucket default encryption HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled ``` @@ -129,25 +113,13 @@ The result has no assets, so `graph_nodes`, `ontology_triples` and `attack_paths ### Map only -`map_inventory()` runs the deep inventory collectors and the [relationship linker](/api/relationshiplinker/). No scanner is involved. Passing `output_dir` writes the map files; passing `findings` as well writes `asset-map.json` and `compliance-map.json`. +`map_inventory()` runs the deep inventory collectors and the [relationship linker](relationshiplinker.md). No scanner is involved. Passing `output_dir` writes the map files; passing `findings` as well writes `asset-map.json` and `compliance-map.json`. ```python title="map_only.py" -from cloudg import CloudGConfig, CloudGEngine - -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["ALL"] -config.inventory.exclude_services = ["kubernetes"] - -engine = CloudGEngine(config) -inventory = engine.map_inventory_sync(output_dir="./inventory", tagging_sweep=False) - -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("internet exposed:", summary["internet_exposed"]) -print("top services:", list(summary["assets_by_service"].items())[:5]) +--8<-- "map/map_only.py" ``` -The return value is an [`InventoryResult`](/api/inventoryresult/). Unlike `collect()`, this method re-raises when the mapping fails, after calling `on_error("inventory_mapping", exc)`. +The return value is an [`InventoryResult`](inventoryresult.md). Unlike `collect()`, this method re-raises when the mapping fails, after calling `on_error("inventory_mapping", exc)`. ### Phases by hand @@ -188,7 +160,6 @@ edges = [ NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), ] - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) engine.on_phase_start = lambda phase: print("phase:", phase) @@ -206,7 +177,6 @@ async def main() -> None: print("RAG chunks:", analysis.rag_chunks_path) print(scan_result.summary["severity_breakdown"]) - asyncio.run(main()) ``` @@ -238,7 +208,7 @@ engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") engine.run_from_reports_sync({"trivy": ["./trivy.json"]}, output_dir="./reports") ``` -[Event hooks](/api/event-hooks/) lists every hook, when it fires and what it receives, with longer examples. +[Event hooks](event-hooks.md) lists every hook, when it fires and what it receives, with longer examples. ## Notes @@ -248,19 +218,55 @@ engine.run_from_reports_sync({"trivy": ["./trivy.json"]}, output_dir="./reports" `on_finding` fires once per finding as each scanner finishes in `scan()` (and when `ingest_reports()` returns); `on_scan_complete` fires once at the end. Both run before normalisation, so the findings have not been deduplicated or rescored yet. They receive deep copies, which normalisation does not change later. For the final list, read `result.findings` after the run. -`map_inventory(findings=...)` stores the overlay on the returned result as `result.asset_map` and `result.compliance_map`, with or without `output_dir`. With `output_dir` set, it also writes `asset-map.json` and `compliance-map.json` there. To overlay findings on a map you already hold, call [`InventoryMapper.export_merged()`](/api/inventorymapper/) yourself. +`map_inventory(findings=...)` stores the overlay on the returned result as `result.asset_map` and `result.compliance_map`, with or without `output_dir`. With `output_dir` set, it also writes `asset-map.json` and `compliance-map.json` there. To overlay findings on a map you already hold, call [`InventoryMapper.export_merged()`](inventorymapper.md) yourself. The Terraform recreation goes to `/terraform`, following the `output_dir` you pass to `run_pipeline()`, `analyze()` or `scan()`. A `terraform.output_dir` set in the config replaces it, except the old default `./reports/terraform`, which counts as unset. `scanners.timeout_seconds` (default 3600) is the subprocess timeout of each scanner job: each Prowler or ScoutSuite run, each Checkov directory, each Trivy image or directory. A job that runs over is killed and reported through `on_error` under its job name, and the findings it produced before that are kept. A scanner that fails to launch is reported the same way. +## Reference + +::: cloudg.api.CloudGEngine + options: + members: + - run_pipeline + - run_pipeline_sync + - collect + - scan + - analyze + - ingest_reports + - normalise_findings + - run_from_reports + - run_from_reports_sync + - map_inventory + - map_inventory_sync + ## Related -:::links -- [Event hooks](/api/event-hooks/) The six callbacks and when each one fires. -- [Result objects](/api/results/) PipelineResult, CollectionResult and AnalysisResult. -- [CloudGConfig](/api/cloudgconfig/) Building the config the engine runs on. -- [The pipeline](/guides/pipeline/) The same phases from the CLI side. -- [Python recipes](/guides/python-recipes/) Longer integration examples. -- [cloudg run](/cli/run/) The command that wraps `run_pipeline()`. -::: +
      + +- [**Event hooks**](event-hooks.md) + + The six callbacks and when each one fires. + +- [**Result objects**](results.md) + + PipelineResult, CollectionResult and AnalysisResult. + +- [**CloudGConfig**](cloudgconfig.md) + + Building the config the engine runs on. + +- [**The pipeline**](../guides/pipeline.md) + + The same phases from the CLI side. + +- [**Python recipes**](../guides/python-recipes.md) + + Longer integration examples. + +- [**cloudg run**](../cli/run.md) + + The command that wraps `run_pipeline()`. + +
      diff --git a/site/content/api/cloudontology.md b/site/content/api/cloudontology.md index 27388f9..a1d5eb1 100644 --- a/site/content/api/cloudontology.md +++ b/site/content/api/cloudontology.md @@ -1,16 +1,19 @@ --- title: CloudOntology -object: cloudg.graph.ontology.CloudOntology -members: [build, graph, triple_count, query, query_internet_exposed, query_by_relation_group, query_asset_neighbourhood, query_compliance_gaps, save, to_turtle, to_jsonld, to_rdfxml, to_ntriples, stats] -lede: Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. -since: "0.6.0" +description: Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. --- -The ontology is the semantic view of an inventory. Where [GraphBuilder](/api/graphbuilder/) keeps the collector's edge types (`SECURITY_GROUP_RULE`, `ATTACHED_TO`), `CloudOntology` reads each edge together with its endpoint types, ports and CIDR and turns it into relations that say what the edge means: `INTERNET_REACHABLE`, `ONLY_SSH`, `PROTECTED_BY_SG`, `VPC_CONTAINS_SUBNET`. `cloudg run` builds it after the scanners finish, so findings are part of the graph, and writes `ontology.ttl` and `ontology.jsonld` by default. +Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + +The ontology is the semantic view of an inventory. Where [GraphBuilder](graphbuilder.md) keeps the collector's edge types (`SECURITY_GROUP_RULE`, `ATTACHED_TO`), `CloudOntology` reads each edge together with its endpoint types, ports and CIDR and turns it into relations that say what the edge means: `INTERNET_REACHABLE`, `ONLY_SSH`, `PROTECTED_BY_SG`, `VPC_CONTAINS_SUBNET`. `cloudg run` builds it after the scanners finish, so findings are part of the graph, and writes `ontology.ttl` and `ontology.jsonld` by default. ## What build() writes -```mermaid caption="Where the triples come from" +```mermaid flowchart LR S["Schema: classes and properties"] --> G["rdflib Graph"] A["Assets"] --> I["Individuals with data properties"] --> G @@ -19,6 +22,8 @@ flowchart LR A --> AR["Asset-level relations"] --> G F["Findings"] --> X["AssetIndex resolves the asset"] --> G ``` +Where the triples come from +{ .diagram-caption } Three namespaces are bound on the graph, so they appear as prefixes in every serialisation and in your SPARQL: @@ -87,103 +92,7 @@ Names are matched without regard to case. `save()` writes Turtle, with a warning Four assets, two edges and one scanner finding that names the database by ARN. The query lists every relation in the graph with readable names on both ends, skipping the data properties. ```python title="ontology_demo.py" -"""Build the RDF ontology for a small estate, save it and query it with SPARQL.""" - -from pathlib import Path - -from rdflib import Graph - -from cloudg.graph.ontology import CloudOntology, RelationGroup -from cloudg.schema.models import ( - AssetType, - CloudAsset, - CloudProvider, - EdgeType, - Finding, - NetworkEdge, - Severity, -) - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY_ARN = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" - - -def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - **extra, - ) - - -assets = [ - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", - tags={"owner": "payments"}), - asset("orders-key", AssetType.KMS_KEY, KEY_ARN), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", - metadata={"storage_encrypted": True, "kms_key_id": KEY_ARN}), -] - -edges = [ - NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range="22", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), -] - -# A scanner finding that names the database by ARN, not by asset id -findings = [ - Finding( - resource_id=f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", - severity=Severity.HIGH, - title="RDS instance has no deletion protection", - description="Deletion protection is disabled.", - source_tool="prowler", - compliance_frameworks=["CIS"], - ) -] - -onto = CloudOntology() -onto.build(assets, edges, findings) -stats = onto.stats() -print(stats["total_triples"], "triples,", stats["classes_used"], "classes,", - stats["individuals"], "individuals") -print("groups:", dict(sorted(stats["relation_group_counts"].items()))) - -# The graph binds the cm:, cmr: and cmp: prefixes, so queries can use them -rows = onto.query(""" - SELECT ?subject ?relation ?object WHERE { - ?s ?p ?o . - FILTER(STRSTARTS(STR(?p), STR(cmp:))) - FILTER(!STRSTARTS(STR(?p), STR(cmp:has))) - FILTER(?p != cmp:isInternetExposed) - ?s cmp:hasName ?subject . - OPTIONAL { ?o cmp:hasName ?oname } - BIND(STRAFTER(STR(?p), STR(cmp:)) AS ?relation) - BIND(COALESCE(?oname, STR(?o)) AS ?object) - } ORDER BY ?subject ?relation -""") -for row in rows: - print(f" {row['subject']} {row['relation']} {row['object']}") - -print("internet reachable:", [r["name"] for r in onto.query_internet_exposed()]) -print("security group:", len(onto.query_by_relation_group(RelationGroup.SECURITY)), "triples") -print("compliance gaps:", [(r["severity"], r["resourceName"]) for r in onto.query_compliance_gaps()]) - -out = Path("ontology-out") -for fmt, name in [("turtle", "ontology.ttl"), ("json-ld", "ontology.jsonld"), - ("xml", "ontology.rdf"), ("nt", "ontology.nt")]: - onto.save(out / name, fmt=fmt) - -# Any RDF tool can read the files back; here rdflib itself -reloaded = Graph().parse(out / "ontology.ttl", format="turtle") -print("reloaded", len(reloaded), "triples from", sorted(p.name for p in out.iterdir())) +--8<-- "graph/ontology_demo.py" ``` ```console @@ -209,7 +118,7 @@ Things to notice: - 636 of the 689 triples are schema. The `stats()` counts cover classes and individuals actually used. - `ENCRYPTED_BY_KMS` points at `orders-key` because the database's `kms_key_id` matched the key asset's ARN. With no collected key, the object would be the raw key reference. - The finding attached to `orders-db` although its `resource_id` was an ARN. -- `query_internet_exposed()` lists the objects of `INTERNET_REACHABLE` triples, which here is the security group. It reads the rule edges, not the reachability walk; for "which workloads can the internet reach" use [ReachabilityAnalyzer](/api/reachabilityanalyzer/). +- `query_internet_exposed()` lists the objects of `INTERNET_REACHABLE` triples, which here is the security group. It reads the rule edges, not the reachability walk; for "which workloads can the internet reach" use [ReachabilityAnalyzer](reachabilityanalyzer.md). - The `OWNED_BY` object is a tag node (`tag_payments`) with no `cmp:hasName`, so the query printed its IRI. Saving N-Triples prints a harmless `UserWarning: NTSerializer always uses UTF-8 encoding` from rdflib. @@ -255,12 +164,48 @@ The same query works against `ontology.jsonld` with `Graph().parse("ontology.jso - Each `build()` call adds to the same graph. Create a new `CloudOntology` per inventory. - The JSON-LD output uses rdflib's serialiser without a framing context, so it is a flat list of nodes. +## Reference + +::: cloudg.graph.ontology.CloudOntology + options: + members: + - build + - graph + - triple_count + - query + - query_internet_exposed + - query_by_relation_group + - query_asset_neighbourhood + - query_compliance_gaps + - save + - to_turtle + - to_jsonld + - to_rdfxml + - to_ntriples + - stats + ## Related -:::links -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) When to use which view of the inventory. -- [RAGExporter](/api/ragexporter/) Relation-group chunks built from the same inference rules. -- [GraphBuilder](/api/graphbuilder/) The NetworkX view of the same edges. -- [cloudg run](/cli/run/) The `--ontology/--no-ontology` switch and the files it writes. -- [Output files](/reference/output-files/) Every file the pipeline writes, ontology included. -::: +
      + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + When to use which view of the inventory. + +- [**RAGExporter**](ragexporter.md) + + Relation-group chunks built from the same inference rules. + +- [**GraphBuilder**](graphbuilder.md) + + The NetworkX view of the same edges. + +- [**cloudg run**](../cli/run.md) + + The `--ontology/--no-ontology` switch and the files it writes. + +- [**Output files**](../reference/output-files.md) + + Every file the pipeline writes, ontology included. + +
      diff --git a/site/content/api/dependencygraph.md b/site/content/api/dependencygraph.md index dc88fe1..e9afaa2 100644 --- a/site/content/api/dependencygraph.md +++ b/site/content/api/dependencygraph.md @@ -1,11 +1,10 @@ --- -object: [cloudg.inventory.dependencies.DependencyGraph, cloudg.inventory.dependencies.AssetIndex] -members: - cloudg.inventory.dependencies.DependencyGraph: [find, depends_on, dependents, direct_counts, tree, shared_dependencies, blast_radius] - cloudg.inventory.dependencies.AssetIndex: [add, find, resolve, resolve_finding] -lede: "What does this asset need, and what breaks if it goes away? `DependencyGraph` answers both over any set of assets and edges; `AssetIndex` is the matcher that finds the asset a reference or a finding is about." +description: What does this asset need, and what breaks if it goes away? DependencyGraph answers both over any set of assets and edges; AssetIndex is the matcher that finds the asset a reference or a finding is about. --- +What does this asset need, and what breaks if it goes away? `DependencyGraph` answers both over any set of assets and edges; `AssetIndex` is the matcher that finds the asset a reference or a finding is about. +{ .lede } + ## From edges to dependencies Edges in an inventory map point in their natural direction: a function `ASSUMES_ROLE` a role, a queue `INVOKES` a function, a WAF `PROTECTS` an API. That direction is not always the direction of dependency. The function needs its role, so the arrow and the dependency agree. The function also needs the queue that triggers it, but the `INVOKES` arrow points from the queue to the function. @@ -18,7 +17,7 @@ Edges in an inventory map point in their natural direction: a function `ASSUMES_ | reverse: the target depends on the source | `CONTAINS`, `INVOKES`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS` | a function depends on the queue that triggers it, an API on the WAF in front of it, a resource on the stack that manages it | | ignored | `SECURITY_GROUP_RULE`, `NACL_RULE`, `INTERNET_EXPOSED` | network rules are not dependencies | -```mermaid caption="Natural edges on the left, dependency arrows on the right" +```mermaid flowchart LR subgraph natural ["inventory edges"] Q1["queue"] -->|INVOKES| F1["function"] @@ -29,12 +28,14 @@ flowchart LR F2 -->|needs| R2["role"] end ``` +Natural edges on the left, dependency arrows on the right +{ .diagram-caption } On that view, `depends_on(x)` walks upstream (everything `x` needs) and `dependents(x)` walks downstream (everything that needs `x`, its blast radius). Both are breadth-first, visit each asset once, stop at `max_depth` (default 10) and return `DependencyLink` records with `asset_id`, `via_edge`, `relationship`, `depth` and `parent_id`, the asset one hop closer to the start. Edges whose source or target is not in `assets` are skipped. With `include_hierarchy=False`, `CONTAINS` edges into `ORGANIZATION`, `ORG_UNIT` and `CLOUD_ACCOUNT` nodes are skipped too, because otherwise every account would top every blast radius list. `InventoryResult.dependency_graph()` builds the graph with this default. -This is an availability and change-impact view: what stops working when something is deleted, disabled or misconfigured. Compromise spreads differently. A stolen function credential exposes the role it assumes, which is the forward direction of `ASSUMES_ROLE` in the map itself, not a dependent of the role. For attack paths use the graph analysis in [`CloudGEngine.analyze()`](/api/cloudgengine/) or walk the identity edges directly. +This is an availability and change-impact view: what stops working when something is deleted, disabled or misconfigured. Compromise spreads differently. A stolen function credential exposes the role it assumes, which is the forward direction of `ASSUMES_ROLE` in the map itself, not a dependent of the role. For attack paths use the graph analysis in [`CloudGEngine.analyze()`](cloudgengine.md) or walk the identity edges directly. ### The reports @@ -66,58 +67,7 @@ Ambiguous names and tails never match at their own tier. Tiers 5 and 6 are also Runs offline on six linked assets. ```python title="blast_radius.py" -from cloudg.inventory import RelationshipLinker -from cloudg.inventory.dependencies import DependencyGraph -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" -ROLE = f"arn:aws:iam::{ACCOUNT}:role/orders-worker-role" -QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" -TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" -FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" -API = f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3d4e5" - - -def aws(arn, name, asset_type, *relations, region=REGION, exposed=False): - return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, - region=region, account_id=ACCOUNT, is_internet_exposed=exposed, - metadata={"relations": list(relations)}) - - -def rel(target, edge, relationship=None, reverse=False): - return {"target": target, "edge": edge, "relationship": relationship, "reverse": reverse} - - -assets = [ - aws(KEY, "orders-key", AssetType.KMS_KEY), - aws(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global"), - aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), - aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), - aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, - rel(ROLE, "ASSUMES_ROLE", "RUNS_ON"), - rel(TABLE, "REFERENCES", "WRITES_TO"), - rel(QUEUE, "INVOKES", "TRIGGERED_BY", reverse=True)), - aws(API, "orders-api", AssetType.API_GATEWAY, rel(FUNCTION, "INVOKES"), exposed=True), -] -edges = RelationshipLinker(assets).link() -graph = DependencyGraph(assets, edges) - -key = graph.find("orders-key") # ID, ARN, unique name or unique ARN tail -print("If", key.name, "is disabled:") -for link in graph.dependents(key.id): - asset = graph.assets[link.asset_id] - parent = graph.assets[link.parent_id].name - print(f" depth {link.depth}: {asset.name} ({asset.asset_type.value}) needs {parent} via {link.via_edge}") - -worker = graph.find(FUNCTION) -print("orders-worker needs:", [graph.assets[l.asset_id].name for l in graph.depends_on(worker.id)]) -print("direct (needs, needed by):", graph.direct_counts(worker.id)) - -for row in graph.blast_radius(top=3): - print(f"{row['name']:20} {row['transitive_dependents']} dependents, " - f"{row['internet_exposed_dependents']} internet-exposed") +--8<-- "map/blast_radius.py" ``` ```console @@ -141,12 +91,10 @@ Two results are worth a second look. The function appears at depth 2 under the k from cloudg.inventory.dependencies import AssetIndex from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, Finding, Severity - def asset(arn, name, asset_type): return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, region="eu-west-1", account_id="123456789012") - assets = [ asset("arn:aws:rds:eu-west-1:123456789012:db:orders", "orders", AssetType.RDS_INSTANCE), asset("arn:aws:dynamodb:eu-west-1:123456789012:table/orders", "orders", AssetType.DYNAMODB_TABLE), @@ -156,12 +104,10 @@ assets = [ index = AssetIndex(assets) names = {a.id: f"{a.asset_type.value}:{a.name}" for a in assets} - def finding(resource_id, resource_arn=None): return Finding(resource_id=resource_id, resource_arn=resource_arn, severity=Severity.MEDIUM, title="example", description="example", source_tool="custom") - cases = [ finding("x", "arn:aws:rds:eu-west-1:123456789012:db:orders"), # exact ARN finding("orders"), # name shared by two assets @@ -194,11 +140,45 @@ The second line shows the limit of tier 3. `orders` is the name of two assets, s The module also has two functions that work on plain asset and edge lists: `cross_account_edges(assets, edges)` (every edge between two different accounts, hierarchy containment excluded) and `security_coverage(assets, edges)` (security services by account and region, the gaps, workloads without vulnerability scanning, internet-facing entry points without a WAF). `InventoryResult.analysis()` calls both. +## Reference + +::: cloudg.inventory.dependencies.DependencyGraph + options: + members: + - find + - depends_on + - dependents + - direct_counts + - tree + - shared_dependencies + - blast_radius + +::: cloudg.inventory.dependencies.AssetIndex + options: + members: + - add + - find + - resolve + - resolve_finding + ## Related -:::links -- [Dependencies and blast radius](/guides/dependencies/) The guide, with `cloudg deps`. -- [cloudg deps](/cli/deps/) The CLI front end to `tree()`. -- [InventoryResult](/api/inventoryresult/) `dependency_graph()` and `analysis()`. -- [RelationshipLinker](/api/relationshiplinker/) Where the edges come from. -::: +
      + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + The guide, with `cloudg deps`. + +- [**cloudg deps**](../cli/deps.md) + + The CLI front end to `tree()`. + +- [**InventoryResult**](inventoryresult.md) + + `dependency_graph()` and `analysis()`. + +- [**RelationshipLinker**](relationshiplinker.md) + + Where the edges come from. + +
      diff --git a/site/content/api/entry-points.md b/site/content/api/entry-points.md index e17f42b..fa0584e 100644 --- a/site/content/api/entry-points.md +++ b/site/content/api/entry-points.md @@ -1,10 +1,11 @@ --- title: Entry points -object: cloudg.registry.PluginRegistry -members: [get_collector, get_scanner, list_collectors, list_scanners] -lede: Ship a collector or a scanner as its own package, and `PluginRegistry` finds it through Python entry points. +description: Ship a collector or a scanner as its own package, and PluginRegistry finds it through Python entry points. --- +Ship a collector or a scanner as its own package, and `PluginRegistry` finds it through Python entry points. +{ .lede } + cloudg looks for plugins in two entry point groups, `cloudg.collectors` and `cloudg.scanners`. A package that declares an entry point in one of them becomes visible to `PluginRegistry` as soon as it is installed in the same environment, with no change to cloudg. cloudg registers its own built-ins the same way in its `pyproject.toml`: ```toml title="pyproject.toml" @@ -27,7 +28,7 @@ Scanner plugins run wherever the built-in scanners run. Name one in `scanners.en Discovery is lazy and happens once per `PluginRegistry` instance, on the first `get_*` or `list_*` call. -```mermaid caption="What the first get or list call does" +```mermaid flowchart TD A["First get_* or list_* call"] --> B["Read entry points in cloudg.collectors"] B --> N{"Built-in name?"} @@ -41,6 +42,8 @@ flowchart TD F --> H["Import the built-ins from the table in registry.py"] H --> I["Cached for this registry"] ``` +What the first get or list call does +{ .diagram-caption } The built-ins always come from the table in `cloudg/registry.py`, which maps the same eight names as the entry points above to the same classes, so they load the same way from an installed package and from a source checkout that was never installed. A built-in whose import fails (a missing optional SDK, say) is left out quietly, at debug log level. @@ -121,7 +124,6 @@ from cloudg.collectors.base import BaseCollector from cloudg.inventory.linker import RelationshipLinker from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, NetworkEdge - class CMDBCollector(BaseCollector): """Turns a CMDB JSON export into CloudAssets and linked edges.""" @@ -174,7 +176,6 @@ from cloudg.schema.models import CloudAsset, Finding, Severity CHECK_ID = "TAGPOLICY_001" - class TagPolicyScanner: """Follows the built-in scanners' shape: is_available, run, parse_report.""" @@ -213,13 +214,17 @@ class TagPolicyScanner: Install it into the environment cloudg runs in, editable while you work on it: -```bash tab="pip" -pip install -e ./cloudg-cmdb -``` +=== "pip" -```bash tab="uv" -uv pip install -e ./cloudg-cmdb -``` + ```bash + pip install -e ./cloudg-cmdb + ``` + +=== "uv" + + ```bash + uv pip install -e ./cloudg-cmdb + ``` ### Use the plugins @@ -296,15 +301,41 @@ To merge your scanner's findings with an equivalent check from Prowler, Checkov - A plugin that fails to import is logged as a warning (`Failed to load scanner plugin ...`) and skipped; the others still load. It goes to the `cloudg.registry` logger, so configure `logging` in your script to see it. - `get_collector()` and `get_scanner()` raise `KeyError` for an unknown name, and the message lists the names that are registered. -- The relation format the linker reads (`target`, `edge`, `relationship`, `reverse`, `properties`) is specified in section 7 of the inventory reference, and the [RelationshipLinker](/api/relationshiplinker/) page shows the linker itself. Unknown `edge` values are linked as `REFERENCES`. -- Assets from a plugin collector work with everything downstream: [GraphBuilder](/api/graphbuilder/), the reachability analysis, the ontology, and `InventoryResult(assets=..., edges=...).export(dir)` for an `inventory-map.json` that `cloudg deps` can read. +- The relation format the linker reads (`target`, `edge`, `relationship`, `reverse`, `properties`) is specified in section 7 of the inventory reference, and the [RelationshipLinker](relationshiplinker.md) page shows the linker itself. Unknown `edge` values are linked as `REFERENCES`. +- Assets from a plugin collector work with everything downstream: [GraphBuilder](graphbuilder.md), the reachability analysis, the ontology, and `InventoryResult(assets=..., edges=...).export(dir)` for an `inventory-map.json` that `cloudg deps` can read. + +## Reference + +::: cloudg.registry.PluginRegistry + options: + members: + - get_collector + - get_scanner + - list_collectors + - list_scanners ## Related -:::links -- [FindingsNormaliser](/api/findingsnormaliser/) Dedupe, scoring and compliance mapping for any findings list. -- [Data models](/api/models/) `CloudAsset`, `NetworkEdge` and `Finding`, field by field. -- [RelationshipLinker](/api/relationshiplinker/) How declared relations become edges. -- [Python recipes](/guides/python-recipes/) More ways to combine the pipeline phases from code. -- [cloudg report](/cli/report/) Render HTML and SVG from a `findings.json`. -::: +
      + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Dedupe, scoring and compliance mapping for any findings list. + +- [**Data models**](models.md) + + `CloudAsset`, `NetworkEdge` and `Finding`, field by field. + +- [**RelationshipLinker**](relationshiplinker.md) + + How declared relations become edges. + +- [**Python recipes**](../guides/python-recipes.md) + + More ways to combine the pipeline phases from code. + +- [**cloudg report**](../cli/report.md) + + Render HTML and SVG from a `findings.json`. + +
      diff --git a/site/content/api/event-hooks.md b/site/content/api/event-hooks.md index 9ecfa07..4762c3a 100644 --- a/site/content/api/event-hooks.md +++ b/site/content/api/event-hooks.md @@ -1,8 +1,11 @@ --- title: Event hooks -lede: "Six optional callbacks on `CloudGEngine` report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it." +description: 'Six optional callbacks on CloudGEngine report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it.' --- +Six optional callbacks on `CloudGEngine` report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it. +{ .lede } + ## The six hooks Every hook starts as `None` in `CloudGEngine.__init__`. Assign a callable before you call a method, and reassign or reset it to `None` between runs as you like. The callback types are defined in `cloudg.api` as `typing.Callable` aliases, so you can annotate your own handlers with them. @@ -30,7 +33,7 @@ on_error: OnError = alert The diagram follows `run_pipeline()`, the one method that can call all six. -```mermaid caption="Hook calls during run_pipeline()" +```mermaid sequenceDiagram participant App as Your code participant E as CloudGEngine @@ -49,6 +52,8 @@ sequenceDiagram E->>App: on_phase_start("reporting") E-->>App: PipelineResult ``` +Hook calls during run_pipeline() +{ .diagram-caption } Other entry points use a subset: @@ -79,7 +84,7 @@ Other entry points use a subset: | `prowler-`, `scoutsuite-` | one scanner run per provider, such as `prowler-aws`, including a run killed at `scanners.timeout_seconds` | | `checkov-` | one Checkov run per IaC directory | | `trivy`, `trivy-fs`, `iam` | the Trivy image scan, the Trivy filesystem scan, the IAM policy linter | -| a plugin's name | a [scanner plugin](/api/entry-points/) run | +| a plugin's name | a [scanner plugin](entry-points.md) run | | `graph_build`, `ontology`, `rag_export` | the steps of `analyze()` | | `normalisation`, `reporting` | the last two pipeline phases | @@ -162,14 +167,12 @@ SPOOL = "./cloudg-findings.jsonl" # tailed by the SIEM agent (Fluent Bit, Splu STOP = object() outbox: queue.Queue = queue.Queue() - def writer() -> None: with open(SPOOL, "a", encoding="utf-8") as spool: while (item := outbox.get()) is not STOP: spool.write(json.dumps(item) + "\n") spool.flush() - thread = threading.Thread(target=writer, daemon=True) thread.start() @@ -189,74 +192,44 @@ thread.join(timeout=30) print(f"forwarded {result.total_findings} findings to {SPOOL}") ``` -Each line is one finding as `Finding.model_dump(mode="json")` produces it (see [Data models](/api/models/) for the fields), plus `"kind": "finding"`. +Each line is one finding as `Finding.model_dump(mode="json")` produces it (see [Data models](models.md) for the fields), plus `"kind": "finding"`. ### Fail fast Raising inside `on_error` does nothing, because the engine swallows it. Record the failure instead, and check between phases you drive yourself. `run_pipeline()` has no such checkpoints, so this uses the individual phase methods. ```python title="fail_fast.py" -import asyncio -import sys - -from cloudg import CloudGConfig, CloudGEngine - - -class PhaseFailed(RuntimeError): - pass +--8<-- "errors/fail_fast.py" +``` +A failed provider never reaches `on_error`; it is a `FAILED` entry in `collection.coverage`. That is why the script checks coverage as well as the failure list after collecting. On a failure the script prints one line in the form `cloudg: failed during : ` to stderr and exits with status 2, so a CI job or a scheduler sees the run as failed. -async def main() -> int: - engine = CloudGEngine(CloudGConfig(providers=["aws"])) - failures: list[tuple[str, Exception]] = [] - engine.on_error = lambda phase, exc: failures.append((phase, exc)) +## Notes - def checkpoint(step: str) -> None: - if failures: - phase, exc = failures[0] - raise PhaseFailed(f"{phase} failed during {step}: {exc}") +Hooks are per engine instance. Two engines in one process do not share them, and there is no global registry. - collection = await engine.collect() - checkpoint("collection") - broken = [c.to_summary() for c in collection.coverage if c.failed_services] - if broken or not collection.assets: - raise PhaseFailed(f"collection incomplete: {broken or 'no assets'}") +`on_collection_complete` does not fire when `collect()` fails as a whole; you get `on_error("collection", exc)` and an empty `CollectionResult` instead. `on_analysis_complete` fires even when every analysis step failed, with whatever was filled in. - findings = await engine.scan(collection.assets, collection.edges, output_dir="./reports") - checkpoint("scanning") +Nothing fires for findings produced by `analyze()`. Its reachability findings reach you through `AnalysisResult.reachability_findings` and, after `run_pipeline()`, through `PipelineResult.findings`. - analysis = await engine.analyze(collection.assets, collection.edges, findings, "./reports") - checkpoint("analysis") +## Related - scan_result = engine.normalise_findings( - findings + analysis.reachability_findings, assets=collection.assets - ) - print(scan_result.summary) - return 0 +
      +- [**CloudGEngine**](cloudgengine.md) -try: - sys.exit(asyncio.run(main())) -except PhaseFailed as exc: - print(f"cloudg: {exc}", file=sys.stderr) - sys.exit(2) -``` + The methods that call the hooks. -A failed provider never reaches `on_error`; it is a `FAILED` entry in `collection.coverage`. That is why the script checks coverage as well as the failure list after collecting. On a failure the script prints one line in the form `cloudg: failed during : ` to stderr and exits with status 2, so a CI job or a scheduler sees the run as failed. +- [**Result objects**](results.md) -## Notes + What `on_collection_complete` and `on_analysis_complete` receive. -Hooks are per engine instance. Two engines in one process do not share them, and there is no global registry. +- [**Data models**](models.md) -`on_collection_complete` does not fire when `collect()` fails as a whole; you get `on_error("collection", exc)` and an empty `CollectionResult` instead. `on_analysis_complete` fires even when every analysis step failed, with whatever was filled in. + The `Finding` that `on_finding` receives. -Nothing fires for findings produced by `analyze()`. Its reachability findings reach you through `AnalysisResult.reachability_findings` and, after `run_pipeline()`, through `PipelineResult.findings`. +- [**CI and automation**](../guides/ci-automation.md) -## Related + Running cloudg unattended. -:::links -- [CloudGEngine](/api/cloudgengine/) The methods that call the hooks. -- [Result objects](/api/results/) What `on_collection_complete` and `on_analysis_complete` receive. -- [Data models](/api/models/) The `Finding` that `on_finding` receives. -- [CI and automation](/guides/ci-automation/) Running cloudg unattended. -::: +
      diff --git a/site/content/api/exceptions.md b/site/content/api/exceptions.md new file mode 100644 index 0000000..18f8b8e --- /dev/null +++ b/site/content/api/exceptions.md @@ -0,0 +1,205 @@ +--- +title: Exceptions +description: Every exception cloudg raises on purpose, where each one comes from, and how to get exceptions from the scanner wrappers instead of recorded failures. +--- + +Every exception cloudg raises on purpose lives in `cloudg.exceptions` and derives from `CloudGError`, so one `except CloudGError` catches anything cloudg itself raised. Errors from the cloud SDKs and the operating system pass through untouched. +{ .lede } + +**Module** `cloudg.exceptions` · **Also exported from** `cloudg` · **Last changed** [v0.6.1](../changelog.md#0.6.1) +{ .page-meta } + +## The class tree + +The narrower classes keep the built-in base they always had, so code written against an older release still works: `ProviderSDKMissingError` is still an `ImportError`, `CredentialError` is still a `RuntimeError`, and `ConfigurationError` is still a `ValueError`. + +```mermaid +flowchart LR + E["CloudGError"] --> CFG["ConfigurationError
      (ValueError)"] + E --> SDK["ProviderSDKMissingError
      (ImportError)"] + E --> COL["CollectionError
      (RuntimeError)"] + COL --> CRED["CredentialError"] + COL --> ORG["OrganizationScopeError
      (ValueError)"] + E --> RES["ResilienceError"] + E --> LOR["LiveOperationRejected"] + E --> SCN["ScannerError"] + SCN --> NI["ScannerNotInstalledError"] + SCN --> TO["ScannerTimeoutError"] + SCN --> EX["ScannerExecutionError"] + SCN --> OUT["ScannerOutputError"] + OUT --> RPE["ReportParseError"] + SCN --> SF["ScannerFailed
      (RuntimeError)"] +``` +The built-in base each class keeps is in brackets. +{ .diagram-caption } + +`ResilienceError` and `LiveOperationRejected` live in `cloudg.resilience`, next to their subclasses (`CircuitOpenError`, `DeadlineExceededError`, `RetryBudgetExhaustedError`, `CooldownActive`, `LiveOperationBusy`, `CallerQuotaExceeded`, `LiveOperationTimeout`). [cloudg.resilience](resilience.md) covers them. Everything else imports from `cloudg.exceptions` or straight from `cloudg`: + +```python +from cloudg import CloudGError, ScannerError +from cloudg.exceptions import OrganizationScopeError, ScannerTimeoutError +from cloudg.resilience import CircuitOpenError, LiveOperationRejected +``` + +## When each one is raised + +| Exception | Raised when | Useful attributes | +|---|---|---| +| `ConfigurationError` | A configuration value is missing, malformed or contradicts another. `cloudg.ingest.parse_report()` raises it for a tool name it does not support. | | +| `ProviderSDKMissingError` | A provider SDK is not installed: boto3 for AWS, azure-identity for Azure, google-auth or google-cloud-asset for GCP. The message names the pip extra. | `provider`, `extra`, `missing` | +| `CollectionError` | Collecting from a provider failed as a whole, for example when GCP rejects both ListAssets and SearchAllResources for a scope. | | +| `CredentialError` | Role assumption (`AssumeRole`, `AssumeRoleWithWebIdentity`) failed, no credentials resolved for `aws.role_arn`, or an account was asked for without a role to reach it. | `provider`, `detail` | +| `OrganizationScopeError` | An `--ou` (`aws.organization.include_ous`) filter matches nothing in the organization. cloudg stops instead of quietly mapping only the caller's account. | | +| `ScannerNotInstalledError` | A scanner wrapper in strict mode finds no executable on `PATH`. | `executable`, `install_hint` | +| `ScannerTimeoutError` | A scanner ran longer than `timeout_seconds` and was killed. | `timeout_seconds`, `target` | +| `ScannerExecutionError` | A scanner process could not be started, or crashed. | `returncode`, `output`, `target` | +| `ScannerOutputError` | A scanner ran, but its output could not be read or parsed. | `path` | +| `ReportParseError` | A report handed to a wrapper's `parse_report(path, strict=True)` is unreadable, or a directory holds no report. | `path` | +| `ScannerFailed` | A scanner did not finish, as reported by `CloudGEngine.scan()` through `on_error`, or by `raise_for_errors()` when a wrapper recorded several failures. | `findings`, `errors`, `failures` | + +Every `ScannerError` also carries `scanner` (the wrapper's short name, such as `"checkov"`), `detail` (the reason in a few words) and `target` (the image, directory or file being scanned, when there is one). `str(exc)` is `": "`. + +## Recorded failures and strict mode + +The scanner wrappers record failures instead of raising them, so one broken scanner never stops a pipeline. Each wrapper keeps two lists: + +- `errors`: why the last run did not finish (timeouts, launch failures), as text; +- `failures`: every `ScannerError` the wrapper met, including output it could not parse. Unparseable output does not stop a run, so it is in `failures` but not in `errors`. + +There are two ways to turn those records into exceptions: + +```mermaid +flowchart TD + R["scanner.run()"] --> Q{"strict=True?"} + Q -->|yes| RAISE["first failure raised at once"] + Q -->|no| REC["failure appended to .failures"] + REC --> D["run finishes, partial findings returned"] + D --> RFE["scanner.raise_for_errors()"] + RFE --> ONE["one failure: raised as it is"] + RFE --> MANY["several: ScannerFailed with .failures"] + RFE --> NONE["none: returns quietly"] +``` + +A missing executable is a warning by default and is not recorded as a failure, because "Trivy is not installed" is a setup choice rather than a broken run. Pass `strict=True` to get a `ScannerNotInstalledError` instead. + +## Examples + +Each tab is a complete script. They run without cloud credentials: with no scanner installed, the strict scanner prints the install hint, and the report parser needs only a file. + +=== "strict=True" + + The first failure is raised as soon as it happens, so the `except` clauses read like an ordinary call. + + ```python title="strict_scanner.py" + --8<-- "errors/strict_scanner.py" + ``` + + Without Checkov on `PATH` it prints: + + ```console + $ python strict_scanner.py + checkov is missing. pip install checkov + ``` + +=== "raise_for_errors()" + + Run everything first and keep the partial results, then decide what counts as a failure. + + ```python title="raise_for_errors.py" + --8<-- "errors/raise_for_errors.py" + ``` + + With a `trivy` on `PATH` that prints something other than JSON, both images fail to parse, so the two failures come back together as one `ScannerFailed`: + + ```console + $ python raise_for_errors.py + 0 findings, 2 failures recorded + - ScannerOutputError output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + - ScannerOutputError output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +=== "Around CloudGEngine" + + `CloudGEngine` reports failures instead of raising them: `collect()` records them per service in the coverage records, and scanner failures reach the `on_error` hook as exception objects. A missing SDK or a refused role is easiest to catch before the run, by building the session yourself. + + ```python title="engine_errors.py" + --8<-- "errors/engine_errors.py" + ``` + + Without boto3 it stops at the first check: + + ```console + $ python engine_errors.py + boto3 is not installed: pip install 'cloudg[aws]' + ``` + + `build_aws_session()` only contacts AWS when it has a role to assume, so a wrong key surfaces later, as failed services in `result.coverage` and lines in `result.errors`. + +=== "parse_report(path, strict=True)" + + Each scanner wrapper has a `parse_report` class method for output that already exists. With `strict=True`, a file that is not a report raises `ReportParseError` instead of being skipped with a warning. + + ```python title="strict_parse_report.py" + --8<-- "errors/strict_parse_report.py" + ``` + + ```console + $ echo "not json" > results_json.json + $ python strict_parse_report.py results_json.json + checkov report results_json.json is unreadable: output is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + + `cloudg.ingest.parse_report(tool, path)`, the function behind `cloudg ingest`, has no `strict` flag: it logs and skips unreadable files, and raises `ConfigurationError` (a `ValueError`) for an unknown tool and `FileNotFoundError` for a missing path. Call the wrapper's own `parse_report` when a bad file should stop the job. + +## Catching everything cloudg raises + +```python title="catch_all.py" +from cloudg import CloudGError +from cloudg.ingest import parse_report + +try: + findings = parse_report("trivvy", "./trivy.json") +except CloudGError as exc: + print("cloudg refused:", exc) +except FileNotFoundError as exc: + print("no such report:", exc) +``` + +```console +$ python catch_all.py +cloudg refused: Unsupported tool 'trivvy'. Supported: prowler, scoutsuite, checkov, trivy +``` + +The misspelt tool name raises `ConfigurationError`, which the first clause catches. A missing file raises the built-in `FileNotFoundError`, which is not a `CloudGError`: `except CloudGError` never hides an operating system error, an SDK exception or a bug in your own code. + +## Reference + +::: cloudg.exceptions + options: + show_root_heading: false + show_source: true + members_order: source + +## Related + +
      + +- [**cloudg.resilience**](resilience.md) + + The rate limiter, circuit breakers and the errors they raise. + +- [**parse_report**](parse-report.md) + + Turning existing scanner output into findings. + +- [**Event hooks**](event-hooks.md) + + `on_error` and the other callbacks on `CloudGEngine`. + +- [**Running scanners**](../guides/running-scanners.md) + + What each scanner needs and how failures show up on the CLI. + +
      diff --git a/site/content/api/findingsnormaliser.md b/site/content/api/findingsnormaliser.md index b145a17..56b6e22 100644 --- a/site/content/api/findingsnormaliser.md +++ b/site/content/api/findingsnormaliser.md @@ -1,16 +1,17 @@ --- -object: cloudg.normaliser.FindingsNormaliser -members: [normalise] -lede: "Takes findings from any number of scanners and returns one `ScanResult`: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped." +description: 'Takes findings from any number of scanners and returns one ScanResult: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped.' --- +Takes findings from any number of scanners and returns one `ScanResult`: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped. +{ .lede } + ## What normalise does `FindingsNormaliser(rules_dir=None, load_external=True)` loads its rules once, at construction: every `*.yaml` file under `rules_dir` (searched recursively) as a compliance ruleset, and `check_equivalence.yaml` from the top of `rules_dir` as the cross-scanner equivalence map. With `rules_dir=None` it uses the directory shipped inside the package, `cloudg/rules/`, which holds CIS, NIST 800-53, PCI DSS, SOC 2, ISO 27001, HIPAA and GDPR rulesets plus the per-provider framework mappings under `frameworks/`. `load_external=False` (the `rulesets.load_external` key) skips the rulesets and keeps only the equivalence map. `normalise(*finding_lists, assets=None, passed_checks=None)` accepts any number of lists, one per source, and runs them through one pass: -```mermaid caption="FindingsNormaliser.normalise()" +```mermaid flowchart TD A["all lists concatenated"] --> B["pass 1: dedupe within a scanner"] B --> C["pass 2: merge across scanners"] @@ -20,6 +21,8 @@ flowchart TD F --> G["one ComplianceResult per framework control"] G --> H["ScanResult"] ``` +FindingsNormaliser.normalise() +{ .diagram-caption } ### Deduplication @@ -31,7 +34,7 @@ A merge keeps the finding with the higher severity, joins the tool names into `s ### Scoring -A `cvss_score` of 9.0 or more sets the severity to `CRITICAL`. A score from 7.0 to 8.9 raises anything below `HIGH` to `HIGH`. Severity never goes down. `risk_score` on the finding is computed from severity and CVSS whenever you read it; see [Data models](/api/models/). +A `cvss_score` of 9.0 or more sets the severity to `CRITICAL`. A score from 7.0 to 8.9 raises anything below `HIGH` to `HIGH`. Severity never goes down. `risk_score` on the finding is computed from severity and CVSS whenever you read it; see [Data models](models.md). ### Compliance mapping @@ -53,37 +56,7 @@ These results have status `FAIL`. Passing checks add `PASS` results: a ruleset c Five findings about one bucket and one image, constructed by hand so each rule shows. Runs offline. ```python title="normalise_demo.py" -from cloudg.normaliser import FindingsNormaliser -from cloudg.schema.models import Finding, Severity - -BUCKET = "arn:aws:s3:::orders-archive" - - -def finding(tool, check, title, severity, **extra): - return Finding(resource_id=BUCKET, resource_arn=BUCKET, source_tool=tool, - source_finding_id=check, title=title, description=title, - severity=severity, **extra) - - -prowler = [ - finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", - "S3 bucket default encryption", Severity.MEDIUM), - # the same check reported twice (two output formats, say): a within-scanner duplicate - finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", - "S3 bucket default encryption", Severity.MEDIUM), -] -trivy = [ - # equivalent check (AVD-AWS-0088), same resource, same normalised title - finding("trivy", "AVD-AWS-0088", "[Trivy] S3 bucket default encryption", Severity.HIGH), - # a CVE with a CVSS score of 9.8 reported as MEDIUM - finding("trivy", "CVE-2024-0001", "openssl: buffer overflow", Severity.MEDIUM, cvss_score=9.8), -] -untagged = [finding("custom", None, "CloudTrail logging disabled in eu-west-1", Severity.LOW)] - -result = FindingsNormaliser().normalise(prowler, trivy, untagged) -for f in result.findings: - print(f"{f.severity.value:8} {f.risk_score:4} {f.source_tool:16} {f.title} {f.compliance_frameworks[:4]}") -print(len(result.compliance), "compliance results") +--8<-- "ingest/normalise_demo.py" ``` ```console @@ -171,11 +144,31 @@ Merging across scanners needs the same resource identifier on both sides. Checko The normaliser keeps the last result on the instance, but nothing reads it back; treat `normalise()` as a function and use its return value. One instance can normalise any number of batches. +## Reference + +::: cloudg.normaliser.FindingsNormaliser + options: + members: + - normalise + ## Related -:::links -- [Compliance frameworks](/guides/compliance/) Which frameworks cloudg maps and how to read the results. -- [parse_report](/api/parse-report/) Getting findings out of scanner output. -- [Data models](/api/models/) Finding, ComplianceResult and ScanResult. -- [Ingesting existing output](/guides/ingesting/) The CLI path through the same normaliser. -::: +
      + +- [**Compliance frameworks**](../guides/compliance.md) + + Which frameworks cloudg maps and how to read the results. + +- [**parse_report**](parse-report.md) + + Getting findings out of scanner output. + +- [**Data models**](models.md) + + Finding, ComplianceResult and ScanResult. + +- [**Ingesting existing output**](../guides/ingesting.md) + + The CLI path through the same normaliser. + +
      diff --git a/site/content/api/graphbuilder.md b/site/content/api/graphbuilder.md index be6e973..e031ba4 100644 --- a/site/content/api/graphbuilder.md +++ b/site/content/api/graphbuilder.md @@ -1,11 +1,14 @@ --- title: GraphBuilder -object: cloudg.graph.builder.GraphBuilder -members: [build, graph, find_attack_paths, find_lateral_movement_paths, compute_centrality, subgraph, to_d3_json, to_cytoscape_json, to_json_str, save_graphml, load_graphml] -lede: Turns a list of `CloudAsset` and `NetworkEdge` objects into a NetworkX `DiGraph`, then searches, scores and exports it. -since: "0.6.0" +description: Turns a list of CloudAsset and NetworkEdge objects into a NetworkX DiGraph, then searches, scores and exports it. --- +Turns a list of `CloudAsset` and `NetworkEdge` objects into a NetworkX `DiGraph`, then searches, scores and exports it. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `GraphBuilder` is the first thing `cloudg run` does after collection, and `InventoryResult.export()` uses it to write `inventory-map.graphml` and `inventory-graph.json`. It needs no credentials and no network: hand it assets and edges from a collector, a saved `inventory-map.json` or your own code, and it gives you a plain `networkx.DiGraph` you can keep working on with the whole NetworkX library. ## What build() does @@ -14,7 +17,7 @@ Each asset becomes a node keyed by `asset.id`. The node carries `name`, `asset_t Each edge then joins `source_id` to `target_id`. Collectors write edges whose endpoints are not assets at all: a CIDR such as `0.0.0.0/0`, an Azure service tag or an unresolved reference. For those, `build()` creates a placeholder node with `asset_type` and `provider` set to `EXTERNAL` and `is_external=True`. -```mermaid caption="How build() places one NetworkEdge" +```mermaid flowchart TD A["NetworkEdge"] --> B{"Endpoints are nodes?"} B -->|no| C["Add EXTERNAL placeholder"] --> D @@ -24,6 +27,8 @@ flowchart TD F -->|yes| G["Merge ports, protocols, descriptions"] F -->|no| H["Replace attributes"] ``` +How build() places one NetworkEdge +{ .diagram-caption } A `DiGraph` holds one edge per ordered pair. Collectors often emit several rules for one pair (two ingress rules from `0.0.0.0/0` to the same security group, for 22 and 443), so since 0.6.0 `SECURITY_GROUP_RULE`, `NACL_RULE` and `INTERNET_EXPOSED` edges on the same pair are merged when their `cidr` and `direction` match: @@ -41,114 +46,10 @@ An empty port list on a TCP, UDP or all-protocol filter rule (GCP writes these) ### Build, search and export a small estate -The script builds eight assets and eleven edges by hand, so it runs offline. It also groups the nodes into communities with python-louvain, which cloudg installs; the [RAG exporter](/api/ragexporter/) runs the same `best_partition` call for its community chunks. +The script builds eight assets and eleven edges by hand, so it runs offline. It also groups the nodes into communities with python-louvain, which cloudg installs; the [RAG exporter](ragexporter.md) runs the same `best_partition` call for its community chunks. ```python title="build_graph.py" -"""Build a graph from a hand-made estate and look at it from a few angles.""" - -import json -from pathlib import Path - -import community as community_louvain # python-louvain, a cloudg dependency - -from cloudg.graph.builder import GraphBuilder -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge - -ACCOUNT = "123456789012" - - -def asset(asset_id: str, name: str, asset_type: AssetType, arn_suffix: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=name, - asset_type=asset_type, - provider=CloudProvider.AWS, - region="eu-west-1", - account_id=ACCOUNT, - arn=f"arn:aws:{arn_suffix}", - **extra, - ) - - -assets = [ - asset("sg-alb", "sg-alb", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-alb"), - asset("sg-web", "sg-web", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), - asset("alb", "web-alb", AssetType.LOAD_BALANCER, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:loadbalancer/app/web-alb/1"), - asset("tg", "web-tg", AssetType.TARGET_GROUP, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:targetgroup/web-tg/1"), - asset("web-1", "web-1", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web1", is_internet_exposed=True), - asset("web-2", "web-2", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web2"), - asset("app-role", "app-role", AssetType.IAM_ROLE, f"iam::{ACCOUNT}:role/app-role"), - asset("orders-db", "orders-db", AssetType.RDS_INSTANCE, f"rds:eu-west-1:{ACCOUNT}:db:orders-db"), -] - - -def rule(source: str, target: str, port: str, cidr: str = "", description: str = "") -> NetworkEdge: - return NetworkEdge( - source_id=source, - target_id=target, - edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range=port, - protocol="TCP", - cidr=cidr or None, - direction="ingress", - description=description or None, - ) - - -edges = [ - # Two rules on the same pair: GraphBuilder merges them into one edge - rule("0.0.0.0/0", "sg-alb", "443", "0.0.0.0/0", "HTTPS from anywhere"), - rule("0.0.0.0/0", "sg-alb", "80", "0.0.0.0/0", "HTTP redirect"), - rule("sg-alb", "sg-web", "8080", description="ALB to web tier"), - NetworkEdge(source_id="alb", target_id="sg-alb", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="alb", target_id="tg", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="tg", target_id="web-1", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="tg", target_id="web-2", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.IAM_TRUST), - NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), -] - -builder = GraphBuilder() -graph = builder.build(assets, edges) -print(f"{graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges from {len(edges)} NetworkEdges") - -merged = graph.edges["0.0.0.0/0", "sg-alb"] -print("merged rule:", merged["port_range"], "|", merged["description"]) -print("placeholder:", graph.nodes["0.0.0.0/0"]["asset_type"], graph.nodes["0.0.0.0/0"]["is_external"]) - - -def names(path: list[str]) -> str: - return " -> ".join(graph.nodes[n]["name"] for n in path) - - -# Paths follow edges in their stored direction -for path in builder.find_attack_paths("0.0.0.0/0", "sg-web"): - print("internet path:", names(path)) -for path in builder.find_attack_paths("web-1", "orders-db"): - print("identity path:", names(path)) -for path in builder.find_lateral_movement_paths(): - print("lateral path:", names(path)) - -centrality = builder.compute_centrality() -top = sorted(centrality.items(), key=lambda kv: kv[1]["betweenness"], reverse=True)[:3] -for node_id, scores in top: - print(f"betweenness {scores['betweenness']:.3f} {graph.nodes[node_id]['name']}") - -# Louvain works on undirected graphs; the RAG exporter does the same -partition = community_louvain.best_partition(graph.to_undirected(), random_state=42) -groups: dict[int, list[str]] = {} -for node_id, community_id in partition.items(): - groups.setdefault(community_id, []).append(graph.nodes[node_id]["name"]) -for community_id, members in sorted(groups.items()): - print(f"community {community_id}: {sorted(members)}") - -out = Path("graph-out") -builder.save_graphml(out / "estate.graphml") -(out / "estate-d3.json").write_text(builder.to_json_str()) -(out / "estate-cytoscape.json").write_text(json.dumps(builder.to_cytoscape_json(), indent=2)) -print("wrote", sorted(p.name for p in out.iterdir())) +--8<-- "graph/build_graph.py" ``` Output with cloudg 0.6.0 and python-louvain 0.16: @@ -171,7 +72,7 @@ community 3: ['web-alb', 'web-tg'] wrote ['estate-cytoscape.json', 'estate-d3.json', 'estate.graphml'] ``` -Eight assets plus the `0.0.0.0/0` placeholder make nine nodes, and the two rules into `sg-alb` became one edge. There is no internet path to `web-1` here even though the load balancer targets it: `alb -> sg-alb` is an `ATTACHED_TO` edge pointing from the resource to its group, and `find_attack_paths` walks edges only in the direction they are stored. The [reachability analysis](/api/reachabilityanalyzer/) knows which way traffic actually moves; walk `network_flow_graph(graph)` instead when you want flow-aware paths. +Eight assets plus the `0.0.0.0/0` placeholder make nine nodes, and the two rules into `sg-alb` became one edge. There is no internet path to `web-1` here even though the load balancer targets it: `alb -> sg-alb` is an `ATTACHED_TO` edge pointing from the resource to its group, and `find_attack_paths` walks edges only in the direction they are stored. The [reachability analysis](reachabilityanalyzer.md) knows which way traffic actually moves; walk `network_flow_graph(graph)` instead when you want flow-aware paths. Leave out `random_state` and Louvain may number and split the communities differently from one run to the next. @@ -205,14 +106,47 @@ Edges whose other end lies outside the filter bring that end back as an `EXTERNA - `compute_centrality()` returns normalised degree, in-degree, out-degree and betweenness per node. Betweenness is exact (`nx.betweenness_centrality`), so it is slow on graphs of tens of thousands of nodes. - `subgraph(node_ids)` returns an independent copy, so changes to it do not touch `builder.graph`. - `save_graphml()` creates missing parent directories and writes with `nx.write_graphml_xml` through `graphml_safe()`, a copy of the graph without `None` attributes and with lists and dicts stored as JSON strings. `builder.graph` itself is not changed. `load_graphml()` reads back through `nx.read_graphml`, which returns a `DiGraph` with string attributes; booleans such as `is_internet_exposed` come back as Python booleans because GraphML stores their type. -- The D3 export (`to_d3_json`, `to_json_str`) has `nodes` and `links`. Cytoscape's has `elements.nodes` and `elements.edges` with a smaller attribute set (no ARN or CIDR). The field reference for the D3 form is [inventory-graph.json](/reference/inventory/inventory-graph-json/). +- The D3 export (`to_d3_json`, `to_json_str`) has `nodes` and `links`. Cytoscape's has `elements.nodes` and `elements.edges` with a smaller attribute set (no ARN or CIDR). The field reference for the D3 form is [inventory-graph.json](../reference/inventory/inventory-graph-json.md). + +## Reference + +::: cloudg.graph.builder.GraphBuilder + options: + members: + - build + - graph + - find_attack_paths + - find_lateral_movement_paths + - compute_centrality + - subgraph + - to_d3_json + - to_cytoscape_json + - to_json_str + - save_graphml + - load_graphml ## Related -:::links -- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) Internet exposure, flow-aware walks and blast radius over this graph. -- [CloudOntology](/api/cloudontology/) The same assets and edges as RDF triples. -- [RAGExporter](/api/ragexporter/) Chunks built from the graph's communities. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) How the three analyses fit together. -- [inventory-map.graphml](/reference/inventory/inventory-map-graphml/) Node and edge attributes, field by field. -::: +
      + +- [**ReachabilityAnalyzer**](reachabilityanalyzer.md) + + Internet exposure, flow-aware walks and blast radius over this graph. + +- [**CloudOntology**](cloudontology.md) + + The same assets and edges as RDF triples. + +- [**RAGExporter**](ragexporter.md) + + Chunks built from the graph's communities. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + How the three analyses fit together. + +- [**inventory-map.graphml**](../reference/inventory/inventory-map-graphml.md) + + Node and edge attributes, field by field. + +
      diff --git a/site/content/api/index.md b/site/content/api/index.md new file mode 100644 index 0000000..776cef5 --- /dev/null +++ b/site/content/api/index.md @@ -0,0 +1,202 @@ +--- +title: Python API +description: cloudg is a library first. The CLI is a thin layer over CloudGEngine, so anything a command does you can do from Python, one phase at a time if you like. +--- + +cloudg is a library first. The CLI is a thin layer over `CloudGEngine`, so anything a command does you can do from Python, one phase at a time if you like. +{ .lede } + +**Package** `cloudg` · **Python** 3.11 or newer · **Install** `pip install cloudg` +{ .page-meta } + +## Start here + +Most programs need three names, all importable from the package root: `CloudGConfig` to say what to collect, `CloudGEngine` to run it, and the result object that comes back. + +=== "Ingest existing reports" + + No credentials and no scanner binaries: point the engine at scanner output produced elsewhere. + + ```python title="ingest_only.py" + from cloudg import CloudGConfig, CloudGEngine + + engine = CloudGEngine(CloudGConfig()) + result = engine.run_from_reports_sync( + {"trivy": ["./trivy.json"], "checkov": ["./results_json.json"]}, + output_dir="./reports", + ) + + print(result.total_findings, result.severity_breakdown) + for f in result.findings: + print(f.severity.value, f.source_tool, f.title) + ``` + +=== "Map an account" + + Read-only credentials and the `aws` extra. No scanner runs. + + ```python title="map_account.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] + + inventory = CloudGEngine(config).map_inventory_sync(output_dir="./reports") + print(inventory.summary["total_assets"], "assets") + print(inventory.summary["internet_exposed"], "internet-exposed") + ``` + +=== "Full pipeline" + + What `cloudg run` does: collect, scan, analyse and report. + + ```python title="pipeline.py" + import asyncio + + from cloudg import CloudGConfig, CloudGEngine + + + async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.scanners.enabled = ["prowler"] + result = await CloudGEngine(config).run_pipeline(output_dir="./reports") + print(result.to_summary()) + + + asyncio.run(main()) + ``` + +## How the pieces fit + +```mermaid +flowchart LR + CFG["CloudGConfig"] --> ENG["CloudGEngine"] + ENG -->|collect| ASSETS["CloudAsset, NetworkEdge"] + ENG -->|scan or ingest| FIND["Finding"] + ASSETS --> GB["GraphBuilder"] + GB --> RA["ReachabilityAnalyzer"] + ASSETS --> ONT["CloudOntology, RAGExporter"] + FIND --> NORM["FindingsNormaliser"] + RA --> NORM + NORM --> RES["PipelineResult"] + ENG -->|map_inventory| IM["InventoryMapper"] + IM --> IR["InventoryResult"] + IR --> DG["DependencyGraph"] +``` + +Each box is a page below. The engine wires them together, but every one of them also works on its own: build a graph from assets you loaded yourself, normalise findings from a CMDB export, or link assets that came from your own collector. + +## Reference + +### Engine + +
      + +- [**CloudGEngine**](cloudgengine.md) + + The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: the whole pipeline in one call, or each phase yourself. + +- [**CloudGConfig**](cloudgconfig.md) + + The pydantic model behind `config.yaml`. Build it in code, load it from a file, or both. + +- [**Result objects**](results.md) + + `PipelineResult`, `CollectionResult` and `AnalysisResult`: what the engine hands back. + +- [**Event hooks**](event-hooks.md) + + Six optional callbacks that report phases, findings, results and failures during a run. + +- [**Exceptions**](exceptions.md) + + The `CloudGError` hierarchy, strict mode for the scanner wrappers and `raise_for_errors()`. + +
      + +### Inventory + +
      + +- [**InventoryMapper**](inventorymapper.md) + + Maps what is deployed and how it connects, then overlays scanner findings on the map. + +- [**InventoryResult**](inventoryresult.md) + + The map as one object: assets, edges, summary, dependency analysis, save and load. + +- [**RelationshipLinker**](relationshiplinker.md) + + Turns a flat list of assets from any source into a connected map. + +- [**DependencyGraph**](dependencygraph.md) + + What an asset needs, and what breaks if it goes away. + +
      + +### Graph + +
      + +- [**GraphBuilder**](graphbuilder.md) + + Assets and edges as a NetworkX `DiGraph`: attack paths, centrality, exports. + +- [**ReachabilityAnalyzer**](reachabilityanalyzer.md) + + What the internet can reach, along the edges traffic can actually take. + +- [**CloudOntology**](cloudontology.md) + + An rdflib graph with 64 typed relations, exported as RDF and queried with SPARQL. + +- [**RAGExporter**](ragexporter.md) + + Text chunks with metadata, ready to embed for retrieval-augmented generation. + +
      + +### Findings + +
      + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Findings from any number of scanners, merged, rescored, sorted and mapped to controls. + +- [**parse_report**](parse-report.md) + + Native Prowler, ScoutSuite, Checkov and Trivy output as `Finding` objects. + +- [**Data models**](models.md) + + The pydantic models every part of cloudg exchanges. + +
      + +### Extend + +
      + +- [**Entry points**](entry-points.md) + + Ship a collector or a scanner as its own package; `PluginRegistry` finds it. + +- [**cloudg.resilience**](resilience.md) + + Your own cloud API calls through cloudg's rate limits, circuit breakers and retry budget. + +- [**CloudGMCPLayer**](mcp-layer.md) + + The MCP catalog as a Python object: call it in-process, mount it, or export it. + +
      + +## Conventions + +- Signatures, parameter tables and the "source" links on these pages are generated from the code at build time, so they always match the release the site was built from. The prose and the examples above each reference are written by hand. +- Methods that end in `_sync` wrap the async method of the same name in `asyncio.run()`. Use them from scripts; inside a running event loop, `await` the async method. +- Everything cloudg raises on purpose derives from `CloudGError`. See [Exceptions](exceptions.md). diff --git a/site/content/api/inventorymapper.md b/site/content/api/inventorymapper.md index 40d2ee3..692362d 100644 --- a/site/content/api/inventorymapper.md +++ b/site/content/api/inventorymapper.md @@ -1,14 +1,15 @@ --- -object: cloudg.inventory.InventoryMapper -members: [map_inventory, map_inventory_sync, build_asset_map, build_compliance_map, export_merged] -lede: "Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up." +description: Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up. --- +Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up. +{ .lede } + ## What a mapping run does `InventoryMapper(config, tagging_sweep=None)` stores the config and reads the `inventory` section from it. `tagging_sweep` overrides `config.inventory.tagging_sweep` for this mapper only. Nothing touches the cloud until you call `map_inventory()`. -```mermaid caption="InventoryMapper.map_inventory(), step by step" +```mermaid flowchart TD A["deep copy of the config"] --> B{"aws.organization.enabled?"} B -->|yes| C["discover the org, pick accounts and member role"] @@ -20,6 +21,8 @@ flowchart TD G --> H["account hierarchy nodes"] H --> I["InventoryResult"] ``` +InventoryMapper.map_inventory(), step by step +{ .diagram-caption } In more detail: @@ -28,12 +31,12 @@ In more detail: 3. With Azure in `providers` and `azure.map_management_groups` on, the management group and Azure Policy hierarchy is read. With GCP, `gcp.organization_id` set and `gcp.map_hierarchy` on, the organization, folders, org policies and VPC Service Controls perimeters are read. 4. The deep collectors run per account and region (AWS), per subscription (Azure) and per project or organization (GCP), with the `inventory` options applied: service families, exclusions, Kubernetes, the tagging and Cloud Control sweeps, IAM resource edges and the rest. 5. Assets that share an ARN (the same resource seen from two regions or from a collector and a sweep) collapse into one. The detailed copy wins over a sweep or placeholder copy, and their declared relations are merged. -6. The [`RelationshipLinker`](/api/relationshiplinker/) resolves declared relations and applies its provider rules, with `inventory.link_references` deciding whether the generic reference scan runs. Placeholder `CLOUD_ACCOUNT` nodes are added for accounts that were referenced but not mapped. +6. The [`RelationshipLinker`](relationshiplinker.md) resolves declared relations and applies its provider rules, with `inventory.link_references` deciding whether the generic reference scan runs. Placeholder `CLOUD_ACCOUNT` nodes are added for accounts that were referenced but not mapped. 7. With `inventory.account_hierarchy` on (the default), account nodes are added and linked to the top-level resources they contain. Cloud API throttling does not fail a map. Calls are rate limited and retried according to `config.ratelimit`; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` in `coverage`, and the run's throttling telemetry lands in `result.throttling`, with each message also logged as a warning. If the code that calls `map_inventory()` already runs inside a `cloudg.resilience.stats_scope()`, the mapper reports into that scope. -`map_inventory()` returns an [`InventoryResult`](/api/inventoryresult/). Calling the engine's `map_inventory()` instead gets you the same result plus an `on_phase_start("inventory_mapping")` hook and optional export. +`map_inventory()` returns an [`InventoryResult`](inventoryresult.md). Calling the engine's `map_inventory()` instead gets you the same result plus an `on_phase_start("inventory_mapping")` hook and optional export. ## Merging findings into a map @@ -41,7 +44,7 @@ The last three methods never call a cloud API or a scanner. They take a finished `build_asset_map(result, findings)` returns one entry per asset in the map: `id`, `arn`, `name`, `type`, `provider`, `region`, `account_id`, `internet_exposed`, `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the most findings first, plus `total_assets` and `assets_with_findings` at the top level. Findings are matched to assets through `AssetIndex.resolve_finding()`, the matcher the ontology and RAG exports use: the asset's internal id, then an ARN taken from `resource_id` or `resource_arn`, then a unique name, then a unique ARN tail. A name that several assets share matches none of them, so a finding for a table named `orders` does not land on a queue also named `orders`. -`build_compliance_map(result, findings)` groups by compliance framework. Each framework gets `findings`, `severity_breakdown`, `affected_assets` (the ARN of the asset the finding matched in the same way, or the finding's own `resource_arn` when no asset matched) and `affected_asset_count`. It reads `finding.compliance_frameworks` as it is, so run the findings through [`FindingsNormaliser`](/api/findingsnormaliser/) first if you want the framework mapping cloudg adds on top of what the scanner reported. +`build_compliance_map(result, findings)` groups by compliance framework. Each framework gets `findings`, `severity_breakdown`, `affected_assets` (the ARN of the asset the finding matched in the same way, or the finding's own `resource_arn` when no asset matched) and `affected_asset_count`. It reads `finding.compliance_frameworks` as it is, so run the findings through [`FindingsNormaliser`](findingsnormaliser.md) first if you want the framework mapping cloudg adds on top of what the scanner reported. `export_merged(result, findings, output_dir)` writes both: `asset-map.json` and `compliance-map.json`, and returns their paths under the keys `asset_map` and `compliance_map`. @@ -81,34 +84,14 @@ for cov in inventory.coverage: inventory.export("./inventory") ``` -[AWS Organizations](/guides/aws-organizations/) covers the role, the account selection and Control Tower in detail. +[AWS Organizations](../guides/aws-organizations.md) covers the role, the account selection and Control Tower in detail. ### Overlay findings that arrive later Mapping and scanning often run on different schedules or different hosts. This script runs offline: it loads a map exported earlier, ingests Prowler and Trivy output, normalises it, and writes the merged maps next to the inventory. ```python title="overlay_findings.py" -from cloudg import CloudGConfig, InventoryMapper, InventoryResult -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser - -# A map written earlier by `cloudg map -o ./inventory` or InventoryResult.export() -inventory = InventoryResult.load("./inventory") - -# Scanner output that arrived later, normalised so compliance mapping is filled in -raw = ingest_reports({"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}) -findings = FindingsNormaliser().normalise(raw).findings - -mapper = InventoryMapper(CloudGConfig(providers=["aws"])) -paths = mapper.export_merged(inventory, findings, "./inventory") -print({name: str(path) for name, path in paths.items()}) - -compliance = mapper.build_compliance_map(inventory, findings) -for framework, entry in list(compliance["frameworks"].items())[:3]: - print(framework, entry["findings"], entry["affected_assets"]) - -asset_map = mapper.build_asset_map(inventory, findings) -print(asset_map["total_assets"], "assets,", asset_map["assets_with_findings"], "with findings") +--8<-- "map/overlay_findings.py" ``` ```console @@ -132,12 +115,39 @@ The mapper needs a config only to exist; the merge methods ignore it. `Inventory `InventoryResult.coverage` is saved in `inventory-map.json` under `coverage`, and `InventoryResult.load()` reads it back. The CLI also prints the failed collectors after a `cloudg map` run. +## Reference + +::: cloudg.inventory.InventoryMapper + options: + members: + - map_inventory + - map_inventory_sync + - build_asset_map + - build_compliance_map + - export_merged + ## Related -:::links -- [InventoryResult](/api/inventoryresult/) What `map_inventory()` returns. -- [RelationshipLinker](/api/relationshiplinker/) The linking step on its own. -- [DependencyGraph](/api/dependencygraph/) Blast radius and dependency walks over the map. -- [Inventory mapping](/guides/inventory-mapping/) The guide, with the CLI side. -- [cloudg map](/cli/map/) The command that wraps this class. -::: +
      + +- [**InventoryResult**](inventoryresult.md) + + What `map_inventory()` returns. + +- [**RelationshipLinker**](relationshiplinker.md) + + The linking step on its own. + +- [**DependencyGraph**](dependencygraph.md) + + Blast radius and dependency walks over the map. + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + The guide, with the CLI side. + +- [**cloudg map**](../cli/map.md) + + The command that wraps this class. + +
      diff --git a/site/content/api/inventoryresult.md b/site/content/api/inventoryresult.md index 698b427..672fe24 100644 --- a/site/content/api/inventoryresult.md +++ b/site/content/api/inventoryresult.md @@ -1,14 +1,15 @@ --- -object: cloudg.inventory.InventoryResult -members: [summary, dependency_graph, analysis, export, load] -lede: "The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from." +description: 'The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from.' --- +The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from. +{ .lede } + ## Where it comes from `InventoryResult` is a dataclass defined in `cloudg/inventory/mapper_result.py` and importable from `cloudg`, `cloudg.inventory` and `cloudg.inventory.mapper`. You get one from `InventoryMapper.map_inventory()`, from `CloudGEngine.map_inventory()`, or from `InventoryResult.load()` on a map saved earlier. You can also build one yourself from any list of `CloudAsset` and `NetworkEdge` objects; every field has a default, and the methods work the same on a hand-built result. -```mermaid caption="An inventory map's round trip through the file system" +```mermaid flowchart LR M["map_inventory()"] --> R["InventoryResult"] Y["your own assets and edges"] --> R @@ -17,12 +18,14 @@ flowchart LR R -->|"dependency_graph()"| G["DependencyGraph"] R -->|"analysis()"| A["dependency and coverage report"] ``` +An inventory map's round trip through the file system +{ .diagram-caption } ## The parts that need explaining -`summary` is a property, not a stored field, and it is recomputed from `assets` and `edges` on every access. On a map with tens of thousands of assets that is a noticeable amount of work, so read it once into a variable. Its keys are counts and breakdowns: `total_assets`, `total_edges`, `providers`, `assets_by_type`, `assets_by_service`, `assets_by_region`, `assets_by_account`, `edges_by_type`, `edges_by_relationship`, `unlinked_assets`, `internet_exposed`, `accounts`, `cross_account_edges`, `external_accounts`, `security_service_gaps`, `unresolved_references`, and `organization` and `throttling` when those exist. The [summary reference](/reference/inventory/summary/) says how each one is computed. `assets_by_service` is derived from the identifier: the service field of an ARN, the provider namespace of an Azure ID, the API host of a GCP name. +`summary` is a property, not a stored field, and it is recomputed from `assets` and `edges` on every access. On a map with tens of thousands of assets that is a noticeable amount of work, so read it once into a variable. Its keys are counts and breakdowns: `total_assets`, `total_edges`, `providers`, `assets_by_type`, `assets_by_service`, `assets_by_region`, `assets_by_account`, `edges_by_type`, `edges_by_relationship`, `unlinked_assets`, `internet_exposed`, `accounts`, `cross_account_edges`, `external_accounts`, `security_service_gaps`, `unresolved_references`, and `organization` and `throttling` when those exist. The [summary reference](../reference/inventory/summary.md) says how each one is computed. `assets_by_service` is derived from the identifier: the service field of an ARN, the provider namespace of an Azure ID, the API host of a GCP name. -`dependency_graph(include_hierarchy=False)` builds a new [`DependencyGraph`](/api/dependencygraph/) on each call. With the default, `CONTAINS` edges into organization, OU and account nodes are left out, so "everything in this account depends on the account" does not dominate every blast radius. +`dependency_graph(include_hierarchy=False)` builds a new [`DependencyGraph`](dependencygraph.md) on each call. With the default, `CONTAINS` edges into organization, OU and account nodes are left out, so "everything in this account depends on the account" does not dominate every blast radius. `analysis(top=25)` runs the dependency graph and the coverage checks and returns a dict with five keys: `shared_dependencies` (assets many others directly depend on), `largest_blast_radius`, `cross_account_edges` (one entry per edge between two accounts, with `external` true when one side is an unmapped account), `security_coverage` (which security services run in which account and region, the gaps, workloads no vulnerability scanner covers, internet-facing entry points without a WAF) and `unresolved_references`. This is what `export()` saves as `inventory-dependencies.json`. @@ -43,59 +46,10 @@ The missing `raw_data` matters if you link a loaded map again: a few linker rule ### Build, export, reload, analyse -Runs offline. Four assets of an orders service are linked by the [`RelationshipLinker`](/api/relationshiplinker/), one of them points at a role in an account that was never mapped, and the result goes through a full save and load. +Runs offline. Four assets of an orders service are linked by the [`RelationshipLinker`](relationshiplinker.md), one of them points at a role in an account that was never mapped, and the result goes through a full save and load. ```python title="inventory_roundtrip.py" -from cloudg import InventoryResult -from cloudg.inventory import RelationshipLinker -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" -QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" -TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" -FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" - - -def aws(arn, name, asset_type, **metadata): - return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, - region=REGION, account_id=ACCOUNT, metadata=metadata) - - -assets = [ - aws(KEY, "orders-key", AssetType.KMS_KEY), - aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, kms_key_id=KEY), - aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, kms_key_id=KEY), - aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, kms_key_id=KEY, relations=[ - {"target": QUEUE, "edge": "INVOKES", "relationship": "TRIGGERED_BY", "reverse": True}, - {"target": TABLE, "edge": "REFERENCES", "relationship": "WRITES_TO"}, - {"target": "arn:aws:iam::999999999999:role/partner-reader", "edge": "ASSUMES_ROLE"}, - ]), -] - -linker = RelationshipLinker(assets) -edges = linker.link() -inventory = InventoryResult( - assets=assets + linker.external_assets, - edges=edges, - providers=["aws"], - regions={"aws": [REGION]}, - unresolved_references=linker.unresolved, -) - -paths = inventory.export("./inventory") -print(sorted(paths)) - -again = InventoryResult.load("./inventory") # the directory or inventory-map.json -s = again.summary -print(s["total_assets"], s["total_edges"], s["edges_by_type"]) -print("external accounts:", s["external_accounts"], "cross-account edges:", s["cross_account_edges"]) - -analysis = again.analysis(top=5) -print(list(analysis)) -print([(d["name"], d["direct_dependents"]) for d in analysis["shared_dependencies"]]) -print(analysis["cross_account_edges"]) +--8<-- "map/inventory_roundtrip.py" ``` ```console @@ -115,19 +69,7 @@ Five assets, not four: the role in account `999999999999` was not mapped, so the Any map exported by `cloudg map -o ./inventory` loads the same way. No credentials are needed. ```python title="whats_exposed.py" -from cloudg import InventoryResult - -inventory = InventoryResult.load("./inventory") -summary = inventory.summary - -print(f"{summary['internet_exposed']} of {summary['total_assets']} assets face the internet") -for asset in inventory.assets: - if asset.is_internet_exposed: - print(f" {asset.asset_type.value:16} {asset.account_id or '-':14} {asset.display_id}") - -print("unresolved references:", summary["unresolved_references"]) -for ref in inventory.unresolved_references[:10]: - print(f" {ref['source_name']} -> {ref['target']} ({ref['edge_type']})") +--8<-- "map/whats_exposed.py" ``` ## Notes @@ -140,11 +82,35 @@ for ref in inventory.unresolved_references[:10]: Building an `InventoryResult` yourself does not deduplicate or link anything. Pass assets that are already unique by ARN and edges you produced (with the linker or otherwise), or the summary counts will include the duplicates. +## Reference + +::: cloudg.inventory.InventoryResult + options: + members: + - summary + - dependency_graph + - analysis + - export + - load + ## Related -:::links -- [InventoryMapper](/api/inventorymapper/) The class that produces this result. -- [DependencyGraph](/api/dependencygraph/) What `dependency_graph()` returns. -- [InventoryResult reference](/reference/inventory/inventoryresult/) Every field and file, specified. -- [Output files](/reference/output-files/) All files cloudg writes, side by side. -::: +
      + +- [**InventoryMapper**](inventorymapper.md) + + The class that produces this result. + +- [**DependencyGraph**](dependencygraph.md) + + What `dependency_graph()` returns. + +- [**InventoryResult reference**](../reference/inventory/inventoryresult.md) + + Every field and file, specified. + +- [**Output files**](../reference/output-files.md) + + All files cloudg writes, side by side. + +
      diff --git a/site/content/api/mcp-layer.md b/site/content/api/mcp-layer.md index 50c015a..dcea76a 100644 --- a/site/content/api/mcp-layer.md +++ b/site/content/api/mcp-layer.md @@ -1,17 +1,14 @@ --- title: CloudGMCPLayer -object: - - cloudg.mcp.CloudGMCPLayer - - cloudg.mcp.register_into - - cloudg.mcp.build_server - - cloudg.mcp.export_definitions -members: - cloudg.mcp.CloudGMCPLayer: [call_tool, read_resource, get_prompt, complete, list_tools, list_resources, list_resource_templates, list_prompts, tools_wire, get_tool, register_into, serve, use, set_prefix, on_change, notify_change, context] -lede: The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. -source: docs/MCP.md -since: "0.6.0" +description: The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. --- +The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `cloudg mcp serve` is a thin wrapper around this class. Everything the server does, from the 74 tools to the policy that hides, redacts and pseudonymises, lives in `CloudGMCPLayer`, so you can embed the same catalog wherever your agent or server already runs. Importing `cloudg.mcp` pulls in nothing beyond cloudg's own dependencies; the official `mcp` SDK (the `cloudg[mcp]` extra) or `fastmcp` is imported only when you mount into one of them. There are three ways in, and all of them end at the same four methods: @@ -22,7 +19,7 @@ There are three ways in, and all of them end at the same four methods: | An MCP server built on the SDK or fastmcp | `layer.register_into(server)` | cloudg's tools, resources and prompts next to the host's own | | A gateway or framework that speaks neither | `export_definitions(layer)` | MCP wire dicts plus async handlers, for one principal | -```mermaid caption="Every route goes through the same layer methods" +```mermaid flowchart LR A["Your code"] --> L["layer.call_tool / read_resource / get_prompt / complete"] B["SDK or fastmcp server via register_into"] --> L @@ -35,6 +32,8 @@ flowchart LR P --> H["Handler"] H --> O["Output transforms"] ``` +Every route goes through the same layer methods +{ .diagram-caption } A tool the policy hides from the caller is indistinguishable from one that does not exist. Every other failure of a tool call (bad arguments, access denied, a handler exception, a timeout) comes back as a `ToolResult` with `is_error=True` that a model can read and recover from, rather than as an exception. Resources and prompts raise `MCPLayerError` subclasses instead. @@ -78,12 +77,10 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, ACCOUNT = "123456789012" - def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: return CloudAsset(id=asset_id, name=asset_id, asset_type=asset_type, provider=CloudProvider.AWS, region="eu-west-1", account_id=ACCOUNT, arn=arn, **extra) - inventory = InventoryResult( assets=[ asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), @@ -99,7 +96,6 @@ inventory = InventoryResult( providers=["aws"], ) - async def main() -> None: workspace = Workspace(allowed_roots=["."]) workspace.add_inventory(inventory, "demo") @@ -132,7 +128,6 @@ async def main() -> None: with open("mcp-audit.jsonl") as fh: print("audit:", fh.readline().strip()) - asyncio.run(main()) ``` @@ -172,13 +167,11 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider server = MCPServer("platform-tools") - @server.tool() def ticket_count(team: str) -> int: """How many open tickets a team has (the host's own tool).""" return 7 - inventory = InventoryResult( assets=[ CloudAsset(id=f"vm-{i}", name=f"vm-{i}", asset_type=AssetType.EC2, @@ -193,7 +186,6 @@ workspace.add_inventory(inventory, "demo") layer = CloudGMCPLayer(policy="standard", workspace=workspace) binding = layer.register_into(server, prefix="cloudg_") # mount after the host's own tools - async def main() -> None: async with Client(server) as client: names = [t.name for t in (await client.list_tools()).tools] @@ -210,7 +202,6 @@ async def main() -> None: "arguments": {"group_by": "type"}}) print("dispatch:", reply["structuredContent"]["groups"], reply["isError"]) - asyncio.run(main()) ``` @@ -240,7 +231,7 @@ layer = CloudGMCPLayer(policy="strict", workspace=workspace, prefix="cloudg_") layer.serve("stdio") # blocks until the client disconnects ``` -For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MCP_TOKEN:analyst:ci"])` takes the same options as the CLI. The token spec format and the other transport settings are on the [cloudg mcp serve](/cli/mcp-serve/) page. +For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MCP_TOKEN:analyst:ci"])` takes the same options as the CLI. The token spec format and the other transport settings are on the [cloudg mcp serve](../cli/mcp-serve.md) page. ## Notes @@ -252,12 +243,57 @@ For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MC - Sync tool handlers run in a worker thread, so a slow graph build does not block the event loop. `notify_change()` is safe to call from those threads. - A policy with `vault.path` set persists its pseudonym vault: `serve()` saves it when serving ends, and any process saves a changed vault at interpreter exit. Call `layer.policy.save_vault()` to save at a moment you choose, for example before a long-running worker is killed. +## Reference + +::: cloudg.mcp.CloudGMCPLayer + options: + members: + - call_tool + - read_resource + - get_prompt + - complete + - list_tools + - list_resources + - list_resource_templates + - list_prompts + - tools_wire + - get_tool + - register_into + - serve + - use + - set_prefix + - on_change + - notify_change + - context + +::: cloudg.mcp.register_into + +::: cloudg.mcp.build_server + +::: cloudg.mcp.export_definitions + ## Related -:::links -- [MCP server overview](/mcp/) What the catalog covers and how clients connect. -- [In-process use](/mcp/server/in-process-use/) Every layer method and the `ToolContext` handlers receive. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each of the seven policies allows and transforms. -- [Tool index](/mcp/tools/tool-index/) All 74 tools by category. -- [cloudg mcp serve](/cli/mcp-serve/) The same layer behind stdio or HTTP. -::: +
      + +- [**MCP server overview**](../mcp/index.md) + + What the catalog covers and how clients connect. + +- [**In-process use**](../mcp/server/in-process-use.md) + + Every layer method and the `ToolContext` handlers receive. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each of the seven policies allows and transforms. + +- [**Tool index**](../mcp/tools/tool-index.md) + + All 74 tools by category. + +- [**cloudg mcp serve**](../cli/mcp-serve.md) + + The same layer behind stdio or HTTP. + +
      diff --git a/site/content/api/models.md b/site/content/api/models.md index b0fc811..790c530 100644 --- a/site/content/api/models.md +++ b/site/content/api/models.md @@ -1,22 +1,15 @@ --- -object: [cloudg.schema.models.Finding, cloudg.schema.models.CloudAsset, cloudg.schema.models.NetworkEdge, cloudg.schema.models.ComplianceResult, cloudg.schema.models.ScanResult, cloudg.schema.models.Severity, cloudg.schema.models.CloudProvider, cloudg.schema.models.ComplianceStatus] -members: - cloudg.schema.models.Finding: [risk_score] - cloudg.schema.models.CloudAsset: [display_id] - cloudg.schema.models.NetworkEdge: [] - cloudg.schema.models.ComplianceResult: [] - cloudg.schema.models.ScanResult: [summary] - cloudg.schema.models.Severity: [] - cloudg.schema.models.CloudProvider: [] - cloudg.schema.models.ComplianceStatus: [] -lede: "The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the `ScanResult` that holds them together." +description: 'The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the ScanResult that holds them together.' --- +The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the `ScanResult` that holds them together. +{ .lede } + ## How the models fit together Every collector produces `CloudAsset` and `NetworkEdge` objects, every scanner parser produces `Finding` objects, and the normaliser bundles findings and their `ComplianceResult` entries into a `ScanResult`. The links between them are plain string ids, not object references: -```mermaid caption="The id fields that connect the models" +```mermaid flowchart LR SR["ScanResult"] -->|assets| CA["CloudAsset"] SR -->|edges| NE["NetworkEdge"] @@ -26,8 +19,10 @@ flowchart LR F -->|"resource_id, resource_arn"| CA CR -->|finding_ids| F ``` +The id fields that connect the models +{ .diagram-caption } -Those references are not checked. An edge may point at an id no asset has (cloudg's own security group rules use the CIDR `0.0.0.0/0` as `source_id`, and the graph builder turns it into a node of its own), and a finding's `resource_id` is often an ARN or a Terraform address instead of an asset id. [`AssetIndex`](/api/dependencygraph/) is the matcher cloudg uses to resolve such references. +Those references are not checked. An edge may point at an id no asset has (cloudg's own security group rules use the CIDR `0.0.0.0/0` as `source_id`, and the graph builder turns it into a node of its own), and a finding's `resource_id` is often an ARN or a Terraform address instead of an asset id. [`AssetIndex`](dependencygraph.md) is the matcher cloudg uses to resolve such references. All five models are configured with `extra="allow"`: a field the model does not define is kept on the instance and written back out by `model_dump()`. That makes it easy to carry your own data (`CloudAsset(..., owner="team-orders")`) through cloudg's pipeline, and it also means a typo in a keyword argument is not an error. @@ -37,9 +32,9 @@ All five models are configured with `extra="allow"`: a field the model does not All of them are `str` enums, so `Severity.HIGH == "HIGH"` is true, and pydantic accepts the plain string wherever the enum is expected. Values outside the enum raise a `ValidationError`. -`AssetType` has 156 values, from `EC2` and `S3_BUCKET` to `K8S_WORKLOAD`, `LANDING_ZONE` and `AI_GUARDRAIL`, plus `OTHER` for anything without a better fit. The [asset types by category](/reference/catalog/asset-types-by-category/) page lists them all, grouped, with the metadata each type carries. +`AssetType` has 156 values, from `EC2` and `S3_BUCKET` to `K8S_WORKLOAD`, `LANDING_ZONE` and `AI_GUARDRAIL`, plus `OTHER` for anything without a better fit. The [asset types by category](../reference/catalog/asset-types-by-category.md) page lists them all, grouped, with the metadata each type carries. -`EdgeType` has 20 values. Eleven describe network and identity structure (`SECURITY_GROUP_RULE`, `NACL_RULE`, `ROUTE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `CONTAINS`, `PEERING`, `LOAD_BALANCER_TARGET`, `INTERNET_EXPOSED`, `ATTACHED_TO`, `REFERENCES`) and nine are the typed inventory relationships (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`), always read "source verb target". The finer `relationship` label on an edge comes from a separate vocabulary, documented under [relationship vocabulary](/reference/catalog/relationship-vocabulary/). +`EdgeType` has 20 values. Eleven describe network and identity structure (`SECURITY_GROUP_RULE`, `NACL_RULE`, `ROUTE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `CONTAINS`, `PEERING`, `LOAD_BALANCER_TARGET`, `INTERNET_EXPOSED`, `ATTACHED_TO`, `REFERENCES`) and nine are the typed inventory relationships (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`), always read "source verb target". The finer `relationship` label on an edge comes from a separate vocabulary, documented under [relationship vocabulary](../reference/catalog/relationship-vocabulary.md). ## Serialised forms @@ -196,7 +191,7 @@ for f in sorted(urgent, key=lambda f: f.risk_score, reverse=True): print(f"{f.risk_score:4} {f.severity.value:8} {f.resource_arn or f.resource_id} {f.title}") ``` -[parse_report](/api/parse-report/) shows how to write such a file from scanner output. +[parse_report](parse-report.md) shows how to write such a file from scanner output. ## Notes @@ -212,12 +207,65 @@ Assignment is not validated. `finding.severity = "SEVERE"` succeeds, and the pla `NetworkEdge.direction` defaults to `"ingress"` and `ports` to an empty list; most collectors fill `port_range` (`"80-443"`, `"22"`) instead of `ports`. Parallel security group rules between the same two nodes are separate edges in the data and are merged into one edge only inside the graph builder. +## Reference + +::: cloudg.schema.models.Finding + options: + members: + - risk_score + +::: cloudg.schema.models.CloudAsset + options: + members: + - display_id + +::: cloudg.schema.models.NetworkEdge + options: + members: false + +::: cloudg.schema.models.ComplianceResult + options: + members: false + +::: cloudg.schema.models.ScanResult + options: + members: + - summary + +::: cloudg.schema.models.Severity + options: + members: false + +::: cloudg.schema.models.CloudProvider + options: + members: false + +::: cloudg.schema.models.ComplianceStatus + options: + members: false + ## Related -:::links -- [Asset types by category](/reference/catalog/asset-types-by-category/) All 156 `AssetType` values, grouped. -- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The `relationship` labels on edges. -- [Result objects](/api/results/) The dataclasses that carry these models out of the engine. -- [FindingsNormaliser](/api/findingsnormaliser/) Where findings get merged, rescored and mapped. -- [Output files](/reference/output-files/) The JSON files these models are written to. -::: +
      + +- [**Asset types by category**](../reference/catalog/asset-types-by-category.md) + + All 156 `AssetType` values, grouped. + +- [**Relationship vocabulary**](../reference/catalog/relationship-vocabulary.md) + + The `relationship` labels on edges. + +- [**Result objects**](results.md) + + The dataclasses that carry these models out of the engine. + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Where findings get merged, rescored and mapped. + +- [**Output files**](../reference/output-files.md) + + The JSON files these models are written to. + +
      diff --git a/site/content/api/parse-report.md b/site/content/api/parse-report.md index 251f770..37df354 100644 --- a/site/content/api/parse-report.md +++ b/site/content/api/parse-report.md @@ -1,8 +1,10 @@ --- -object: [cloudg.ingest.parse_report, cloudg.ingest.ingest_reports] -lede: "Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg `Finding` objects. No engine, no credentials, no scanner binaries." +description: Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg Finding objects. No engine, no credentials, no scanner binaries. --- +Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg `Finding` objects. No engine, no credentials, no scanner binaries. +{ .lede } + ## Two functions, one dispatcher `cloudg.ingest` is a small module with two public functions and one constant. `SUPPORTED_TOOLS` is the tuple `("prowler", "scoutsuite", "checkov", "trivy")`. @@ -11,7 +13,7 @@ lede: "Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into clo `ingest_reports(reports)` takes a dict of tool name to a list of paths and calls `parse_report` for each pair, collecting everything into one list. It is what `CloudGEngine.ingest_reports()` and `cloudg ingest` use underneath. -```mermaid caption="How ingest_reports handles each path" +```mermaid flowchart TD R["reports dict"] --> L["for each tool, for each path"] L --> P["parse_report(tool, path)"] @@ -23,6 +25,8 @@ flowchart TD N --> K F --> C["combined list"] ``` +How ingest_reports handles each path +{ .diagram-caption } `parse_report` raises both errors. `ingest_reports` catches exactly those two, logs them on the `cloudg.ingest` logger at ERROR level, and carries on with the next path. A file that exists but does not parse (truncated JSON, the wrong tool's output) is handled one level down: the scanner wrapper logs a warning and returns what it could read, often an empty list. Neither function raises for bad content. @@ -35,9 +39,9 @@ flowchart TD | `checkov` | the JSON report | `results_json.json` anywhere below it, else every `*.json` | `checkov -d ./iac --output json > results_json.json` | | `trivy` | an `image` or `fs` JSON report | every `*.json` below it | `trivy image --format json myrepo/app:latest > trivy-image.json` | -A few rules per tool decide what becomes a finding. Prowler records are read as OCSF or ASFF, record by record. Passing checks (ASFF `Compliance.Status` of `PASSED`, OCSF `status_code` of `PASS`) give no finding, and the check name inside the finding id (`prowler--...` or `prowler---...`) becomes the deduplication key. ScoutSuite yields one finding per flagged item, with `danger`, `warning` and `caution` mapped to CRITICAL, HIGH and MEDIUM. Checkov findings come from `results.failed_checks`, whether the report is one object or a list of them (several frameworks). Trivy reports are split by their `ArtifactType`: `container_image` reports give vulnerability and secret findings, anything else also gives misconfigurations. [Input requirements](/guides/ingesting/#input-requirements) lists every field read. +A few rules per tool decide what becomes a finding. Prowler records are read as OCSF or ASFF, record by record. Passing checks (ASFF `Compliance.Status` of `PASSED`, OCSF `status_code` of `PASS`) give no finding, and the check name inside the finding id (`prowler--...` or `prowler---...`) becomes the deduplication key. ScoutSuite yields one finding per flagged item, with `danger`, `warning` and `caution` mapped to CRITICAL, HIGH and MEDIUM. Checkov findings come from `results.failed_checks`, whether the report is one object or a list of them (several frameworks). Trivy reports are split by their `ArtifactType`: `container_image` reports give vulnerability and secret findings, anything else also gives misconfigurations. [Input requirements](../guides/ingesting.md#input-requirements) lists every field read. -The findings come back raw. Their severities are what the scanner said, duplicates are still there, and only scanner-native compliance tags are set. Pass them through [`FindingsNormaliser`](/api/findingsnormaliser/) (or `CloudGEngine.normalise_findings()`) before you count or report on them. +The findings come back raw. Their severities are what the scanner said, duplicates are still there, and only scanner-native compliance tags are set. Pass them through [`FindingsNormaliser`](findingsnormaliser.md) (or `CloudGEngine.normalise_findings()`) before you count or report on them. For Prowler, the list returned is a `FindingList`: a plain `list` subclass with a `passed_checks` set naming the checks that passed. `ingest_reports` returns a `FindingList` too, with the passed checks of every Prowler report in it. Hand the list to `FindingsNormaliser.normalise()` as it is and the compliance controls those checks map to get PASS results. `+=` keeps the left list's attribute (as in the example below), but `a + b` and `list(a)` build a plain list without it; pass `passed_checks=` to `normalise()` yourself in that case. @@ -48,29 +52,7 @@ For Prowler, the list returned is a `FindingList`: a plain `list` subclass with Runs offline against one Prowler directory, one Checkov file and one Trivy file. Logging is on so the skipped paths are visible. ```python title="parse_first.py" -import logging - -from cloudg.ingest import SUPPORTED_TOOLS, ingest_reports, parse_report -from cloudg.normaliser import FindingsNormaliser - -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - -print(SUPPORTED_TOOLS) -findings = parse_report("prowler", "./prowler-output/") -findings += parse_report("checkov", "./results_json.json") -print(len(findings), "findings from two reports") - -# A typo'd tool name or a missing path is skipped by ingest_reports, not raised -more = ingest_reports({"trivy": ["./trivy.json", "./missing.json"], "trivvy": ["./trivy.json"]}) -print(len(more), "findings from the batch") - -try: - parse_report("grype", "./trivy.json") -except ValueError as exc: - print("ValueError:", exc) - -scan_result = FindingsNormaliser().normalise(findings, more) -print(scan_result.summary["severity_breakdown"], len(scan_result.compliance), "control results") +--8<-- "ingest/parse_first.py" ``` ```console @@ -97,22 +79,7 @@ The log lines appear first because logging writes to stderr and `print` to stdou `cloudg map --findings` reads a JSON list of findings. This writes one from scanner output collected in CI, so a later map run on another host can overlay it. ```python title="export_findings.py" -import json -from pathlib import Path - -from cloudg.ingest import ingest_reports - -findings = ingest_reports( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy.json"], - } -) -out = Path("./reports/raw-findings.json") -out.parent.mkdir(parents=True, exist_ok=True) -out.write_text(json.dumps([f.model_dump(mode="json") for f in findings], indent=2)) -print(f"wrote {len(findings)} findings to {out}") +--8<-- "ingest/export_findings.py" ``` ```bash @@ -136,13 +103,32 @@ Directories are searched recursively, and for Prowler and Trivy every `*.json` f `parse_report` passes the path to the scanner wrappers as a string. A `pathlib.Path` is fine as input to both functions. -The `resource_id` of a finding is whatever the scanner identified: an ARN for Prowler, a Terraform address such as `aws_s3_bucket.data` for Checkov, an image name for Trivy image scans. Matching those to collected assets is the job of [`AssetIndex`](/api/dependencygraph/) and of the asset map in [`InventoryMapper`](/api/inventorymapper/). +The `resource_id` of a finding is whatever the scanner identified: an ARN for Prowler, a Terraform address such as `aws_s3_bucket.data` for Checkov, an image name for Trivy image scans. Matching those to collected assets is the job of [`AssetIndex`](dependencygraph.md) and of the asset map in [`InventoryMapper`](inventorymapper.md). + +## Reference + +::: cloudg.ingest.parse_report + +::: cloudg.ingest.ingest_reports ## Related -:::links -- [Ingesting existing output](/guides/ingesting/) The guide, with the CLI and the input formats in detail. -- [cloudg ingest](/cli/ingest/) The command built on these functions. -- [FindingsNormaliser](/api/findingsnormaliser/) The step that should follow parsing. -- [CloudGEngine](/api/cloudgengine/) `ingest_reports()` and `run_from_reports()` with hooks. -::: +
      + +- [**Ingesting existing output**](../guides/ingesting.md) + + The guide, with the CLI and the input formats in detail. + +- [**cloudg ingest**](../cli/ingest.md) + + The command built on these functions. + +- [**FindingsNormaliser**](findingsnormaliser.md) + + The step that should follow parsing. + +- [**CloudGEngine**](cloudgengine.md) + + `ingest_reports()` and `run_from_reports()` with hooks. + +
      diff --git a/site/content/api/ragexporter.md b/site/content/api/ragexporter.md index d49156b..ffaa7a7 100644 --- a/site/content/api/ragexporter.md +++ b/site/content/api/ragexporter.md @@ -1,11 +1,14 @@ --- title: RAGExporter -object: cloudg.graph.rag_export.RAGExporter -members: [export_all, export_entity_chunks, export_community_chunks, export_relation_chunks] -lede: Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. -since: "0.6.0" +description: Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. --- +Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + A language model answering questions about your cloud needs the right slice of the inventory in its context window, not the whole `inventory-map.json`. `RAGExporter` writes three kinds of chunk, each aimed at a different kind of question, as one JSON object per line. Embed the `content` field, keep `metadata` for filtering, and you have a retrieval corpus. Nothing in the exporter calls a model or an embedding API. ## Chunk kinds @@ -26,7 +29,7 @@ Every chunk has the same five keys: `chunk_id`, `chunk_type`, `content` (the tex `severity_max` is the worst severity among the matched findings, or `"NONE"`. `compliance_frameworks` lists the frameworks of every matched finding, including findings cut from the chunk's text. A community's `risk_score` is `0.3` per member plus `0.5` per finding plus `2.0` per internet-exposed member, capped at 10. -The relations are the ones the [ontology](/api/cloudontology/) uses: `infer_relations()` on each edge (so a security group rule from `0.0.0.0/0` on port 443 becomes `INGRESS_ALLOWED`, `INTERNET_REACHABLE` and `ONLY_HTTPS`), plus the asset-level relations from metadata (`ENCRYPTED_BY_KMS`, VPC containment from `vpc_id`, tag ownership, key rotation, EC2 security group membership). Relation-group chunks carry both; entity chunks carry the edge relations only. +The relations are the ones the [ontology](cloudontology.md) uses: `infer_relations()` on each edge (so a security group rule from `0.0.0.0/0` on port 443 becomes `INGRESS_ALLOWED`, `INTERNET_REACHABLE` and `ONLY_HTTPS`), plus the asset-level relations from metadata (`ENCRYPTED_BY_KMS`, VPC containment from `vpc_id`, tag ownership, key rotation, EC2 security group membership). Relation-group chunks carry both; entity chunks carry the edge relations only. Findings attach to assets through `AssetIndex`, so a finding whose `resource_id` is an ARN or a unique name still counts toward the right entity and community. @@ -39,7 +42,7 @@ Findings attach to assets through `AssetIndex`, so a finding whose `resource_id` | `rag_chunks.jsonl` | one chunk per line: entity chunks first, then community, then relation-group | | `rag_metadata_index.json` | `total_chunks`, `entity_chunks`, `community_chunks`, `relation_group_chunks`, `chunk_ids` (in file order) and `chunk_types` | -```mermaid caption="What export_all() does" +```mermaid flowchart LR A["assets, edges"] --> E["export_entity_chunks"] G["graph"] --> L["Louvain communities"] --> C["export_community_chunks"] @@ -51,6 +54,8 @@ flowchart LR R --> J J --> I["rag_metadata_index.json"] ``` +What export_all() does +{ .diagram-caption } `cloudg run` writes these files twice. Phase 2c exports right after the graph is built, when the only findings are the reachability ones. After the scanners finish, a silent second pass rewrites both files with every finding. If that second pass fails, the files from phase 2c stay, with only reachability findings in them. `--no-rag-export` or `rag.enabled: false` turns the export off. @@ -61,120 +66,7 @@ flowchart LR The script needs nothing beyond cloudg. It exports the chunks, then turns every chunk into an `{id, text, metadata}` record with plain-text content and scalar metadata, which is the shape most vector stores accept. ```python title="rag_demo.py" -"""Export RAG chunks for a small estate and reshape them for a vector store.""" - -import json -from pathlib import Path - -from cloudg.graph.builder import GraphBuilder -from cloudg.graph.rag_export import RAGExporter -from cloudg.schema.models import ( - AssetType, - CloudAsset, - CloudProvider, - EdgeType, - Finding, - NetworkEdge, - Severity, -) - -ACCOUNT = "123456789012" -REGION = "eu-west-1" - - -def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - **extra, - ) - - -assets = [ - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", - is_internet_exposed=True), - asset("web-2", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web2"), - asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), -] -edges = [ - NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range="443", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), - NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), - NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.ASSUMES_ROLE), - NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), -] -findings = [ - Finding( - resource_id="web-1", - severity=Severity.HIGH, - title="Instance metadata service v1 enabled", - description="IMDSv2 is not enforced.", - source_tool="prowler", - compliance_frameworks=["CIS"], - ) -] - -graph = GraphBuilder().build(assets, edges) -paths = RAGExporter().export_all(assets, edges, graph, findings, output_dir="rag-out") -print({kind: path.name for kind, path in paths.items()}) - -index = json.loads(paths["index"].read_text()) -print({k: v for k, v in index.items() if k.endswith("chunks")}) - -chunks = [json.loads(line) for line in paths["chunks"].read_text().splitlines()] - -# The content uses a few symbols; plain words embed and display more reliably -MARKERS = { - "\N{WARNING SIGN}": "WARNING:", - "\N{RIGHTWARDS ARROW}": "->", - "\N{LEFTWARDS ARROW}": "<-", - "\N{BULLET}": "-", -} - - -def clean(text: str) -> str: - for symbol, word in MARKERS.items(): - text = text.replace(symbol, word) - return text - - -def scalar(value): - """Most vector stores accept only str, int, float and bool metadata.""" - if isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, list): - return ",".join(map(str, value)) - return json.dumps(value, sort_keys=True) - - -records = [ - { - "id": c["chunk_id"], - "text": clean(c["content"]), - "metadata": {"chunk_type": c["chunk_type"], - **{k: scalar(v) for k, v in c["metadata"].items()}}, - } - for c in chunks -] -out = Path("rag-out") / "vector-records.jsonl" -out.write_text("".join(json.dumps(r) + "\n" for r in records)) -print(len(records), "records ->", out) - -web_1 = next(r for r in records if r["id"] == "entity::web-1") -print(web_1["text"]) -print(web_1["metadata"]) - -# Pre-filter on metadata before any similarity search -urgent = [r["id"] for r in records if r["metadata"].get("severity_max") in ("CRITICAL", "HIGH")] -print("high or critical:", urgent) +--8<-- "graph/rag_demo.py" ``` ```console @@ -239,12 +131,38 @@ for chunk in RAGExporter().export_relation_chunks(result.edges, assets_by_id): - `content` uses four non-ASCII symbols: a warning sign (U+26A0) before exposure lines, arrows (U+2192, U+2190) before outgoing and incoming relations, and a bullet (U+2022) before community members. `rag_chunks.jsonl` stores them as `\u` escapes, and they come back as the symbols when parsed. The example swaps them for words before embedding. - `chunk_types` in the index comes from a set, so its order varies between runs. +## Reference + +::: cloudg.graph.rag_export.RAGExporter + options: + members: + - export_all + - export_entity_chunks + - export_community_chunks + - export_relation_chunks + ## Related -:::links -- [CloudOntology](/api/cloudontology/) The same relations as RDF. -- [GraphBuilder](/api/graphbuilder/) Builds the graph the community chunks are cut from. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) The analysis phase end to end. -- [cloudg run](/cli/run/) The `--rag-export` switch. -- [Output files](/reference/output-files/) Where `rag_chunks.jsonl` sits among the other outputs. -::: +
      + +- [**CloudOntology**](cloudontology.md) + + The same relations as RDF. + +- [**GraphBuilder**](graphbuilder.md) + + Builds the graph the community chunks are cut from. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + The analysis phase end to end. + +- [**cloudg run**](../cli/run.md) + + The `--rag-export` switch. + +- [**Output files**](../reference/output-files.md) + + Where `rag_chunks.jsonl` sits among the other outputs. + +
      diff --git a/site/content/api/reachabilityanalyzer.md b/site/content/api/reachabilityanalyzer.md index 2022c3d..c0970a2 100644 --- a/site/content/api/reachabilityanalyzer.md +++ b/site/content/api/reachabilityanalyzer.md @@ -1,15 +1,14 @@ --- title: ReachabilityAnalyzer -object: - - cloudg.graph.reachability.ReachabilityAnalyzer - - cloudg.graph.reachability.finding_id - - cloudg.graph.reachability.network_flow_graph -members: - cloudg.graph.reachability.ReachabilityAnalyzer: [internet_entry_points, find_internet_exposed, flow_hops, flow_successors, generate_findings, compute_blast_radius] -lede: Walks a `GraphBuilder` graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. -since: "0.6.0" +description: Walks a GraphBuilder graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. --- +Walks a `GraphBuilder` graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `cloudg run` calls `ReachabilityAnalyzer(graph).generate_findings()` right after it builds the graph, and the results show up in `findings.json` with `source_tool` set to `cloudg-reachability`. The class answers two different questions, and they follow different edges: - What can the internet reach? `find_internet_exposed()` walks network-flow edges only, with the rules below. @@ -21,7 +20,7 @@ The analyzer works on the graph you pass in and changes it: `find_internet_expos Release 0.6.0 changed the walk. Before it, the analysis followed every edge type out of an exposed node, so a database whose only link to the internet was an IAM role granted access to it came out as a CRITICAL "Internet-exposed RDS_INSTANCE". Now an edge is followed only when traffic can travel along it, and in the direction it travels. -```mermaid caption="Edges the exposure walk follows, in the direction traffic moves" +```mermaid flowchart LR I["0.0.0.0/0, ::/0, Internet tag"] -->|"SG or NACL rule, ingress"| G["Security group, NSG, NACL"] G -->|"ATTACHED_TO, walked backwards"| R["Resource in the group"] @@ -32,6 +31,8 @@ flowchart LR N["VPC, VNet, subnet"] -->|"CONTAINS"| M["Resource placed in it"] S["Route or peering source"] -->|"ROUTE, PEERING"| D["Destination"] ``` +Edges the exposure walk follows, in the direction traffic moves +{ .diagram-caption } | Edge type | Followed | Direction | |---|---|---| @@ -80,107 +81,7 @@ The key is the node's ARN, or the node id when there is no ARN (CIDR placeholder Ten assets, fourteen edges, no credentials. The estate has a public load balancer in front of `web-1`, an SSH rule left open on the web tier's group, a database reachable only from a private subnet and through an IAM grant, and a Lambda function behind a public API Gateway. ```python title="exposure.py" -"""Internet exposure, flow hops, finding ids and blast radius on a small estate.""" - -from cloudg.graph.builder import GraphBuilder -from cloudg.graph.reachability import ( - RULE_UNEXPECTED_EXPOSURE, - ReachabilityAnalyzer, - asset_key, - finding_id, - network_flow_graph, -) -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge - -ACCOUNT = "123456789012" -REGION = "eu-west-1" - - -def asset(asset_id: str, asset_type: AssetType, arn: str) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - ) - - -assets = [ - asset("sg-alb", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-alb"), - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("sg-db", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-db"), - asset("web-alb", AssetType.LOAD_BALANCER, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:loadbalancer/app/web-alb/1"), - asset("web-tg", AssetType.TARGET_GROUP, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:targetgroup/web-tg/1"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1"), - asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), - asset("orders-api", AssetType.API_GATEWAY, f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3"), - asset("orders-fn", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-fn"), -] - - -def ingress(source: str, target: str, ports: str, cidr: str | None = None) -> NetworkEdge: - return NetworkEdge( - source_id=source, - target_id=target, - edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range=ports, - protocol="TCP", - cidr=cidr, - direction="ingress", - ) - - -def edge(source: str, target: str, edge_type: EdgeType, **extra) -> NetworkEdge: - return NetworkEdge(source_id=source, target_id=target, edge_type=edge_type, **extra) - - -edges = [ - ingress("0.0.0.0/0", "sg-alb", "443", cidr="0.0.0.0/0"), - ingress("0.0.0.0/0", "sg-web", "22", cidr="0.0.0.0/0"), # SSH left open - ingress("sg-alb", "sg-web", "8080"), - ingress("10.0.1.0/24", "sg-db", "5432", cidr="10.0.1.0/24"), # private source - # The default egress rule points from the group to the internet - edge("sg-web", "0.0.0.0/0", EdgeType.SECURITY_GROUP_RULE, port_range="0-65535", - protocol="ALL", cidr="0.0.0.0/0", direction="egress"), - edge("web-alb", "sg-alb", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("web-1", "sg-web", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("orders-db", "sg-db", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("web-alb", "web-tg", EdgeType.LOAD_BALANCER_TARGET), - edge("web-tg", "web-1", EdgeType.LOAD_BALANCER_TARGET), - edge("web-1", "app-role", EdgeType.ASSUMES_ROLE), - edge("app-role", "orders-db", EdgeType.GRANTS_ACCESS), - edge("0.0.0.0/0", "orders-api", EdgeType.INTERNET_EXPOSED, cidr="0.0.0.0/0"), - edge("orders-api", "orders-fn", EdgeType.INVOKES), -] - -graph = GraphBuilder().build(assets, edges) -analyzer = ReachabilityAnalyzer(graph) - -print("entry points:", sorted(analyzer.internet_entry_points())) -print("exposed:", sorted(analyzer.find_internet_exposed())) - -print("hops out of sg-web:") -for nxt, data, walked_back in analyzer.flow_hops("sg-web"): - print(f" -> {nxt} via {data['edge_type']}{' (reversed)' if walked_back else ''}") - -for finding in analyzer.generate_findings(): - print(f"{finding.severity.value:8} {finding.title} id={finding.id[:8]}") - -# The id can be recomputed from the rule name and the asset's ARN -expected = finding_id(RULE_UNEXPECTED_EXPOSURE, asset_key(graph, "web-1")) -print("web-1 exposure id:", expected[:8]) - -flow = network_flow_graph(graph) -print("flow edges:", flow.number_of_edges(), "of", graph.number_of_edges()) -print("flow reversed:", sorted((u, v) for u, v, d in flow.edges(data=True) if d["reversed"])) - -radius = analyzer.compute_blast_radius("web-1") -print("blast radius of web-1:", sorted(radius["reachable_nodes"]), - "depth", radius["depth"], "risk", radius["risk_score"]) +--8<-- "graph/exposure.py" ``` ```console @@ -244,12 +145,44 @@ The first path is the over-approximation at work: `sg-alb` admits 443 from the i - The open-port title keeps the historical `0.0.0.0/0` wording for both `0.0.0.0/0` and `::/0`, and names the Azure tag as written (`Internet`, `Any`, `*`) otherwise. - Stored findings from 0.5.x carry random ids. They will not match the 0.6.0 ids; re-baseline suppressions after upgrading. +## Reference + +::: cloudg.graph.reachability.ReachabilityAnalyzer + options: + members: + - internet_entry_points + - find_internet_exposed + - flow_hops + - flow_successors + - generate_findings + - compute_blast_radius + +::: cloudg.graph.reachability.finding_id + +::: cloudg.graph.reachability.network_flow_graph + ## Related -:::links -- [GraphBuilder](/api/graphbuilder/) Builds the graph this class walks. -- [Dependencies and blast radius](/guides/dependencies/) The inventory-level blast radius behind `cloudg deps`. -- [DependencyGraph](/api/dependencygraph/) Dependency walks over a saved map. -- [Data models](/api/models/) The `Finding` model these results use. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) Where reachability sits in the analysis phase. -::: +
      + +- [**GraphBuilder**](graphbuilder.md) + + Builds the graph this class walks. + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + The inventory-level blast radius behind `cloudg deps`. + +- [**DependencyGraph**](dependencygraph.md) + + Dependency walks over a saved map. + +- [**Data models**](models.md) + + The `Finding` model these results use. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + Where reachability sits in the analysis phase. + +
      diff --git a/site/content/api/relationshiplinker.md b/site/content/api/relationshiplinker.md index 91fd651..03decd6 100644 --- a/site/content/api/relationshiplinker.md +++ b/site/content/api/relationshiplinker.md @@ -1,16 +1,17 @@ --- -object: cloudg.inventory.linker.RelationshipLinker -members: [link, seed_existing, resolve] -lede: "Turns a flat list of `CloudAsset` objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg's collectors, your own plugin, a CMDB export, a previous run." +description: 'Turns a flat list of CloudAsset objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg''s collectors, your own plugin, a CMDB export, a previous run.' --- +Turns a flat list of `CloudAsset` objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg's collectors, your own plugin, a CMDB export, a previous run. +{ .lede } + ## How linking works `RelationshipLinker(assets, materialize_external=True)` indexes every asset by the identifiers other resources use to point at it: ARN or resource ID, name, the short ID at the end of the ARN (`i-0b1c...`, `sg-0a1b...`), metadata keys such as `queue_url`, `repository_uri` or `zone_id`, every entry in `metadata["aliases"]`, and a few derived forms (an S3 bucket's `.s3.amazonaws.com`, a load balancer's DNS name). Building the index makes no API calls and keeps no reference to a cloud session. `link()` then makes three passes and returns the new edges: -```mermaid caption="The three passes of RelationshipLinker.link()" +```mermaid flowchart TD I["identifier index over all assets"] --> P1["pass 1: declared relations in metadata.relations"] P1 --> P2["pass 2: provider rules on metadata keys"] @@ -22,6 +23,8 @@ flowchart TD P1 -.->|"target in an unmapped account"| X["external CLOUD_ACCOUNT placeholder"] P1 -.->|"target not found"| U["linker.unresolved"] ``` +The three passes of RelationshipLinker.link() +{ .diagram-caption } Pass 1 reads `metadata["relations"]`, the list each collector fills with what an asset talks to. Pass 2 applies provider-aware rules to well-known metadata keys: `security_groups` and `nsg_id` (attachment), `subnet_id` and `vpc_config` (containment), `role_arn` (execution roles), `kms_key_id` (encryption), `origins` and `web_acl_id` (CloudFront and WAF), route table routes, ENI and volume attachments, VPC peering, Azure NIC wiring and the GCP `parent_full_resource_name`. Pass 3 walks every other string in the metadata, up to six levels deep, and links anything that looks like an ARN, an Azure resource ID, a GCP full resource name or an AWS short ID and resolves to another asset. Pairs already joined in either direction are skipped, so the generic pass only adds links nothing else found. @@ -34,8 +37,8 @@ Each entry in `metadata["relations"]` is a dict: | Key | Required | Meaning | |---|---|---| | `target` | yes | Any identifier of the other asset: ARN, resource ID, name, short ID, alias. | -| `edge` | no | An [`EdgeType`](/api/models/) value such as `INVOKES` or `ASSUMES_ROLE`. Missing or unknown values become `REFERENCES`. | -| `relationship` | no | A finer label, copied to the edge's `relationship` (`TRIGGERED_BY`, `ENCRYPTED_BY_KMS`, `RUNS_ON`). The [relationship vocabulary](/reference/catalog/relationship-vocabulary/) lists the names cloudg's collectors use. | +| `edge` | no | An [`EdgeType`](models.md) value such as `INVOKES` or `ASSUMES_ROLE`. Missing or unknown values become `REFERENCES`. | +| `relationship` | no | A finer label, copied to the edge's `relationship` (`TRIGGERED_BY`, `ENCRYPTED_BY_KMS`, `RUNS_ON`). The [relationship vocabulary](../reference/catalog/relationship-vocabulary.md) lists the names cloudg's collectors use. | | `reverse` | no | When true, the edge runs from the target to the declaring asset. Use it when the declaring side is the passive one: "my queue triggers me", "this account is trusted by me". | | `description` | no | Copied to the edge's `description`. | | `properties` | no | Copied to the edge's `properties`. | @@ -71,13 +74,11 @@ QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" - def cmdb_row(arn, name, asset_type, region=REGION, **metadata): """One record from a CMDB export, turned into a CloudAsset.""" return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, region=region, account_id=ACCOUNT, metadata=metadata) - assets = [ cmdb_row(KEY, "orders-key", AssetType.KMS_KEY), cmdb_row(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global", relations=[ @@ -133,7 +134,7 @@ Reading the output line by line: - `role_arn` on the function produced `ASSUMES_ROLE` / `RUNS_ON` through a pass 2 rule, with no declared relation. - The log group is not in the export, so that relation is in `unresolved`. -To feed the result into the rest of cloudg, wrap it in an [`InventoryResult`](/api/inventoryresult/) and call `export()`, or hand `assets` and `edges` to [`CloudGEngine.analyze()`](/api/cloudgengine/). +To feed the result into the rest of cloudg, wrap it in an [`InventoryResult`](inventoryresult.md) and call `export()`, or hand `assets` and `edges` to [`CloudGEngine.analyze()`](cloudgengine.md). ### Link only what was declared @@ -193,12 +194,37 @@ The linker never modifies the assets you pass in. It only reads `metadata`, `arn Errors inside a pass are caught per asset and logged at DEBUG on `cloudg.inventory.linker`, so one malformed asset cannot stop the run. If edges you expect are missing, turn on DEBUG for that logger. +## Reference + +::: cloudg.inventory.linker.RelationshipLinker + options: + members: + - link + - seed_existing + - resolve + ## Related -:::links -- [InventoryMapper](/api/inventorymapper/) Runs the linker as part of every map. -- [DependencyGraph](/api/dependencygraph/) Dependency questions over the linked edges. -- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The relationship names and what they mean. -- [Asset types by category](/reference/catalog/asset-types-by-category/) Which types exist and what they carry. -- [Inventory mapping](/guides/inventory-mapping/) The linker in the context of a full map. -::: +
      + +- [**InventoryMapper**](inventorymapper.md) + + Runs the linker as part of every map. + +- [**DependencyGraph**](dependencygraph.md) + + Dependency questions over the linked edges. + +- [**Relationship vocabulary**](../reference/catalog/relationship-vocabulary.md) + + The relationship names and what they mean. + +- [**Asset types by category**](../reference/catalog/asset-types-by-category.md) + + Which types exist and what they carry. + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + The linker in the context of a full map. + +
      diff --git a/site/content/api/resilience.md b/site/content/api/resilience.md index f04deec..3cac7d9 100644 --- a/site/content/api/resilience.md +++ b/site/content/api/resilience.md @@ -1,29 +1,19 @@ --- title: cloudg.resilience -object: - - cloudg.resilience.call_with_resilience - - cloudg.resilience.call_with_resilience_sync - - cloudg.resilience.resilient - - cloudg.resilience.Scope - - cloudg.resilience.RetryPolicy - - cloudg.resilience.stats_scope - - cloudg.resilience.configure - - cloudg.resilience.get_governor - - cloudg.resilience.Governor - - cloudg.resilience.classify -members: - cloudg.resilience.Scope: [service_scope, account_scope] - cloudg.resilience.Governor: [summary, budget, try_retry, breaker_scope, provider_enabled, reset] -lede: Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. -source: docs/RESILIENCE.md -since: "0.6.0" +description: Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. --- +Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every cloud call cloudg makes goes through one process-wide governor: adaptive token buckets per provider, account, region and service, a concurrency cap per account, a circuit breaker per bucket and a retry budget per provider. If your own code calls the same APIs in the same process (extra enumeration next to an `InventoryMapper` run, an MCP tool of your own, a CI job that maps and then queries), wrap those calls with `call_with_resilience` and they queue behind cloudg's instead of competing with them. A throttle your call hits slows cloudg's calls to that scope too, and the other way round. ## What happens to one call -```mermaid caption="call_with_resilience, step by step" +```mermaid flowchart TD A["call_with_resilience(fn, scope)"] --> B{"Breaker open?"} B -->|yes| X["CircuitOpenError, nothing sent"] @@ -38,6 +28,8 @@ flowchart TD F -->|yes| G["Back off, never under Retry-After"] --> B F -->|no| Z["Re-raise, breaker counts a failure"] ``` +call_with_resilience, step by step +{ .diagram-caption } A throttling error also halves the scope's rate (at most once per second) and pauses the scope for the server's Retry-After, capped at 300 seconds. After five seconds without a throttle each success gives back 5% of the configured rate. The backoff before a retry is decorrelated jitter, `min(cap, uniform(0.5, 3 x previous))`. Retries stop at the first of `max_retries` (8 by default), the per-call deadline (900 seconds) or an empty retry budget (500 per provider; the first two retries of each call are free, and each success refunds 0.1). When retrying stops, your original exception is re-raised, not a wrapper. @@ -92,7 +84,6 @@ configure(RateLimitConfig.model_validate( attempts = {"describe": 0} - def describe_vpcs() -> dict: """Stands in for ec2.describe_vpcs(): throttled twice, then answers.""" attempts["describe"] += 1 @@ -101,14 +92,12 @@ def describe_vpcs() -> dict: "ResponseMetadata": {"HTTPStatusCode": 503}}, "DescribeVpcs") return {"Vpcs": [{"VpcId": "vpc-0a1b2c3d"}]} - @resilient(lambda name, **_: Scope("aws", ACCOUNT, "eu-west-1", "secretsmanager", "DescribeSecret"), policy=RetryPolicy(max_retries=2, deadline=30.0)) async def describe_secret(name: str) -> str: """Stands in for an async SDK call; the decorator paces every call.""" return f"arn:aws:secretsmanager:eu-west-1:{ACCOUNT}:secret:{name}" - async def main() -> None: ec2 = Scope("aws", ACCOUNT, "eu-west-1", "ec2", "DescribeVpcs") loop = asyncio.get_running_loop() @@ -125,7 +114,6 @@ async def main() -> None: print("totals:", summary["totals"]) print("slowed:", {k: v["rate_rps"] for k, v in summary["slowed_buckets"].items()}) - asyncio.run(main()) ``` @@ -157,7 +145,6 @@ import boto3 from cloudg.resilience import Scope, call_with_resilience, stats_scope - async def main() -> None: ec2 = boto3.client("ec2", region_name="eu-west-1") scope = Scope("aws", "123456789012", "eu-west-1", "ec2", "DescribeVpcs") @@ -165,7 +152,6 @@ async def main() -> None: vpcs = await call_with_resilience(ec2.describe_vpcs, scope=scope, MaxResults=50) print(len(vpcs["Vpcs"]), stats.messages()) - asyncio.run(main()) ``` @@ -181,12 +167,62 @@ boto3 also retries inside the call, so a throttled request can be retried both b - `reset_governor()` replaces the governor with a fresh one and forgets every learned rate. It exists for tests. `get_governor().reset()` clears the same learned state on the current governor and keeps its configuration. - `cloudg.retry.with_retry` is the older decorator. It keeps its signature and also retries provider throttling recognised from codes and statuses, but it does not use the shared buckets or breakers. +## Reference + +::: cloudg.resilience.call_with_resilience + +::: cloudg.resilience.call_with_resilience_sync + +::: cloudg.resilience.resilient + +::: cloudg.resilience.Scope + options: + members: + - service_scope + - account_scope + +::: cloudg.resilience.RetryPolicy + +::: cloudg.resilience.stats_scope + +::: cloudg.resilience.configure + +::: cloudg.resilience.get_governor + +::: cloudg.resilience.Governor + options: + members: + - summary + - budget + - try_retry + - breaker_scope + - provider_enabled + - reset + +::: cloudg.resilience.classify + ## Related -:::links -- [Rate limits and throttling](/guides/rate-limits/) What cloudg does under throttling, and how to tune it. -- [ratelimit settings](/reference/config/ratelimit/) Every `ratelimit` key in `config.yaml`. -- [How one call is paced](/reference/resilience/how-one-call-is-paced/) Buckets, AIMD, breakers and the retry budget in depth. -- [Python API](/reference/resilience/python-api/) `LiveOperationGuard`, the bulkhead and other lower-level pieces. -- [CloudGMCPLayer](/api/mcp-layer/) Live MCP tools run under the same governor. -::: +
      + +- [**Rate limits and throttling**](../guides/rate-limits.md) + + What cloudg does under throttling, and how to tune it. + +- [**ratelimit settings**](../reference/config/ratelimit.md) + + Every `ratelimit` key in `config.yaml`. + +- [**How one call is paced**](../reference/resilience/how-one-call-is-paced.md) + + Buckets, AIMD, breakers and the retry budget in depth. + +- [**Python API**](../reference/resilience/python-api.md) + + `LiveOperationGuard`, the bulkhead and other lower-level pieces. + +- [**CloudGMCPLayer**](mcp-layer.md) + + Live MCP tools run under the same governor. + +
      diff --git a/site/content/api/results.md b/site/content/api/results.md index 48033b3..a3ca933 100644 --- a/site/content/api/results.md +++ b/site/content/api/results.md @@ -1,12 +1,10 @@ --- -object: [cloudg.api.PipelineResult, cloudg.api.CollectionResult, cloudg.api.AnalysisResult] -members: - cloudg.api.PipelineResult: [total_assets, total_findings, severity_breakdown, to_summary] - cloudg.api.CollectionResult: [] - cloudg.api.AnalysisResult: [] -lede: "Three plain dataclasses carry what the engine produces. `PipelineResult` is the one most callers keep; the other two come back from single phases." +description: Three plain dataclasses carry what the engine produces. PipelineResult is the one most callers keep; the other two come back from single phases. --- +Three plain dataclasses carry what the engine produces. `PipelineResult` is the one most callers keep; the other two come back from single phases. +{ .lede } + ## Which method returns which | Returned by | Type | Holds | @@ -21,7 +19,7 @@ All three are `@dataclass` classes with defaults for every field, so you can als `run_pipeline()` gets a `CollectionResult` from `collect()` and an `AnalysisResult` from `analyze()`, normalises the findings, writes the reports, and copies a subset of fields into one `PipelineResult`. -```mermaid caption="Where each PipelineResult field comes from" +```mermaid flowchart LR C["CollectionResult"] -->|"assets, edges, coverage, providers_scanned, regions_scanned"| P["PipelineResult"] A["AnalysisResult"] -->|"graph_nodes, graph_edges, ontology_triples, rag_chunks_path, terraform_paths, attack_paths"| P @@ -29,6 +27,8 @@ flowchart LR R["report writers"] -->|"report_paths"| P E["on_error calls, failed collection targets"] -->|errors| P ``` +Where each PipelineResult field comes from +{ .diagram-caption } Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_count` and `reachability_findings` have no counterpart on `PipelineResult` (the reachability findings are merged into `findings` instead), and `duration_ms` on the pipeline result is the whole run, not the collection time. If you need those, drive the phases yourself with `collect()` and `analyze()`. @@ -38,7 +38,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou `findings` holds the normalised list when normalisation succeeded: deduplicated, rescored, sorted from CRITICAL to INFO. If normalisation failed, it holds the raw findings instead, `scan_result` is `None`, and `errors` says why. -`scan_result` is the [`ScanResult`](/api/models/) the normaliser produced. It is where the compliance mapping lives (`scan_result.compliance`, one `ComplianceResult` per framework control), and its `summary` adds `total_edges` and `compliance_frameworks` to the counts. In `run_pipeline()` its `edges` are set to the collected edges. +`scan_result` is the [`ScanResult`](models.md) the normaliser produced. It is where the compliance mapping lives (`scan_result.compliance`, one `ComplianceResult` per framework control), and its `summary` adds `total_edges` and `compliance_frameworks` to the counts. In `run_pipeline()` its `edges` are set to the collected edges. `severity_breakdown` counts `finding.severity.value` over `findings`, recomputed on every access. Only severities that occur appear as keys, so read it with `.get("CRITICAL", 0)`. `total_assets` and `total_findings` are `len()` of the lists. @@ -50,7 +50,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou `attack_paths` is a `list[list[str]]`. Each inner list is a path of graph node ids, starting at an internet-exposed or external node and ending at the target of an `IAM_TRUST` edge, at most eight hops long. Node ids are asset ids, so map them back with `{a.id: a for a in result.assets}`. The search stops after about a hundred paths. -`errors` holds one string of the form `": "` per failure of the run: everything reported through the `on_error` hook (scanners, scanner credentials, graph, ontology, RAG, Terraform, normalisation, reporting; see [Event hooks](/api/event-hooks/)), plus, for `run_pipeline()`, one `collection: : ` line per target that could not be collected. An empty list means nothing failed. +`errors` holds one string of the form `": "` per failure of the run: everything reported through the `on_error` hook (scanners, scanner credentials, graph, ontology, RAG, Terraform, normalisation, reporting; see [Event hooks](event-hooks.md)), plus, for `run_pipeline()`, one `collection: : ` line per target that could not be collected. An empty list means nothing failed. `AnalysisResult.ontology_path` is the last file the ontology was saved to. With the default `export_formats: [turtle, json-ld]` that is `ontology.jsonld`; `ontology.ttl` is written next to it. `rag_chunk_count` is the number of chunks written to `rag_chunks_path`, one per line. @@ -61,37 +61,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou Runs offline against existing Checkov and Trivy output. ```python title="save_result.py" -import json -import sys -from pathlib import Path - -from cloudg import CloudGConfig, CloudGEngine - -result = CloudGEngine(CloudGConfig()).run_from_reports_sync( - {"checkov": ["./results_json.json"], "trivy": ["./trivy.json"]}, - output_dir="./reports", -) - -# Headline numbers: plain JSON types only, safe to json.dumps -summary = result.to_summary() -print(json.dumps(summary, indent=2)) - -# Everything else: dump the pydantic models and stringify the paths yourself -payload = { - "summary": summary, - "findings": [f.model_dump(mode="json") for f in result.findings], - "compliance": [c.model_dump(mode="json") for c in result.scan_result.compliance] - if result.scan_result - else [], - "report_paths": {kind: str(path) for kind, path in result.report_paths.items()}, -} -Path("./reports/run-summary.json").write_text(json.dumps(payload, indent=2)) - -breakdown = result.severity_breakdown # only severities that occur -blocking = breakdown.get("CRITICAL", 0) + breakdown.get("HIGH", 0) -if blocking: - print(f"{blocking} critical/high findings, failing the build", file=sys.stderr) - sys.exit(1) +--8<-- "ingest/save_result.py" ``` ```console @@ -129,12 +99,10 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, ACCOUNT = "123456789012" - def aws(name, asset_type, arn, exposed=False): return CloudAsset(name=name, asset_type=asset_type, arn=arn, provider=CloudProvider.AWS, region="eu-west-1", account_id=ACCOUNT, is_internet_exposed=exposed) - api = aws("orders-api", AssetType.API_GATEWAY, "arn:aws:apigateway:eu-west-1::/restapis/a1b2c3d4e5", exposed=True) fn = aws("orders-worker", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:eu-west-1:{ACCOUNT}:function:orders-worker") role = aws("orders-worker", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/orders-worker") @@ -148,7 +116,6 @@ edges = [ NetworkEdge(source_id=role.id, target_id=admin.id, edge_type=EdgeType.IAM_TRUST), ] - async def main() -> None: config = CloudGConfig() config.ontology.enabled = False @@ -160,7 +127,6 @@ async def main() -> None: for path in analysis.attack_paths: print(" -> ".join(names.get(node, node) for node in path)) - asyncio.run(main()) ``` @@ -193,11 +159,42 @@ The dataclasses are mutable and hold the same objects the engine worked on. Chan `total_assets` counts what was collected, not what was scanned: a scanner like Checkov reports findings against IaC resources that may not be in `assets` at all. +## Reference + +::: cloudg.api.PipelineResult + options: + members: + - total_assets + - total_findings + - severity_breakdown + - to_summary + +::: cloudg.api.CollectionResult + options: + members: false + +::: cloudg.api.AnalysisResult + options: + members: false + ## Related -:::links -- [CloudGEngine](/api/cloudgengine/) The methods that return these objects. -- [Data models](/api/models/) Finding, CloudAsset, ScanResult and the enums. -- [Event hooks](/api/event-hooks/) The `on_error` calls that `errors` records. -- [Output files](/reference/output-files/) What the report paths point at. -::: +
      + +- [**CloudGEngine**](cloudgengine.md) + + The methods that return these objects. + +- [**Data models**](models.md) + + Finding, CloudAsset, ScanResult and the enums. + +- [**Event hooks**](event-hooks.md) + + The `on_error` calls that `errors` records. + +- [**Output files**](../reference/output-files.md) + + What the report paths point at. + +
      diff --git a/site/static/img/favicon.svg b/site/content/assets/images/favicon.svg similarity index 100% rename from site/static/img/favicon.svg rename to site/content/assets/images/favicon.svg diff --git a/site/content/assets/images/logo-mark.svg b/site/content/assets/images/logo-mark.svg new file mode 100644 index 0000000..23afee1 --- /dev/null +++ b/site/content/assets/images/logo-mark.svg @@ -0,0 +1 @@ + diff --git a/site/content/assets/stylesheets/extra.css b/site/content/assets/stylesheets/extra.css new file mode 100644 index 0000000..4fe6f18 --- /dev/null +++ b/site/content/assets/stylesheets/extra.css @@ -0,0 +1,264 @@ +/* cloudg docs: additions to the Material theme. + Colours come from Material's own variables, so both palettes work. */ + +:root { + --cg-brand: #ec3013; + --cg-radius: 0.4rem; +} + +/* ---- page lede and the metadata strip under it ---------------------------- */ + +.md-typeset .lede { + font-size: 0.92rem; + line-height: 1.6; + color: var(--md-default-fg-color--light); + margin-top: -0.4rem; +} + +.md-typeset .page-meta, +.md-typeset .release-date { + font-size: 0.72rem; + color: var(--md-default-fg-color--light); + border-left: 0.15rem solid var(--md-accent-fg-color); + padding-left: 0.6rem; +} + +.md-typeset .table-note { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .diagram-caption { + margin-top: -0.6rem; + text-align: center; + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .mermaid { + text-align: center; +} + +/* ---- numbered steps --------------------------------------------------------- */ + +.md-typeset .steps { + counter-reset: step; + margin-left: 1rem; + padding-left: 1.6rem; + border-left: 0.1rem solid var(--md-default-fg-color--lightest); +} + +.md-typeset .steps > h3 { + counter-increment: step; + position: relative; +} + +.md-typeset .steps > h3::before { + content: counter(step); + position: absolute; + left: -2.75rem; + top: -0.1rem; + width: 1.7rem; + height: 1.7rem; + line-height: 1.7rem; + border-radius: 50%; + text-align: center; + font-size: 0.8rem; + font-weight: 700; + color: var(--md-accent-bg-color); + background: var(--md-accent-fg-color); +} + +/* ---- cards ------------------------------------------------------------------ */ + +.md-typeset .grid.cells > .card h3 { + margin-top: 0; + font-size: 0.85rem; +} + +.md-typeset .grid.cells > .card p:last-child { + margin-bottom: 0; +} + +.md-typeset .link-cards > ul > li > p:first-child a { + display: block; +} + +.md-typeset .link-cards > ul > li > p:first-child a::after { + content: " \2192"; + color: var(--md-accent-fg-color); +} + +.md-typeset .link-cards > ul > li > p:not(:first-child) { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +/* ---- API reference ------------------------------------------------------------ */ + +.md-typeset .doc-heading .doc-source-link { + float: right; + margin-left: 0.6rem; + padding: 0 0.45rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + font-family: var(--md-text-font-family); + font-size: 0.6rem; + font-weight: 400; + line-height: 1.5; + color: var(--md-default-fg-color--light); +} + +.md-typeset .doc-heading .doc-source-link:hover { + color: var(--md-accent-fg-color); + border-color: var(--md-accent-fg-color); +} + +.md-typeset .doc-object > .doc-contents { + padding-left: 0.2rem; +} + +.md-typeset h2.doc-heading, +.md-typeset h3.doc-heading { + border-top: 0.05rem solid var(--md-default-fg-color--lightest); + padding-top: 0.8rem; +} + +.md-typeset h4.doc-heading { + margin-top: 1.6rem; +} + +/* ---- config reference ------------------------------------------------------------ */ + +.md-typeset a.config-key { + white-space: nowrap; +} + +.md-typeset a.config-key:target code { + outline: 0.1rem solid var(--md-accent-fg-color); +} + +/* ---- home page ------------------------------------------------------------------- */ + +.md-typeset .cg-hero { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(0, 1fr); + gap: 2rem; + align-items: center; + margin: 0.5rem 0 2rem; +} + +.md-typeset .cg-hero h1 { + margin: 0.4rem 0 0.8rem; + font-size: 2.1rem; + font-weight: 800; + line-height: 1.15; + letter-spacing: -0.02em; + color: var(--md-default-fg-color); +} + +.md-typeset .cg-kicker { + font-family: var(--md-code-font-family); + font-size: 0.65rem; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--md-accent-fg-color); +} + +.md-typeset .cg-hero .lede { + margin-top: 0; +} + +.md-typeset .cg-hero .md-button { + margin: 0.3rem 0.4rem 0.3rem 0; +} + +.md-typeset .cg-hero-art { + text-align: center; +} + +.md-typeset .cg-hero-art img { + width: 100%; + max-width: 22rem; +} + +.md-typeset .cg-stats { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.8rem; + margin: 0 0 2rem; +} + +.md-typeset .cg-stat { + display: block; + padding: 0.8rem 1rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + color: inherit; +} + +.md-typeset a.cg-stat:hover { + border-color: var(--md-accent-fg-color); +} + +.md-typeset .cg-stat strong { + display: block; + font-size: 1.4rem; + font-weight: 800; + line-height: 1.2; + color: var(--md-default-fg-color); +} + +.md-typeset .cg-stat span { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .cards .cg-num { + font-family: var(--md-code-font-family); + font-size: 0.6rem; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--md-accent-fg-color); +} + +.md-typeset .sample-report-frame { + width: 100%; + height: calc(100vh - 9rem); + min-height: 32rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + background: #fff; +} + +@media screen and (max-width: 59.9375em) { + .md-typeset .cg-hero { + grid-template-columns: minmax(0, 1fr); + } + + .md-typeset .cg-hero-art { + display: none; + } + + .md-typeset .cg-stats { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .md-typeset .cg-hero h1 { + font-size: 1.6rem; + } +} + +/* ---- the example map on the home page ------------------------------------------- */ + +.cg-hero-art .hero-graph { width: 100%; height: auto; } +.hg-cloud { fill: var(--md-default-bg-color); stroke: var(--md-default-fg-color--lighter); stroke-width: 0.5; } +.hg-edges line { stroke: var(--md-default-fg-color--light); stroke-width: 1.5; } +.hg-edges .hg-dash { stroke-dasharray: 4 4; } +.hg-hot line { stroke: var(--cg-brand); stroke-width: 3; } +.hg-nodes circle, .hg-nodes rect { fill: var(--md-default-bg-color); stroke: var(--md-default-fg-color); stroke-width: 2.5; } +.hg-nodes .hg-red { fill: var(--cg-brand); stroke: var(--cg-brand); } +.hg-labels text { font: 600 13px var(--md-code-font-family); fill: var(--md-default-fg-color); } +.hg-elabel text { font: 10px var(--md-code-font-family); fill: var(--md-default-fg-color--light); letter-spacing: 0.04em; } +.hg-elabel .hg-hot-t { fill: var(--cg-brand); font-weight: 600; } +.md-typeset .cg-hero-art figcaption { font-size: 0.65rem; color: var(--md-default-fg-color--light); } diff --git a/site/content/cli/auth-flags.md b/site/content/cli/auth-flags.md index 3d4542d..4d54d36 100644 --- a/site/content/cli/auth-flags.md +++ b/site/content/cli/auth-flags.md @@ -1,13 +1,16 @@ --- title: Auth flags -lede: "Every `--aws-*`, `--azure-*` and `--gcp-*` flag of `cloudg run`, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set." +description: Every --aws-*, --azure-* and --gcp-* flag of cloudg run, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set. --- -The credential flags exist on `cloudg run` only. Each one writes a single field of the loaded config and changes nothing else, so you can mix flags with values from `config.yaml`. `cloudg map` takes the same settings from `config.yaml` (it has `--profile` and nothing else). `cloudg collect` uses neither and works from the environment and default chains alone. The [Authentication](/guides/authentication/) guide covers the identities, roles and permissions each method needs; this page is about the flags. +Every `--aws-*`, `--azure-*` and `--gcp-*` flag of `cloudg run`, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set. +{ .lede } -:::warning Secrets on the command line -Values passed as flags can be seen in your shell history and, while cloudg runs, in the process list of the machine. Prefer the environment variables or a `config.yaml` with tight file permissions for secrets such as `--aws-secret` and `--azure-client-secret`. -::: +The credential flags exist on `cloudg run` only. Each one writes a single field of the loaded config and changes nothing else, so you can mix flags with values from `config.yaml`. `cloudg map` takes the same settings from `config.yaml` (it has `--profile` and nothing else). `cloudg collect` uses neither and works from the environment and default chains alone. The [Authentication](../guides/authentication.md) guide covers the identities, roles and permissions each method needs; this page is about the flags. + +!!! warning "Secrets on the command line" + + Values passed as flags can be seen in your shell history and, while cloudg runs, in the process list of the machine. Prefer the environment variables or a `config.yaml` with tight file permissions for secrets such as `--aws-secret` and `--azure-client-secret`. ## AWS @@ -23,7 +26,7 @@ Values passed as flags can be seen in your shell history and, while cloudg runs, cloudg builds the base session from the first method that is complete, then optionally assumes a role on top: -```mermaid caption="How cloudg picks AWS credentials" +```mermaid flowchart TD A{"Key and secret both set?"} -->|yes| K["Direct keys, plus session token if set"] A -->|no| B{"Role ARN and token file set?"} @@ -36,6 +39,8 @@ flowchart TD D --> R R -->|yes| S["sts:AssumeRole, with ExternalId if set"] ``` +How cloudg picks AWS credentials +{ .diagram-caption } 1. Direct keys need both `--aws-key` and `--aws-secret`. One without the other is ignored and the next method is tried. `--aws-session-token` is added when present, for temporary keys. 2. Web identity federation needs a role ARN and a token file. The token file comes from `--aws-web-identity-token-file`, or from `AWS_WEB_IDENTITY_TOKEN_FILE` when the flag is absent. This is the pattern for GitHub Actions, GitLab CI and EKS, with no stored keys. @@ -168,9 +173,22 @@ cloudg -c config.yaml map -p all --regions all ## Related -:::links -- [Authentication](/guides/authentication/) Identities, roles and least-privilege permissions per provider. -- [Provider and region flags](/cli/provider-flags/) `--profile`, `--subscription-id` and `--project-id`. -- [cloudg run](/cli/run/) The command these flags belong to. -- [AWS Organizations](/guides/aws-organizations/) Member-account roles for `cloudg map --org`. -::: +
      + +- [**Authentication**](../guides/authentication.md) + + Identities, roles and least-privilege permissions per provider. + +- [**Provider and region flags**](provider-flags.md) + + `--profile`, `--subscription-id` and `--project-id`. + +- [**cloudg run**](run.md) + + The command these flags belong to. + +- [**AWS Organizations**](../guides/aws-organizations.md) + + Member-account roles for `cloudg map --org`. + +
      diff --git a/site/content/cli/collect.md b/site/content/cli/collect.md index 8d4a008..bdedea5 100644 --- a/site/content/cli/collect.md +++ b/site/content/cli/collect.md @@ -1,15 +1,20 @@ --- +description: Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. command: collect -lede: Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. -intro: | - `cloudg collect` is phase 1 of `cloudg run` on its own, for one provider and, on AWS, one region. It is a quick way to test credentials and permissions before a long run. For a full inventory with relationships and dependency analysis, use [cloudg map](/cli/map/). --- +Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. +{ .lede } + +`cloudg collect` is phase 1 of `cloudg run` on its own, for one provider and, on AWS, one region. It is a quick way to test credentials and permissions before a long run. For a full inventory with relationships and dependency analysis, use [cloudg map](map.md). + + + ## How it works `-p/--provider` is required and takes exactly one of `aws`, `azure` or `gcp` (case-insensitive). There is no `all` here. -Credentials and scope come from the flags on this page, then from the provider's section of the config passed with `cloudg -c` (`aws`, `azure` or `gcp`), then from the environment and each provider's default chain, through `CredentialResolver`. Every method the config supports works here: direct keys, `role_arn` and web identity on AWS, a service principal, workload identity or managed identity on Azure, a credentials file or impersonation on GCP. See [Auth flags](/cli/auth-flags/) for the order within each provider. +Credentials and scope come from the flags on this page, then from the provider's section of the config passed with `cloudg -c` (`aws`, `azure` or `gcp`), then from the environment and each provider's default chain, through `CredentialResolver`. Every method the config supports works here: direct keys, `role_arn` and web identity on AWS, a service principal, workload identity or managed identity on Azure, a credentials file or impersonation on GCP. See [Auth flags](auth-flags.md) for the order within each provider. How each provider resolves its identity: @@ -91,9 +96,22 @@ The file is written to `-o/--output` (default `./reports`), and a second run for ## Related -:::links -- [cloudg map](/cli/map/) The full inventory map with relationships and dependencies. -- [cloudg run](/cli/run/) Collection as part of the complete pipeline. -- [Authentication](/guides/authentication/) Every supported identity per provider. -- [Provider and region flags](/cli/provider-flags/) How `--region` and `--regions` differ. -::: +
      + +- [**cloudg map**](map.md) + + The full inventory map with relationships and dependencies. + +- [**cloudg run**](run.md) + + Collection as part of the complete pipeline. + +- [**Authentication**](../guides/authentication.md) + + Every supported identity per provider. + +- [**Provider and region flags**](provider-flags.md) + + How `--region` and `--regions` differ. + +
      diff --git a/site/content/cli/deps.md b/site/content/cli/deps.md index 62a36f3..0c10576 100644 --- a/site/content/cli/deps.md +++ b/site/content/cli/deps.md @@ -1,17 +1,22 @@ --- +description: Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? command: deps -lede: Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? -intro: | - `cloudg deps` reads the files `cloudg map` wrote. It never calls a cloud API, so it runs offline and needs no credentials. --- +Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? +{ .lede } + +`cloudg deps` reads the files `cloudg map` wrote. It never calls a cloud API, so it runs offline and needs no credentials. + + + ## How it works The command loads `inventory-map.json` from `-m/--map`. You can pass the file itself or the folder that holds it; the default is `./reports`. If an `inventory-organization.json` sits next to the map, it is loaded too. From the assets and edges it builds a dependency graph in which every edge reads "dependent needs dependency". A Lambda function needs its role, its image, its KMS key and the queue that triggers it. A load balancer needs the WAF in front of it. A resource needs the CloudFormation stack that manages it. What happens next depends on whether you name an asset. -```mermaid caption="The two modes of cloudg deps" +```mermaid flowchart TD A["Load inventory-map.json"] --> B{"ASSET given?"} B -->|yes| C["Resolve the reference"] @@ -22,6 +27,8 @@ flowchart TD D -->|both| G["Both trees"] B -->|no| H["Overview: shared deps, blast radius, cross-account edges"] ``` +The two modes of cloudg deps +{ .diagram-caption } ### With an asset @@ -103,28 +110,32 @@ cloudg deps jobs-queue --direction down --depth 5 Feed a function's dependencies into another tool: -```bash tab="CLI" -cloudg deps api-handler --direction up --json > api-handler-deps.json -``` +=== "CLI" -```python tab="Python" title="deps_tree.py" -import json + ```bash + cloudg deps api-handler --direction up --json > api-handler-deps.json + ``` -from cloudg.inventory import InventoryResult +=== "Python" -result = InventoryResult.load("./reports") -graph = result.dependency_graph() + ```python title="deps_tree.py" + import json -asset = graph.find("api-handler") -if asset is None: - raise SystemExit("no unique asset matches 'api-handler'") + from cloudg.inventory import InventoryResult -tree = graph.tree(asset.id, direction="up", max_depth=3) -print(json.dumps(tree, indent=2, default=str)) + result = InventoryResult.load("./reports") + graph = result.dependency_graph() -for row in graph.blast_radius(top=5): - print(row["name"], row["transitive_dependents"]) -``` + asset = graph.find("api-handler") + if asset is None: + raise SystemExit("no unique asset matches 'api-handler'") + + tree = graph.tree(asset.id, direction="up", max_depth=3) + print(json.dumps(tree, indent=2, default=str)) + + for row in graph.blast_radius(top=5): + print(row["name"], row["transitive_dependents"]) + ``` ## Exit codes @@ -136,10 +147,26 @@ for row in graph.blast_radius(top=5): ## Related -:::links -- [Dependencies and blast radius](/guides/dependencies/) How to read the trees and what the edge types mean. -- [cloudg map](/cli/map/) Produces the inventory map this command reads. -- [DependencyGraph](/api/dependencygraph/) The Python class behind `deps`. -- [InventoryResult](/api/inventoryresult/) Loading and exporting saved maps. -- [MCP server](/mcp/) Ask the same questions from an AI assistant. -::: +
      + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + How to read the trees and what the edge types mean. + +- [**cloudg map**](map.md) + + Produces the inventory map this command reads. + +- [**DependencyGraph**](../api/dependencygraph.md) + + The Python class behind `deps`. + +- [**InventoryResult**](../api/inventoryresult.md) + + Loading and exporting saved maps. + +- [**MCP server**](../mcp/index.md) + + Ask the same questions from an AI assistant. + +
      diff --git a/site/content/cli/index.md b/site/content/cli/index.md new file mode 100644 index 0000000..38bcb4f --- /dev/null +++ b/site/content/cli/index.md @@ -0,0 +1,11 @@ +--- +title: CLI reference +description: Every cloudg command and the flags it takes, generated from the click definitions so the tables match --help. +--- + +Every `cloudg` command and the flags it takes. The tables are generated from the click definitions at build time, so they match `--help`. +{ .lede } + +Options that belong to the root command go before the subcommand: `cloudg -c config.yaml map ...`. Without `-c`, cloudg runs on its built-in defaults and does not look for a `config.yaml` on its own. The flags shared by several commands have pages of their own: [provider and region flags](provider-flags.md), [auth flags](auth-flags.md) and [output flags](output-flags.md). + + diff --git a/site/content/cli/ingest.md b/site/content/cli/ingest.md index 0b108d2..65a6860 100644 --- a/site/content/cli/ingest.md +++ b/site/content/cli/ingest.md @@ -1,15 +1,20 @@ --- +description: Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. command: ingest -lede: Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. -intro: | - Use `cloudg ingest` when Prowler, ScoutSuite, Checkov or Trivy already ran somewhere else: in a CI job, on another host, on a schedule. cloudg does the second half of the pipeline on their files: cross-scanner deduplication, scoring, compliance mapping and report rendering. --- +Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. +{ .lede } + +Use `cloudg ingest` when Prowler, ScoutSuite, Checkov or Trivy already ran somewhere else: in a CI job, on another host, on a schedule. cloudg does the second half of the pipeline on their files: cross-scanner deduplication, scoring, compliance mapping and report rendering. + + + ## How it works You pass each tool's native output with its own flag. Every flag is repeatable, and any mix of tools works, from one Trivy file to all four scanners at once. A path can be a single file or the directory the tool wrote. -```mermaid caption="From scanner files to reports" +```mermaid flowchart LR A["--prowler / --scoutsuite / --checkov / --trivy"] --> B["Parse each path"] B -->|"missing or unreadable"| W["Warn and skip"] @@ -18,6 +23,8 @@ flowchart LR D --> E["findings.json"] D --> F["report.html"] ``` +From scanner files to reports +{ .diagram-caption } 1. cloudg groups the paths by tool and prints them in an "Ingest Configuration" panel. With no input flag at all it stops with "No inputs" and exit code 1. 2. Each path goes through that tool's parser. A path that does not exist, or that the parser rejects, is skipped with a warning; the other paths still count. The per-tool totals are printed as they come in. @@ -33,7 +40,7 @@ What each flag accepts: | `--checkov` | A `checkov --output json` file, or a directory holding `results_json.json` | `checkov -d ./iac --output json > results_json.json` | | `--trivy` | A `trivy image` or `trivy fs` JSON file, or a directory of them | `trivy image --format json myrepo/app:latest > trivy-image.json` | -[Ingesting existing output](/guides/ingesting/) lists the exact fields cloudg reads from each format. +[Ingesting existing output](../guides/ingesting.md) lists the exact fields cloudg reads from each format. Prowler's two JSON formats both work. Each record is read as OCSF when it has OCSF keys (`finding_info`, `class_uid` and so on) and as ASFF otherwise, so one directory can hold both. When it does, Prowler reports the same check twice, and normalisation merges the pair into one finding. A muted OCSF finding comes in with `is_suppressed` set. The MCP server's `load_dataset` and `ingest_reports` tools read OCSF the same way. @@ -101,21 +108,25 @@ cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./repo The same aggregation in Python, with no files written: -```bash tab="CLI" -cloudg ingest --prowler ./prowler-output/ --trivy ./trivy-image.json --format json -``` +=== "CLI" -```python tab="Python" title="ingest_reports.py" -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser + ```bash + cloudg ingest --prowler ./prowler-output/ --trivy ./trivy-image.json --format json + ``` -findings = ingest_reports({ - "prowler": ["./prowler-output/"], - "trivy": ["./trivy-image.json"], -}) -result = FindingsNormaliser().normalise(findings) -print(len(findings), "parsed,", len(result.findings), "after deduplication") -``` +=== "Python" + + ```python title="ingest_reports.py" + from cloudg.ingest import ingest_reports + from cloudg.normaliser import FindingsNormaliser + + findings = ingest_reports({ + "prowler": ["./prowler-output/"], + "trivy": ["./trivy-image.json"], + }) + result = FindingsNormaliser().normalise(findings) + print(len(findings), "parsed,", len(result.findings), "after deduplication") + ``` ## Output files @@ -139,10 +150,26 @@ There is no SVG format here, because ingest has no inventory to draw. `raw-findi ## Related -:::links -- [Ingesting existing output](/guides/ingesting/) Input formats in detail and how deduplication works. -- [cloudg scan](/cli/scan/) Run the scanners from cloudg instead. -- [cloudg report](/cli/report/) Re-render reports from a `findings.json`. -- [parse_report](/api/parse-report/) The parser behind each flag. -- [FindingsNormaliser](/api/findingsnormaliser/) Deduplication, scoring and compliance mapping. -::: +
      + +- [**Ingesting existing output**](../guides/ingesting.md) + + Input formats in detail and how deduplication works. + +- [**cloudg scan**](scan.md) + + Run the scanners from cloudg instead. + +- [**cloudg report**](report.md) + + Re-render reports from a `findings.json`. + +- [**parse_report**](../api/parse-report.md) + + The parser behind each flag. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + Deduplication, scoring and compliance mapping. + +
      diff --git a/site/content/cli/map.md b/site/content/cli/map.md index 225934a..ea18767 100644 --- a/site/content/cli/map.md +++ b/site/content/cli/map.md @@ -1,10 +1,15 @@ --- +description: Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for cloudg deps, the MCP server or your own code. command: map -lede: Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for `cloudg deps`, the MCP server or your own code. -intro: | - `cloudg map` needs read access to the cloud APIs and nothing else. No scanner runs and none has to be installed. The [Inventory mapping](/guides/inventory-mapping/) guide explains what the map contains; this page covers the command itself. --- +Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for `cloudg deps`, the MCP server or your own code. +{ .lede } + +`cloudg map` needs read access to the cloud APIs and nothing else. No scanner runs and none has to be installed. The [Inventory mapping](../guides/inventory-mapping.md) guide explains what the map contains; this page covers the command itself. + + + ## How it works Like `cloudg run`, the command starts from the config loaded by the root command (`cloudg -c config.yaml map ...`), or from built-in defaults when there is no `-c`. It then applies three groups of flags. A flag you leave out never clears a value the config set. @@ -36,7 +41,7 @@ $ cloudg map -p aws --org --org-role cloudg-readonly --ou Workloads --regions al The mapping run itself goes through these steps: -```mermaid caption="What cloudg map does" +```mermaid flowchart TD A["Config + flags"] --> B{"--org?"} B -->|yes| C["Discover org, OUs, accounts, Control Tower"] @@ -51,6 +56,8 @@ flowchart TD J --> K{"--findings?"} K -->|yes| L["asset-map.json, compliance-map.json"] ``` +What cloudg map does +{ .diagram-caption } With `--org`, cloudg first lists the Organization from the account your credentials belong to (the management account, or a delegated administrator for the Organizations APIs). It picks the member accounts under `--ou`, drops `--exclude-account` entries, and, if you also passed `--accounts`, keeps only the accounts in both lists. The role assumed in each member account is `--org-role`, else `--role-name`, else `AWSControlTowerExecution`. If Control Tower is present and you asked for `--regions all`, only its governed regions are mapped (`aws.organization.use_governed_regions`, on by default). @@ -58,7 +65,7 @@ Without `--org`, `--accounts` plus `--role-name` gives you the same fan-out over Deep collection runs per account and region. Account-wide services (IAM, S3, CloudFront, Route 53, Shield and others) run once per account, in the primary region, so `--regions all` does not repeat them. After the deep collectors, the breadth sweeps fill gaps: Cloud Control lists every resource type with a list handler, and the Resource Groups Tagging API finds tagged resources. When the same resource turns up twice, the dedicated collector wins, then Cloud Control, then the tagging API. -The linker then resolves every identifier the collectors recorded into typed edges (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `PROTECTS`, `GOVERNS` and so on), adds an account node above each account's top-level resources, and computes the dependency analysis. The [Inventory mapping](/guides/inventory-mapping/) guide lists the edge types. +The linker then resolves every identifier the collectors recorded into typed edges (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `PROTECTS`, `GOVERNS` and so on), adds an account node above each account's top-level resources, and computes the dependency analysis. The [Inventory mapping](../guides/inventory-mapping.md) guide lists the edge types. The command prints an organization table (with `--org`), an inventory summary, the 15 services with the most assets, the 10 most shared dependencies, the 10 largest blast radii and up to 15 security service gaps. Here is the summary part for the synthetic estate in the test suite (`tests/mcp/fixtures/sample_estate.py`), so the numbers are small: @@ -86,11 +93,11 @@ If some collectors failed (an API you lack permission for, a region that is not `--findings` takes a cloudg findings file: a `raw-findings.json` from `cloudg scan` or `cloudg ingest`, or a `findings.json` report. Both shapes work, a bare list or an object with a `findings` key. The option is repeatable. Files that cannot be read or validated are skipped with a warning. Findings attach to assets by ARN, resource ID or name. -The merge happens after a fresh mapping run, because `--findings` belongs to `map`. To overlay findings on a map you already saved without calling the cloud again, use `InventoryMapper.export_merged()` from Python (see [InventoryMapper](/api/inventorymapper/)) or load both files into the MCP server. +The merge happens after a fresh mapping run, because `--findings` belongs to `map`. To overlay findings on a map you already saved without calling the cloud again, use `InventoryMapper.export_merged()` from Python (see [InventoryMapper](../api/inventorymapper.md)) or load both files into the MCP server. ### Credentials -`map` has no `--aws-key`, `--azure-*` or `--gcp-*` flags. It takes credentials from the config file, the standard environment variables, `--profile`, or the provider's default chain, in the order described under [Auth flags](/cli/auth-flags/). Put role ARNs, service principals and credential files in `config.yaml` when you map with something other than your default identity. +`map` has no `--aws-key`, `--azure-*` or `--gcp-*` flags. It takes credentials from the config file, the standard environment variables, `--profile`, or the provider's default chain, in the order described under [Auth flags](auth-flags.md). Put role ARNs, service principals and credential files in `config.yaml` when you map with something other than your default identity. ## Examples @@ -144,25 +151,29 @@ cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./repo The same mapping from Python: -```bash tab="CLI" -cloudg map -p aws --profile audit --regions all -o ./reports -``` +=== "CLI" -```python tab="Python" title="map_inventory.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg map -p aws --profile audit --regions all -o ./reports + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "audit" -config.aws.regions = ["ALL"] +=== "Python" -engine = CloudGEngine(config) -result = engine.map_inventory_sync("./reports") -print(result.summary["total_assets"], "assets") -``` + ```python title="map_inventory.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] + + engine = CloudGEngine(config) + result = engine.map_inventory_sync("./reports") + print(result.summary["total_assets"], "assets") + ``` ## Output files -Written to `-o/--output` (default `./reports`). The [output files reference](/reference/output-files/) describes each one in more detail. +Written to `-o/--output` (default `./reports`). The [output files reference](../reference/output-files.md) describes each one in more detail. | File | Written when | Contents | |---|---|---| @@ -186,11 +197,30 @@ Written to `-o/--output` (default `./reports`). The [output files reference](/re ## Related -:::links -- [Inventory mapping](/guides/inventory-mapping/) What the map collects and how relationships are built. -- [AWS Organizations](/guides/aws-organizations/) Roles and permissions for `--org`. -- [cloudg deps](/cli/deps/) Query the saved map: what depends on what. -- [Dependencies and blast radius](/guides/dependencies/) How to read dependency trees. -- [Provider and region flags](/cli/provider-flags/) `--regions all` and discovery. -- [InventoryMapper](/api/inventorymapper/) The Python class behind the command. -::: +
      + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + What the map collects and how relationships are built. + +- [**AWS Organizations**](../guides/aws-organizations.md) + + Roles and permissions for `--org`. + +- [**cloudg deps**](deps.md) + + Query the saved map: what depends on what. + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + How to read dependency trees. + +- [**Provider and region flags**](provider-flags.md) + + `--regions all` and discovery. + +- [**InventoryMapper**](../api/inventorymapper.md) + + The Python class behind the command. + +
      diff --git a/site/content/cli/mcp-call.md b/site/content/cli/mcp-call.md index 7b27f89..265b126 100644 --- a/site/content/cli/mcp-call.md +++ b/site/content/cli/mcp-call.md @@ -1,13 +1,18 @@ --- +description: Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. command: mcp call -lede: Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. -intro: | - `cloudg mcp call` is the quickest way to try a tool, debug a policy, or use a tool from a shell script. No server runs and no client is needed. --- +Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. +{ .lede } + +`cloudg mcp call` is the quickest way to try a tool, debug a policy, or use a tool from a shell script. No server runs and no client is needed. + + + ## How it works -Each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), calls `TOOL` once as the chosen principal, prints the result and exits. Nothing carries over between invocations. A `load_dataset` call in one command is gone by the next, so pass the data you need with `--dataset` every time. +Each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), calls `TOOL` once as the chosen principal, prints the result and exits. Nothing carries over between invocations. A `load_dataset` call in one command is gone by the next, so pass the data you need with `--dataset` every time. Arguments go in `--args` as a JSON object. You have three ways to supply it: @@ -19,7 +24,7 @@ Arguments go in `--args` as a JSON object. You have three ways to supply it: Anything that is not a JSON object is refused before the tool runs, with `Invalid value for --args: not valid JSON: ...` or `must be a JSON object` and exit code 2. Without `--args`, the tool is called with no arguments. -The principal is the local user unless you pass `--role` or `--principal-id`, as on [cloudg mcp tools](/cli/mcp-tools/). A tool the policy hides from that principal behaves as if it did not exist. +The principal is the local user unless you pass `--role` or `--principal-id`, as on [cloudg mcp tools](mcp-tools.md). A tool the policy hides from that principal behaves as if it did not exist. What gets printed depends on the result and on `--raw`: @@ -140,10 +145,26 @@ fi ## Related -:::links -- [Tool index](/mcp/tools/tool-index/) Every tool's arguments and results. -- [Conventions shared by every tool](/mcp/tools/conventions-shared-by-every-tool/) References, pagination, fields and errors. -- [Errors](/mcp/server/errors/) Error codes and what the error data holds. -- [cloudg mcp read](/cli/mcp-read/) The same thing for resources. -- [Live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/) What happens before a live tool calls the cloud. -::: +
      + +- [**Tool index**](../mcp/tools/tool-index.md) + + Every tool's arguments and results. + +- [**Conventions shared by every tool**](../mcp/tools/conventions-shared-by-every-tool.md) + + References, pagination, fields and errors. + +- [**Errors**](../mcp/server/errors.md) + + Error codes and what the error data holds. + +- [**cloudg mcp read**](mcp-read.md) + + The same thing for resources. + +- [**Live tools and the credential preflight**](../mcp/server/live-tools-and-the-credential-preflight.md) + + What happens before a live tool calls the cloud. + +
      diff --git a/site/content/cli/mcp-config.md b/site/content/cli/mcp-config.md index 427a03d..cda0412 100644 --- a/site/content/cli/mcp-config.md +++ b/site/content/cli/mcp-config.md @@ -1,10 +1,15 @@ --- +description: Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. command: mcp config -lede: Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. -intro: | - Each client stores MCP servers in a slightly different JSON shape. `cloudg mcp config` writes the right one, with the launch command and arguments filled in, so you don't have to work out paths by hand. --- +Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. +{ .lede } + +Each client stores MCP servers in a slightly different JSON shape. `cloudg mcp config` writes the right one, with the launch command and arguments filled in, so you don't have to work out paths by hand. + + + ## How it works The command builds a JSON snippet, prints it on stdout, and prints a one-line hint on stderr saying where the snippet goes. Because the hint is on stderr, you can redirect stdout straight into a file. @@ -134,9 +139,22 @@ The command exits 0 after printing. Click returns 2 for an unknown `--client` or ## Related -:::links -- [Connect a client](/mcp/connect/) Where each client keeps its configuration, step by step. -- [cloudg mcp serve](/cli/mcp-serve/) Every option the generated command can take. -- [Quick start with a desktop client](/mcp/server/quick-start-with-a-desktop-client/) From install to the first question. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) Which `--policy` to put in the launch arguments. -::: +
      + +- [**Connect a client**](../mcp/connect.md) + + Where each client keeps its configuration, step by step. + +- [**cloudg mcp serve**](mcp-serve.md) + + Every option the generated command can take. + +- [**Quick start with a desktop client**](../mcp/server/quick-start-with-a-desktop-client.md) + + From install to the first question. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + Which `--policy` to put in the launch arguments. + +
      diff --git a/site/content/cli/mcp-prompts.md b/site/content/cli/mcp-prompts.md index a9ea061..ec997ba 100644 --- a/site/content/cli/mcp-prompts.md +++ b/site/content/cli/mcp-prompts.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP prompts a server would expose, with the arguments each one takes. command: mcp prompts -lede: Lists the MCP prompts a server would expose, with the arguments each one takes. -intro: | - cloudg's prompts are ready-made workflows, such as a security posture review or an incident triage, that a client can offer its user as one-click tasks. `cloudg mcp prompts` shows which of them a principal can see and what arguments they expect. --- +Lists the MCP prompts a server would expose, with the arguments each one takes. +{ .lede } + +cloudg's prompts are ready-made workflows, such as a security posture review or an incident triage, that a client can offer its user as one-click tasks. `cloudg mcp prompts` shows which of them a principal can see and what arguments they expect. + + + ## How it works -The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)) and prints the prompts visible to one principal: the local user by default, or the one described by `--role` and `--principal-id` (see [cloudg mcp tools](/cli/mcp-tools/) for how those work). `--prefix` is applied to prompt names as well as tool names. +The command builds the layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)) and prints the prompts visible to one principal: the local user by default, or the one described by `--role` and `--principal-id` (see [cloudg mcp tools](mcp-tools.md) for how those work). `--prefix` is applied to prompt names as well as tool names. The table has four columns. Arguments are listed in order, and a `*` marks the required ones. In 0.6.0 there are 11 prompts, all in the `prompts` category, and every built-in policy lists all of them for the local user: @@ -110,9 +115,22 @@ A missing required argument raises `Missing required prompt arguments: ref`, and ## Related -:::links -- [Prompts](/mcp/tools/prompts/) Each prompt with its arguments and rendered output. -- [cloudg mcp tools](/cli/mcp-tools/) The tools the prompts tell the assistant to call. -- [In-process use](/mcp/server/in-process-use/) Driving the layer from Python. -- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. -::: +
      + +- [**Prompts**](../mcp/tools/prompts.md) + + Each prompt with its arguments and rendered output. + +- [**cloudg mcp tools**](mcp-tools.md) + + The tools the prompts tell the assistant to call. + +- [**In-process use**](../mcp/server/in-process-use.md) + + Driving the layer from Python. + +- [**cloudg mcp serve**](mcp-serve.md) + + The shared layer options in full. + +
      diff --git a/site/content/cli/mcp-read.md b/site/content/cli/mcp-read.md index 87fa26d..f920fd6 100644 --- a/site/content/cli/mcp-read.md +++ b/site/content/cli/mcp-read.md @@ -1,20 +1,25 @@ --- +description: Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. command: mcp read -lede: Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. -intro: | - Resources are fetched by URI, such as `cloudg://findings/summary` or `cloudg://assets/`. `cloudg mcp read` lets you look at one from the shell, which helps when you are writing a policy or checking what an assistant will be shown. --- +Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. +{ .lede } + +Resources are fetched by URI, such as `cloudg://findings/summary` or `cloudg://assets/`. `cloudg mcp read` lets you look at one from the shell, which helps when you are writing a policy or checking what an assistant will be shown. + + + ## How it works -Like [cloudg mcp call](/cli/mcp-call/), each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), reads `URI` once as the chosen principal, prints the contents and exits. Most resources read the active dataset, so load one with `--dataset` on the same command line. Without a dataset, those resources fail: +Like [cloudg mcp call](mcp-call.md), each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), reads `URI` once as the chosen principal, prints the contents and exits. Most resources read the active dataset, so load one with `--dataset` on the same command line. Without a dataset, those resources fail: ```console $ cloudg mcp read cloudg://findings/summary Error: No dataset is loaded. Call load_dataset(path=...) with an inventory-map.json / findings.json / scanner report, or map_inventory to collect live data. ``` -The URI can be a fixed resource or a filled-in template. [cloudg mcp resources](/cli/mcp-resources/) lists both. For the asset templates, the reference can be the internal ID, a unique name, or a full ARN, and the ARN can be written as is or URL-encoded. Quote the URI in the shell, since ARNs contain `:` and `/` and some templates contain characters the shell treats specially. +The URI can be a fixed resource or a filled-in template. [cloudg mcp resources](mcp-resources.md) lists both. For the asset templates, the reference can be the internal ID, a unique name, or a full ARN, and the ARN can be written as is or URL-encoded. Quote the URI in the shell, since ARNs contain `:` and `/` and some templates contain characters the shell treats specially. Text contents are printed as they are, which for most resources is indented JSON and for some is another format: `cloudg://ontology/turtle` prints Turtle, `cloudg://graph/graphml` prints GraphML XML, and `cloudg://docs/{topic}` prints Markdown. Binary contents, if a resource returns any, are printed as their wire JSON with the base64 blob. @@ -25,7 +30,7 @@ $ cloudg mcp read cloudg://ontology/turtle --policy strict --dataset prod=invent Error: Unknown resource: cloudg://ontology/turtle ``` -The principal options (`--role`, `--principal-id`) work as on [cloudg mcp tools](/cli/mcp-tools/). stdout carries only the contents, so the output can be piped or redirected. +The principal options (`--role`, `--principal-id`) work as on [cloudg mcp tools](mcp-tools.md). stdout carries only the contents, so the output can be piped or redirected. ## Examples @@ -101,9 +106,22 @@ cloudg mcp read cloudg://policy --policy ./mcp-policy.yaml --role analyst ## Related -:::links -- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with example contents. -- [cloudg mcp resources](/cli/mcp-resources/) List the URIs and templates. -- [cloudg mcp call](/cli/mcp-call/) Call a tool instead of reading a resource. -- [The workspace](/mcp/server/the-workspace/) How datasets are loaded and named. -::: +
      + +- [**Resources and resource templates**](../mcp/tools/resources-and-resource-templates.md) + + Every resource with example contents. + +- [**cloudg mcp resources**](mcp-resources.md) + + List the URIs and templates. + +- [**cloudg mcp call**](mcp-call.md) + + Call a tool instead of reading a resource. + +- [**The workspace**](../mcp/server/the-workspace.md) + + How datasets are loaded and named. + +
      diff --git a/site/content/cli/mcp-resources.md b/site/content/cli/mcp-resources.md index a4bbd91..4b8a707 100644 --- a/site/content/cli/mcp-resources.md +++ b/site/content/cli/mcp-resources.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. command: mcp resources -lede: Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. -intro: | - Resources are the read-only side of the MCP server: summaries, graphs, schemas and single assets that a client can fetch by URI without calling a tool. `cloudg mcp resources` shows which ones the active policy lets a principal see. --- +Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. +{ .lede } + +Resources are the read-only side of the MCP server: summaries, graphs, schemas and single assets that a client can fetch by URI without calling a tool. `cloudg mcp resources` shows which ones the active policy lets a principal see. + + + ## How it works -The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the resources and resource templates visible to one principal, and prints them. The principal options work as on [cloudg mcp tools](/cli/mcp-tools/): the local user by default, or `--role` and `--principal-id` to see what a token holder gets. +The command builds the layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), asks it for the resources and resource templates visible to one principal, and prints them. The principal options work as on [cloudg mcp tools](mcp-tools.md): the local user by default, or `--role` and `--principal-id` to see what a token holder gets. There are two kinds of entry. A resource has a fixed URI, such as `cloudg://findings/summary`. A resource template has placeholders in braces that the client fills in, such as `cloudg://assets/{+ref}`. The `{+ref}` form means the value may contain `/` and `:`, so a full ARN fits. The table lists resources first, then templates, under one "URI / template" column: @@ -33,7 +38,7 @@ In 0.6.0, the default policy shows the local user 14 resources and 11 templates: | `cloudg://docs` | `cloudg://docs/{topic}` | | `cloudg://policy`, `cloudg://privacy/detectors`, `cloudg://ratelimit`, `cloudg://metrics` | | -The list does not depend on loaded datasets. Resources that read data, like `cloudg://findings/summary`, use the active dataset when they are read, so preload one with `--dataset` before you [read](/cli/mcp-read/) them. +The list does not depend on loaded datasets. Resources that read data, like `cloudg://findings/summary`, use the active dataset when they are read, so preload one with `--dataset` before you [read](mcp-read.md) them. `--include-category` and `--exclude-category` apply to resources and templates as well as tools. `--include-tool` and `--exclude-tool` do not touch them. @@ -92,9 +97,22 @@ cloudg mcp resources --exclude-category ontology --exclude-category privacy ## Related -:::links -- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with an example of its content. -- [cloudg mcp read](/cli/mcp-read/) Fetch one of these URIs from the shell. -- [cloudg mcp tools](/cli/mcp-tools/) The same listing for tools. -- [The workspace](/mcp/server/the-workspace/) Datasets and which one is active. -::: +
      + +- [**Resources and resource templates**](../mcp/tools/resources-and-resource-templates.md) + + Every resource with an example of its content. + +- [**cloudg mcp read**](mcp-read.md) + + Fetch one of these URIs from the shell. + +- [**cloudg mcp tools**](mcp-tools.md) + + The same listing for tools. + +- [**The workspace**](../mcp/server/the-workspace.md) + + Datasets and which one is active. + +
      diff --git a/site/content/cli/mcp-serve.md b/site/content/cli/mcp-serve.md index 61424b5..777882d 100644 --- a/site/content/cli/mcp-serve.md +++ b/site/content/cli/mcp-serve.md @@ -1,15 +1,20 @@ --- +description: Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. command: mcp serve -lede: Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. -intro: | - Desktop clients (Claude Desktop, Claude Code, Cursor, VS Code) launch `cloudg mcp serve` themselves over stdio, and you rarely type it by hand. You do type it when you serve over HTTP, test a policy, or preload datasets. [cloudg mcp config](/cli/mcp-config/) writes the client snippet for you. The [MCP server](/mcp/) section covers the server in depth. --- +Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. +{ .lede } + +Desktop clients (Claude Desktop, Claude Code, Cursor, VS Code) launch `cloudg mcp serve` themselves over stdio, and you rarely type it by hand. You do type it when you serve over HTTP, test a policy, or preload datasets. [cloudg mcp config](mcp-config.md) writes the client snippet for you. The [MCP server](../mcp/index.md) section covers the server in depth. + + + ## How it works The command builds one MCP layer from its options, picks a server implementation, and serves until the transport closes: end of input on stdio, Ctrl+C on HTTP. -```mermaid caption="From the command line to a running server" +```mermaid flowchart TD A["Options and env vars"] --> B["Validate tokens and option combinations"] B -->|invalid| X["Error before start"] @@ -22,6 +27,8 @@ flowchart TD G --> I["Stop: save pseudonym vault"] H --> I ``` +From the command line to a running server +{ .diagram-caption } ### Building the layer @@ -29,7 +36,7 @@ These options are shared with `tools`, `resources`, `prompts`, `call` and `read` The registry is the built-in catalog unless `--registry MODULE:ATTR` names your own. `--read-only` then drops every tool that writes files, calls cloud APIs, runs scanners or is marked destructive. `--include-category`, `--exclude-category`, `--include-tool` and `--exclude-tool` narrow it further, and `--prefix` is put in front of every tool and prompt name (at most 64 characters from `A-Z a-z 0-9 _ . -`). -The policy comes from `--policy`, which accepts a profile name (`open`, `standard`, `strict`, `read_only`, `airgapped`, `audit`, `soc-analyst`), a YAML or JSON file, or inline JSON. Without the option, cloudg reads `CLOUDG_MCP_POLICY`, and without that it uses `standard`. The policy decides which tools each principal sees and which privacy transforms run on results. See [Built-in profiles](/mcp/privacy/built-in-profiles/). +The policy comes from `--policy`, which accepts a profile name (`open`, `standard`, `strict`, `read_only`, `airgapped`, `audit`, `soc-analyst`), a YAML or JSON file, or inline JSON. Without the option, cloudg reads `CLOUDG_MCP_POLICY`, and without that it uses `standard`. The policy decides which tools each principal sees and which privacy transforms run on results. See [Built-in profiles](../mcp/privacy/built-in-profiles.md). Middleware is added from the options. A metrics middleware is always on, which is why `cloudg://metrics` is always listed. `--audit-log` (or `CLOUDG_MCP_AUDIT_LOG`) appends a JSONL record per call with argument values hashed. `--max-concurrency` caps parallel tool calls, and `--cache-ttl` caches results of read-only, idempotent tools for that many seconds. `--timeout` is the default per-call limit, 300 seconds; tools with their own limit, such as the live collection tools, keep theirs. @@ -48,7 +55,7 @@ $ cloudg mcp serve --transport sse --flavor fastmcp --json-response Error: --json-response / --stateless apply to Streamable HTTP; the fastmcp flavor cannot combine them with --transport sse ``` -`--page-size` (items per page in list results) only applies to the native flavor. [Flavors](/mcp/server/flavors/) compares them. +`--page-size` (items per page in list results) only applies to the native flavor. [Flavors](../mcp/server/flavors.md) compares them. ### stdio @@ -94,9 +101,9 @@ $ cloudg mcp serve --transport http --auth-token env:CLOUDG_MCP_TOKEN Error: Invalid value for --auth-token: Environment variable 'CLOUDG_MCP_TOKEN' for an auth token is empty ``` -:::warning Binding beyond loopback -Keep `--host 127.0.0.1` unless you also set `--auth-token`. On a non-loopback bind the `Host` header is not checked unless you pass `--allowed-host`, so set that too. -::: +!!! warning "Binding beyond loopback" + + Keep `--host 127.0.0.1` unless you also set `--auth-token`. On a non-loopback bind the `Host` header is not checked unless you pass `--allowed-host`, so set that too. `--cors-origin` grants CORS to a browser origin. Only the native flavor supports it. @@ -172,11 +179,30 @@ cloudg mcp serve --log-level DEBUG 2> ./mcp-server.log ## Related -:::links -- [Connect a client](/mcp/connect/) Step-by-step setup for each desktop client. -- [cloudg mcp config](/cli/mcp-config/) Generate the client configuration snippet. -- [Transports and protocol versions](/mcp/server/transports-and-protocol-versions/) stdio, Streamable HTTP and SSE in detail. -- [Security model in brief](/mcp/server/security-model-in-brief/) What the policy, tokens and Origin checks protect. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each `--policy` profile shows and hides. -- [cloudg mcp tools](/cli/mcp-tools/) Check what a policy exposes before you serve it. -::: +
      + +- [**Connect a client**](../mcp/connect.md) + + Step-by-step setup for each desktop client. + +- [**cloudg mcp config**](mcp-config.md) + + Generate the client configuration snippet. + +- [**Transports and protocol versions**](../mcp/server/transports-and-protocol-versions.md) + + stdio, Streamable HTTP and SSE in detail. + +- [**Security model in brief**](../mcp/server/security-model-in-brief.md) + + What the policy, tokens and Origin checks protect. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each `--policy` profile shows and hides. + +- [**cloudg mcp tools**](mcp-tools.md) + + Check what a policy exposes before you serve it. + +
      diff --git a/site/content/cli/mcp-tools.md b/site/content/cli/mcp-tools.md index 9479e89..dd95454 100644 --- a/site/content/cli/mcp-tools.md +++ b/site/content/cli/mcp-tools.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP tools a server would expose, as a table or as the exact tools/list JSON a client receives. command: mcp tools -lede: Lists the MCP tools a server would expose, as a table or as the exact `tools/list` JSON a client receives. -intro: | - Run `cloudg mcp tools` with the same options you plan to give `cloudg mcp serve`. What it prints is what the client will see, after the policy, the filters and `--read-only` have done their work. --- +Lists the MCP tools a server would expose, as a table or as the exact `tools/list` JSON a client receives. +{ .lede } + +Run `cloudg mcp tools` with the same options you plan to give `cloudg mcp serve`. What it prints is what the client will see, after the policy, the filters and `--read-only` have done their work. + + + ## How it works -The command builds the same layer `serve` would build from the shared options (`--policy`, `--read-only`, `--include-category` and the rest, listed on [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the tool list of one principal, and prints it. No server starts and no client is involved. +The command builds the same layer `serve` would build from the shared options (`--policy`, `--read-only`, `--include-category` and the rest, listed on [cloudg mcp serve](mcp-serve.md#building-the-layer)), asks it for the tool list of one principal, and prints it. No server starts and no client is involved. The principal is the local user unless you pass `--role` or `--principal-id`. The local user has the id `local` and the roles `default` and `local`. With either option, the caller becomes a principal with the id you gave (default `cli`) and the roles you gave (default `default`). This is how you check what a token holder will see before you hand out the token: give the same roles you put in its `--auth-token` spec. @@ -100,9 +105,22 @@ cloudg mcp tools --json --policy open > tools-0.6.0.json ## Related -:::links -- [Tool index](/mcp/tools/tool-index/) Every tool with its arguments and an example result. -- [cloudg mcp call](/cli/mcp-call/) Call one of the listed tools from the shell. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy hides and transforms. -- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. -::: +
      + +- [**Tool index**](../mcp/tools/tool-index.md) + + Every tool with its arguments and an example result. + +- [**cloudg mcp call**](mcp-call.md) + + Call one of the listed tools from the shell. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each policy hides and transforms. + +- [**cloudg mcp serve**](mcp-serve.md) + + The shared layer options in full. + +
      diff --git a/site/content/cli/output-flags.md b/site/content/cli/output-flags.md index bc630ee..dd6d7cd 100644 --- a/site/content/cli/output-flags.md +++ b/site/content/cli/output-flags.md @@ -1,8 +1,11 @@ --- title: Output flags -lede: "Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: `-o`, `--format`, `--ontology`, `--rag-export`, `--terraform` and `--json`." +description: 'Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: -o, --format, --ontology, --rag-export, --terraform and --json.' --- +Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: `-o`, `--format`, `--ontology`, `--rag-export`, `--terraform` and `--json`. +{ .lede } + ## -o, --output Every command that writes files takes `-o/--output`, and it defaults to `./reports` everywhere. The directory is created if it doesn't exist, parents included. @@ -55,7 +58,7 @@ reports/ └── terraform/ provider.tf.json, variables.tf.json, main.tf.json, import_commands.sh (run, --terraform) ``` -The [output files reference](/reference/output-files/) describes each file's structure. +The [output files reference](../reference/output-files.md) describes each file's structure. The Terraform recreation goes to `/terraform`, so it follows `-o`. A `terraform.output_dir` in your config sends it somewhere else, except the old default value `./reports/terraform`, which counts as unset. @@ -98,7 +101,7 @@ Turn off the ontology and the RAG export when you only need findings: cloudg run -p aws --no-ontology --no-rag-export ``` -`--terraform` has a second effect: when no IaC directory is given by `--iac-dir` or `scanners.iac_directories`, Checkov and the Trivy filesystem scan target the generated Terraform. See [cloudg run](/cli/run/#how-it-works). +`--terraform` has a second effect: when no IaC directory is given by `--iac-dir` or `scanners.iac_directories`, Checkov and the Trivy filesystem scan target the generated Terraform. See [cloudg run](run.md#how-it-works). ## --json @@ -131,9 +134,22 @@ cloudg -v --log-file ./cloudg.log run -p aws ## Related -:::links -- [Output files](/reference/output-files/) The structure of every file in the output directory. -- [Reports](/guides/reports/) Reading the HTML report and `findings.json`. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) What the ontology and RAG exports are for. -- [Provider and region flags](/cli/provider-flags/) What gets collected in the first place. -::: +
      + +- [**Output files**](../reference/output-files.md) + + The structure of every file in the output directory. + +- [**Reports**](../guides/reports.md) + + Reading the HTML report and `findings.json`. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + What the ontology and RAG exports are for. + +- [**Provider and region flags**](provider-flags.md) + + What gets collected in the first place. + +
      diff --git a/site/content/cli/provider-flags.md b/site/content/cli/provider-flags.md index 225ba78..be19cf6 100644 --- a/site/content/cli/provider-flags.md +++ b/site/content/cli/provider-flags.md @@ -1,8 +1,11 @@ --- title: Provider and region flags -lede: "`-p`, `--region`, `--regions`, `--subscription-id`, `--project-id` and `--profile` decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere." +description: -p, --region, --regions, --subscription-id, --project-id and --profile decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere. --- +`-p`, `--region`, `--regions`, `--subscription-id`, `--project-id` and `--profile` decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere. +{ .lede } + ## Which command takes which flag | Flag | `run` | `map` | `collect` | `scan` | @@ -43,7 +46,7 @@ Without either flag, `run` and `map` use the config: `aws.regions` defaults to ` ### How discovery works -```mermaid caption="Resolving the AWS region list" +```mermaid flowchart TD A["aws.regions after flags"] --> B{"Is it ALL?"} B -->|no| C["Use the list as given"] @@ -53,6 +56,8 @@ flowchart TD F -->|ok| G["Enabled regions, sorted"] F -->|"fails"| H["The 17 regions enabled by default"] ``` +Resolving the AWS region list +{ .diagram-caption } | Provider | API | Filter | Fallback | |---|---|---|---| @@ -89,7 +94,7 @@ To cover a whole GCP organization in one Cloud Asset Inventory listing, set `gcp ## --profile -An AWS CLI profile name, SSO profiles included. On `run` and `map` it sets `aws.profile`, which sits third in the AWS credential order: direct keys and OIDC federation win over it. See [Auth flags](/cli/auth-flags/). +An AWS CLI profile name, SSO profiles included. On `run` and `map` it sets `aws.profile`, which sits third in the AWS credential order: direct keys and OIDC federation win over it. See [Auth flags](auth-flags.md). On `run`, `collect` and `scan` it replaces `aws.profile` the same way. `run` and `scan` also hand the profile to the scanners, for AWS only: Prowler gets `-p ` and ScoutSuite gets `--profile `, unless direct keys or a `role_arn` win, in which case they get the resulting keys instead. On `collect`, with no profile from the flag or the config, `AWS_PROFILE` and then the default chain apply. @@ -127,9 +132,22 @@ cloudg collect -p aws --profile audit --region ap-southeast-2 ## Related -:::links -- [Auth flags](/cli/auth-flags/) Credentials for each provider and which method wins. -- [Output flags](/cli/output-flags/) Where results go. -- [cloudg map](/cli/map/) Organization-wide AWS mapping with `--org`. -- [Provider settings](/reference/config/providers/) The matching `config.yaml` keys. -::: +
      + +- [**Auth flags**](auth-flags.md) + + Credentials for each provider and which method wins. + +- [**Output flags**](output-flags.md) + + Where results go. + +- [**cloudg map**](map.md) + + Organization-wide AWS mapping with `--org`. + +- [**Provider settings**](../reference/config/providers.md) + + The matching `config.yaml` keys. + +
      diff --git a/site/content/cli/report.md b/site/content/cli/report.md index d8bd666..94da181 100644 --- a/site/content/cli/report.md +++ b/site/content/cli/report.md @@ -1,10 +1,15 @@ --- +description: Re-renders the HTML, JSON and SVG reports from a findings.json you already have. No cloud access and no scanners. command: report -lede: Re-renders the HTML, JSON and SVG reports from a `findings.json` you already have. No cloud access and no scanners. -intro: | - Use `cloudg report` after upgrading cloudg to get the newer HTML template, or to rebuild one format you skipped. It also turns a `raw-findings.json` into reports. --- +Re-renders the HTML, JSON and SVG reports from a `findings.json` you already have. No cloud access and no scanners. +{ .lede } + +Use `cloudg report` after upgrading cloudg to get the newer HTML template, or to rebuild one format you skipped. It also turns a `raw-findings.json` into reports. + + + ## How it works `-i/--input` names a `findings.json` written by `cloudg run`, `cloudg ingest` or `CloudGEngine`, or a `raw-findings.json` from `cloudg scan` or `cloudg ingest`. @@ -101,9 +106,22 @@ cloudg report -i ./reports/raw-findings.json -o ./reports/scan-report ## Related -:::links -- [Reports](/guides/reports/) What is in the HTML report and how to read it. -- [Output files](/reference/output-files/) The `findings.json` format. -- [cloudg ingest](/cli/ingest/) Build reports from scanner output. -- [Output flags](/cli/output-flags/) `-o` and `--format` across commands. -::: +
      + +- [**Reports**](../guides/reports.md) + + What is in the HTML report and how to read it. + +- [**Output files**](../reference/output-files.md) + + The `findings.json` format. + +- [**cloudg ingest**](ingest.md) + + Build reports from scanner output. + +- [**Output flags**](output-flags.md) + + `-o` and `--format` across commands. + +
      diff --git a/site/content/cli/run.md b/site/content/cli/run.md index 19fd512..615f258 100644 --- a/site/content/cli/run.md +++ b/site/content/cli/run.md @@ -1,10 +1,15 @@ --- +description: The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. command: run -lede: The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. -intro: | - `cloudg run` is the command most people start with. It is also the only command that takes the full set of credential flags, so it works the same way on a laptop, in CI and on cloud compute. If you only want to know what exists, [cloudg map](/cli/map/) is faster and needs no scanners. --- +The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. +{ .lede } + +`cloudg run` is the command most people start with. It is also the only command that takes the full set of credential flags, so it works the same way on a laptop, in CI and on cloud compute. If you only want to know what exists, [cloudg map](map.md) is faster and needs no scanners. + + + ## How it works `cloudg run` builds one `CloudGConfig` and runs a fixed sequence of phases against it. The config comes from the file you pass to the root command (`cloudg -c config.yaml run ...`). Without `-c`, cloudg uses its built-in defaults. It does not look for a `config.yaml` in the current directory on its own. @@ -14,11 +19,11 @@ On top of that config, the command applies its flags: 1. `-p/--provider` replaces `providers`. `all` expands to `aws`, `azure` and `gcp`. 2. `--regions` replaces the region list of all three providers at once. `all` becomes the `ALL` sentinel, which triggers region discovery. When `--regions` is absent, `--region` sets `aws.regions` to that single region and leaves Azure and GCP alone. 3. `--subscription-id`, `--project-id` and `--profile` set `azure.subscription_ids`, `gcp.project_ids` and `aws.profile`. -4. Every `--aws-*`, `--azure-*` and `--gcp-*` flag you give overwrites the matching credential field. Flags you leave out keep whatever the config file says. See [Auth flags](/cli/auth-flags/). +4. Every `--aws-*`, `--azure-*` and `--gcp-*` flag you give overwrites the matching credential field. Flags you leave out keep whatever the config file says. See [Auth flags](auth-flags.md). Then the phases run: -```mermaid caption="Phases of cloudg run" +```mermaid flowchart TD A["Config file + flags"] --> B["1: collect assets"] B --> C["2: graph and reachability"] @@ -30,6 +35,8 @@ flowchart TD F --> H["4: normalise"] H --> I["5: JSON, SVG, HTML"] ``` +Phases of cloudg run +{ .diagram-caption } Phase 1 is collection. The multi-provider collector walks every configured account, subscription or project, and every region, concurrently. An `aws.accounts` list in the config needs `aws.role_name` too (unless `aws.organization.enabled` is on); without it the command stops with exit status 2 before collecting anything. If collection throws, cloudg prints a "Collection failed" panel and carries on with an empty inventory, so the scanners still run. Each account, region or provider that could not be collected gets a "Not collected: ..." line at the end of the phase. @@ -39,7 +46,7 @@ Phase 2c, the RAG export, runs when `--rag-export` is on (the default) and `rag. Phase 2d, the Terraform recreation, runs when you pass `--terraform` or set `terraform.enabled: true`. Either one is enough. -Phase 3 runs the scanners. Every scanner job gets its own thread, so they all start at once. The scanner list comes from `--scanners`, or from `scanners.enabled` in config when the flag is absent. The default config enables all five: `prowler`, `scoutsuite`, `checkov`, `trivy` and `iam`. A [scanner plugin](/api/entry-points/) runs when its name is in the list, and a name that matches nothing is skipped with a warning. Prowler and ScoutSuite run once per provider. Checkov runs once per IaC directory. Trivy scans the images from `--images` (or `scanners.trivy_images`), and falls back to a filesystem scan of the IaC directories when there are no images. +Phase 3 runs the scanners. Every scanner job gets its own thread, so they all start at once. The scanner list comes from `--scanners`, or from `scanners.enabled` in config when the flag is absent. The default config enables all five: `prowler`, `scoutsuite`, `checkov`, `trivy` and `iam`. A [scanner plugin](../api/entry-points.md) runs when its name is in the list, and a name that matches nothing is skipped with a warning. Prowler and ScoutSuite run once per provider. Checkov runs once per IaC directory. Trivy scans the images from `--images` (or `scanners.trivy_images`), and falls back to a filesystem scan of the IaC directories when there are no images. The IaC directory is resolved in this order: `--iac-dir`, then `scanners.iac_directories`, then the Terraform recreation from phase 2d if it produced any `*.tf.json` files. There is no fallback to the current directory. A scan of whatever folder you happened to run cloudg from would report zero findings and look like a clean result, so cloudg warns "nothing to scan" instead. @@ -130,24 +137,28 @@ Skip the semantic exports when you only want findings and the HTML report: cloudg run -p aws --no-ontology --no-rag-export ``` -Drive the pipeline from Python when you want result objects as well as files. `CloudGEngine.run_pipeline_sync()` covers the same ground, though it orders the analysis steps a little differently and writes no `topology.graphml` (see [CloudGEngine](/api/cloudgengine/)): +Drive the pipeline from Python when you want result objects as well as files. `CloudGEngine.run_pipeline_sync()` covers the same ground, though it orders the analysis steps a little differently and writes no `topology.graphml` (see [CloudGEngine](../api/cloudgengine.md)): -```bash tab="CLI" -cloudg run -p aws --region us-east-1 --scanners prowler,checkov,iam -o ./reports -``` +=== "CLI" -```python tab="Python" title="run_pipeline.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg run -p aws --region us-east-1 --scanners prowler,checkov,iam -o ./reports + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] -config.scanners.enabled = ["prowler", "checkov", "iam"] +=== "Python" -engine = CloudGEngine(config) -result = engine.run_pipeline_sync(output_dir="./reports") -print(len(result.findings), "findings") -print(result.report_paths) -``` + ```python title="run_pipeline.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + config.scanners.enabled = ["prowler", "checkov", "iam"] + + engine = CloudGEngine(config) + result = engine.run_pipeline_sync(output_dir="./reports") + print(len(result.findings), "findings") + print(result.report_paths) + ``` ## Output files @@ -168,7 +179,7 @@ All paths are relative to `-o/--output` (default `./reports`). The Terraform files go to `/terraform`, so `cloudg run -o /tmp/scan --terraform` writes them to `/tmp/scan/terraform`. A `terraform.output_dir` in your config replaces that directory, except the old default `./reports/terraform`, which counts as unset. -`cloudg run` does not write `raw-findings.json`. Use [cloudg scan](/cli/scan/) or [cloudg ingest](/cli/ingest/) when you need the pre-normalisation findings. +`cloudg run` does not write `raw-findings.json`. Use [cloudg scan](scan.md) or [cloudg ingest](ingest.md) when you need the pre-normalisation findings. ## Exit codes @@ -181,15 +192,34 @@ The Terraform files go to `/terraform`, so `cloudg run -o /tmp/scan --te Exit 3 means no target could be collected at all: a profile that does not exist, a role that cannot be assumed, or no credentials anywhere in the default chain, where every service of the region fails on its own. A run where some services or regions worked exits 0; the coverage table shows what was missed. -To fail a CI job on findings, read `findings.json` after the run. [CI and automation](/guides/ci-automation/) has a severity gate. +To fail a CI job on findings, read `findings.json` after the run. [CI and automation](../guides/ci-automation.md) has a severity gate. ## Related -:::links -- [The pipeline](/guides/pipeline/) What each phase does and why it runs in that order. -- [Running scanners](/guides/running-scanners/) Installing Prowler, ScoutSuite, Checkov and Trivy. -- [Auth flags](/cli/auth-flags/) Every credential flag and which one wins. -- [Provider and region flags](/cli/provider-flags/) `-p`, `--region`, `--regions` and discovery. -- [Output flags](/cli/output-flags/) The output directory and what lands in it. -- [cloudg map](/cli/map/) Inventory only, no scanners. -::: +
      + +- [**The pipeline**](../guides/pipeline.md) + + What each phase does and why it runs in that order. + +- [**Running scanners**](../guides/running-scanners.md) + + Installing Prowler, ScoutSuite, Checkov and Trivy. + +- [**Auth flags**](auth-flags.md) + + Every credential flag and which one wins. + +- [**Provider and region flags**](provider-flags.md) + + `-p`, `--region`, `--regions` and discovery. + +- [**Output flags**](output-flags.md) + + The output directory and what lands in it. + +- [**cloudg map**](map.md) + + Inventory only, no scanners. + +
      diff --git a/site/content/cli/scan.md b/site/content/cli/scan.md index 55bd99a..db8af44 100644 --- a/site/content/cli/scan.md +++ b/site/content/cli/scan.md @@ -1,15 +1,20 @@ --- +description: Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. command: scan -lede: Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. -intro: | - `cloudg scan` is the scanner phase of `cloudg run` cut loose from the rest. It is handy for checking that your scanner binaries work, or for producing a `raw-findings.json` to merge into a map later. For a complete assessment with deduplication and reports, use [cloudg run](/cli/run/). --- +Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. +{ .lede } + +`cloudg scan` is the scanner phase of `cloudg run` cut loose from the rest. It is handy for checking that your scanner binaries work, or for producing a `raw-findings.json` to merge into a map later. For a complete assessment with deduplication and reports, use [cloudg run](run.md). + + + ## How it works `scan` reads the config loaded with `cloudg -c`, like `run` and `map`, and plans its scanner jobs with the same code as `cloudg run`. From the config it takes `scanners.enabled` (the default for `--scanners`), the `*_extra_args` lists, `checkov_frameworks`, `iac_directories`, `trivy_images` and `timeout_seconds`, plus the AWS credentials and regions (`aws.profile`, keys, `role_arn`, web identity, `aws.regions`). Flags replace the matching config values. Without `-c`, the built-in defaults apply: all five scanners, a 3600-second timeout. -`--scanners` is a comma-separated list of built-in names or [scanner plugin](/api/entry-points/) names; an unknown name is skipped with a warning. Each scanner becomes one or more jobs, and all jobs run in parallel threads with a progress display: +`--scanners` is a comma-separated list of built-in names or [scanner plugin](../api/entry-points.md) names; an unknown name is skipped with a warning. Each scanner becomes one or more jobs, and all jobs run in parallel threads with a progress display: | Scanner | What it runs | |---|---| @@ -104,7 +109,7 @@ cloudg map -p aws --profile audit --regions all --findings ./reports/raw-finding | `prowler//` | Prowler's native ASFF output, when Prowler ran | | `scoutsuite//` | ScoutSuite's native report, when ScoutSuite ran | -`scan` writes no `findings.json` and no HTML report. To get them, run `cloudg report -i ./reports/raw-findings.json -o ./reports/html`, which normalises the raw findings first, or feed the native files to [cloudg ingest](/cli/ingest/), for example `cloudg ingest --prowler ./reports/prowler/aws/`. +`scan` writes no `findings.json` and no HTML report. To get them, run `cloudg report -i ./reports/raw-findings.json -o ./reports/html`, which normalises the raw findings first, or feed the native files to [cloudg ingest](ingest.md), for example `cloudg ingest --prowler ./reports/prowler/aws/`. ## Exit codes @@ -118,9 +123,22 @@ A missing scanner next to one that worked still exits 0, so check the `not insta ## Related -:::links -- [Running scanners](/guides/running-scanners/) Install and configure Prowler, ScoutSuite, Checkov and Trivy. -- [cloudg run](/cli/run/) Scanners plus collection, graph, normalisation and reports. -- [cloudg ingest](/cli/ingest/) Normalise and report on scanner output. -- [Provider and region flags](/cli/provider-flags/) How `-p` differs between commands. -::: +
      + +- [**Running scanners**](../guides/running-scanners.md) + + Install and configure Prowler, ScoutSuite, Checkov and Trivy. + +- [**cloudg run**](run.md) + + Scanners plus collection, graph, normalisation and reports. + +- [**cloudg ingest**](ingest.md) + + Normalise and report on scanner output. + +- [**Provider and region flags**](provider-flags.md) + + How `-p` differs between commands. + +
      diff --git a/site/content/credits.md b/site/content/credits.md new file mode 100644 index 0000000..b5e5195 --- /dev/null +++ b/site/content/credits.md @@ -0,0 +1,53 @@ +--- +title: Credits +description: The projects, data and standards cloudg builds on, and the licenses that come with them. +--- + +cloudg builds on other people's work, and some of it ships inside the package. This page lists where each piece comes from and under which license. +{ .lede } + +## At a glance + +| Project | License | What cloudg uses | +|---|---|---| +| [Prowler](https://github.com/prowler-cloud/prowler) | Apache-2.0 | 22 compliance framework files generated from its compliance JSON; its ASFF and OCSF output | +| [Checkov](https://github.com/bridgecrewio/checkov) | Apache-2.0 | Run as a separate program; reports parsed, `CKV_*` check IDs | +| [Trivy](https://github.com/aquasecurity/trivy) | Apache-2.0 | Run as a separate program; reports parsed, `AVD-*` check IDs | +| [ScoutSuite](https://github.com/nccgroup/ScoutSuite) | GPL-2.0 | Run as a separate program; reports parsed | +| [Cloud Custodian](https://github.com/cloud-custodian/cloud-custodian) | Apache-2.0 | Policy format and the `custodian` CLI for `cloudg/policies/` | +| [parliament](https://github.com/duo-labs/parliament) | BSD-3-Clause | IAM policy linting | +| [MITRE ATT&CK®](https://attack.mitre.org/) | Used with permission | Technique IDs and descriptions | +| [Chart.js](https://www.chartjs.org) 4.4.0 | MIT | Charts in report.html | +| [D3](https://d3js.org) 7.9.0 | ISC | The graph view in report.html | + +## Prowler + +[Prowler](https://github.com/prowler-cloud/prowler) (Apache-2.0). The 22 framework files in `cloudg/rules/frameworks/` are generated from Prowler's compliance JSON by `scripts/import_prowler_compliance.py`. That covers the control IDs, the control titles and descriptions, and the mapping from each control to Prowler check names. Every generated file says so in its header and in its `source` field. Prowler check names also appear in `cloudg/rules/check_equivalence.yaml`, and cloudg runs Prowler and parses its ASFF and OCSF output. + +## Scanners + +[Checkov](https://github.com/bridgecrewio/checkov) (Apache-2.0), [Trivy](https://github.com/aquasecurity/trivy) (Apache-2.0) and [ScoutSuite](https://github.com/nccgroup/ScoutSuite) (GPL-2.0) are run as separate programs, and their reports are parsed. Their check IDs (`CKV_*`, `AVD-*`) are used in the check-equivalence map. None of their code is part of cloudg. The Docker image installs Prowler, Checkov and Trivy, so their own licenses apply to those copies. + +## Cloud Custodian + +[Cloud Custodian](https://github.com/cloud-custodian/cloud-custodian) (Apache-2.0). The policy packs in `cloudg/policies/` are written in Cloud Custodian's policy format and run with its `custodian` CLI. + +## parliament + +[parliament](https://github.com/duo-labs/parliament) (BSD-3-Clause) from Duo Labs does the IAM policy linting. + +## MITRE ATT&CK + +[MITRE ATT&CK®](https://attack.mitre.org/). The `mitre_attack_*.yaml` files contain ATT&CK technique IDs and descriptions. © The MITRE Corporation. This work is reproduced and distributed with the permission of The MITRE Corporation. ATT&CK is a registered trademark of The MITRE Corporation. + +## Standards + +The hand-written rulesets in `cloudg/rules/` reference public control numbering from the CIS Benchmarks for AWS, Azure and GCP, NIST SP 800-53, NIST CSF, PCI DSS, the AICPA SOC 2 Trust Services Criteria, the HIPAA Security Rule, GDPR, ISO/IEC 27001:2022 and the AWS Foundational Security Best Practices. The standards belong to their publishers. CIS Benchmarks is a trademark of the Center for Internet Security, and cloudg is not affiliated with or endorsed by CIS or any other publisher named here. + +## Python libraries + +The Python libraries cloudg depends on (NetworkX, rdflib, pydantic, Click, Rich and the rest) are listed in [`pyproject.toml`](https://github.com/morpheuslord/cloudg/blob/main/pyproject.toml). + +## License + +cloudg itself is released under the [MIT License](https://github.com/morpheuslord/cloudg/blob/main/LICENSE). report.html embeds unmodified builds of Chart.js and D3 when `report.inline_js` is true (the default), so the report works offline. Their notices are in [`cloudg/templates/vendor/NOTICE`](https://github.com/morpheuslord/cloudg/blob/main/cloudg/templates/vendor/NOTICE). diff --git a/site/content/examples/errors.md b/site/content/examples/errors.md new file mode 100644 index 0000000..4a1bbc3 --- /dev/null +++ b/site/content/examples/errors.md @@ -0,0 +1,126 @@ +--- +title: Errors and strict mode +description: Get exceptions from the cloudg scanner wrappers instead of recorded failures, check credentials before a run, stop a pipeline at the first failed phase and reject unreadable reports. +--- + +cloudg is built to keep going: a scanner that times out, a service that refuses a call or a report that will not parse is recorded, and the rest of the run carries on. These scripts show how to turn those records into exceptions when your job should stop instead. +{ .lede } + +Every exception cloudg raises on purpose derives from `CloudGError`. [Exceptions](../api/exceptions.md) has the class tree and when each one is raised. + +## A scanner that raises + +**Offline.** With `strict=True` the first failure is raised at once. Without Checkov on `PATH`, that is a `ScannerNotInstalledError`. + +=== "Python" + + ```python title="strict_scanner.py" + --8<-- "errors/strict_scanner.py" + ``` + +=== "Output" + + ```console + $ python strict_scanner.py + checkov is missing. pip install checkov + ``` + +## Decide after the run + +`raise_for_errors()` raises what the last run recorded: one failure as it is, several as a `ScannerFailed` that lists each of them. Nothing recorded, nothing raised. + +=== "Python" + + ```python title="raise_for_errors.py" + --8<-- "errors/raise_for_errors.py" + ``` + +=== "Output" + + With a `trivy` on `PATH` that prints something other than JSON: + + ```console + $ python raise_for_errors.py + 0 findings, 2 failures recorded + - ScannerOutputError output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + - ScannerOutputError output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +## Check credentials before a run + +`CloudGEngine.collect()` records failures per service instead of raising. Build the session yourself first and a missing SDK or a refused role stops the job before anything runs. + +=== "Python" + + ```python title="engine_errors.py" + --8<-- "errors/engine_errors.py" + ``` + +=== "Output" + + ```console + $ pip uninstall -y boto3 + $ python engine_errors.py + boto3 is not installed: pip install 'cloudg[aws]' + ``` + +=== "CLI" + + On the CLI, check the exit status instead. `cloudg run` exits 3 when collection failed for every target, and 0 with a warning when only some failed: + + ```bash + cloudg run -p aws --profile audit --regions eu-west-1 + echo $? + ``` + +## Stop at the first failed phase + +Drive the phases yourself and check the `on_error` records between them. Exit status 2 means a phase failed, 0 a complete run. + +```python title="fail_fast.py" +--8<-- "errors/fail_fast.py" +``` + +Without credentials, every AWS service fails and the script stops after the collection with exit status 2, before any scanner starts. + +## Reject an unreadable report + +**Offline.** Each wrapper's `parse_report(path, strict=True)` raises `ReportParseError` for a file that is not a report, instead of skipping it with a warning. + +=== "Python" + + ```python title="strict_parse_report.py" + --8<-- "errors/strict_parse_report.py" + ``` + +=== "Output" + + ```console + $ echo "not json" > results_json.json + $ python strict_parse_report.py results_json.json + checkov report results_json.json is unreadable: output is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +## Related + +
      + +- [**Exceptions**](../api/exceptions.md) + + The `CloudGError` hierarchy, attribute by attribute. + +- [**Event hooks**](../api/event-hooks.md) + + `on_error` and the other callbacks. + +- [**cloudg.resilience**](../api/resilience.md) + + Circuit breakers, retry budgets and the errors they raise. + +- [**CI and automation**](../guides/ci-automation.md) + + Exit codes and gates in a pipeline. + +
      diff --git a/site/content/examples/graph-and-exports.md b/site/content/examples/graph-and-exports.md new file mode 100644 index 0000000..56c704a --- /dev/null +++ b/site/content/examples/graph-and-exports.md @@ -0,0 +1,188 @@ +--- +title: Graph, ontology and RAG +description: Build the cloudg graph from your own assets, find what the internet reaches, query an RDF ontology with SPARQL and cut text chunks for retrieval-augmented generation. +--- + +The graph, the ontology and the RAG export all start from the same two lists, `CloudAsset` and `NetworkEdge`. These scripts build a small estate by hand, so every one of them is **offline**: swap the hand-made lists for `collection.assets` and `collection.edges` from `CloudGEngine.collect()`, or for an `InventoryResult` you loaded, and nothing else changes. +{ .lede } + +```mermaid +flowchart LR + A["CloudAsset + NetworkEdge"] --> G["GraphBuilder"] + G --> P["attack paths, centrality"] + G --> R["ReachabilityAnalyzer"] + R --> F["exposure findings"] + A --> O["CloudOntology"] + O --> Q["SPARQL, Turtle, JSON-LD"] + A --> X["RAGExporter"] + X --> C["rag_chunks.jsonl"] +``` + +## Build and explore the graph + +`GraphBuilder` turns the lists into a NetworkX `DiGraph`, merges parallel rules between the same pair of nodes, and adds a placeholder node for `0.0.0.0/0`. + +=== "Python" + + ```python title="build_graph.py" + --8<-- "graph/build_graph.py" + ``` + +=== "Output" + + ```console + $ python build_graph.py + 9 nodes, 10 edges from 11 NetworkEdges + merged rule: 443,80 | HTTPS from anywhere; HTTP redirect + placeholder: EXTERNAL True + internet path: 0.0.0.0/0 -> sg-alb -> sg-web + identity path: web-1 -> app-role -> orders-db + lateral path: web-1 -> app-role + betweenness 0.080 web-1 + betweenness 0.071 web-tg + betweenness 0.054 app-role + community 0: ['0.0.0.0/0', 'sg-alb'] + community 1: ['sg-web', 'web-2'] + community 2: ['app-role', 'orders-db', 'web-1'] + community 3: ['web-alb', 'web-tg'] + wrote ['estate-cytoscape.json', 'estate-d3.json', 'estate.graphml'] + ``` + +=== "CLI" + + `cloudg run` builds the same graph from a live collection and writes it next to the reports: + + ```bash + cloudg run -p aws --regions eu-west-1 -o ./reports + ls reports/topology.graphml + ``` + + `graph.persist_graphml` (on by default) writes `topology.graphml`, and `graph.export_cytoscape` (off by default) adds `topology-cytoscape.json`. Both are in the [graph section](../reference/config/graph.md) of config.yaml. + +## What can the internet reach + +`ReachabilityAnalyzer` walks the graph from the internet along the edges traffic can actually take, and gives each finding an id that stays the same from run to run. + +=== "Python" + + ```python title="exposure.py" + --8<-- "graph/exposure.py" + ``` + +=== "Output" + + ```console + $ python exposure.py + entry points: ['0.0.0.0/0'] + exposed: ['orders-api', 'sg-alb', 'sg-web', 'web-1', 'web-alb', 'web-tg'] + hops out of sg-web: + -> web-1 via ATTACHED_TO (reversed) + HIGH Unexpected internet-exposed resource: orders-api id=b5fbe235 + HIGH Unexpected internet-exposed resource: web-1 id=4ec3a339 + CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 id=17cd5168 + web-1 exposure id: 4ec3a339 + flow edges: 10 of 14 + flow reversed: [('sg-alb', 'web-alb'), ('sg-db', 'orders-db'), ('sg-web', 'web-1')] + blast radius of web-1: ['0.0.0.0/0', 'app-role', 'orders-db', 'sg-db', 'sg-web'] depth 3 risk 4.5 + ``` + +## Query it with SPARQL + +`CloudOntology` writes the estate into an rdflib graph with typed relations, then exports it in four RDF serialisations and answers SPARQL queries. + +=== "Python" + + ```python title="ontology_demo.py" + --8<-- "graph/ontology_demo.py" + ``` + +=== "Output" + + ```console + $ python ontology_demo.py + 689 triples, 9 classes, 9 individuals + groups: {'GOVERNANCE': 2, 'NETWORK': 3, 'SECURITY': 4} + 0.0.0.0/0 INGRESS_ALLOWED sg-web + 0.0.0.0/0 INTERNET_REACHABLE sg-web + 0.0.0.0/0 ONLY_SSH sg-web + RDS instance has no deletion protection FINDING_AFFECTS orders-db + orders-db ENCRYPTED_BY_KMS orders-key + web-1 OWNED_BY https://cloudg.io/resource/tag_payments + web-1 PROTECTED_BY_SG sg-web + web-1 TAGGED_WITH https://cloudg.io/resource/tag_owner_payments + internet reachable: ['sg-web'] + security group: 4 triples + compliance gaps: [('HIGH', 'orders-db')] + reloaded 689 triples from ['ontology.jsonld', 'ontology.nt', 'ontology.rdf', 'ontology.ttl'] + ``` + +## Chunks for retrieval-augmented generation + +`RAGExporter` cuts the estate into one chunk per asset, per community and per relation group, each with metadata you can filter on before a similarity search. + +=== "Python" + + ```python title="rag_demo.py" + --8<-- "graph/rag_demo.py" + ``` + +=== "Output" + + ```console + $ python rag_demo.py + {'chunks': 'rag_chunks.jsonl', 'index': 'rag_metadata_index.json'} + {'total_chunks': 11, 'entity_chunks': 5, 'community_chunks': 2, 'relation_group_chunks': 4} + 11 records -> rag-out/vector-records.jsonl + Resource: web-1 + Type: EC2 + Provider: AWS + Region: eu-west-1 + ARN: arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1 + Account: 123456789012 + WARNING: INTERNET EXPOSED + + Relations (2): + -> PROTECTED_BY_SG: sg-web + -> RUNS_ON: app-role + + Findings (1): + [HIGH] Instance metadata service v1 enabled + {'chunk_type': 'entity', 'asset_type': 'EC2', 'provider': 'AWS', 'region': 'eu-west-1', 'account_id': '123456789012', 'is_internet_exposed': True, 'relation_types': 'PROTECTED_BY_SG,RUNS_ON', 'severity_max': 'HIGH', 'compliance_frameworks': 'CIS', 'neighbour_count': 2, 'finding_count': 1, 'arn': 'arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1'} + high or critical: ['entity::web-1', 'community::1'] + ``` + +=== "config.yaml" + + `cloudg run` writes the ontology and the chunks when these are on: + + ```yaml title="config.yaml" + ontology: + enabled: true + export_formats: [turtle, json-ld] + rag: + enabled: true + chunk_strategy: hybrid + max_chunk_tokens: 2000 + ``` + +## Related + +
      + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + The guide: what each export contains and how to load it elsewhere. + +- [**GraphBuilder**](../api/graphbuilder.md) + + Every method, from `build()` to `save_graphml()`. + +- [**CloudOntology**](../api/cloudontology.md) + + The relations, the query helpers and the serialisations. + +- [**RAGExporter**](../api/ragexporter.md) + + Chunk types, metadata fields and size limits. + +
      diff --git a/site/content/examples/index.md b/site/content/examples/index.md new file mode 100644 index 0000000..9c2fdef --- /dev/null +++ b/site/content/examples/index.md @@ -0,0 +1,55 @@ +--- +title: Examples +description: Complete cloudg scripts for mapping, ingesting, graph exports and error handling, each shown next to the CLI command that does the same and the output it prints. +--- + +Complete scripts you can save and run, each next to the CLI command that does the same job and the output it printed when these pages were written. They are the files in [`site/examples/`](https://github.com/morpheuslord/cloudg/tree/main/site/examples), included into the pages as they are. +{ .lede } + +Most of them need no cloud account. Those marked **offline** build a small estate by hand or read sample scanner reports, so they run on any machine with `pip install cloudg`. The others say which credentials they need. + +## Sample reports to work with + +The ingest examples read native scanner output. If you have none at hand, this script writes one small report per scanner (Prowler ASFF, ScoutSuite, Checkov and two Trivy reports), shaped like the real tools' output: + +??? example "make_sample_reports.py" + + ```python title="make_sample_reports.py" + --8<-- "make_sample_reports.py" + ``` + +Run it in an empty directory, then run the examples from the same directory. + +## By task + +
      + +- :material-map-search-outline:{ .lg .middle } **[Map and query](map-and-query.md)** + + --- + + Map an account, save the map, load it again and ask what depends on what. + +- :material-file-import-outline:{ .lg .middle } **[Ingest and report](ingest-and-report.md)** + + --- + + Merge Prowler, Checkov and Trivy output, gate a CI job on severity and export the findings. + +- :material-graph-outline:{ .lg .middle } **[Graph, ontology and RAG](graph-and-exports.md)** + + --- + + Build the graph from your own assets, find what the internet reaches, query it with SPARQL, cut RAG chunks. + +- :material-alert-octagon-outline:{ .lg .middle } **[Errors and strict mode](errors.md)** + + --- + + Strict scanner wrappers, `raise_for_errors()`, credential checks and strict report parsing. + +
      + +## More examples + +Every page of the [Python API](../api/index.md) reference has its own examples above the generated reference, and [Python recipes](../guides/python-recipes.md) collects the longer integrations: streaming findings to a SIEM, combining a live inventory with ingested findings, and linking assets from a CMDB. diff --git a/site/content/examples/ingest-and-report.md b/site/content/examples/ingest-and-report.md new file mode 100644 index 0000000..f521012 --- /dev/null +++ b/site/content/examples/ingest-and-report.md @@ -0,0 +1,209 @@ +--- +title: Ingest and report +description: Merge Prowler, Checkov and Trivy output into one deduplicated result, render the reports, gate a CI job on severity and export the findings. +--- + +Already ran the scanners somewhere else? Feed cloudg their native output. No cloud credentials and no scanner binaries are needed, and the core package is enough. +{ .lede } + +Every script on this page is **offline**. Run [`make_sample_reports.py`](index.md#sample-reports-to-work-with) in an empty directory first, then copy its Trivy image report to the name the scripts read: + +```bash +python make_sample_reports.py +cp trivy-image.json trivy.json +``` + +## Merge reports and render them + +=== "CLI" + + ```bash + cloudg ingest --prowler ./prowler-output/ --checkov ./results_json.json --trivy ./trivy.json -o ./reports + ``` + +=== "Python" + + ```python title="ingest_only.py" + --8<-- "ingest/ingest_only.py" + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD:/app/input:ro" \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + ingest --prowler /app/input/prowler-output/ --checkov /app/input/results_json.json --trivy /app/input/trivy.json + ``` + +=== "Output" + + ```console + $ python ingest_only.py + phase: ingest + phase: normalisation + phase: reporting + 3 {'CRITICAL': 1, 'HIGH': 2} + {'json': PosixPath('reports/findings.json'), 'html': PosixPath('reports/report.html')} + CRITICAL 9.7 trivy [Trivy] CVE-2024-0001: openssl (alpine) + HIGH 7.5 prowler S3 bucket default encryption + HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled + ``` + +`reports/report.html` is the page shown in the [sample report](../sample-report.md), filled with these three findings. + +## Parse first, decide later + +`parse_report()` reads one report into `Finding` objects without normalising anything, so you can filter or enrich them before the merge. `ingest_reports()` does several at once and skips files it cannot read. + +=== "Python" + + ```python title="parse_first.py" + --8<-- "ingest/parse_first.py" + ``` + +=== "Output" + + ```console + $ python parse_first.py + INFO cloudg.scanners.prowler: Parsed 1 findings from Prowler (1 passed checks) + INFO cloudg.scanners.checkov: Parsed 1 findings from Checkov + INFO cloudg.scanners.trivy: Parsed 1 findings from Trivy for myrepo/app:latest + INFO cloudg.ingest: [trivy] 1 findings ingested from ./trivy.json + ERROR cloudg.ingest: Skipping trivy report ./missing.json: Report path does not exist: ./missing.json + ERROR cloudg.ingest: Skipping trivvy report ./trivy.json: Unsupported tool 'trivvy'. Supported: prowler, scoutsuite, checkov, trivy + INFO cloudg.normaliser: Normalising 3 total findings from 2 sources + INFO cloudg.normaliser: After deduplication: 3 findings + ('prowler', 'scoutsuite', 'checkov', 'trivy') + 2 findings from two reports + 1 findings from the batch + ValueError: Unsupported tool 'grype'. Supported: prowler, scoutsuite, checkov, trivy + {'CRITICAL': 1, 'HIGH': 2} 31 control results + ``` + +The unknown tool raises `ConfigurationError`, which is a `ValueError`, so the `except ValueError` in the script catches it. [Exceptions](../api/exceptions.md) has the strict variant that raises on an unreadable file as well. + +## Merge findings from your own tools + +`FindingsNormaliser` takes lists of `Finding` objects from any source. Findings from two scanners about the same check on the same resource are merged into one, with both tools listed. + +=== "Python" + + ```python title="normalise_demo.py" + --8<-- "ingest/normalise_demo.py" + ``` + +=== "Output" + + ```console + $ python normalise_demo.py + CRITICAL 9.7 trivy openssl: buffer overflow ['HIPAA', 'NIST-800-53', 'PCI-DSS', 'SOC2'] + HIGH 7.5 trivy, prowler [Trivy] S3 bucket default encryption ['GDPR-AWS', 'HIPAA-AWS', 'ISO27001-AWS', 'MITRE-ATTACK-AWS'] + LOW 2.5 custom CloudTrail logging disabled in eu-west-1 ['GDPR', 'HIPAA', 'ISO-27001', 'NIST-800-53'] + 38 compliance results + ``` + +## Gate a CI job on severity + +Write a summary the pipeline can archive, and exit non-zero when anything critical or high turns up. + +=== "Python" + + ```python title="save_result.py" + --8<-- "ingest/save_result.py" + ``` + +=== "Output" + + ```console + $ python save_result.py + { + "total_assets": 0, + "total_findings": 2, + "severity_breakdown": { + "CRITICAL": 1, + "HIGH": 1 + }, + "providers_scanned": [], + "regions_scanned": {}, + "graph_nodes": 0, + "graph_edges": 0, + "ontology_triples": 0, + "attack_paths_count": 0, + "duration_ms": 1219, + "errors": [] + } + 2 critical/high findings, failing the build + $ echo $? + 1 + ``` + +=== "GitHub Actions" + + ```yaml title=".github/workflows/scan-gate.yml" + jobs: + gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install cloudg + # results_json.json and trivy.json come from earlier steps or artifacts + - run: python save_result.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: cloudg-reports + path: reports/ + ``` + +[CI and automation](../guides/ci-automation.md) has complete workflows for GitHub Actions, GitLab and scheduled scans. + +## Re-render a report, or export the raw findings + +=== "CLI" + + ```bash + cloudg report -i ./reports/findings.json -o ./rendered --format all + cloudg report -i ./reports/raw-findings.json -o ./rendered + ``` + +=== "Python" + + ```python title="export_findings.py" + --8<-- "ingest/export_findings.py" + ``` + +=== "Output" + + ```console + $ python export_findings.py + wrote 3 findings to reports/raw-findings.json + ``` + +`cloudg report` reads either file: a `findings.json` keeps its scan ID, compliance results and edges, and a `raw-findings.json` is normalised first. + +## Related + +
      + +- [**Ingesting existing output**](../guides/ingesting.md) + + What each scanner's output must look like, and how deduplication works. + +- [**parse_report**](../api/parse-report.md) + + `parse_report()` and `ingest_reports()` in detail. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + Merging, CVSS rescoring and compliance mapping. + +- [**cloudg ingest**](../cli/ingest.md) + + Every flag of the command. + +
      diff --git a/site/content/examples/map-and-query.md b/site/content/examples/map-and-query.md new file mode 100644 index 0000000..34bdeed --- /dev/null +++ b/site/content/examples/map-and-query.md @@ -0,0 +1,162 @@ +--- +title: Map and query +description: Map an AWS account into an inventory, save it, load it again, ask what depends on what and overlay scanner findings later. +--- + +Map an account once, then ask questions of the saved map as often as you like: what faces the internet, what depends on a key, what breaks if a queue goes away. +{ .lede } + +```mermaid +flowchart LR + M["cloudg map / map_inventory()"] --> F["./inventory/inventory-map.json"] + F --> L["InventoryResult.load()"] + L --> Q["summary, exposure"] + L --> D["DependencyGraph: depends_on, blast_radius"] + L --> O["export_merged(): asset-map.json, compliance-map.json"] + S["scanner findings, any time later"] --> O +``` + +## Map an account + +Needs read-only AWS credentials and `pip install "cloudg[aws]"`. No scanner runs. + +=== "CLI" + + ```bash + cloudg map -p aws --regions all --no-sweep -o ./inventory + ``` + +=== "Python" + + ```python title="map_only.py" + --8<-- "map/map_only.py" + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/inventory:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --regions all --no-sweep + ``` + +`tagging_sweep=False` and `--no-sweep` both skip the Resource Groups Tagging API sweep. [InventoryMapper](../api/inventorymapper.md) has the other switches and [Inventory mapping](../guides/inventory-mapping.md) explains what each layer collects. + +## Save, load and summarise a map + +**Offline.** An `InventoryResult` built from hand-made assets, exported to disk and loaded back. The same `load()` reads a map that `cloudg map` wrote. + +=== "Python" + + ```python title="inventory_roundtrip.py" + --8<-- "map/inventory_roundtrip.py" + ``` + +=== "Output" + + ```console + $ python inventory_roundtrip.py + ['dependencies', 'graph', 'graphml', 'map'] + 5 6 {'INVOKES': 1, 'REFERENCES': 4, 'ASSUMES_ROLE': 1} + external accounts: 1 cross-account edges: 1 + ['shared_dependencies', 'largest_blast_radius', 'cross_account_edges', 'security_coverage', 'unresolved_references'] + [('orders-key', 3)] + [{'source': 'arn:aws:lambda:eu-west-1:123456789012:function:orders-worker', 'source_account': '123456789012', 'target': 'arn:aws:iam::999999999999:root', 'target_account': '999999999999', 'edge_type': 'ASSUMES_ROLE', 'relationship': None, 'external': True}] + ``` + +Then read the saved map without calling the cloud: + +```python title="whats_exposed.py" +--8<-- "map/whats_exposed.py" +``` + +## What breaks if this goes away + +**Offline.** The script builds its assets by hand. `DependencyGraph` answers "what does this need" and "what needs this", with the edge each step follows. + +=== "CLI" + + Against the map `inventory_roundtrip.py` saved in `./inventory`: + + ```bash + cloudg deps orders-key --direction down --map ./inventory + cloudg deps orders-worker --direction up --map ./inventory + ``` + +=== "Python" + + ```python title="blast_radius.py" + --8<-- "map/blast_radius.py" + ``` + +=== "Output" + + ```console + $ python blast_radius.py + If orders-key is disabled: + depth 1: orders-inbound (MESSAGE_QUEUE) needs orders-key via REFERENCES + depth 1: orders (DYNAMODB_TABLE) needs orders-key via REFERENCES + depth 2: orders-worker (LAMBDA_FUNCTION) needs orders-inbound via INVOKES + orders-worker needs: ['orders-worker-role', 'orders', 'orders-inbound', 'orders-api', 'orders-key'] + direct (needs, needed by): (4, 0) + orders-key 3 dependents, 0 internet-exposed + orders-api 1 dependents, 0 internet-exposed + orders 1 dependents, 0 internet-exposed + ``` + +[Dependencies and blast radius](../guides/dependencies.md) explains how direction works and how to read the tables `cloudg deps` prints. + +## Overlay findings on a saved map + +Scanner findings can arrive days after the map. Load both and write the overlay next to the map. The script reads the map in `./inventory` (from `inventory_roundtrip.py` above, or `cloudg map -o ./inventory`) and the sample reports: run [`make_sample_reports.py`](index.md#sample-reports-to-work-with) in the same directory first, and copy its `trivy-image.json` to `trivy.json`. + +=== "CLI" + + ```bash + cloudg map -p aws --regions all -o ./inventory --findings ./reports/raw-findings.json + ``` + + On the CLI the flag belongs to `cloudg map`, so it maps again. The Python call reuses the saved map. + +=== "Python" + + ```python title="overlay_findings.py" + --8<-- "map/overlay_findings.py" + ``` + +=== "Output" + + ```console + $ python overlay_findings.py + {'asset_map': 'inventory/asset-map.json', 'compliance_map': 'inventory/compliance-map.json'} + CIS 1 ['arn:aws:s3:::my-bucket'] + CVE 1 ['myrepo/app:latest'] + GDPR-AWS 1 ['arn:aws:s3:::my-bucket'] + 5 assets, 0 with findings + ``` + +The sample findings name resources that are not on this map, so no asset picks one up, but the compliance map still lists every affected resource. Against a map of the account the scanners looked at, each asset in `asset-map.json` carries its own findings. + +## Related + +
      + +- [**InventoryResult**](../api/inventoryresult.md) + + Every field and method of the saved map. + +- [**DependencyGraph**](../api/dependencygraph.md) + + `depends_on`, `dependents`, `tree`, `blast_radius` and the asset matcher. + +- [**cloudg deps**](../cli/deps.md) + + The same questions from the command line. + +- [**Inventory structures**](../reference/inventory/index.md) + + Every file the map writes, field by field. + +
      diff --git a/site/content/guides/authentication.md b/site/content/guides/authentication.md index faf849a..5e98686 100644 --- a/site/content/guides/authentication.md +++ b/site/content/guides/authentication.md @@ -1,25 +1,25 @@ --- title: Authentication -lede: "Every provider has several ways in, tried in a fixed order, so one `config.yaml` works on a laptop, in CI and on cloud compute." -meta: - - [Code, "`cloudg/credentials.py`"] - - [Providers, "AWS, Azure, GCP"] - - [Flags, "[Auth flags](/cli/auth-flags/)"] -source: cloudg/credentials.py -since: "0.6.0" +description: Every provider has several ways in, tried in a fixed order, so one config.yaml works on a laptop, in CI and on cloud compute. --- +Every provider has several ways in, tried in a fixed order, so one `config.yaml` works on a laptop, in CI and on cloud compute. +{ .lede } + +**Code** `cloudg/credentials.py` · **Providers** AWS, Azure, GCP · **Flags** [Auth flags](../cli/auth-flags.md) · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + All credential handling lives in three functions in `cloudg/credentials.py`: `build_aws_session`, `build_azure_credential` and `build_gcp_credentials`. `cloudg run`, `cloudg map`, `CloudGEngine`, `InventoryMapper` and the credential preflight of the MCP live tools all call them, so the precedence on this page holds everywhere. Each function walks its list from the top and stops at the first method whose settings are complete. It never tries the next method after a failure, which matters when you debug: if direct keys are set but wrong, cloudg will not fall back to your profile. You can supply each setting three ways. CLI flags (on `cloudg run`) are copied onto the loaded config before anything resolves, `config.yaml` holds the same keys under `aws`, `azure` and `gcp`, and a handful of environment variables are read directly. The tabs below show all three where they exist. -:::note Which commands take which flags -`cloudg run` has the full set of credential flags listed under [auth flags](/cli/auth-flags/). `cloudg map` has only `--profile`, `--accounts`, `--role-name` and `--org-role`; put everything else in `config.yaml` and pass it with `cloudg -c config.yaml map ...`. `cloudg collect` is the exception: it builds credentials from `--profile` (or `AWS_PROFILE`) and the environment only, and ignores the auth keys in `config.yaml`. -::: +!!! note "Which commands take which flags" + + `cloudg run` has the full set of credential flags listed under [auth flags](../cli/auth-flags.md). `cloudg map` has only `--profile`, `--accounts`, `--role-name` and `--org-role`; put everything else in `config.yaml` and pass it with `cloudg -c config.yaml map ...`. `cloudg collect` is the exception: it builds credentials from `--profile` (or `AWS_PROFILE`) and the environment only, and ignores the auth keys in `config.yaml`. ## AWS -```mermaid caption="How build_aws_session picks the base credentials, then the optional AssumeRole hop" +```mermaid flowchart TD A["access_key_id and secret_access_key set?"] -->|yes| K[Direct keys] A -->|no| B["role_arn and a web identity token file?"] @@ -38,6 +38,8 @@ flowchart TD M -->|no| Done[Session ready] T --> Done ``` +How build_aws_session picks the base credentials, then the optional AssumeRole hop +{ .diagram-caption } | Order | Method | Settings that select it | |---|---|---| @@ -47,33 +49,39 @@ flowchart TD | 4 | Default provider chain | nothing set: env vars, shared config, container credentials, instance role | | then | STS AssumeRole | `role_arn` (when not already used by step 2), then, for each account in `accounts`, `role_name` in that account | -Whatever the base method, the session is instrumented with cloudg's rate limiter before it is used, so STS, Organizations and every collector call are paced. See [rate limits](/guides/rate-limits/). +Whatever the base method, the session is instrumented with cloudg's rate limiter before it is used, so STS, Organizations and every collector call are paced. See [rate limits](rate-limits.md). ### Direct keys and session tokens The third-party auditor case: someone hands you an access key pair, or a temporary key pair with a session token. Keys win over everything else, including a profile set in the same config. -```bash tab="CLI" -cloudg run -p aws --regions us-east-1 \ - --aws-key AKIAIOSFODNN7EXAMPLE \ - --aws-secret "$AUDIT_SECRET" \ - --aws-session-token "$AUDIT_SESSION_TOKEN" -``` +=== "CLI" -```yaml tab="config.yaml" title="config.yaml" -aws: - regions: [us-east-1] - access_key_id: AKIAIOSFODNN7EXAMPLE - secret_access_key: REPLACE_ME # keep this file out of version control - session_token: null # only for temporary credentials -``` + ```bash + cloudg run -p aws --regions us-east-1 \ + --aws-key AKIAIOSFODNN7EXAMPLE \ + --aws-secret "$AUDIT_SECRET" \ + --aws-session-token "$AUDIT_SESSION_TOKEN" + ``` -```bash tab="Environment" -export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -export AWS_SECRET_ACCESS_KEY="$AUDIT_SECRET" -export AWS_SESSION_TOKEN="$AUDIT_SESSION_TOKEN" -cloudg run -p aws --regions us-east-1 -``` +=== "config.yaml" + + ```yaml title="config.yaml" + aws: + regions: [us-east-1] + access_key_id: AKIAIOSFODNN7EXAMPLE + secret_access_key: REPLACE_ME # keep this file out of version control + session_token: null # only for temporary credentials + ``` + +=== "Environment" + + ```bash + export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE + export AWS_SECRET_ACCESS_KEY="$AUDIT_SECRET" + export AWS_SESSION_TOKEN="$AUDIT_SESSION_TOKEN" + cloudg run -p aws --regions us-east-1 + ``` The environment tab works through a different door. cloudg does not copy `AWS_ACCESS_KEY_ID` into its config, so resolution falls through to step 4 and boto3's default chain picks the variables up. The effect is the same, and it is the variant to prefer: nothing secret lands in a file or in your shell history. @@ -81,31 +89,37 @@ The environment tab works through a different door. cloudg does not copy `AWS_AC For GitHub Actions, GitLab CI and EKS service accounts: no long-lived keys, only a short-lived token file exchanged for role credentials with `AssumeRoleWithWebIdentity`. cloudg reads the token file itself and names the session after `role_session_name` (default `cloudg-scan`). -```bash tab="CLI" -cloudg run -p aws --regions all \ - --aws-role-arn arn:aws:iam::123456789012:role/cloudg-readonly \ - --aws-web-identity-token-file /var/run/secrets/oidc/token -``` +=== "CLI" -```yaml tab="config.yaml" title="config.yaml" -aws: - role_arn: arn:aws:iam::123456789012:role/cloudg-readonly - web_identity_token_file: /var/run/secrets/oidc/token - role_session_name: cloudg-ci -``` + ```bash + cloudg run -p aws --regions all \ + --aws-role-arn arn:aws:iam::123456789012:role/cloudg-readonly \ + --aws-web-identity-token-file /var/run/secrets/oidc/token + ``` -```bash tab="Environment" -# Read by boto3's default chain (step 4), not by cloudg's step 2 -export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/cloudg-readonly -export AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/oidc/token -cloudg run -p aws --regions all -``` +=== "config.yaml" + + ```yaml title="config.yaml" + aws: + role_arn: arn:aws:iam::123456789012:role/cloudg-readonly + web_identity_token_file: /var/run/secrets/oidc/token + role_session_name: cloudg-ci + ``` + +=== "Environment" + + ```bash + # Read by boto3's default chain (step 4), not by cloudg's step 2 + export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/cloudg-readonly + export AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/oidc/token + cloudg run -p aws --regions all + ``` When step 2 succeeds, `role_arn` has been used up as the OIDC target and no second AssumeRole follows. A cross-account hop after federation needs `accounts` plus `role_name` instead. -:::warning The token file variable on EKS -cloudg also reads `$AWS_WEB_IDENTITY_TOKEN_FILE` from the environment, and EKS (IRSA) sets it in every pod. On such a pod, a `role_arn` meant as a second hop becomes the OIDC target instead, and the exchange fails unless that role trusts the cluster's OIDC provider. Either let the pod's own role do the work (no `role_arn`), or reach other accounts with `accounts` plus `role_name`. -::: +!!! warning "The token file variable on EKS" + + cloudg also reads `$AWS_WEB_IDENTITY_TOKEN_FILE` from the environment, and EKS (IRSA) sets it in every pod. On such a pod, a `role_arn` meant as a second hop becomes the OIDC target instead, and the exchange fails unless that role trusts the cluster's OIDC provider. Either let the pod's own role do the work (no `role_arn`), or reach other accounts with `accounts` plus `role_name`. A failed exchange raises `RuntimeError: AssumeRoleWithWebIdentity failed for : ...`. The usual causes are a trust policy whose `sub` condition does not match the branch or environment, or a token minted for the wrong audience (AWS expects `sts.amazonaws.com`). @@ -113,26 +127,32 @@ A failed exchange raises `RuntimeError: AssumeRoleWithWebIdentity failed for `, `AWS auth: default provider chain`, `AWS auth: assuming role `, `Azure auth: workload identity federation (client ...)`, `Azure auth: DefaultAzureCredential chain`, `GCP auth: file-based identity (...)`, `GCP auth: application default credentials` and `GCP auth: impersonating `. -:::links -- [Auth flags](/cli/auth-flags/) Every credential flag of `cloudg run`. -- [AWS Organizations](/guides/aws-organizations/) Mapping every account from the management account. -- [CI and automation](/guides/ci-automation/) OIDC from GitHub Actions and GitLab CI. -- [Docker](/guides/docker/) Mounting credentials into the container. -::: +
      + +- [**Auth flags**](../cli/auth-flags.md) + + Every credential flag of `cloudg run`. + +- [**AWS Organizations**](aws-organizations.md) + + Mapping every account from the management account. + +- [**CI and automation**](ci-automation.md) + + OIDC from GitHub Actions and GitLab CI. + +- [**Docker**](docker.md) + + Mounting credentials into the container. + +
      diff --git a/site/content/guides/aws-organizations.md b/site/content/guides/aws-organizations.md index 150ae0e..f8bdc93 100644 --- a/site/content/guides/aws-organizations.md +++ b/site/content/guides/aws-organizations.md @@ -1,19 +1,21 @@ --- title: AWS Organizations -lede: "With `--org`, `cloudg map` starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them." -meta: - - [Command, "`cloudg map --org`"] - - [Runs from, "Management account or delegated administrator"] -source: cloudg/inventory/organization.py +description: With --org, cloudg map starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them. --- -Mapping one account at a time misses what holds an organization together: which OU an account sits in, which SCPs restrict it, which Control Tower controls apply, and which roles in one account trust another. `cloudg map --org` reads the organization first, picks the accounts you asked for, assumes a role in each one, and maps all of them into a single [inventory map](/guides/inventory-mapping/). Accounts hang under their OUs, SCPs and controls point at what they govern, and cross-account trust becomes an edge you can query with [`cloudg deps`](/guides/dependencies/). +With `--org`, `cloudg map` starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them. +{ .lede } + +**Command** `cloudg map --org` · **Runs from** Management account or delegated administrator +{ .page-meta } + +Mapping one account at a time misses what holds an organization together: which OU an account sits in, which SCPs restrict it, which Control Tower controls apply, and which roles in one account trust another. `cloudg map --org` reads the organization first, picks the accounts you asked for, assumes a role in each one, and maps all of them into a single [inventory map](inventory-mapping.md). Accounts hang under their OUs, SCPs and controls point at what they govern, and cross-account trust becomes an edge you can query with [`cloudg deps`](dependencies.md). Every call cloudg makes here is read-only: `Describe*`, `List*` and `Get*` against Organizations and Control Tower, then the usual read calls inside each member account. ## What happens during discovery -```mermaid caption="Organization discovery, then one role assumption per member account and region" +```mermaid sequenceDiagram participant C as cloudg participant S as STS @@ -33,6 +35,8 @@ sequenceDiagram C->>M: Deep collectors (read-only) end ``` +Organization discovery, then one role assumption per member account and region +{ .diagram-caption } In order: @@ -80,13 +84,14 @@ The caller's own account is mapped too, with the caller's credentials and not th cloudg assumes one role, by name, in every member account. The default is `AWSControlTowerExecution`, because Control Tower creates it in every account it enrolls. -:::warning AWSControlTowerExecution has admin rights -`AWSControlTowerExecution` carries administrator permissions. cloudg only reads, but anything that can run cloudg with management-account credentials could do more with that role. The read-only role Control Tower creates, `aws-controltower-ReadOnlyExecutionRole`, can't be used instead: it only trusts a role in the audit account. For production, deploy your own read-only role to every account and pass it with `--org-role`. -::: +!!! warning "AWSControlTowerExecution has admin rights" + + `AWSControlTowerExecution` carries administrator permissions. cloudg only reads, but anything that can run cloudg with management-account credentials could do more with that role. The read-only role Control Tower creates, `aws-controltower-ReadOnlyExecutionRole`, can't be used instead: it only trusts a role in the audit account. For production, deploy your own read-only role to every account and pass it with `--org-role`. A service-managed CloudFormation StackSet deploys the role to every account in the OUs you target, and to accounts that join those OUs later. -::::steps +
      + ### Save the template ```yaml title="cloudg-readonly-role.yaml" @@ -157,48 +162,55 @@ aws cloudformation list-stack-instances --stack-set-name cloudg-readonly ``` Service-managed StackSets never deploy to the management account itself. cloudg doesn't need them to, since it maps the management account with the caller's own credentials. -:::: -To map Kubernetes workloads inside EKS clusters as well, each cluster needs an access entry for `cloudg-readonly` with the `AmazonEKSViewPolicy`. The [inventory mapping guide](/guides/inventory-mapping/#kubernetes-inside-eks) has the two `aws eks` commands. +
      + +To map Kubernetes workloads inside EKS clusters as well, each cluster needs an access entry for `cloudg-readonly` with the `AmazonEKSViewPolicy`. The [inventory mapping guide](inventory-mapping.md#kubernetes-inside-eks) has the two `aws eks` commands. ## Map the organization -```bash tab="CLI" -cloudg map -p aws --profile management --org --org-role cloudg-readonly --regions all -``` +=== "CLI" -```python tab="Python" title="map_org.py" -from cloudg import CloudGConfig -from cloudg.inventory import InventoryMapper - -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "management" -config.aws.regions = ["ALL"] - -org = config.aws.organization -org.enabled = True -org.role_name = "cloudg-readonly" -org.include_ous = ["Workloads"] -org.exclude_accounts = ["111122223333"] - -mapper = InventoryMapper(config) -inventory = mapper.map_inventory_sync() -paths = inventory.export("./reports") - -topology = mapper.organization # None when discovery failed -if topology is not None: - print(topology.organization_id, len(topology.accounts), "accounts") - print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) -print(paths.get("organization")) -``` + ```bash + cloudg map -p aws --profile management --org --org-role cloudg-readonly --regions all + ``` -```bash tab="Docker" -docker run --rm \ - -v ~/.aws:/home/cloudg/.aws:ro \ - -v "$PWD/reports:/app/reports" \ - ghcr.io/morpheuslord/cloudg:latest \ - map -p aws --profile management --org --org-role cloudg-readonly --regions all -``` +=== "Python" + + ```python title="map_org.py" + from cloudg import CloudGConfig + from cloudg.inventory import InventoryMapper + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "management" + config.aws.regions = ["ALL"] + + org = config.aws.organization + org.enabled = True + org.role_name = "cloudg-readonly" + org.include_ous = ["Workloads"] + org.exclude_accounts = ["111122223333"] + + mapper = InventoryMapper(config) + inventory = mapper.map_inventory_sync() + paths = inventory.export("./reports") + + topology = mapper.organization # None when discovery failed + if topology is not None: + print(topology.organization_id, len(topology.accounts), "accounts") + print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) + print(paths.get("organization")) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile management --org --org-role cloudg-readonly --regions all + ``` With `--org`, the CLI's configuration panel shows "Organization: discover all accounts" (plus the OU filter, if any) and the member role it will use. After the run an "Organization" table lists the organization ID, the number of accounts and OUs, whether Control Tower was found and the governed regions, ahead of the usual inventory summary. @@ -385,9 +397,22 @@ It raises `RuntimeError` when the caller can't describe or list the organization | Far fewer accounts than expected | an `--ou` value matched a different OU of the same name | use OU IDs | | Fewer regions than `--regions all` would give | Control Tower governed regions replaced `all` | set `use_governed_regions: false`, or pass the regions explicitly | -:::links -- [Inventory mapping](/guides/inventory-mapping/) What a mapping run collects in each account. -- [Dependencies and blast radius](/guides/dependencies/) Query cross-account edges on the finished map. -- [Authentication](/guides/authentication/) Profiles, OIDC and role assumption for the base credentials. -- [cloudg map](/cli/map/) Every option of the command. -::: +
      + +- [**Inventory mapping**](inventory-mapping.md) + + What a mapping run collects in each account. + +- [**Dependencies and blast radius**](dependencies.md) + + Query cross-account edges on the finished map. + +- [**Authentication**](authentication.md) + + Profiles, OIDC and role assumption for the base credentials. + +- [**cloudg map**](../cli/map.md) + + Every option of the command. + +
      diff --git a/site/content/guides/ci-automation.md b/site/content/guides/ci-automation.md index 037d310..080117a 100644 --- a/site/content/guides/ci-automation.md +++ b/site/content/guides/ci-automation.md @@ -1,23 +1,43 @@ --- title: CI and automation -lede: "Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts." -meta: - - [Commands, "`cloudg run`, `cloudg map`, `cloudg ingest`"] - - [CI systems, "GitHub Actions, GitLab CI"] -source: .github/workflows/cloudg.yml -since: "0.6.0" +description: Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts. --- +Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts. +{ .lede } + +**Commands** `cloudg run`, `cloudg map`, `cloudg ingest` · **CI systems** GitHub Actions, GitLab CI · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + There are three useful shapes for cloudg in a pipeline, and most teams end up with two of them: -:::cells +
      + +
      + ### Scheduled posture run + `cloudg run` against the live account on a cron, with OIDC credentials, Prowler and the IAM linter. Slow, thorough, read-only. + +
      + +
      + ### Pull request gate + Checkov and Trivy over the IaC in the branch, aggregated with `cloudg ingest`. No cloud access at all, so it is safe for forks. + +
      + +
      + ### Inventory snapshot + `cloudg map` on a cron. No scanners, just the map, kept as an artifact for diffing and for the MCP server. -::: + +
      + +
      Before wiring any of them up, know what cloudg's exit code does and does not tell you. @@ -39,7 +59,7 @@ cloudg exits non-zero when it cannot do its job, never because of what it found. ## A severity gate -Every run writes `findings.json` (see [reports](/guides/reports/)). Its `findings` list holds the normalised, deduplicated findings, each with `severity`, `risk_score`, `source_tool`, `is_suppressed` and the affected resource, and `summary` holds the totals. Save this as `ci/severity_gate.py` in the repository that runs the job: +Every run writes `findings.json` (see [reports](reports.md)). Its `findings` list holds the normalised, deduplicated findings, each with `severity`, `risk_score`, `source_tool`, `is_suppressed` and the affected resource, and `summary` holds the totals. Save this as `ci/severity_gate.py` in the repository that runs the job: ```python title="ci/severity_gate.py" """Fail a CI job when a cloudg findings.json crosses a severity threshold. @@ -57,7 +77,6 @@ from collections import Counter ORDER = ["INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"] - def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("findings", help="findings.json written by cloudg run or cloudg ingest") @@ -96,7 +115,6 @@ def main() -> int: print("gate: passed") return 0 - if __name__ == "__main__": sys.exit(main()) ``` @@ -119,13 +137,14 @@ Use `--min-assets 1` (or a realistic floor for your account) on live runs, and l jq -e '[.findings[] | select(.severity == "CRITICAL" and (.is_suppressed | not))] | length == 0' reports/findings.json ``` -When the gate should run inside Python, for example in a job that already uses the engine, see the gate recipe in [Python recipes](/guides/python-recipes/). +When the gate should run inside Python, for example in a job that already uses the engine, see the gate recipe in [Python recipes](python-recipes.md). ## GitHub Actions with OIDC to AWS No access keys in repository secrets. GitHub mints an OIDC token per job, AWS exchanges it for a role session, and the role only trusts this repository. -::::steps +
      + ### Create the role In the target account, add GitHub as an identity provider (`token.actions.githubusercontent.com`, audience `sts.amazonaws.com`) once, then create a role with `SecurityAudit` and `ViewOnlyAccess` attached and this trust policy. Replace the account ID and repository: @@ -151,9 +170,10 @@ In the target account, add GitHub as an identity provider (`token.actions.github } ``` -:::warning Scope the subject -A `sub` of `repo:acme/cloud-posture:*` lets every branch and every pull request from a branch assume the role. Pin it to `main` or to a deployment environment (`repo:acme/cloud-posture:environment:cloud-audit`). -::: +!!! warning "Scope the subject" + + A `sub` of `repo:acme/cloud-posture:*` lets every branch and every pull request from a branch assume the role. Pin it to `main` or to a deployment environment (`repo:acme/cloud-posture:environment:cloud-audit`). + ### Add the config and the gate Commit `ci/severity_gate.py` from above and a config next to it: @@ -166,6 +186,7 @@ scanners: enabled: [prowler, iam] timeout_seconds: 3600 # per scanner ``` + ### Add the workflow ```yaml title=".github/workflows/cloud-posture.yml" @@ -240,9 +261,10 @@ jobs: path: reports/ retention-days: 30 ``` -:::: -Credentials go through `configure-aws-credentials` here, which exports `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` for the whole job; cloudg's collectors and Prowler pick them up through the default chain. cloudg's own `--aws-role-arn` with `--aws-web-identity-token-file` works as well: cloudg then assumes the role itself and hands Prowler and ScoutSuite the temporary keys. The action has the advantage that every other step of the job gets the same credentials. [Authentication](/guides/authentication/#what-the-scanners-receive) has the details. +
      + +Credentials go through `configure-aws-credentials` here, which exports `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` for the whole job; cloudg's collectors and Prowler pick them up through the default chain. cloudg's own `--aws-role-arn` with `--aws-web-identity-token-file` works as well: cloudg then assumes the role itself and hands Prowler and ScoutSuite the temporary keys. The action has the advantage that every other step of the job gets the same credentials. [Authentication](authentication.md#what-the-scanners-receive) has the details. The action's session lasts one hour by default. A long organization-wide run can outlive it; raise `role-duration-seconds` (and the role's maximum session duration) or split the run per account. @@ -341,7 +363,7 @@ Do not also pass `--aws-role-arn` here. With `AWS_WEB_IDENTITY_TOKEN_FILE` set i ## Scheduled inventory snapshots -A map is cheaper than a full run and changes less often than findings. A nightly `cloudg map` gives you `inventory-map.json`, `inventory-dependencies.json` and, with `--org`, `inventory-organization.json`, ready for `cloudg deps`, for diffing, or for an [MCP server](/cli/mcp-serve/) to load. +A map is cheaper than a full run and changes less often than findings. A nightly `cloudg map` gives you `inventory-map.json`, `inventory-dependencies.json` and, with `--org`, `inventory-organization.json`, ready for `cloudg deps`, for diffing, or for an [MCP server](../cli/mcp-serve.md) to load. ```yaml title=".github/workflows/inventory.yml" name: Inventory snapshot @@ -375,7 +397,7 @@ jobs: retention-days: 90 ``` -Throttling during a map is recorded in coverage and in the `throttling` block of `inventory-map.json` rather than failing the job, so check that block when a snapshot looks thin. Several pipelines mapping the same account at once share its API quota; stagger their schedules, or lower `ratelimit.aws.max_rps` for the CI config as described in [rate limits](/guides/rate-limits/). +Throttling during a map is recorded in coverage and in the `throttling` block of `inventory-map.json` rather than failing the job, so check that block when a snapshot looks thin. Several pipelines mapping the same account at once share its API quota; stagger their schedules, or lower `ratelimit.aws.max_rps` for the CI config as described in [rate limits](rate-limits.md). ## Caching and artifacts @@ -391,11 +413,24 @@ Do not cache `reports/`. Publish it as an artifact instead. A run's useful files | `inventory-map.json` | the map from `cloudg map`, loadable with `InventoryResult.load` | | `ontology.ttl`, `rag_chunks.jsonl` | feed downstream analysis or a retrieval index | -The reports describe your estate in detail: account IDs, ARNs, open ports, IAM grants. Keep artifact retention short and the repository private, and use the Docker image or a private runner when the reports must not leave your network. See [Docker](/guides/docker/) for the image, which bundles Prowler, Checkov and Trivy. +The reports describe your estate in detail: account IDs, ARNs, open ports, IAM grants. Keep artifact retention short and the repository private, and use the Docker image or a private runner when the reports must not leave your network. See [Docker](docker.md) for the image, which bundles Prowler, Checkov and Trivy. + +
      + +- [**Authentication**](authentication.md) + + Every credential method and its precedence. + +- [**Python recipes**](python-recipes.md) + + The gate and the pipeline from Python. + +- [**Ingesting existing output**](ingesting.md) + + What `cloudg ingest` accepts. + +- [**cloudg run**](../cli/run.md) + + Every flag of the full pipeline. -:::links -- [Authentication](/guides/authentication/) Every credential method and its precedence. -- [Python recipes](/guides/python-recipes/) The gate and the pipeline from Python. -- [Ingesting existing output](/guides/ingesting/) What `cloudg ingest` accepts. -- [cloudg run](/cli/run/) Every flag of the full pipeline. -::: +
      diff --git a/site/content/guides/compliance.md b/site/content/guides/compliance.md index 8873c84..5def4e2 100644 --- a/site/content/guides/compliance.md +++ b/site/content/guides/compliance.md @@ -1,14 +1,14 @@ --- title: Compliance frameworks -lede: "cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package." -meta: - - [Rulesets, "`cloudg/rules/`"] - - [Frameworks, "28, with 4,271 controls"] - - [Config, "`rulesets.rules_dir`"] -source: cloudg/normaliser.py -since: "0.6.0" +description: cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package. --- +cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package. +{ .lede } + +**Rulesets** `cloudg/rules/` · **Frameworks** 28, with 4,271 controls · **Config** `rulesets.rules_dir` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Compliance mapping is the last step of normalisation, so it runs wherever findings are normalised: phase 4 of `cloudg run`, `cloudg ingest`, `cloudg report` on raw findings, `CloudGEngine.normalise_findings()`, `scan()` followed by normalisation, `run_pipeline()`, `run_from_reports()` and the MCP server's dataset loaders. Its output is a list of `ComplianceResult` records in `findings.json` and the Compliance tab of `report.html`. One thing to understand before reading any of that output: a control only appears when cloudg has evidence about it. A control that a finding maps to is `FAIL`. A control that only a passing Prowler check maps to is `PASS`: Prowler reports the checks it ran and passed (ASFF `Compliance.Status: PASSED`, OCSF `status_code: PASS`), and cloudg keeps their names. Every other control was not assessed and gets no result at all. Checkov, Trivy, ScoutSuite and the IAM linter report failures only, so without Prowler the output is all `FAIL`. Read it as "these controls have open findings, and these passed Prowler's checks", not as a full audit score. @@ -56,13 +56,13 @@ Loaded together, that is 28 framework names and 4,271 controls. The README's fig The merged CIS frameworks mix two benchmark versions under one name: the generated controls use CIS 5.0 numbering (`1.4`) and the hand-written ones CIS 3.0 or 2.0 numbering with a prefix (`CIS-1.4`). They don't collide, but don't expect one numbering scheme inside `CIS-AWS`. -`rules/check_equivalence.yaml` sits in the same directory but isn't a ruleset. It tells the deduplicator which checks from different scanners mean the same thing; see [Ingesting existing output](/guides/ingesting/#normalisation-and-deduplication). +`rules/check_equivalence.yaml` sits in the same directory but isn't a ruleset. It tells the deduplicator which checks from different scanners mean the same thing; see [Ingesting existing output](ingesting.md#normalisation-and-deduplication). ## How a finding gets its controls Mapping works in four tiers, most precise first. Each finding carries a list of framework names, `compliance_frameworks`, which the tiers add to. -```mermaid caption="From a finding to framework controls" +```mermaid flowchart TD F["Finding"] --> T1["Tier 1: tags set by the scanner parser"] T1 --> T2["Tier 2: exact check-ID lookup in ruleset checks"] @@ -76,6 +76,8 @@ flowchart TD G --> CR["ComplianceResult per control: FAIL"] P["Prowler passing checks"] --> PC["PASS for ruleset controls no finding fails"] ``` +From a finding to framework controls +{ .diagram-caption } ### Tier 1: what the scanner said @@ -122,7 +124,7 @@ After tagging, cloudg groups findings by framework and control ID and writes one `ComplianceStatus` defines four values: `PASS`, `FAIL`, `NOT_APPLICABLE` and `MANUAL`. The normaliser produces `FAIL` and `PASS`. The other two are there for your own code, or for results you build yourself. -A `PASS` result needs an exact match: the passing check's name has to appear in a ruleset control's `checks` list (tier 2), and no finding may map to that control. Its `control_id` and `control_title` come from the ruleset. Passing checks travel from the Prowler parser to the normaliser on the parsed list (see [FindingsNormaliser](/api/findingsnormaliser/)); `cloudg run`, `cloudg ingest`, `CloudGEngine.scan()`, `run_pipeline()`, `run_from_reports()`, `ingest_reports()`, `parse_report()` and the MCP loaders all carry them through. +A `PASS` result needs an exact match: the passing check's name has to appear in a ruleset control's `checks` list (tier 2), and no finding may map to that control. Its `control_id` and `control_title` come from the ruleset. Passing checks travel from the Prowler parser to the normaliser on the parsed list (see [FindingsNormaliser](../api/findingsnormaliser.md)); `cloudg run`, `cloudg ingest`, `CloudGEngine.scan()`, `run_pipeline()`, `run_from_reports()`, `ingest_reports()`, `parse_report()` and the MCP loaders all carry them through. The `control_id` depends on how the framework was attached: @@ -167,7 +169,7 @@ for result in controls.get("CIS-AWS", []) + controls.get("NIST-CSF-AWS", []): print(" ", f["severity"], f["source_tool"], f["resource_arn"]) ``` -Against the [sample reports from the ingest guide](/guides/ingesting/#try-it-without-any-scanner), the start of the output is: +Against the [sample reports from the ingest guide](ingesting.md#try-it-without-any-scanner), the start of the output is: ```console $ python compliance_summary.py @@ -188,7 +190,7 @@ The coarse tier 1 tags and the ruleset frameworks appear side by side: `CIS` nex ### report.html {#report-html} -The Compliance tab shows one card per framework with Pass and Fail counts and a percentage bar. Fail is the number of distinct controls with findings, and Pass the number of controls that only passing Prowler checks map to. When the input had no passing-check data (no Prowler output, or only failing records), Pass is 0 and the bar reads 0%; treat such a card as a count of affected controls. The Analytics tab charts the same counts, and the detail panel of each finding lists its frameworks. [Reports](/guides/reports/) covers the rest of the page. +The Compliance tab shows one card per framework with Pass and Fail counts and a percentage bar. Fail is the number of distinct controls with findings, and Pass the number of controls that only passing Prowler checks map to. When the input had no passing-check data (no Prowler output, or only failing records), Pass is 0 and the bar reads 0%; treat such a card as a count of affected controls. The Analytics tab charts the same counts, and the detail panel of each finding lists its frameworks. [Reports](reports.md) covers the rest of the page. ### compliance-map.json @@ -267,28 +269,32 @@ cp acme_baseline.yaml ./my-rules/ Then point cloudg at it. -```yaml tab="CLI" title="config.yaml" -rulesets: - rules_dir: ./my-rules -``` +=== "CLI" -```python tab="Python" title="custom_rules.py" -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser + ```yaml title="config.yaml" + rulesets: + rules_dir: ./my-rules + ``` -findings = ingest_reports( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "scoutsuite": ["./scoutsuite-report/"], - } -) -scan_result = FindingsNormaliser(rules_dir="./my-rules").normalise(findings) +=== "Python" -for result in scan_result.compliance: - if result.framework == "ACME-BASELINE": - print(result.control_id, result.control_title, len(result.finding_ids)) -``` + ```python title="custom_rules.py" + from cloudg.ingest import ingest_reports + from cloudg.normaliser import FindingsNormaliser + + findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "scoutsuite": ["./scoutsuite-report/"], + } + ) + scan_result = FindingsNormaliser(rules_dir="./my-rules").normalise(findings) + + for result in scan_result.compliance: + if result.framework == "ACME-BASELINE": + print(result.control_id, result.control_title, len(result.finding_ids)) + ``` With the CLI, pass the config with `-c`: `cloudg -c config.yaml ingest ...` or `cloudg -c config.yaml run ...`. From `CloudGEngine`, set `config.rulesets.rules_dir`. On the sample reports, the result is: @@ -323,9 +329,22 @@ The list of frameworks it imports is the `FRAMEWORKS` constant at the top of the ## Next -:::links -- [Ingesting existing output](/guides/ingesting/) Deduplication and the check-equivalence map. -- [Reports](/guides/reports/) The Compliance tab and the rest of `report.html`. -- [Data models](/api/models/) `ComplianceResult`, `ComplianceStatus` and `ScanResult`. -- [FindingsNormaliser](/api/findingsnormaliser/) The class that does the mapping. -::: +
      + +- [**Ingesting existing output**](ingesting.md) + + Deduplication and the check-equivalence map. + +- [**Reports**](reports.md) + + The Compliance tab and the rest of `report.html`. + +- [**Data models**](../api/models.md) + + `ComplianceResult`, `ComplianceStatus` and `ScanResult`. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + The class that does the mapping. + +
      diff --git a/site/content/guides/dependencies.md b/site/content/guides/dependencies.md index bc6e872..8720015 100644 --- a/site/content/guides/dependencies.md +++ b/site/content/guides/dependencies.md @@ -1,13 +1,15 @@ --- title: Dependencies and blast radius -lede: "`cloudg deps` asks two questions of any asset on a saved map: what does it need, and what breaks with it?" -meta: - - [Command, "`cloudg deps`"] - - [Input, "`inventory-map.json` from `cloudg map`"] -source: cloudg/inventory/dependencies.py +description: 'cloudg deps asks two questions of any asset on a saved map: what does it need, and what breaks with it?' --- -A map from [`cloudg map`](/guides/inventory-mapping/) records edges the way you would describe them out loud: the queue invokes the function, the function assumes its role, the WAF protects the API. That is the right shape for drawing, but not for impact questions. To ask "what breaks if I rotate this KMS key?" you need every edge turned into one arrow that reads "dependent needs dependency". `cloudg deps` and the `DependencyGraph` class behind it do that conversion and then walk the result. +`cloudg deps` asks two questions of any asset on a saved map: what does it need, and what breaks with it? +{ .lede } + +**Command** `cloudg deps` · **Input** `inventory-map.json` from `cloudg map` +{ .page-meta } + +A map from [`cloudg map`](inventory-mapping.md) records edges the way you would describe them out loud: the queue invokes the function, the function assumes its role, the WAF protects the API. That is the right shape for drawing, but not for impact questions. To ask "what breaks if I rotate this KMS key?" you need every edge turned into one arrow that reads "dependent needs dependency". `cloudg deps` and the `DependencyGraph` class behind it do that conversion and then walk the result. `cloudg deps` reads a saved `inventory-map.json` and nothing else. It makes no cloud calls, so you can run it on a laptop against a map someone produced in CI last night. @@ -25,7 +27,7 @@ Two more rules shape the graph. `CONTAINS` edges whose container is an organizat Here is a small map as `cloudg map` would record it: -```mermaid caption="A slice of a map, with edges in their natural direction" +```mermaid flowchart LR WAF["orders-api-acl (WAF)"] -->|PROTECTS| API["orders-api (API Gateway)"] API -->|INVOKES| FN["orders-worker (Lambda)"] @@ -42,10 +44,12 @@ flowchart LR TABLE -->|REFERENCES| KEY EXT["account 999999999999"] -->|IAM_TRUST| ROLE ``` +A slice of a map, with edges in their natural direction +{ .diagram-caption } And the same slice after the conversion. Every arrow now points from the asset that needs something to the thing it needs. Note how the `INVOKES` and `PROTECTS` arrows flipped: -```mermaid caption="The same slice as dependency arrows. Walking with the arrows is depends_on (up); walking against them is dependents (down)." +```mermaid flowchart LR FN["orders-worker (Lambda)"] --> API["orders-api (API Gateway)"] API --> WAF["orders-api-acl (WAF)"] @@ -62,10 +66,12 @@ flowchart LR TABLE --> KEY EXT["account 999999999999"] --> ROLE ``` +The same slice as dependency arrows. Walking with the arrows is depends_on (up); walking against them is dependents (down). +{ .diagram-caption } Upstream of the function sits everything it needs to run, including the API and queue that feed it work. Downstream of the KMS key sits everything that stops working if the key is disabled. -This is an availability and change-impact view. Compromise runs the other way along identity edges: a compromised function exposes its role, not the reverse. For that question, walk the `IAM_TRUST`, `ASSUMES_ROLE` and `GRANTS_ACCESS` edges of the map directly, or use the attack path analysis in [GraphBuilder](/api/graphbuilder/). +This is an availability and change-impact view. Compromise runs the other way along identity edges: a compromised function exposes its role, not the reverse. For that question, walk the `IAM_TRUST`, `ASSUMES_ROLE` and `GRANTS_ACCESS` edges of the map directly, or use the attack path analysis in [GraphBuilder](../api/graphbuilder.md). ## Query one asset @@ -311,9 +317,22 @@ for ref in refs: Against the example map, the ambiguous `orders-worker` stays unresolved in both modes, `function:orders-worker` and `ORDERS-KEY` resolve only with the opt-in tiers, and the other two resolve either way. The opt-in tiers never pick one of several assets that share an exact name. `index.resolve_finding(finding)` tries a finding's `resource_id` and `resource_arn` through the same tiers. -:::links -- [Inventory mapping](/guides/inventory-mapping/) Produce the map that `cloudg deps` reads. -- [cloudg deps](/cli/deps/) The command reference. -- [DependencyGraph](/api/dependencygraph/) Every method and return type. -- [InventoryResult](/api/inventoryresult/) Loading, analysis and export. -::: +
      + +- [**Inventory mapping**](inventory-mapping.md) + + Produce the map that `cloudg deps` reads. + +- [**cloudg deps**](../cli/deps.md) + + The command reference. + +- [**DependencyGraph**](../api/dependencygraph.md) + + Every method and return type. + +- [**InventoryResult**](../api/inventoryresult.md) + + Loading, analysis and export. + +
      diff --git a/site/content/guides/docker.md b/site/content/guides/docker.md index 224ce3c..4a6976e 100644 --- a/site/content/guides/docker.md +++ b/site/content/guides/docker.md @@ -1,13 +1,14 @@ --- title: Docker -lede: "The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host." -meta: - - [Image, "`ghcr.io/morpheuslord/cloudg`"] - - [Base, "`python:3.12-slim`"] - - [Entrypoint, "`cloudg`"] -source: Dockerfile +description: The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host. --- +The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host. +{ .lede } + +**Image** `ghcr.io/morpheuslord/cloudg` · **Base** `python:3.12-slim` · **Entrypoint** `cloudg` +{ .page-meta } + The image is the quickest route to a full `cloudg run`, because the scanners are the hard part of a local install and the image has them preinstalled. It is also a clean way to run the MCP server on a host where you would rather not install Python packages. ## What is in the image @@ -42,16 +43,20 @@ At run time the container works like this: ## Get the image -```bash tab="Pull" -docker pull ghcr.io/morpheuslord/cloudg:latest -docker tag ghcr.io/morpheuslord/cloudg:latest cloudg:latest -``` +=== "Pull" -```bash tab="Build" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -docker build -t cloudg:latest . -``` + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + docker tag ghcr.io/morpheuslord/cloudg:latest cloudg:latest + ``` + +=== "Build" + + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + docker build -t cloudg:latest . + ``` Releases are tagged with their version (`ghcr.io/morpheuslord/cloudg:0.6.0`) as well as `latest`. The examples on this page use the local name `cloudg:latest`, which is what `docker compose` builds too. @@ -89,25 +94,31 @@ The container sees no credentials until you give it some. Each provider has a fe ### AWS -```bash tab="Profile" -docker run --rm \ - -v "$HOME/.aws:/home/cloudg/.aws:ro" \ - -e AWS_PROFILE=audit \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p aws --regions us-east-1 -``` +=== "Profile" -```bash tab="Environment" -docker run --rm \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p aws --regions us-east-1 -``` + ```bash + docker run --rm \ + -v "$HOME/.aws:/home/cloudg/.aws:ro" \ + -e AWS_PROFILE=audit \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 + ``` -```bash tab="Instance role" -# on EC2 or ECS: credentials come from the instance or task role -docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest run -p aws --regions all -``` +=== "Environment" + + ```bash + docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 + ``` + +=== "Instance role" + + ```bash + # on EC2 or ECS: credentials come from the instance or task role + docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest run -p aws --regions all + ``` Mount `~/.aws` at `/home/cloudg/.aws`, not `/root/.aws`; the container does not run as root. With SSO profiles, run `aws sso login` on the host first. A read-only mount can use the cached token but cannot store a refreshed one, so log in again on the host when it expires. @@ -119,26 +130,32 @@ On EC2 with IMDSv2, a container on Docker's default bridge network is one networ The usual container setup is a service principal in environment variables. cloudg reads `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` when the matching flags are not given: -```bash tab="Client secret" -docker run --rm \ - -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --subscription-id 00000000-0000-0000-0000-000000000000 -``` +=== "Client secret" -```bash tab="Certificate" -docker run --rm \ - -e AZURE_TENANT_ID -e AZURE_CLIENT_ID \ - -v "$PWD/sp-cert.pem:/secrets/sp-cert.pem:ro" \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --azure-cert-path /secrets/sp-cert.pem -``` + ```bash + docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --subscription-id 00000000-0000-0000-0000-000000000000 + ``` -```bash tab="Managed identity" -# on an Azure VM or container host with a managed identity -docker run --rm -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --azure-managed-identity -``` +=== "Certificate" + + ```bash + docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID \ + -v "$PWD/sp-cert.pem:/secrets/sp-cert.pem:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-cert-path /secrets/sp-cert.pem + ``` + +=== "Managed identity" + + ```bash + # on an Azure VM or container host with a managed identity + docker run --rm -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-managed-identity + ``` Without `--subscription-id`, cloudg collects every enabled subscription the identity can see. @@ -146,21 +163,25 @@ Without `--subscription-id`, cloudg collects every enabled subscription the iden Mount a key file (or a workload identity federation config) and point `GOOGLE_APPLICATION_CREDENTIALS` at the path inside the container: -```bash tab="Key file" -docker run --rm \ - -v "$PWD/scanner-sa.json:/secrets/gcp.json:ro" \ - -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p gcp --project-id my-project -``` +=== "Key file" -```bash tab="gcloud ADC" -# after `gcloud auth application-default login` on the host -docker run --rm \ - -v "$HOME/.config/gcloud/application_default_credentials.json:/home/cloudg/.config/gcloud/application_default_credentials.json:ro" \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p gcp --project-id my-project -``` + ```bash + docker run --rm \ + -v "$PWD/scanner-sa.json:/secrets/gcp.json:ro" \ + -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project + ``` + +=== "gcloud ADC" + + ```bash + # after `gcloud auth application-default login` on the host + docker run --rm \ + -v "$HOME/.config/gcloud/application_default_credentials.json:/home/cloudg/.config/gcloud/application_default_credentials.json:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project + ``` google-auth finds the application default credentials file at its standard path under `HOME` without needing gcloud in the container. @@ -294,13 +315,15 @@ docker compose run --rm cloudg run -p aws -p gcp --project-id my-project --iac-d The MCP server runs from the same image. Over stdio, the AI client starts the container itself; over HTTP, the container runs as a long-lived service. -```mermaid caption="Two ways a client reaches cloudg in a container" +```mermaid flowchart LR D["Desktop client"] -->|"docker run -i (stdio)"| S1["cloudg mcp serve"] W["HTTP client"] -->|"127.0.0.1:8765/mcp, bearer token"| S2["cloudg mcp serve --transport http"] S1 --> V[("/app/reports")] S2 --> V ``` +Two ways a client reaches cloudg in a container +{ .diagram-caption } ### stdio for a desktop client @@ -358,4 +381,4 @@ curl -s http://127.0.0.1:8765/healthz The MCP live tools (`map_inventory`, `collect_assets`, `run_scanners`, `run_pipeline`) call cloud APIs with the server's credentials, whoever is asking. With no credentials mounted, their credential preflight (at most 25 seconds) fails and nothing is collected. If you do mount credentials, restrict who can use them: the `strict`, `read_only`, `airgapped`, `audit` and `soc-analyst` policies hide those tools, and `--read-only` drops them along with anything that writes files. A server that only answers questions about saved maps should run with `--read-only` and a read-only reports mount, as above. -[Connect a client](/mcp/connect/) has configuration for the other MCP clients, and [cloudg mcp serve](/cli/mcp-serve/) lists every server option. +[Connect a client](../mcp/connect.md) has configuration for the other MCP clients, and [cloudg mcp serve](../cli/mcp-serve.md) lists every server option. diff --git a/site/content/guides/graph-ontology-rag.md b/site/content/guides/graph-ontology-rag.md index 998da6f..31e69ff 100644 --- a/site/content/guides/graph-ontology-rag.md +++ b/site/content/guides/graph-ontology-rag.md @@ -1,17 +1,17 @@ --- title: Graph, ontology and RAG -lede: "Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM." -meta: - - [Modules, "`cloudg.graph`"] - - [Config, "`graph`, `ontology`, `rag`, `terraform`"] - - [Changed in, "0.6.0"] -source: cloudg/graph/reachability.py -since: "0.6.0" +description: Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM. --- +Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM. +{ .lede } + +**Modules** `cloudg.graph` · **Config** `graph`, `ontology`, `rag`, `terraform` · **Changed in** 0.6.0 · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every collection ends with two lists: `CloudAsset` objects and directed `NetworkEdge` objects between their ids. Everything on this page is derived from those two lists, plus findings when you have them. Nothing here calls a cloud API, so you can rebuild any of it offline from a saved map or a `findings.json`. -```mermaid caption="From collected data to the four analysis outputs" +```mermaid flowchart LR A["Assets and edges"] --> G["GraphBuilder: nx.DiGraph"] F[Findings] --> O @@ -26,6 +26,8 @@ flowchart LR C --> J["rag_chunks.jsonl"] A --> TF["TerraformExporter: .tf.json"] ``` +From collected data to the four analysis outputs +{ .diagram-caption } `cloudg run` does all of it in order: graph and reachability in phase 2, RAG export in 2c, Terraform in 2d, then after the scanners the ontology and a second RAG pass so both include the scanner findings. `CloudGEngine.analyze()` does the graph, ontology, RAG and Terraform steps for assets you collected yourself. The sections below use a small hand-built estate so every output is reproducible; the full script is at the end. @@ -103,7 +105,7 @@ Only `sg-web` admits the internet, yet the database is flagged. That is because blast radius of web-1: {'reachable_nodes': ['sg-web', 'role-app', 'sg-db', 'db-1'], 'depth': 2, 'risk_score': 4.0} ``` -The score is `0.5` per reachable node plus `2.0` per sensitive data store, capped at 10. It is a quick ranking signal. For dependency questions on a full inventory (what breaks if this KMS key goes, which shared resources have the most dependents) use `DependencyGraph` and `cloudg deps`, described in [dependencies and blast radius](/guides/dependencies/). +The score is `0.5` per reachable node plus `2.0` per sensitive data store, capped at 10. It is a quick ranking signal. For dependency questions on a full inventory (what breaks if this KMS key goes, which shared resources have the most dependents) use `DependencyGraph` and `cloudg deps`, described in [dependencies and blast radius](dependencies.md). ## The ontology @@ -345,7 +347,6 @@ from cloudg.schema.models import ( ACCOUNT, REGION = "123456789012", "eu-west-1" OUT = Path("./graph-out") - def aws(asset_id: str, name: str, kind: AssetType, arn_tail: str, **metadata) -> CloudAsset: service = arn_tail.split(":", 1)[0] return CloudAsset( @@ -359,11 +360,9 @@ def aws(asset_id: str, name: str, kind: AssetType, arn_tail: str, **metadata) -> metadata=metadata, ) - def edge(src: str, dst: str, kind: EdgeType, **attrs) -> NetworkEdge: return NetworkEdge(source_id=src, target_id=dst, edge_type=kind, **attrs) - assets = [ aws("vpc-1", "prod-vpc", AssetType.VPC, "ec2:vpc/vpc-1", vpc_id="vpc-1"), aws("subnet-a", "public-a", AssetType.SUBNET, "ec2:subnet/subnet-a", vpc_id="vpc-1"), @@ -444,9 +443,22 @@ print("terraform preview:", terraform.preview(assets)["resource_types"]) print({name: path.name for name, path in terraform.export(assets, edges).items()}) ``` -:::links -- [Dependencies and blast radius](/guides/dependencies/) `DependencyGraph` and `cloudg deps` on a full inventory. -- [GraphBuilder](/api/graphbuilder/) Every method of the builder. -- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) The walk and its helpers. -- [CloudOntology](/api/cloudontology/) and [RAGExporter](/api/ragexporter/) The exporters' APIs. -::: +
      + +- [**Dependencies and blast radius**](dependencies.md) + + `DependencyGraph` and `cloudg deps` on a full inventory. + +- [**GraphBuilder**](../api/graphbuilder.md) + + Every method of the builder. + +- [**ReachabilityAnalyzer**](../api/reachabilityanalyzer.md) + + The walk and its helpers. + +- [**CloudOntology**](../api/cloudontology.md) + + and [RAGExporter](../api/ragexporter.md) The exporters' APIs. + +
      diff --git a/site/content/guides/ingesting.md b/site/content/guides/ingesting.md index 7d011e5..d9b2a78 100644 --- a/site/content/guides/ingesting.md +++ b/site/content/guides/ingesting.md @@ -1,19 +1,19 @@ --- title: Ingesting existing output -lede: "`cloudg ingest` is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports." -meta: - - [Command, "`cloudg ingest`"] - - [Python, "`parse_report`, `ingest_reports`, `CloudGEngine.run_from_reports()`"] - - [Needs, "No credentials, no scanner binaries"] -source: cloudg/ingest.py -since: "0.6.0" +description: cloudg ingest is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports. --- +`cloudg ingest` is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports. +{ .lede } + +**Command** `cloudg ingest` · **Python** `parse_report`, `ingest_reports`, `CloudGEngine.run_from_reports()` · **Needs** No credentials, no scanner binaries · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Ingest exists for the common case where the scanners already run somewhere you don't control from cloudg: a CI job that runs Checkov on every pull request, a nightly Prowler container in the security account, a Trivy step in the image build. Those jobs produce files. `cloudg ingest` reads the files and does the rest. Nothing is executed and no cloud API is called, so it runs on a laptop with no AWS profile, in an air-gapped review environment, or in a CI step after the scan step. ## What ingest does -```mermaid caption="cloudg ingest, from native files to reports" +```mermaid flowchart LR P["--prowler files"] --> PR["parse_report"] S["--scoutsuite files"] --> PR @@ -24,6 +24,8 @@ flowchart LR N --> J["findings.json"] N --> H["report.html"] ``` +cloudg ingest, from native files to reports +{ .diagram-caption } Each path goes through `cloudg.ingest.parse_report(tool, path)`, which hands it to that scanner's own parser (the same code `cloudg run` uses on fresh output). Every parsed `Finding` is collected into one list and written, unchanged, to `raw-findings.json`. The list then goes through the same `FindingsNormaliser` that `cloudg run` uses: two-pass deduplication, CVSS-based rescoring and compliance mapping. The result is written as `findings.json` and `report.html`. @@ -33,44 +35,50 @@ What ingest doesn't do is anything that needs live assets. There is no collectio Every input flag is repeatable, and any combination of tools works. -```bash tab="CLI" -cloudg ingest \ - --prowler ./prowler-output/ \ - --scoutsuite ./scoutsuite-report/ \ - --checkov ./results_json.json \ - --trivy ./trivy-image.json \ - --trivy ./trivy-fs.json \ - -o ./reports -``` - -```python tab="Python" title="ingest_reports_demo.py" -from cloudg import CloudGConfig, CloudGEngine - -engine = CloudGEngine(CloudGConfig()) -result = engine.run_from_reports_sync( - { - "prowler": ["./prowler-output/"], - "scoutsuite": ["./scoutsuite-report/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy-image.json", "./trivy-fs.json"], - }, - output_dir="./reports", -) - -print(result.total_findings, result.severity_breakdown) -print(result.report_paths) -print(result.errors) -``` - -```bash tab="Docker" -docker run --rm \ - -v "$PWD:/data" \ - ghcr.io/morpheuslord/cloudg:latest ingest \ - --prowler /data/prowler-output/ \ - --checkov /data/results_json.json \ - --trivy /data/trivy-image.json \ - -o /data/reports -``` +=== "CLI" + + ```bash + cloudg ingest \ + --prowler ./prowler-output/ \ + --scoutsuite ./scoutsuite-report/ \ + --checkov ./results_json.json \ + --trivy ./trivy-image.json \ + --trivy ./trivy-fs.json \ + -o ./reports + ``` + +=== "Python" + + ```python title="ingest_reports_demo.py" + from cloudg import CloudGConfig, CloudGEngine + + engine = CloudGEngine(CloudGConfig()) + result = engine.run_from_reports_sync( + { + "prowler": ["./prowler-output/"], + "scoutsuite": ["./scoutsuite-report/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + }, + output_dir="./reports", + ) + + print(result.total_findings, result.severity_breakdown) + print(result.report_paths) + print(result.errors) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD:/data" \ + ghcr.io/morpheuslord/cloudg:latest ingest \ + --prowler /data/prowler-output/ \ + --checkov /data/results_json.json \ + --trivy /data/trivy-image.json \ + -o /data/reports + ``` | Flag | Accepts | |---|---| @@ -81,7 +89,7 @@ docker run --rm \ | `--format` | `html`, `json` or `all` (default). `raw-findings.json` is written in every case. | | `-o, --output` | Output directory, `./reports` by default | -The global `-c` option matters here for one reason: `rulesets.rules_dir` decides which compliance rulesets the normaliser loads. See [Compliance frameworks](/guides/compliance/#adding-a-custom-ruleset). +The global `-c` option matters here for one reason: `rulesets.rules_dir` decides which compliance rulesets the normaliser loads. See [Compliance frameworks](compliance.md#adding-a-custom-ruleset). ```bash cloudg -c config.yaml ingest --prowler ./prowler-output/ -o ./reports @@ -92,86 +100,7 @@ cloudg -c config.yaml ingest --prowler ./prowler-output/ -o ./reports The script below writes one small file per tool in each tool's native shape, the same shapes the test suite uses. Run it in an empty directory, then run the `cloudg ingest` command above. ```python title="make_sample_reports.py" -"""Write one small native report per scanner, shaped like the real tools' output.""" -import json -from pathlib import Path - -Path("prowler-output").mkdir(exist_ok=True) -Path("prowler-output/prowler-output-123456789012.asff.json").write_text(json.dumps([ - { - "Id": "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-abc", - "Title": "S3 bucket default encryption", - "Description": "Bucket has no default encryption", - "Severity": {"Label": "HIGH"}, - "Resources": [{"Id": "arn:aws:s3:::my-bucket"}], - "Compliance": {"Status": "FAILED", "RelatedRequirements": ["CIS 2.1.1"]}, - "Remediation": {"Recommendation": {"Text": "Enable default encryption"}}, - } -])) - -results = Path("scoutsuite-report/scoutsuite-results") -results.mkdir(parents=True, exist_ok=True) -(results / "scoutsuite_results_aws-123456789012.js").write_text( - "scoutsuite_results =\n" + json.dumps({ - "services": {"s3": {"findings": {"s3-bucket-no-encryption": { - "description": "Bucket without encryption", - "rationale": "Data at rest should be encrypted", - "remediation": "Enable encryption", - "level": "danger", - "flagged_items": 1, - "items": ["s3.buckets.my-bucket"], - "references": [], - }}}} - }) -) - -Path("results_json.json").write_text(json.dumps({ - "check_type": "terraform", - "results": {"failed_checks": [{ - "check_id": "CKV_AWS_19", - "check_name": "Ensure S3 bucket has server-side encryption enabled", - "severity": "HIGH", - "resource": "aws_s3_bucket.data", - "file_path": "/main.tf", - "file_line_range": [1, 10], - "guideline": "https://docs.example/ckv-aws-19", - }]}, -})) - -Path("trivy-image.json").write_text(json.dumps({ - "ArtifactName": "myrepo/app:latest", - "ArtifactType": "container_image", - "Results": [{ - "Target": "myrepo/app:latest (alpine 3.19)", - "Type": "alpine", - "Vulnerabilities": [{ - "VulnerabilityID": "CVE-2024-0001", - "PkgName": "openssl", - "InstalledVersion": "3.1.0", - "FixedVersion": "3.1.1", - "Severity": "CRITICAL", - "Description": "Example vulnerability", - "CVSS": {"nvd": {"V3Score": 9.8}}, - }], - }], -})) - -Path("trivy-fs.json").write_text(json.dumps({ - "ArtifactName": "./iac", - "ArtifactType": "filesystem", - "Results": [{ - "Target": "main.tf", - "Type": "terraform", - "Misconfigurations": [{ - "ID": "AVD-AWS-0088", - "Title": "S3 bucket encryption not enabled", - "Description": "Bucket does not have encryption enabled", - "Severity": "HIGH", - "Resolution": "Enable encryption", - }], - }], -})) -print("sample reports written") +--8<-- "make_sample_reports.py" ``` ```console @@ -231,7 +160,7 @@ From each OCSF record: | `remediation.desc` | `remediation` | | `status_detail` (else `message`) | `evidence` | -The passing checks travel with the parsed findings (`passed_checks` on the list `parse_report` returns) into normalisation, where the compliance controls that only passing checks map to get a PASS result. See [Compliance frameworks](/guides/compliance/). +The passing checks travel with the parsed findings (`passed_checks` on the list `parse_report` returns) into normalisation, where the compliance controls that only passing checks map to get a PASS result. See [Compliance frameworks](compliance.md). ### ScoutSuite @@ -306,7 +235,7 @@ If you feed the same report twice, or two overlapping directories, the duplicate ## Normalisation and deduplication {#normalisation-and-deduplication} -The normaliser runs two deduplication passes, then rescoring, then compliance mapping. Compliance mapping has [its own page](/guides/compliance/); this is what happens to the findings themselves. +The normaliser runs two deduplication passes, then rescoring, then compliance mapping. Compliance mapping has [its own page](compliance.md); this is what happens to the findings themselves. Pass 1 works inside each scanner. The key is the tool, the check ID (or the normalised title when there is no check ID) and the resource. Two different checks that happen to share a title on the same bucket stay separate. @@ -390,7 +319,7 @@ HIGH 7.5 trivy [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enab The functions differ in how they fail. `parse_report` raises `ValueError` for a tool name it doesn't know (only `prowler`, `scoutsuite`, `checkov` and `trivy` are accepted, case-insensitive) and `FileNotFoundError` for a missing path. `ingest_reports` catches both, logs them and moves on. The per-scanner parsers are public too, as class methods: `ProwlerScanner.parse_report(path)`, `ScoutSuiteScanner.parse_report(path)`, `CheckovScanner.parse_report(path)` and `TrivyScanner.parse_report(path)`. -On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires the `on_phase_start("ingest")`, `on_finding` and `on_scan_complete` [event hooks](/api/event-hooks/). `run_from_reports()` and its sync wrapper go further and write `findings.json` and `report.html` as `report.formats` allows (not `raw-findings.json`), returning a `PipelineResult` whose `report_paths` holds both paths and whose `errors` lists any phase that failed. +On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires the `on_phase_start("ingest")`, `on_finding` and `on_scan_complete` [event hooks](../api/event-hooks.md). `run_from_reports()` and its sync wrapper go further and write `findings.json` and `report.html` as `report.formats` allows (not `raw-findings.json`), returning a `PipelineResult` whose `report_paths` holds both paths and whose `errors` lists any phase that failed. ## What comes out @@ -400,11 +329,11 @@ On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires | `findings.json` | `metadata`, `summary`, `findings`, `compliance`, and empty `assets`, `edges` and `graph` | | `report.html` | The interactive report: findings table, analytics, compliance cards | -With no assets, the report's Infrastructure Map tab falls back to a force graph of finding nodes linked to compliance framework nodes. The [Reports](/guides/reports/) guide covers both files in detail, and the field-level layout of `findings.json` is on [Output files](/reference/output-files/). +With no assets, the report's Infrastructure Map tab falls back to a force graph of finding nodes linked to compliance framework nodes. The [Reports](reports.md) guide covers both files in detail, and the field-level layout of `findings.json` is on [Output files](../reference/output-files.md). ## Combining with a live inventory {#combining-with-a-live-inventory} -Ingest needs no credentials; mapping needs read-only ones. Run them separately, at different times if you like, and merge at the end. [`cloudg map`](/guides/inventory-mapping/) accepts a cloudg findings file and writes `asset-map.json` (each asset with its findings) and `compliance-map.json` (each framework with its affected assets): +Ingest needs no credentials; mapping needs read-only ones. Run them separately, at different times if you like, and merge at the end. [`cloudg map`](inventory-mapping.md) accepts a cloudg findings file and writes `asset-map.json` (each asset with its findings) and `compliance-map.json` (each framework with its affected assets): ```bash cloudg ingest --prowler ./ci/prowler-output/ --checkov ./ci/results_json.json -o ./reports @@ -418,7 +347,6 @@ import asyncio from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -427,15 +355,27 @@ async def main() -> None: scan_result = engine.normalise_findings(ingested, assets=collection.assets) print(len(scan_result.assets), "assets,", len(scan_result.findings), "findings") - asyncio.run(main()) ``` ## Next -:::links -- [Compliance frameworks](/guides/compliance/) How findings are tagged with controls, and how to add your own. -- [Reports](/guides/reports/) Reading `findings.json` and `report.html`, and re-rendering them. -- [cloudg ingest](/cli/ingest/) The command reference. -- [parse_report](/api/parse-report/) The Python parser entry point. -::: +
      + +- [**Compliance frameworks**](compliance.md) + + How findings are tagged with controls, and how to add your own. + +- [**Reports**](reports.md) + + Reading `findings.json` and `report.html`, and re-rendering them. + +- [**cloudg ingest**](../cli/ingest.md) + + The command reference. + +- [**parse_report**](../api/parse-report.md) + + The Python parser entry point. + +
      diff --git a/site/content/guides/installation.md b/site/content/guides/installation.md index c0b2880..0de139a 100644 --- a/site/content/guides/installation.md +++ b/site/content/guides/installation.md @@ -1,13 +1,14 @@ --- title: Installation -lede: "cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image." -meta: - - [Python, "3.11 or newer"] - - [Package, "`cloudg` on PyPI"] - - [Image, "`ghcr.io/morpheuslord/cloudg`"] -source: pyproject.toml +description: cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image. --- +cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image. +{ .lede } + +**Python** 3.11 or newer · **Package** `cloudg` on PyPI · **Image** `ghcr.io/morpheuslord/cloudg` +{ .page-meta } + There are two things to install, and it helps to keep them apart. The first is cloudg itself: the CLI, the graph engine, the normaliser, the report renderers and, through extras, the cloud SDKs. The second is the set of scanners (Prowler, ScoutSuite, Checkov, Trivy) that `cloudg run` and `cloudg scan` call as subprocesses. cloudg works with any subset of them, including none. ## Requirements @@ -18,27 +19,33 @@ Credentials are not needed to install, and not needed at all for `cloudg ingest` ## Install cloudg -```bash tab="pip" -python3 -m venv .venv -source .venv/bin/activate -pip install "cloudg[all]" -``` +=== "pip" -```bash tab="uv" -# as a standalone CLI in its own environment -uv tool install "cloudg[all]" -# or into the current project's virtualenv -uv pip install "cloudg[all]" -``` + ```bash + python3 -m venv .venv + source .venv/bin/activate + pip install "cloudg[all]" + ``` -```bash tab="Docker" -docker pull ghcr.io/morpheuslord/cloudg:latest -docker run --rm ghcr.io/morpheuslord/cloudg:latest --version -``` +=== "uv" + + ```bash + # as a standalone CLI in its own environment + uv tool install "cloudg[all]" + # or into the current project's virtualenv + uv pip install "cloudg[all]" + ``` + +=== "Docker" + + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + docker run --rm ghcr.io/morpheuslord/cloudg:latest --version + ``` Quote the package spec. In zsh, unquoted square brackets are a glob pattern and `pip install cloudg[all]` fails with "no matches found". -The Docker image already contains cloudg, the AWS, Azure and GCP SDKs, Prowler, Checkov, Trivy and Parliament. It is the shortest way to the full pipeline; [Docker](/guides/docker/) covers running it with credentials and volumes. +The Docker image already contains cloudg, the AWS, Azure and GCP SDKs, Prowler, Checkov, Trivy and Parliament. It is the shortest way to the full pipeline; [Docker](docker.md) covers running it with credentials and volumes. ### From a source checkout @@ -71,9 +78,9 @@ The cloud SDKs are optional, so a machine that only aggregates scanner output do Extras combine: `pip install "cloudg[aws,gcp]"` gets exactly two providers. -:::note What the core package can do on its own -Without any extra you still get the graph builder, the ontology, the RAG exporter, the findings normaliser with every compliance ruleset, all four scanner parsers, the HTML, JSON and SVG renderers, and the MCP layer with its native server. That is enough for `cloudg ingest` on an air-gapped analysis host. -::: +!!! note "What the core package can do on its own" + + Without any extra you still get the graph builder, the ontology, the RAG exporter, the findings normaliser with every compliance ruleset, all four scanner parsers, the HTML, JSON and SVG renderers, and the MCP layer with its native server. That is enough for `cloudg ingest` on an air-gapped analysis host. About `full`: in 0.6.0 the IAM linter only checks whether `policy_sentry` can be imported, and nothing in the package imports `diagrams` or `weasyprint`. Installing `full` does no harm, but it changes no output today. @@ -103,17 +110,23 @@ Both tools put the executables in `~/.local/bin`, which must be on `PATH` for cl Trivy is a Go binary. Use your package manager or Aqua Security's install script: -```bash tab="macOS" -brew install trivy -``` +=== "macOS" -```bash tab="Linux" -curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin -``` + ```bash + brew install trivy + ``` -```bash tab="Windows" -winget install AquaSecurity.Trivy -``` +=== "Linux" + + ```bash + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin + ``` + +=== "Windows" + + ```bash + winget install AquaSecurity.Trivy + ``` The scanners authenticate on their own. cloudg passes Prowler the AWS profile (`-p`) and, when you gave direct keys, the key pair through the environment; ScoutSuite gets the AWS profile. For Azure and GCP they read the same environment and credential files the SDKs do. If a scanner works when you run it by hand, it works under cloudg. @@ -121,17 +134,21 @@ The scanners authenticate on their own. cloudg passes Prowler the AWS profile (` `install.sh` (Linux and macOS) and `install.bat` (Windows) set up a complete workstation in one go. Run them from the root of a source checkout, because they install cloudg in editable mode from the current directory: -```bash tab="Linux / macOS" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -./install.sh -``` +=== "Linux / macOS" -```bash tab="Windows" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -install.bat -``` + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + ./install.sh + ``` + +=== "Windows" + + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + install.bat + ``` `install.sh` detects the package manager (apt, dnf, yum, pacman, zypper or Homebrew) and then, in order: @@ -216,23 +233,29 @@ You should see "Ingested 1 findings from 1 tool(s), 1 after deduplication" and t ## Upgrading -```bash tab="pip" -pip install --upgrade "cloudg[all]" -``` +=== "pip" -```bash tab="uv" -uv tool upgrade cloudg -``` + ```bash + pip install --upgrade "cloudg[all]" + ``` -```bash tab="Docker" -docker pull ghcr.io/morpheuslord/cloudg:latest -``` +=== "uv" + + ```bash + uv tool upgrade cloudg + ``` + +=== "Docker" + + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + ``` For a source checkout, `git pull` and then reinstall: `uv pip install -e ".[all,dev]"`. Reinstalling matters when the entry points change. The `cloudg-mcp` console script arrived in 0.6.0, so an editable install made before that does not have it until you reinstall. Image tags follow the release version (`ghcr.io/morpheuslord/cloudg:0.6.0`), and `latest` moves with each release. Pin the version tag in CI so an upgrade happens when you choose. -Configuration files carry over. A `config.yaml` that still uses the old single `provider:` key is migrated to `providers: [...]` when it loads. Read the [changelog](/changelog/) before upgrading across a minor version; it lists every changed default and output field. +Configuration files carry over. A `config.yaml` that still uses the old single `provider:` key is migrated to `providers: [...]` when it loads. Read the [changelog](../changelog.md) before upgrading across a minor version; it lists every changed default and output field. ## Troubleshooting @@ -242,4 +265,4 @@ If pip says no matching distribution was found, or mentions "a different Python" A scanner that works in your shell but is "not installed" under cloudg is on a different `PATH`. That happens with scanners installed in a second virtualenv that is not active. Use `uv tool` or `pipx`, or symlink the executable somewhere on `PATH`. -Next: run your first scan in the [quickstart](/guides/quickstart/). +Next: run your first scan in the [quickstart](quickstart.md). diff --git a/site/content/guides/introduction.md b/site/content/guides/introduction.md index 3b6dd9d..c76b2ef 100644 --- a/site/content/guides/introduction.md +++ b/site/content/guides/introduction.md @@ -1,14 +1,14 @@ --- title: Introduction -lede: "cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result." -meta: - - [Version, "0.6.0"] - - [Python, "3.11 or newer"] - - [Providers, "AWS, Azure, GCP"] - - [License, "MIT"] -source: README.md +description: cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result. --- +cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result. +{ .lede } + +**Version** 0.6.0 · **Python** 3.11 or newer · **Providers** AWS, Azure, GCP · **License** MIT +{ .page-meta } + cloudg is a Python command-line tool and a library. You point it at one or more cloud accounts and it builds an inventory of what is deployed there: instances, buckets, roles, security groups, load balancers, functions, key vaults, projects. Those assets and the network edges between them go into a directed graph. From the graph cloudg works out what the internet can reach, which paths an attacker could take sideways, and how much of the estate each node could touch if it were compromised. The same inventory then goes to the security scanners you already know (Prowler, ScoutSuite, Checkov and Trivy), plus a built-in IAM policy linter. Each scanner reports the same problem in its own words, so cloudg normalises every finding into one model, merges duplicates within and across tools, rescores them against CVSS, and maps them onto controls from 28 compliance frameworks. What you open at the end is a single HTML report, not four. @@ -19,16 +19,35 @@ None of that requires the whole pipeline. Each half of cloudg works without the Most people arrive with one of three questions, and cloudg has a command for each. -:::cells +
      + +
      + ### cloudg run + "What is wrong with my cloud?" Collects every configured provider, builds the graph, runs the scanners in parallel, normalises the findings and renders the reports. Needs credentials and, for full coverage, the scanner binaries. + +
      + +
      + ### cloudg ingest + "I already ran the scanners, can you make sense of the output?" Reads native Prowler, ScoutSuite, Checkov and Trivy files, then deduplicates, maps compliance and renders reports. No cloud credentials and no scanner binaries. + +
      + +
      + ### cloudg map + "What do I have, and how is it wired?" A scanner-free inventory mapper: deep collectors, a sweep across every service, typed relationships and dependency analysis. No scanner runs or needs to be installed. -::: -They are independent but they compose. You can `cloudg map` an account today, run Prowler in CI next week, and merge that output into the map later with `cloudg map --findings`. You can collect live assets from Python and normalise findings you ingested from someone else's pipeline in the same script. The [quickstart](/guides/quickstart/) walks through all three. +
      + +
      + +They are independent but they compose. You can `cloudg map` an account today, run Prowler in CI next week, and merge that output into the map later with `cloudg map --findings`. You can collect live assets from Python and normalise findings you ingested from someone else's pipeline in the same script. The [quickstart](quickstart.md) walks through all three. ### Choosing between them @@ -44,7 +63,7 @@ They are independent but they compose. You can `cloudg map` an account today, ru ## How the pieces fit -```mermaid caption="From cloud APIs to the files and tools you use" +```mermaid flowchart LR P["AWS, Azure, GCP APIs"] --> C[Collectors] C --> G["Graph (NetworkX)"] @@ -58,6 +77,8 @@ flowchart LR N --> M["MCP server"] G --> M ``` +From cloud APIs to the files and tools you use +{ .diagram-caption } Collectors talk to the provider APIs through a shared rate limiter, so a big account slows down instead of failing. In the NetworkX graph, edge endpoints that are not assets (the `0.0.0.0/0` on a security group rule, for example) become external nodes, and a breadth-first search from the internet sources `0.0.0.0/0` and `::/0` over network-flow edges is how cloudg decides what is internet-reachable. Scanners run as separate processes, each in its own worker thread, and anything not installed is skipped with a warning. The normaliser is the meeting point: scanner findings, the graph's reachability findings and the IAM linter's results all go through the same deduplication and compliance mapping. @@ -71,7 +92,7 @@ The MCP server sits beside all of this. It loads the files a run or a map produc | Azure | `cloudg[azure]` (azure-identity and the management SDKs) | Per subscription; every enabled subscription the credential can see when none is set | all locations | | GCP | `cloudg[gcp]` (google-cloud-asset, google-auth) | Cloud Asset Inventory, per project or once at organization scope | all regions | -Every provider supports several authentication methods, resolved in a fixed order: direct keys, OIDC federation, profiles and service principals, managed identities, and finally each SDK's default chain. The same configuration works on a laptop, in CI and on cloud compute. [Authentication](/guides/authentication/) has the details. +Every provider supports several authentication methods, resolved in a fixed order: direct keys, OIDC federation, profiles and service principals, managed identities, and finally each SDK's default chain. The same configuration works on a laptop, in CI and on cloud compute. [Authentication](authentication.md) has the details. ## Scanners @@ -83,7 +104,7 @@ Every provider supports several authentication methods, resolved in a fixed orde | Trivy | Container images, or a filesystem when no images are given | `trivy` | | IAM linter | IAM policies on collected roles and policies, using Parliament | built in | -The scanners are executables, not Python dependencies. cloudg checks `PATH` when it runs each one and skips the missing ones. [Installation](/guides/installation/#scanner-binaries) shows how to install each, and [Running scanners](/guides/running-scanners/) covers what cloudg passes to them. +The scanners are executables, not Python dependencies. cloudg checks `PATH` when it runs each one and skips the missing ones. [Installation](installation.md#scanner-binaries) shows how to install each, and [Running scanners](running-scanners.md) covers what cloudg passes to them. ## What you get @@ -98,7 +119,7 @@ A `cloudg run` writes everything into `./reports` (change it with `-o`): | `rag_chunks.jsonl`, `rag_metadata_index.json` | Retrieval-ready chunks of the infrastructure for LLM pipelines | | `terraform/*.tf.json` | Terraform recreation of the live infrastructure, with `--terraform` | -`cloudg ingest` writes `findings.json`, `report.html` and `raw-findings.json`. `cloudg map` writes `inventory-map.json`, `inventory-map.graphml`, `inventory-graph.json` and `inventory-dependencies.json`, plus `inventory-organization.json` with `--org` and `asset-map.json` and `compliance-map.json` with `--findings`. [Output files](/reference/output-files/) describes each one field by field. +`cloudg ingest` writes `findings.json`, `report.html` and `raw-findings.json`. `cloudg map` writes `inventory-map.json`, `inventory-map.graphml`, `inventory-graph.json` and `inventory-dependencies.json`, plus `inventory-organization.json` with `--org` and `asset-map.json` and `compliance-map.json` with `--findings`. [Output files](../reference/output-files.md) describes each one field by field. ## Using it from Python @@ -114,15 +135,34 @@ result = engine.run_pipeline_sync(output_dir="./reports") print(result.to_summary()) ``` -The graph builder, ontology, RAG exporter, scanner parsers, normaliser, inventory mapper and MCP layer are all importable on their own. [The pipeline](/guides/pipeline/) shows the phases from both the CLI and Python, and the [Python API](/api/) tab documents every class. +The graph builder, ontology, RAG exporter, scanner parsers, normaliser, inventory mapper and MCP layer are all importable on their own. [The pipeline](pipeline.md) shows the phases from both the CLI and Python, and the [Python API](../api/index.md) tab documents every class. ## Where to go next -:::links -- [Installation](/guides/installation/) pip, uv or Docker, the provider extras, and the scanner binaries. -- [Quickstart](/guides/quickstart/) A first report in a few minutes, with or without credentials. -- [The pipeline](/guides/pipeline/) What each phase of cloudg run reads, writes and does when something fails. -- [Inventory mapping](/guides/inventory-mapping/) The scanner-free map of everything deployed. -- [Ingesting existing output](/guides/ingesting/) Reports from scanner files you already have. -- [MCP server](/mcp/) Hand the inventory and findings to an AI assistant. -::: +
      + +- [**Installation**](installation.md) + + pip, uv or Docker, the provider extras, and the scanner binaries. + +- [**Quickstart**](quickstart.md) + + A first report in a few minutes, with or without credentials. + +- [**The pipeline**](pipeline.md) + + What each phase of cloudg run reads, writes and does when something fails. + +- [**Inventory mapping**](inventory-mapping.md) + + The scanner-free map of everything deployed. + +- [**Ingesting existing output**](ingesting.md) + + Reports from scanner files you already have. + +- [**MCP server**](../mcp/index.md) + + Hand the inventory and findings to an AI assistant. + +
      diff --git a/site/content/guides/inventory-mapping.md b/site/content/guides/inventory-mapping.md index 0e9358e..ac7bfa5 100644 --- a/site/content/guides/inventory-mapping.md +++ b/site/content/guides/inventory-mapping.md @@ -1,55 +1,63 @@ --- title: Inventory mapping -lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" -meta: - - [Command, "`cloudg map`"] - - [Providers, "AWS, Azure, GCP"] -source: docs/DOCUMENTATION.md +description: 'cloudg map asks a different question from cloudg run: what exists, and how is it wired together?' --- +`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together? +{ .lede } + +**Command** `cloudg map` · **Providers** AWS, Azure, GCP +{ .page-meta } + `cloudg map` reads your cloud accounts and writes down everything it finds, deployed resources and defaults alike, then links them into one graph of typed relationships. A Lambda function points at the role it runs as, the queue that triggers it, the image it was built from and the KMS key that encrypts its environment. A WAF points at the load balancer it protects. An SCP points at the accounts it restricts. No scanner runs during a map, and none needs to be installed. The command only needs read access to the cloud APIs. -:::note -The map doesn't depend on the scanners. You can map today and pull in next week's scan findings with `--findings ./reports/raw-findings.json`. On the CLI that flag belongs to `cloudg map`, so it maps again; to overlay findings onto a map you already saved without calling the cloud, use the Python call in [Overlay scanner findings](#overlay-scanner-findings). -::: +!!! note + + The map doesn't depend on the scanners. You can map today and pull in next week's scan findings with `--findings ./reports/raw-findings.json`. On the CLI that flag belongs to `cloudg map`, so it maps again; to overlay findings onto a map you already saved without calling the cloud, use the Python call in [Overlay scanner findings](#overlay-scanner-findings). ## Map an account The smallest useful run maps one AWS account in every enabled region: -```bash tab="CLI" -cloudg map -p aws --profile audit --regions all -``` +=== "CLI" -```python tab="Python" title="map_account.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg map -p aws --profile audit --regions all + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "audit" -config.aws.regions = ["ALL"] +=== "Python" -engine = CloudGEngine(config) -inventory = engine.map_inventory_sync(output_dir="./reports") + ```python title="map_account.py" + from cloudg import CloudGConfig, CloudGEngine -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("internet-exposed:", summary["internet_exposed"]) -print("unlinked:", summary["unlinked_assets"]) -``` + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] -```bash tab="Docker" -docker run --rm \ - -v ~/.aws:/home/cloudg/.aws:ro \ - -v "$PWD/reports:/app/reports" \ - ghcr.io/morpheuslord/cloudg:latest \ - map -p aws --profile audit --regions all -``` + engine = CloudGEngine(config) + inventory = engine.map_inventory_sync(output_dir="./reports") + + summary = inventory.summary + print(summary["total_assets"], "assets,", summary["total_edges"], "edges") + print("internet-exposed:", summary["internet_exposed"]) + print("unlinked:", summary["unlinked_assets"]) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile audit --regions all + ``` The image runs as the user `cloudg` with its home at `/home/cloudg`, which is why the AWS config directory is mounted there. Reports land in `/app/reports`, the container's `./reports`. -While it runs, the CLI prints a "Map Configuration" panel (providers, regions, selected services, whether the Kubernetes, Cloud Control and catch-all sweeps are on) and then an "Inventory Summary" table: assets, interconnections, accounts, services, internet-exposed assets, cross-account edges, external accounts referenced, security service gaps, unlinked assets and unresolved references. Two ranked tables follow, "Most shared dependencies" and "Largest blast radius", and a warning line for each security service that is not enabled. [Dependencies and blast radius](/guides/dependencies/) explains how to read them. +While it runs, the CLI prints a "Map Configuration" panel (providers, regions, selected services, whether the Kubernetes, Cloud Control and catch-all sweeps are on) and then an "Inventory Summary" table: assets, interconnections, accounts, services, internet-exposed assets, cross-account edges, external accounts referenced, security service gaps, unlinked assets and unresolved references. Two ranked tables follow, "Most shared dependencies" and "Largest blast radius", and a warning line for each security service that is not enabled. [Dependencies and blast radius](dependencies.md) explains how to read them. Pass a config file with the group option, before the subcommand: `cloudg -c config.yaml map ...`. Without `-c`, cloudg uses its built-in defaults and does not look for a `config.yaml` on its own. @@ -59,7 +67,7 @@ Pass a config file with the group option, before the subcommand: `cloudg -c conf With `all`, cloudg discovers the enabled regions first and runs one collection unit per account and region. Account-wide services (IAM, Route 53, ECR Public, Direct Connect gateways and the other global tasks) run once per account, in the primary region: `us-east-1` when it is part of the run, otherwise the first region in the list. So `--regions all` doesn't collect IAM once per region. -With `--org` and Control Tower, `--regions all` is replaced by the landing zone's governed regions. An explicit list is always used as given. The [AWS Organizations](/guides/aws-organizations/) guide covers that case. +With `--org` and Control Tower, `--regions all` is replaced by the landing zone's governed regions. An explicit list is always used as given. The [AWS Organizations](aws-organizations.md) guide covers that case. ## Many accounts @@ -69,9 +77,9 @@ With `--org` and Control Tower, `--regions all` is replaced by the landing zone' cloudg map -p aws --org --org-role cloudg-readonly --regions all ``` -:::warning AWSControlTowerExecution has admin rights -Without `--org-role` (or `--role-name`), cloudg assumes `AWSControlTowerExecution` in every member account. That role exists wherever Control Tower enrolled an account, but it carries administrator permissions. Deploy a read-only role to every account and pass it with `--org-role`. [AWS Organizations](/guides/aws-organizations/) has a StackSet template for it. -::: +!!! warning "AWSControlTowerExecution has admin rights" + + Without `--org-role` (or `--role-name`), cloudg assumes `AWSControlTowerExecution` in every member account. That role exists wherever Control Tower enrolled an account, but it carries administrator permissions. Deploy a read-only role to every account and pass it with `--org-role`. [AWS Organizations](aws-organizations.md) has a StackSet template for it. Always pair `--accounts` with `--role-name`. Without a role name cloudg doesn't assume anything and maps your own account once per listed ID, labelled with each of those IDs. @@ -79,14 +87,33 @@ Always pair `--accounts` with `--role-name`. Without a role name cloudg doesn't Coverage comes in layers. Each one catches what the previous one missed. -:::cells +
      + +
      + ### Deep collectors + 131 dedicated AWS collector tasks in 19 service families, each reading one service in depth and declaring what every asset talks to. Azure goes through Resource Graph, GCP through Cloud Asset Inventory. + +
      + +
      + ### Breadth sweep + On AWS, a Cloud Control sweep lists every CloudFormation resource type that has a list handler, tagged or not, so services without a dedicated collector still show up. The tagging API sweep adds tagged resources on top. + +
      + +
      + ### Every account + Every account gets a `CLOUD_ACCOUNT` node that contains its top-level resources. With `--org`, accounts hang under the OU tree next to SCPs and Control Tower controls. -::: + +
      + +
      When the same resource turns up twice, the copy from a dedicated collector wins over a Cloud Control hit, which wins over a tagging API hit. Their declared relationships and aliases are merged into the copy that is kept. @@ -125,9 +152,9 @@ cloudg map -p aws --regions all --services containers,serverless,sweep cloudg map -p aws --regions all --exclude-services data,cloudformation ``` -:::warning The sweeps are a family too -The two sweeps belong to the family `sweep`. Once you pass `--services`, only what you listed runs, so `--services containers,serverless` also drops the Cloud Control and tagging API sweeps. Add `sweep` to the list if you want them. A family name that matches nothing (a typo, say) is not an error; it selects no tasks. -::: +!!! warning "The sweeps are a family too" + + The two sweeps belong to the family `sweep`. Once you pass `--services`, only what you listed runs, so `--services containers,serverless` also drops the Cloud Control and tagging API sweeps. Add `sweep` to the list if you want them. A family name that matches nothing (a typo, say) is not an error; it selects no tasks. Narrowing the run has a side effect on the links. A function whose role lives in the `identity` family can't be linked to that role when `identity` is excluded, so the reference shows up under `unresolved_references` instead of as an edge. @@ -158,9 +185,9 @@ Two sweeps catch what no deep collector reads. They are separate flags. `--sweep / --no-sweep` (on by default) controls the Resource Groups Tagging API sweep. AWS only returns resources that have been tagged at some point, so this sweep mostly enriches; the Cloud Control sweep is the one that finds untagged resources. Hits carry `discovered_via: "tagging-api"`. -:::note -`cloudg map` always hands the value of `--sweep` to the mapper, and that value defaults to on, so the CLI overrides `inventory.tagging_sweep: false` from your config file even when you don't type the flag. Use `--no-sweep` on the command line to turn the tagging sweep off. -::: +!!! note + + `cloudg map` always hands the value of `--sweep` to the mapper, and that value defaults to on, so the CLI overrides `inventory.tagging_sweep: false` from your config file even when you don't type the flag. Use `--no-sweep` on the command line to turn the tagging sweep off. ## Azure and GCP @@ -178,7 +205,7 @@ cloudg map -p all --regions all ## How a run fits together -```mermaid caption="One mapping run, from discovery to the files on disk" +```mermaid flowchart TD A["Org discovery (--org)"] --> B["Deep collectors, per account and region"] B --> C["Cloud Control and tagging sweeps"] @@ -188,6 +215,8 @@ flowchart TD F --> G["Dependency analysis"] G --> H["Exports in ./reports"] ``` +One mapping run, from discovery to the files on disk +{ .diagram-caption } Collectors never create edges across services themselves. They declare what each asset talks to, by any identifier they have (an ARN, a queue URL, an image URI, a bare `sg-` ID), and the relationship linker resolves those identifiers across services, regions and accounts. It runs three passes: declared relations, provider rules for older metadata conventions, and a generic pass that turns any identifier-looking string in an asset's metadata into a `REFERENCES` edge when it resolves to another asset. @@ -240,7 +269,7 @@ if inventory.throttling: print("throttled:", line) ``` -Throttling never fails a map. Calls are paced and retried; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` with an error starting `throttled:`. The run's throttling telemetry is attached as `inventory.throttling` and, when present, written under a top-level `throttling` key in `inventory-map.json`. The [rate limits guide](/guides/rate-limits/) covers tuning. +Throttling never fails a map. Calls are paced and retried; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` with an error starting `throttled:`. The run's throttling telemetry is attached as `inventory.throttling` and, when present, written under a top-level `throttling` key in `inventory-map.json`. The [rate limits guide](rate-limits.md) covers tuning. ## Output files @@ -256,7 +285,7 @@ Everything goes to `-o/--output`, `./reports` by default. | `asset-map.json` | with `--findings` | per-asset finding counts and severity breakdown, riskiest first | | `compliance-map.json` | with `--findings` | framework to findings and affected assets | -`inventory-map.json` is self-contained, and `cloudg deps` reads it back without cloud access. The graph files merge parallel rule edges, so count edges in `inventory-map.json`. The [output files reference](/reference/output-files/) has the field-level detail. +`inventory-map.json` is self-contained, and `cloudg deps` reads it back without cloud access. The graph files merge parallel rule edges, so count edges in `inventory-map.json`. The [output files reference](../reference/output-files.md) has the field-level detail. ## Overlay scanner findings @@ -333,11 +362,24 @@ graph = GraphBuilder().build(assets + linker.external_assets, edges) print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges in the graph") ``` -Your own collectors can declare relationships the same way the built-in ones do, by putting entries such as `{"target": "", "edge": "INVOKES", "relationship": "TRIGGERED_BY"}` in `metadata["relations"]`. The [RelationshipLinker](/api/relationshiplinker/) page has the details. +Your own collectors can declare relationships the same way the built-in ones do, by putting entries such as `{"target": "", "edge": "INVOKES", "relationship": "TRIGGERED_BY"}` in `metadata["relations"]`. The [RelationshipLinker](../api/relationshiplinker.md) page has the details. + +
      + +- [**Dependencies and blast radius**](dependencies.md) + + Query the saved map with `cloudg deps`. + +- [**AWS Organizations**](aws-organizations.md) + + Map every account under the management account. + +- [**cloudg map**](../cli/map.md) + + Every option of the command. + +- [**InventoryMapper**](../api/inventorymapper.md) + + The Python class behind it. -:::links -- [Dependencies and blast radius](/guides/dependencies/) Query the saved map with `cloudg deps`. -- [AWS Organizations](/guides/aws-organizations/) Map every account under the management account. -- [cloudg map](/cli/map/) Every option of the command. -- [InventoryMapper](/api/inventorymapper/) The Python class behind it. -::: +
      diff --git a/site/content/guides/pipeline.md b/site/content/guides/pipeline.md index d613554..b8f511f 100644 --- a/site/content/guides/pipeline.md +++ b/site/content/guides/pipeline.md @@ -1,16 +1,17 @@ --- title: The pipeline -lede: "`cloudg run` is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails." -meta: - - [Command, "`cloudg run`"] - - [Python, "`CloudGEngine.run_pipeline()`"] - - [Output, "`./reports` by default"] -source: cloudg/cli_run_helpers.py +description: 'cloudg run is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails.' --- +`cloudg run` is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails. +{ .lede } + +**Command** `cloudg run` · **Python** `CloudGEngine.run_pipeline()` · **Output** `./reports` by default +{ .page-meta } + Each phase hands its results to the next in memory. Files are written along the way, but no phase reads back what an earlier one wrote, so you can delete `./reports` between runs without breaking anything. The graph phase and the scan phase both start from the collected inventory; the normaliser is where their findings meet. -```mermaid caption="Phases of cloudg run and what each one writes" +```mermaid flowchart TD A["1 Collect: MultiAccountCollector"] --> B["2 Graph: GraphBuilder, reachability"] B --> B2["2c RAG chunks / 2d Terraform"] @@ -24,6 +25,8 @@ flowchart TD C2 -. "ontology.ttl, ontology.jsonld" .-> F E -. "findings.json, report.html" .-> F ``` +Phases of cloudg run and what each one writes +{ .diagram-caption } The CLI prints each phase as it starts, with the same numbering: @@ -52,7 +55,7 @@ Collection reads the configuration (providers, regions, accounts, credentials) a `--regions all` turns into `["ALL"]` for all three providers, and region discovery replaces it with the enabled regions before collection starts. Without `--regions`, AWS uses `us-east-1` (or `--region`), while Azure and GCP default to all. -Two limits keep collection polite. An `asyncio.Semaphore` sized by `concurrency_limit` (default 5) caps how many tasks run at once: AWS account and region pairs, Azure subscriptions and GCP projects or organizations, all providers together. Below that, every API call passes through the shared rate limiter configured under `ratelimit`, which paces calls per service, honours Retry-After, and opens a circuit breaker on an API that stays throttled. [Rate limits](/guides/rate-limits/) explains the knobs. +Two limits keep collection polite. An `asyncio.Semaphore` sized by `concurrency_limit` (default 5) caps how many tasks run at once: AWS account and region pairs, Azure subscriptions and GCP projects or organizations, all providers together. Below that, every API call passes through the shared rate limiter configured under `ratelimit`, which paces calls per service, honours Retry-After, and opens a circuit breaker on an API that stays throttled. [Rate limits](rate-limits.md) explains the knobs. ### When collection fails @@ -130,7 +133,7 @@ Three renderers write the final files: ## The whole run, step by step -```mermaid caption="Who calls whom during cloudg run" +```mermaid sequenceDiagram participant CLI as cloudg run participant Col as MultiAccountCollector @@ -149,6 +152,8 @@ sequenceDiagram N-->>CLI: ScanResult CLI->>R: findings.json, topology.svg, report.html ``` +Who calls whom during cloudg run +{ .diagram-caption } ## Running it from the CLI @@ -171,48 +176,50 @@ cloudg run -p aws --iac-dir ./infra --images nginx:1.27,myrepo/api:2.3 --no-onto cloudg -c config.yaml run -p all --regions all -o ./reports/2026-10-09 ``` -`config.yaml` is only read when you pass `-c`; there is no automatic lookup in the current directory. The [cloudg run reference](/cli/run/) lists every flag. +`config.yaml` is only read when you pass `-c`; there is no automatic lookup in the current directory. The [cloudg run reference](../cli/run.md) lists every flag. ## Running it from Python `CloudGEngine` implements the same phases. `run_pipeline()` is the async version and `run_pipeline_sync()` wraps it in `asyncio.run` for plain scripts. Inside a running event loop (Jupyter runs one) the sync wrapper raises `RuntimeError`; `await engine.run_pipeline()` there instead: -```python tab="Sync" title="run_pipeline.py" -from cloudg import CloudGConfig, CloudGEngine +=== "Sync" -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] -config.scanners.enabled = ["prowler", "iam"] + ```python title="run_pipeline.py" + from cloudg import CloudGConfig, CloudGEngine -engine = CloudGEngine(config) -engine.on_phase_start = lambda phase: print(f"-> {phase}") -engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + config.scanners.enabled = ["prowler", "iam"] -result = engine.run_pipeline_sync(output_dir="./reports") + engine = CloudGEngine(config) + engine.on_phase_start = lambda phase: print(f"-> {phase}") + engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") -print(result.to_summary()) -print(result.report_paths) # {'json': ..., 'html': ...} -for cov in result.coverage: - failed = [s.service for s in cov.services if s.status.value == "FAILED"] - if failed: - print(cov.provider, cov.region, "failed:", failed) -``` + result = engine.run_pipeline_sync(output_dir="./reports") -```python tab="Async" title="run_pipeline_async.py" -import asyncio + print(result.to_summary()) + print(result.report_paths) # {'json': ..., 'html': ...} + for cov in result.coverage: + failed = [s.service for s in cov.services if s.status.value == "FAILED"] + if failed: + print(cov.provider, cov.region, "failed:", failed) + ``` -from cloudg import CloudGConfig, CloudGEngine +=== "Async" + ```python title="run_pipeline_async.py" + import asyncio -async def main() -> None: - engine = CloudGEngine(CloudGConfig(providers=["aws", "gcp"])) - result = await engine.run_pipeline(output_dir="./reports") - print(result.total_assets, "assets,", result.total_findings, "findings") - print(result.severity_breakdown) + from cloudg import CloudGConfig, CloudGEngine + async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws", "gcp"])) + result = await engine.run_pipeline(output_dir="./reports") + print(result.total_assets, "assets,", result.total_findings, "findings") + print(result.severity_breakdown) -asyncio.run(main()) -``` + asyncio.run(main()) + ``` CLI flags become config fields or method arguments: @@ -258,7 +265,6 @@ import asyncio from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -275,7 +281,6 @@ async def main() -> None: print(analysis.graph_nodes, "nodes,", len(analysis.attack_paths), "attack paths") print(scan_result.summary) - asyncio.run(main()) ``` @@ -317,7 +322,6 @@ edges = [ NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), ] - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -339,7 +343,6 @@ async def main() -> None: for finding in scan_result.findings: print(finding.severity.value, finding.title) - asyncio.run(main()) ``` @@ -353,4 +356,4 @@ findings.json ontology.jsonld ontology.ttl rag_chunks.jsonl rag_metadata_ind Three nodes for two assets: the third is the external `0.0.0.0/0` node the security group rule points from. -For scanner output that already exists, `run_from_reports()` replaces collection and scanning with ingest and runs normalisation and reporting on top; [Ingesting existing output](/guides/ingesting/) covers it. [Python recipes](/guides/python-recipes/) has more combinations, and [CloudGEngine](/api/cloudgengine/) documents every method. +For scanner output that already exists, `run_from_reports()` replaces collection and scanning with ingest and runs normalisation and reporting on top; [Ingesting existing output](ingesting.md) covers it. [Python recipes](python-recipes.md) has more combinations, and [CloudGEngine](../api/cloudgengine.md) documents every method. diff --git a/site/content/guides/python-recipes.md b/site/content/guides/python-recipes.md index aac266e..cec826e 100644 --- a/site/content/guides/python-recipes.md +++ b/site/content/guides/python-recipes.md @@ -1,18 +1,19 @@ --- title: Python recipes -lede: "Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI." -meta: - - [Entry point, "`cloudg.CloudGEngine`"] - - [Needs credentials, "recipes 3 to 6 only"] -source: docs/DOCUMENTATION.md -since: "0.6.0" +description: 'Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI.' --- -The CLI is a thin layer over a public API, so anything `cloudg run` does you can do from Python, in pieces, with your own decisions in between. Each recipe below is one file you can save and run. The ones that only read scanner output work on any machine with `pip install cloudg`; the ones that collect need the provider extra (`pip install "cloudg[aws]"`) and credentials resolved as described in [authentication](/guides/authentication/). +Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI. +{ .lede } -The sample inputs used in the offline recipes are ordinary scanner output: a Prowler output directory (OCSF or `-M json-asff`), a `checkov --output json` file and `trivy image` / `trivy fs` JSON. [Ingesting existing output](/guides/ingesting/) lists exactly which fields each parser reads. +**Entry point** `cloudg.CloudGEngine` · **Needs credentials** recipes 3 to 6 only · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } -```mermaid caption="Where each recipe enters the pipeline" +The CLI is a thin layer over a public API, so anything `cloudg run` does you can do from Python, in pieces, with your own decisions in between. Each recipe below is one file you can save and run. The ones that only read scanner output work on any machine with `pip install cloudg`; the ones that collect need the provider extra (`pip install "cloudg[aws]"`) and credentials resolved as described in [authentication](authentication.md). + +The sample inputs used in the offline recipes are ordinary scanner output: a Prowler output directory (OCSF or `-M json-asff`), a `checkov --output json` file and `trivy image` / `trivy fs` JSON. [Ingesting existing output](ingesting.md) lists exactly which fields each parser reads. + +```mermaid flowchart LR C["collect()"] --> S["scan()"] I["ingest_reports()"] --> N["normalise_findings()"] @@ -23,6 +24,8 @@ flowchart LR M["map_inventory()"] --> O["export_merged()"] I --> O ``` +Where each recipe enters the pipeline +{ .diagram-caption } ## 1. Aggregate existing scan results @@ -128,7 +131,7 @@ trivy image only: ['[Trivy] CVE-2024-0001: openssl (alpine)'] `on_finding` is called once per finding, as each scanner finishes during `scan()` (or once the reports are parsed, for ingest), with a copy of the finding. Hooks run inline in the engine and any exception they raise is swallowed, so the shipper below only puts findings on a queue and lets a worker thread batch them to an HTTP collector, keeping a local NDJSON spool either way. -```python title="stream_to_siem.py" hl="35-36,82" +```python title="stream_to_siem.py" hl_lines="35-36 82" """Ship every cloudg finding to a SIEM over HTTP while the pipeline runs.""" import asyncio @@ -148,7 +151,6 @@ SIEM_URL = os.environ.get("SIEM_URL") # e.g. https://siem.example.c SIEM_TOKEN = os.environ.get("SIEM_TOKEN", "") SPOOL_FILE = "cloudg-findings.ndjson" # local copy, also the fallback - class FindingShipper: """on_finding callback: queue the finding, send batches from a worker thread. @@ -203,7 +205,6 @@ class FindingShipper: except OSError as exc: # URLError and timeouts; the spool file keeps the batch log.warning("SIEM rejected a batch of %d: %s", len(batch), exc) - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1", "eu-west-1"] @@ -222,7 +223,6 @@ async def main() -> None: print(f"shipped {shipper.sent} raw findings; {result.total_findings} after normalisation") print("summary:", json.dumps(result.to_summary(), default=str)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") asyncio.run(main()) @@ -253,7 +253,6 @@ from cloudg.renderers.json_export import JSONExporter OUT = Path("./reports") - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1"] @@ -293,7 +292,6 @@ async def main() -> None: print(JSONExporter(output_dir=str(OUT)).export(scan_result)) print(HTMLReportGenerator(output_dir=str(OUT)).generate(scan_result)) - asyncio.run(main()) ``` @@ -328,7 +326,6 @@ from cloudg.renderers.json_export import JSONExporter OUT = Path("./reports") - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1"] @@ -356,7 +353,6 @@ async def main() -> None: print(f"{len(collection.assets)} assets, {analysis.ontology_triples} ontology triples") print("findings by tool:", by_tool) - asyncio.run(main()) ``` @@ -372,54 +368,60 @@ Passing the ingested findings to `analyze()` puts them into the ontology and the Inventory mapping needs read credentials and nothing else. Scanners can run elsewhere and on another schedule, and their output merges in whenever it arrives, even in a different process, because the map round-trips through `inventory-map.json`. -```python tab="Map" title="map_inventory.py" -"""Map the inventory now; no scanners involved.""" +=== "Map" -from cloudg import CloudGConfig, CloudGEngine + ```python title="map_inventory.py" + """Map the inventory now; no scanners involved.""" -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] # or ["ALL"] + from cloudg import CloudGConfig, CloudGEngine -inventory = CloudGEngine(config).map_inventory_sync(output_dir="./map-reports") + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] # or ["ALL"] -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("by service:", summary["assets_by_service"]) -print("internet exposed:", summary["internet_exposed"]) -print("unlinked:", summary["unlinked_assets"]) -if inventory.throttling: - print("throttled:", inventory.throttling["messages"]) -``` + inventory = CloudGEngine(config).map_inventory_sync(output_dir="./map-reports") -```python tab="Overlay" title="overlay_findings.py" -"""Overlay scanner findings on an inventory map written earlier.""" + summary = inventory.summary + print(summary["total_assets"], "assets,", summary["total_edges"], "edges") + print("by service:", summary["assets_by_service"]) + print("internet exposed:", summary["internet_exposed"]) + print("unlinked:", summary["unlinked_assets"]) + if inventory.throttling: + print("throttled:", inventory.throttling["messages"]) + ``` -import json +=== "Overlay" -from cloudg import CloudGConfig, CloudGEngine -from cloudg.inventory import InventoryMapper, InventoryResult + ```python title="overlay_findings.py" + """Overlay scanner findings on an inventory map written earlier.""" -inventory = InventoryResult.load("./map-reports") # directory or inventory-map.json -print(len(inventory.assets), "assets in the saved map") + import json -engine = CloudGEngine(CloudGConfig()) -findings = engine.ingest_reports({"prowler": ["./prowler-output/"]}) + from cloudg import CloudGConfig, CloudGEngine + from cloudg.inventory import InventoryMapper, InventoryResult -mapper = InventoryMapper(CloudGConfig()) -paths = mapper.export_merged(inventory, findings, "./map-reports") -print({name: str(path) for name, path in paths.items()}) + inventory = InventoryResult.load("./map-reports") # directory or inventory-map.json + print(len(inventory.assets), "assets in the saved map") -asset_map = json.loads(paths["asset_map"].read_text()) -for entry in asset_map["assets"]: - if entry["finding_count"]: - print(entry["name"], entry["type"], entry["severity_breakdown"]) -``` + engine = CloudGEngine(CloudGConfig()) + findings = engine.ingest_reports({"prowler": ["./prowler-output/"]}) -```bash tab="CLI" -cloudg map -p aws --regions all -o ./map-reports -# later, with scanner output that cloudg already normalised: -cloudg map -p aws --regions all -o ./map-reports --findings ./reports/raw-findings.json -``` + mapper = InventoryMapper(CloudGConfig()) + paths = mapper.export_merged(inventory, findings, "./map-reports") + print({name: str(path) for name, path in paths.items()}) + + asset_map = json.loads(paths["asset_map"].read_text()) + for entry in asset_map["assets"]: + if entry["finding_count"]: + print(entry["name"], entry["type"], entry["severity_breakdown"]) + ``` + +=== "CLI" + + ```bash + cloudg map -p aws --regions all -o ./map-reports + # later, with scanner output that cloudg already normalised: + cloudg map -p aws --regions all -o ./map-reports --findings ./reports/raw-findings.json + ``` The overlay step needs no credentials. `asset-map.json` lists every asset with `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the worst first; `compliance-map.json` groups affected assets by framework. Both maps match a finding to an asset by asset id, then ARN, then a name or ARN tail that only one asset has, so scanner output that names resources by ARN (Prowler, ScoutSuite) overlays well, and IaC findings keyed by a Terraform address (Checkov) do not match a live asset at all. The CLI variant re-maps the account before merging; the Python one reuses the saved map. @@ -453,7 +455,6 @@ from cloudg.graph.builder import GraphBuilder from cloudg.inventory import RelationshipLinker from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - def to_asset(row: dict) -> CloudAsset: relations = [] if row.get("role_arn"): @@ -478,7 +479,6 @@ def to_asset(row: dict) -> CloudAsset: metadata={"relations": relations}, ) - with open("cmdb_assets.json") as fh: assets = [to_asset(row) for row in json.load(fh)] @@ -509,7 +509,7 @@ Each relation is `{"target", "edge", "relationship", "reverse", "properties", "d ## 8. Gate CI on severity -For a CI job that already runs Python, the gate can live next to the aggregation. This version fails on HIGH and CRITICAL, honours suppressions and a list of accepted check IDs, and refuses to pass on partial data. For a gate that only reads `findings.json`, with no cloudg import, see [CI and automation](/guides/ci-automation/#a-severity-gate). +For a CI job that already runs Python, the gate can live next to the aggregation. This version fails on HIGH and CRITICAL, honours suppressions and a list of accepted check IDs, and refuses to pass on partial data. For a gate that only reads `findings.json`, with no cloudg import, see [CI and automation](ci-automation.md#a-severity-gate). ```python title="gate_on_severity.py" """Aggregate IaC scanner output and fail the build on blocking findings.""" @@ -557,9 +557,22 @@ exit 1 After a cross-scanner merge, `source_finding_id` holds one scanner's check ID and `source_tool` becomes a comma list such as `"prowler, checkov"`, so an allow-list keyed on one tool's ID can miss a merged finding. When that matters, filter on the parsed findings before normalising, as in recipe 2. -:::links -- [CloudGEngine](/api/cloudgengine/) Every method and its signature. -- [Event hooks](/api/event-hooks/) When each hook fires. -- [Result objects](/api/results/) `PipelineResult`, `CollectionResult`, `AnalysisResult`. -- [RelationshipLinker](/api/relationshiplinker/) Resolution rules for declared relations. -::: +
      + +- [**CloudGEngine**](../api/cloudgengine.md) + + Every method and its signature. + +- [**Event hooks**](../api/event-hooks.md) + + When each hook fires. + +- [**Result objects**](../api/results.md) + + `PipelineResult`, `CollectionResult`, `AnalysisResult`. + +- [**RelationshipLinker**](../api/relationshiplinker.md) + + Resolution rules for declared relations. + +
      diff --git a/site/content/guides/quickstart.md b/site/content/guides/quickstart.md index e9ad24b..77a675f 100644 --- a/site/content/guides/quickstart.md +++ b/site/content/guides/quickstart.md @@ -1,17 +1,20 @@ --- title: Quickstart -lede: "Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners." -meta: - - [Time, "about 10 minutes, plus scanner run time"] - - [Needs, "Python 3.11+, read-only AWS credentials"] -source: site/content/guides/quickstart.md +description: Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners. --- +Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners. +{ .lede } + +**Time** about 10 minutes, plus scanner run time · **Needs** Python 3.11+, read-only AWS credentials +{ .page-meta } + This page takes the shortest path to each of cloudg's three entry points. It sticks to AWS and one region so the first run is quick; every command takes `-p azure`, `-p gcp` or `-p all` the same way. ## Scan an AWS account -::::steps +
      + ### Install cloudg and a scanner ```bash @@ -19,7 +22,7 @@ pip install "cloudg[aws]" pip install prowler ``` -Prowler is enough for a first run. cloudg skips scanners it cannot find on `PATH`, so you will see warnings for ScoutSuite, Checkov and Trivy; that is expected. [Installation](/guides/installation/) covers the rest, and the [Docker image](/guides/docker/) has Prowler, Checkov and Trivy built in. +Prowler is enough for a first run. cloudg skips scanners it cannot find on `PATH`, so you will see warnings for ScoutSuite, Checkov and Trivy; that is expected. [Installation](installation.md) covers the rest, and the [Docker image](docker.md) has Prowler, Checkov and Trivy built in. ### Authenticate @@ -31,9 +34,9 @@ export AWS_PROFILE=audit aws sts get-caller-identity ``` -:::tip Use read-only credentials -cloudg only reads. The AWS managed policies `SecurityAudit` and `ViewOnlyAccess` are the usual starting point for the role or user you scan with. -::: +!!! tip "Use read-only credentials" + + cloudg only reads. The AWS managed policies `SecurityAudit` and `ViewOnlyAccess` are the usual starting point for the role or user you scan with. ### Run the pipeline @@ -53,7 +56,8 @@ xdg-open reports/report.html # Linux ``` `report.html` is a single file with the findings, assets and graph data embedded in it: the topology graph, the findings table and the compliance matrix. Chart.js and D3 are embedded too, so it opens without network access. -:::: + +
      ### What lands in ./reports @@ -84,7 +88,8 @@ cloudg report -i reports/findings.json --format html If someone else runs the scanners (in CI, on a schedule, on another network), you only need their output files. `cloudg ingest` needs no cloud access and no scanner binaries, and the core package without extras is enough. -::::steps +
      + ### Install the core package ```bash @@ -113,15 +118,17 @@ findings.json raw-findings.json report.html ``` `raw-findings.json` holds every parsed finding before deduplication. `findings.json` and `report.html` hold the merged, compliance-mapped set. The last line of the output tells you how many duplicates were merged, for example "Ingested 3 findings from 3 tool(s), 3 after deduplication". -:::: -There is no topology in an ingest report, because nothing was collected. [Ingesting existing output](/guides/ingesting/) covers the exact input formats and how to combine ingested findings with a live collection. +
      + +There is no topology in an ingest report, because nothing was collected. [Ingesting existing output](ingesting.md) covers the exact input formats and how to combine ingested findings with a live collection. ## No scanners: map the inventory `cloudg map` answers "what is deployed, and what depends on what". It needs read-only credentials and the provider extra, but no scanners. -::::steps +
      + ### Map one region ```bash @@ -146,15 +153,29 @@ cloudg deps arn:aws:iam::123456789012:role/app-role ``` `cloudg deps` reads the saved map from `./reports` and prints what that asset depends on and what would break without it. `-d down` shows only the blast radius. -:::: -When you do run scanners later, merge their findings into the map with `cloudg map --findings ./reports/findings.json` (or the `raw-findings.json` that `cloudg ingest` and `cloudg scan` write), which adds `asset-map.json` and `compliance-map.json`. [Inventory mapping](/guides/inventory-mapping/) goes through the map in depth. +
      + +When you do run scanners later, merge their findings into the map with `cloudg map --findings ./reports/findings.json` (or the `raw-findings.json` that `cloudg ingest` and `cloudg scan` write), which adds `asset-map.json` and `compliance-map.json`. [Inventory mapping](inventory-mapping.md) goes through the map in depth. ## Next steps -:::links -- [The pipeline](/guides/pipeline/) What each phase of cloudg run does, and how to drive it from Python. -- [Authentication](/guides/authentication/) Profiles, role assumption, OIDC in CI, Azure and GCP. -- [Running scanners](/guides/running-scanners/) IaC directories, container images and scanner arguments. -- [Reports](/guides/reports/) Reading report.html and findings.json. -::: +
      + +- [**The pipeline**](pipeline.md) + + What each phase of cloudg run does, and how to drive it from Python. + +- [**Authentication**](authentication.md) + + Profiles, role assumption, OIDC in CI, Azure and GCP. + +- [**Running scanners**](running-scanners.md) + + IaC directories, container images and scanner arguments. + +- [**Reports**](reports.md) + + Reading report.html and findings.json. + +
      diff --git a/site/content/guides/rate-limits.md b/site/content/guides/rate-limits.md index caaad8b..4a7d2b6 100644 --- a/site/content/guides/rate-limits.md +++ b/site/content/guides/rate-limits.md @@ -1,23 +1,23 @@ --- title: Rate limits and throttling -lede: "Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map." -meta: - - [Package, "`cloudg.resilience`"] - - [Config, "[`ratelimit`](/reference/config/ratelimit/)"] - - [Since, "0.6.0"] -source: docs/RESILIENCE.md -since: "0.6.0" +description: Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map. --- +Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map. +{ .lede } + +**Package** `cloudg.resilience` · **Config** [`ratelimit`](../reference/config/ratelimit.md) · **Since** 0.6.0 · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + A single `cloudg map -p aws --org --regions all` runs 133 deep collector tasks, a Cloud Control sweep and a tagging sweep in every region of every member account, and some collectors fan out per item (one `DescribeImageAttribute` per AMI). Those APIs have quotas, and the quotas are shared with your CI pipelines, Terraform runs and the console. Up to 0.5.x cloudg trusted each SDK's own retries. Forty EC2 clients in one account then backed off independently and kept the account's bucket empty, a service that stayed throttled looked the same in coverage as an access error, and nothing remembered that an API had just said no. -Since 0.6.0 the `cloudg.resilience` package sits between cloudg and every SDK. You rarely need to touch it. This page explains what it does so you can read its output and tune it when an estate needs it. Every key and default is in the [ratelimit reference](/reference/config/ratelimit/), and the Python side in the [cloudg.resilience API](/api/resilience/). +Since 0.6.0 the `cloudg.resilience` package sits between cloudg and every SDK. You rarely need to touch it. This page explains what it does so you can read its output and tune it when an estate needs it. Every key and default is in the [ratelimit reference](../reference/config/ratelimit.md), and the Python side in the [cloudg.resilience API](../api/resilience.md). ## How one call is paced Every call is attributed to a scope, `Scope(provider, account, region, service, operation)`, which prints as `aws/123456789012/us-east-1/ec2` (with `/DescribeSubnets` appended where the operation matters). Before the call goes out it passes four gates, and after it comes back the outcome feeds the same state. -```mermaid caption="call_with_resilience: the full sequence. The SDK integrations reuse the same parts and let the SDK do the retrying." +```mermaid flowchart TD A[Call arrives with a scope] --> B{"Breaker open or deadline spent?"} B -->|yes| X["Fail fast, nothing sent"] @@ -33,6 +33,8 @@ flowchart TD J -->|no| Z["Breaker counts a failure, record give-up"] G -->|FATAL| R[Re-raise at once] ``` +call_with_resilience: the full sequence. The SDK integrations reuse the same parts and let the SDK do the retrying. +{ .diagram-caption } Errors are sorted by `classify(exc)` into three kinds. THROTTLED means the provider asked cloudg to slow down: AWS `Throttling`, `RequestLimitExceeded`, `TooManyRequestsException` and the rest of botocore's list, Azure `SubscriptionRequestsThrottled`, GCP `ResourceExhausted`, or any bare HTTP 429 or 503. TRANSIENT covers 408, 500, 502, 504 and connection resets. FATAL is everything else with a provider error code, `AccessDenied` included. The classifier reads error codes, exception classes and HTTP statuses first and only falls back to message text when none of them says anything. @@ -83,7 +85,7 @@ A bulkhead caps requests in flight per provider and account: 128 for AWS, 16 for Each leaf scope has a breaker with the same granularity as its bucket: per EC2 action, per service elsewhere. A throttled `DescribeSubnets` therefore does not block `DescribeVpcs` in the same account and region. -```mermaid caption="One breaker. Rejected calls never reach the provider and never count as failures." +```mermaid stateDiagram-v2 [*] --> Closed Closed --> Closed: success resets the count @@ -93,6 +95,8 @@ stateDiagram-v2 HalfOpen --> Open: probe fails, cooldown doubled up to 600 s HalfOpen --> HalfOpen: probe lease expires, next call probes ``` +One breaker. Rejected calls never reach the provider and never count as failures. +{ .diagram-caption } A failure here means a call that still failed with throttling or a transient error after all its retries. The defaults are 5 consecutive failures (`breaker_threshold`) and a 60-second cooldown (`breaker_cooldown_seconds`). While open, calls fail at once with `circuit open for after repeated throttling; retry in s`. @@ -186,7 +190,7 @@ People rarely run `cloudg map` twice in a minute. An AI agent might. The MCP liv Scopes come from the config: `aws/` per configured account, else `aws/profile:`, else `aws`, and the equivalents for Azure subscriptions and GCP projects. A refusal is a tool error with code -31029 and `reason` `cooldown`, `busy` or `quota`, plus `retry_after_seconds` and a `use_dataset` pointer to the newest loaded live dataset covering the same providers. Principals with the `admin` or `operator` role can pass `force=true` to skip the cooldown, though not the concurrency caps. -The tool `rate_limit_status` and the resource `cloudg://ratelimit` show cooldowns, operations in flight and the governor's throttling state without calling any cloud API. Point agents at them before they reach for a live tool. These guard rails are separate from the MCP policy's per-principal rate limits, which decide how often someone may call a tool at all; see the [MCP server](/mcp/) section. +The tool `rate_limit_status` and the resource `cloudg://ratelimit` show cooldowns, operations in flight and the governor's throttling state without calling any cloud API. Point agents at them before they reach for a live tool. These guard rails are separate from the MCP policy's per-principal rate limits, which decide how often someone may call a tool at all; see the [MCP server](../mcp/index.md) section. ## Tuning the ratelimit block @@ -208,9 +212,9 @@ ratelimit: ec2.DescribeImageAttribute: {max_rps: 5, burst: 10} # AMI-heavy region ``` -:::warning An override replaces the whole built-in entry -`ec2: {max_rps: 10}` drops the built-in burst of 100 to 20 (twice the rate). Repeat the burst if you want to keep it: `ec2: {max_rps: 10, burst: 50}`. EC2 stays per action either way. -::: +!!! warning "An override replaces the whole built-in entry" + + `ec2: {max_rps: 10}` drops the built-in burst of 100 to 20 (twice the rate). Repeat the burst if you want to keep it: `ec2: {max_rps: 10, burst: 50}`. EC2 stays per action either way. ### A shared or large estate @@ -292,8 +296,18 @@ print(get_governor().budget("aws").remaining) Learned rates and breaker state live in memory, so each new `cloudg map` process starts from the configured rates. A few calls are not instrumented yet: the GCP organization-policy and access-context-manager clients outside the asset listing, the urllib fallback that lists Azure subscriptions, and the single region-discovery call. A blocking boto3 or Azure call made on an event-loop thread goes ahead without a bulkhead slot when none is free, so `max_concurrency` is not a hard cap on that path. And per-item fan-out is now held to the documented rates, which makes some maps slower than in 0.5.x: 1,000 `DescribeImageAttribute` calls in one region take about 50 seconds at 20 rps. -:::links -- [cloudg.resilience](/api/resilience/) `call_with_resilience`, `@resilient`, the governor and the guard from Python. -- [ratelimit reference](/reference/config/ratelimit/) Every key with its default. -- [Inventory mapping](/guides/inventory-mapping/) Where coverage records come from. -::: +
      + +- [**cloudg.resilience**](../api/resilience.md) + + `call_with_resilience`, `@resilient`, the governor and the guard from Python. + +- [**ratelimit reference**](../reference/config/ratelimit.md) + + Every key with its default. + +- [**Inventory mapping**](inventory-mapping.md) + + Where coverage records come from. + +
      diff --git a/site/content/guides/reports.md b/site/content/guides/reports.md index d9f2c97..2a9ded6 100644 --- a/site/content/guides/reports.md +++ b/site/content/guides/reports.md @@ -1,19 +1,19 @@ --- title: Reports -lede: "Every path through cloudg ends in the same two files: `findings.json` for machines and `report.html` for people. `cloudg report` rebuilds them from an earlier run without touching the cloud." -meta: - - [Command, "`cloudg report`"] - - [Formats, "HTML, JSON, SVG"] - - [Python, "`JSONExporter`, `HTMLReportGenerator`, `SVGRenderer`"] -source: cloudg/renderers/html_report.py -since: "0.6.0" +description: 'Every path through cloudg ends in the same two files: findings.json for machines and report.html for people. cloudg report rebuilds them from an earlier run without touching the cloud.' --- +Every path through cloudg ends in the same two files: `findings.json` for machines and `report.html` for people. `cloudg report` rebuilds them from an earlier run without touching the cloud. +{ .lede } + +**Command** `cloudg report` · **Formats** HTML, JSON, SVG · **Python** `JSONExporter`, `HTMLReportGenerator`, `SVGRenderer` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + The renderers in `cloudg/renderers/` take a normalised `ScanResult` (assets, findings, compliance results) plus, when there is one, the D3 graph of the collected assets. They don't call any cloud API or scanner, which is why `cloudg report` can re-render an old run anywhere. ## Which command writes what -```mermaid caption="Report inputs and outputs" +```mermaid flowchart LR RUN["cloudg run"] --> SR["ScanResult and graph"] ING["cloudg ingest"] --> SR @@ -23,6 +23,8 @@ flowchart LR SR --> SVG["topology.svg"] SCAN["cloudg scan, cloudg ingest"] --> RAW["raw-findings.json"] ``` +Report inputs and outputs +{ .diagram-caption } | Command | `findings.json` | `report.html` | `topology.svg` | `raw-findings.json` | |---|---|---|---|---| @@ -35,7 +37,7 @@ flowchart LR `cloudg run`, `run_pipeline()` and `run_from_reports()` write the formats listed in `report.formats`, all three by default. `run_from_reports()` never writes `topology.svg`, because it has no collected assets to draw. `cloudg ingest` and `cloudg report` go by `--format` instead. -`cloudg run` writes more than these (GraphML, Cytoscape JSON when `graph.export_cytoscape` is on, the ontology, RAG chunks, Terraform); those come from the graph and analysis phases and are listed on [Output files](/reference/output-files/). +`cloudg run` writes more than these (GraphML, Cytoscape JSON when `graph.export_cytoscape` is on, the ontology, RAG chunks, Terraform); those come from the graph and analysis phases and are listed on [Output files](../reference/output-files.md). Those three are the only report formats. There is no CSV, SARIF or PDF writer in 0.6.0. `findings.json` is easy to convert, though; there's a [CSV recipe](#exporting-to-csv) further down. @@ -68,7 +70,7 @@ Those three are the only report formats. There is no CSV, SARIF or PDF writer in } ``` -`assets`, `findings`, `compliance` and `edges` are the Pydantic models from `cloudg.schema.models`, serialised in JSON mode. Assets leave out `raw_data`. Findings include the computed `risk_score`. `edges` holds the collected `NetworkEdge` objects, so `cloudg report` can draw the topology again. `graph` holds D3 nodes and links for `cloudg run`, and is empty for ingest. In the `metadata` block, the normaliser doesn't fill in `provider`, `account_id` or `region` in 0.6.0, so expect `null` there. Both timestamps are ISO 8601 strings set at normalisation time, and a `completed_at` that was never set is `null`. The field-by-field reference is on [Output files](/reference/output-files/), and the models are on [Data models](/api/models/). +`assets`, `findings`, `compliance` and `edges` are the Pydantic models from `cloudg.schema.models`, serialised in JSON mode. Assets leave out `raw_data`. Findings include the computed `risk_score`. `edges` holds the collected `NetworkEdge` objects, so `cloudg report` can draw the topology again. `graph` holds D3 nodes and links for `cloudg run`, and is empty for ingest. In the `metadata` block, the normaliser doesn't fill in `provider`, `account_id` or `region` in 0.6.0, so expect `null` there. Both timestamps are ISO 8601 strings set at normalisation time, and a `completed_at` that was never set is `null`. The field-by-field reference is on [Output files](../reference/output-files.md), and the models are on [Data models](../api/models.md). `raw-findings.json` is different: a bare JSON list of `Finding` objects as the parsers produced them, before deduplication, rescoring or compliance mapping. It has no `summary`, `compliance` or `graph`. @@ -84,7 +86,7 @@ The Findings tab lists every finding, sorted by severity, with dropdown filters Analytics has four Chart.js charts: severity distribution, asset types, findings by source tool, and compliance per framework. -Compliance shows one card per framework with Pass and Fail counts. As explained in [Compliance frameworks](/guides/compliance/#report-html), Fail counts the controls that have findings, and Pass counts the controls that only Prowler's passing checks map to. Without passing-check data in the input (no Prowler, or scanners that only report failures), Pass is 0 and the percentage bar reads 0%. +Compliance shows one card per framework with Pass and Fail counts. As explained in [Compliance frameworks](compliance.md#report-html), Fail counts the controls that have findings, and Pass counts the controls that only Prowler's passing checks map to. Without passing-check data in the input (no Prowler, or scanners that only report failures), Pass is 0 and the percentage bar reads 0%. The download button saves a file named `findings.json` that holds only `findings`, `graph` and the tree data. It isn't the same file cloudg writes, but `cloudg report -i` and the viewer both accept it. @@ -102,17 +104,21 @@ If the template can't be found or fails to render, cloudg logs a warning and wri `cloudg report` reads a `findings.json` and renders it again. Use it to regenerate the HTML after upgrading cloudg, to produce just the SVG, or to rebuild a report on a machine that never had cloud access. -```bash tab="CLI" -cloudg report -i ./reports/findings.json -o ./reports-rerendered -cloudg report -i ./reports/findings.json -o ./reports-rerendered --format html -cloudg report -i ./archive/2026-09-30/findings.json -o ./archive/2026-09-30/svg --format svg -``` +=== "CLI" -```bash tab="Docker" -docker run --rm \ - -v "$PWD/reports:/data" \ - ghcr.io/morpheuslord/cloudg:latest report -i /data/findings.json -o /data/rerendered -``` + ```bash + cloudg report -i ./reports/findings.json -o ./reports-rerendered + cloudg report -i ./reports/findings.json -o ./reports-rerendered --format html + cloudg report -i ./archive/2026-09-30/findings.json -o ./archive/2026-09-30/svg --format svg + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD/reports:/data" \ + ghcr.io/morpheuslord/cloudg:latest report -i /data/findings.json -o /data/rerendered + ``` | Flag | Meaning | |---|---| @@ -229,13 +235,26 @@ report: output_dir: ./reports ``` -`formats` lists the reports that `cloudg run`, `run_pipeline()` and `run_from_reports()` write; unknown names are dropped with a warning. `cloudg ingest` and `cloudg report` use `--format` instead. `inline_js` embeds Chart.js and D3 in `report.html` (`true`, the default) or loads them from the CDN (`false`); every command that writes the HTML report reads it. `output_dir` is where the `CloudGEngine` methods write when they get no `output_dir`, and the MCP server's default workspace directory. The CLI commands use `-o` (default `./reports`) and ignore it. The keys are listed on the [report config page](/reference/config/report/). +`formats` lists the reports that `cloudg run`, `run_pipeline()` and `run_from_reports()` write; unknown names are dropped with a warning. `cloudg ingest` and `cloudg report` use `--format` instead. `inline_js` embeds Chart.js and D3 in `report.html` (`true`, the default) or loads them from the CDN (`false`); every command that writes the HTML report reads it. `output_dir` is where the `CloudGEngine` methods write when they get no `output_dir`, and the MCP server's default workspace directory. The CLI commands use `-o` (default `./reports`) and ignore it. The keys are listed on the [report config page](../reference/config/report.md). ## Next -:::links -- [cloudg report](/cli/report/) The command reference. -- [Output files](/reference/output-files/) Every file cloudg writes, field by field. -- [Compliance frameworks](/guides/compliance/) What the compliance output means. -- [Ingesting existing output](/guides/ingesting/) Reports from scans that ran elsewhere. -::: +
      + +- [**cloudg report**](../cli/report.md) + + The command reference. + +- [**Output files**](../reference/output-files.md) + + Every file cloudg writes, field by field. + +- [**Compliance frameworks**](compliance.md) + + What the compliance output means. + +- [**Ingesting existing output**](ingesting.md) + + Reports from scans that ran elsewhere. + +
      diff --git a/site/content/guides/running-scanners.md b/site/content/guides/running-scanners.md index 068e307..6622b2c 100644 --- a/site/content/guides/running-scanners.md +++ b/site/content/guides/running-scanners.md @@ -1,17 +1,17 @@ --- title: Running scanners -lede: "cloudg doesn't ship its own cloud checks. It drives Prowler, ScoutSuite, Checkov and Trivy as subprocesses, adds a built-in IAM linter, and turns everything they report into one list of findings." -meta: - - [Commands, "`cloudg scan`, `cloudg run`"] - - [Python, "`CloudGEngine.scan()`"] - - [Needs, "Scanner binaries on `PATH`"] -source: cloudg/api_scanners.py -since: "0.6.0" +description: cloudg doesn't ship its own cloud checks. It drives Prowler, ScoutSuite, Checkov and Trivy as subprocesses, adds a built-in IAM linter, and turns everything they report into one list of findings. --- +cloudg doesn't ship its own cloud checks. It drives Prowler, ScoutSuite, Checkov and Trivy as subprocesses, adds a built-in IAM linter, and turns everything they report into one list of findings. +{ .lede } + +**Commands** `cloudg scan`, `cloudg run` · **Python** `CloudGEngine.scan()` · **Needs** Scanner binaries on `PATH` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + There are two ways to run the scanners. `cloudg scan` runs them on their own, with no collection, and writes the raw findings to disk. `cloudg run` runs them as phase 3 of the full pipeline, after collection and graph analysis, and passes the findings through the normaliser into the reports. The two commands, and `CloudGEngine.scan()`, plan the scanner jobs with the same code, so they pick scanners, credentials, regions, IaC directories and timeouts the same way. What differs is what they have to work with: `cloudg scan` collects nothing. [The table near the end](#scan-and-run-compared) lists every difference. -If the scans already ran somewhere else (a CI job, another host, a nightly cron) you don't need either command. Hand the output files to [`cloudg ingest`](/guides/ingesting/) instead. +If the scans already ran somewhere else (a CI job, another host, a nightly cron) you don't need either command. Hand the output files to [`cloudg ingest`](ingesting.md) instead. ## The scanners @@ -25,7 +25,7 @@ Each scanner has a name you use in `--scanners` and in `scanners.enabled`. Four | `trivy` | `trivy` | Binary from [trivy.dev](https://trivy.dev/) | Container images (CVEs, secrets), or a directory (vulnerabilities, misconfigurations, secrets) | Only for private registries | | `iam` | none | Parliament ships with cloudg | IAM role trust policies and policy documents from the collected inventory | No (reads collected assets) | -Installing each external tool into its own virtual environment keeps its dependencies away from cloudg's and from the other scanners. The [Docker image](/guides/docker/) does the same: Prowler and Checkov live in separate venvs under `/opt`, with the binaries symlinked onto `PATH`. That image bundles Prowler, Checkov and Trivy. It does not include ScoutSuite. +Installing each external tool into its own virtual environment keeps its dependencies away from cloudg's and from the other scanners. The [Docker image](docker.md) does the same: Prowler and Checkov live in separate venvs under `/opt`, with the binaries symlinked onto `PATH`. That image bundles Prowler, Checkov and Trivy. It does not include ScoutSuite. ```bash python -m venv ~/.venvs/prowler && ~/.venvs/prowler/bin/pip install prowler @@ -53,7 +53,7 @@ An explicit region list in `aws.regions` is passed as `-f `, unless `sc Prowler and ScoutSuite audit the account those credentials belong to: the caller's, or `role_arn`'s. They do not run once per member account of `aws.accounts` or `--org`; only collection fans out over those. -Prowler's stdout is streamed into the cloudg log, with a progress line every 25 checks. A non-zero exit code is logged as a warning and the output is parsed anyway, since Prowler exits non-zero whenever a check fails. Records with `Compliance.Status` of `PASSED` give no finding; their check names are kept so the compliance controls they cover can be marked `PASS` (see [Compliance frameworks](/guides/compliance/)). +Prowler's stdout is streamed into the cloudg log, with a progress line every 25 checks. A non-zero exit code is logged as a warning and the output is parsed anyway, since Prowler exits non-zero whenever a check fails. Records with `Compliance.Status` of `PASSED` give no finding; their check names are kept so the compliance controls they cover can be marked `PASS` (see [Compliance frameworks](compliance.md)). ### ScoutSuite @@ -112,49 +112,55 @@ custodian run --output-dir ./custodian-output cloudg/policies/custodian-aws-secu ## Choosing scanners -`--scanners` takes a comma-separated list of the names above, or the name of an installed [scanner plugin](/api/entry-points/). Spaces are trimmed and names are lowercased. A name that matches nothing is skipped with a warning listing the available scanners. +`--scanners` takes a comma-separated list of the names above, or the name of an installed [scanner plugin](../api/entry-points.md). Spaces are trimmed and names are lowercased. A name that matches nothing is skipped with a warning listing the available scanners. -```bash tab="CLI" -cloudg run -p aws --regions us-east-1,eu-west-1 --scanners prowler,trivy --images 123456789012.dkr.ecr.us-east-1.amazonaws.com/api:1.4.2 -cloudg scan -p aws --profile audit --scanners prowler,scoutsuite -o ./scan-2026-10-09 -``` +=== "CLI" + + ```bash + cloudg run -p aws --regions us-east-1,eu-west-1 --scanners prowler,trivy --images 123456789012.dkr.ecr.us-east-1.amazonaws.com/api:1.4.2 + cloudg scan -p aws --profile audit --scanners prowler,scoutsuite -o ./scan-2026-10-09 + ``` + +=== "Python" -```python tab="Python" title="scan_only.py" -import asyncio + ```python title="scan_only.py" + import asyncio -from cloudg import CloudGConfig, CloudGEngine -from cloudg.scanners.checkov import CheckovScanner -from cloudg.scanners.prowler import ProwlerScanner -from cloudg.scanners.trivy import TrivyScanner + from cloudg import CloudGConfig, CloudGEngine + from cloudg.scanners.checkov import CheckovScanner + from cloudg.scanners.prowler import ProwlerScanner + from cloudg.scanners.trivy import TrivyScanner -for tool in (ProwlerScanner, CheckovScanner, TrivyScanner): - print(tool.__name__, "installed" if tool.is_available() else "missing") + for tool in (ProwlerScanner, CheckovScanner, TrivyScanner): + print(tool.__name__, "installed" if tool.is_available() else "missing") -config = CloudGConfig(providers=["aws"]) -config.scanners.enabled = ["checkov", "trivy"] -config.scanners.checkov_frameworks = ["terraform"] + config = CloudGConfig(providers=["aws"]) + config.scanners.enabled = ["checkov", "trivy"] + config.scanners.checkov_frameworks = ["terraform"] -engine = CloudGEngine(config) -engine.on_error = lambda name, exc: print("scanner error:", name, exc) + engine = CloudGEngine(config) + engine.on_error = lambda name, exc: print("scanner error:", name, exc) -findings = asyncio.run( - engine.scan( - assets=[], - edges=[], - iac_dir="./terraform", - images=["python:3.12-slim"], - output_dir="./reports", + findings = asyncio.run( + engine.scan( + assets=[], + edges=[], + iac_dir="./terraform", + images=["python:3.12-slim"], + output_dir="./reports", + ) ) -) -print(len(findings), "findings") -``` + print(len(findings), "findings") + ``` -```bash tab="Docker" -docker run --rm \ - -v ~/.aws:/home/cloudg/.aws:ro \ - -v "$PWD/reports:/app/reports" \ - ghcr.io/morpheuslord/cloudg:latest scan -p aws --scanners prowler -o /app/reports -``` +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest scan -p aws --scanners prowler -o /app/reports + ``` Without `--scanners`, both commands use `scanners.enabled` from the config passed with `-c`, which defaults to all five names. The IAM linter runs only when `iam` is in the list, like every other scanner. @@ -162,7 +168,7 @@ Without `--scanners`, both commands use `scanners.enabled` from the config passe Checkov, and Trivy when it has no images, need a directory. `cloudg run`, `cloudg scan` and `CloudGEngine.scan()` resolve it in a fixed order and stop at the first hit. -```mermaid caption="How cloudg run picks the directories for Checkov and Trivy fs" +```mermaid flowchart TD A{"--iac-dir given?"} -->|yes| X["Scan that directory"] A -->|no| B{"scanners.iac_directories set?"} @@ -171,6 +177,8 @@ flowchart TD C -->|yes| Z["Scan the recreation of the live cloud"] C -->|no| S["Skip Checkov, warn"] ``` +How cloudg run picks the directories for Checkov and Trivy fs +{ .diagram-caption } The last step is the interesting one. With `--terraform` (or `terraform.enabled: true`) and nothing else configured, cloudg writes a Terraform recreation of the collected infrastructure and points Checkov at it, so Checkov audits your real cloud through its Terraform representation. There is no fallback to the current directory: scanning whatever folder cloudg was started from would produce a clean-looking result that says nothing about the cloud. @@ -186,7 +194,7 @@ cloudg run -p aws --regions us-east-1 --terraform --scanners checkov,trivy ## Configuration -Every key below lives under `scanners:` in `config.yaml`. They apply to `cloudg run`, `cloudg scan` (pass the file with `cloudg -c config.yaml scan ...`) and `CloudGEngine.scan()`. The full reference is on the [scanners config page](/reference/config/scanners/). +Every key below lives under `scanners:` in `config.yaml`. They apply to `cloudg run`, `cloudg scan` (pass the file with `cloudg -c config.yaml scan ...`) and `CloudGEngine.scan()`. The full reference is on the [scanners config page](../reference/config/scanners.md). ```yaml title="config.yaml" scanners: @@ -274,9 +282,9 @@ From Python, set `engine.on_error` to see failures as they happen. It receives t | Trivy | Not saved (read from stdout) | Not saved | | All findings | Normalised into `findings.json` and `report.html` | `raw-findings.json`, before normalisation | -:::warning Reusing an output directory -Prowler names each output file after the account and a timestamp, and cloudg parses every `*.json` under the Prowler output directory. Point a second run at the same `-o` and the older ASFF files are parsed again. Repeated checks on the same resource are merged during normalisation, but a problem you fixed after the first run still shows up from the old file. Use a fresh output directory per run. -::: +!!! warning "Reusing an output directory" + + Prowler names each output file after the account and a timestamp, and cloudg parses every `*.json` under the Prowler output directory. Point a second run at the same `-o` and the older ASFF files are parsed again. Repeated checks on the same resource are merged during normalisation, but a problem you fixed after the first run still shows up from the old file. Use a fresh output directory per run. The Prowler and ScoutSuite directories are in the native formats that `cloudg ingest` reads, so you can re-aggregate them later without scanning again: @@ -284,7 +292,7 @@ The Prowler and ScoutSuite directories are in the native formats that `cloudg in cloudg ingest --prowler ./reports/prowler/aws/ --scoutsuite ./reports/scoutsuite/aws/ -o ./reports-again ``` -`raw-findings.json` from `cloudg scan` is a JSON list of cloudg `Finding` objects, not a scanner format. `cloudg report -i raw-findings.json` normalises it and writes the reports, and [`cloudg map --findings`](/guides/inventory-mapping/) merges it into an inventory map. +`raw-findings.json` from `cloudg scan` is a JSON list of cloudg `Finding` objects, not a scanner format. `cloudg report -i raw-findings.json` normalises it and writes the reports, and [`cloudg map --findings`](inventory-mapping.md) merges it into an inventory map. ## Scan and run compared {#scan-and-run-compared} @@ -303,9 +311,22 @@ cloudg ingest --prowler ./reports/prowler/aws/ --scoutsuite ./reports/scoutsuite ## Next -:::links -- [Ingesting existing output](/guides/ingesting/) Aggregate scans that ran elsewhere, with no binaries and no credentials. -- [Compliance frameworks](/guides/compliance/) How findings are tagged with framework controls after the scan. -- [Reports](/guides/reports/) What `findings.json` and `report.html` contain. -- [cloudg scan](/cli/scan/) Every flag of the standalone scan command. -::: +
      + +- [**Ingesting existing output**](ingesting.md) + + Aggregate scans that ran elsewhere, with no binaries and no credentials. + +- [**Compliance frameworks**](compliance.md) + + How findings are tagged with framework controls after the scan. + +- [**Reports**](reports.md) + + What `findings.json` and `report.html` contain. + +- [**cloudg scan**](../cli/scan.md) + + Every flag of the standalone scan command. + +
      diff --git a/site/content/index.md b/site/content/index.md new file mode 100644 index 0000000..5df8d5a --- /dev/null +++ b/site/content/index.md @@ -0,0 +1,244 @@ +--- +title: cloudg documentation +description: cloudg maps AWS, Azure and GCP into a graph, runs the security scanners you already use over the same inventory, and merges every finding into one deduplicated, compliance-mapped result. +hide: + - navigation + - toc +--- + +
      +
      +Cloud graphing for AWS, Azure and GCP + +

      Map the cloud. Scan it. Merge everything.

      + +cloudg collects your infrastructure into a graph, runs the security scanners you already trust over the same accounts and merges every finding against 28 compliance frameworks. You can also skip scanning entirely and just map what you have. +{ .lede } + +[Get started](guides/quickstart.md){ .md-button .md-button--primary } +[Python API](api/index.md){ .md-button } +[Sample report](sample-report.md){ .md-button } + +```bash +pip install "cloudg[aws]" +``` +
      +
      + +
      An example map. Red marks the path in from the internet.
      +
      +
      + + + +## Three ways in + +
      + +- 01 · full pipeline + + **Collect, scan, report** + + --- + + Every provider in parallel, all four scanners and every report format, in one command. + + ```bash + cloudg run -p all --regions all + ``` + + [:octicons-arrow-right-24: The pipeline](guides/pipeline.md) + +- 02 · ingest only + + **Merge output you already have** + + --- + + No cloud credentials and no scanner binaries. Point it at the files and get the same reports. + + ```bash + cloudg ingest --prowler ./out/ + ``` + + [:octicons-arrow-right-24: Ingesting existing output](guides/ingesting.md) + +- 03 · map only + + **Inventory, every service linked** + + --- + + Typed edges such as `INVOKES`, `ASSUMES_ROLE` and `PROTECTS`, with no scanner involved. + + ```bash + cloudg map -p aws --org + ``` + + [:octicons-arrow-right-24: Inventory mapping](guides/inventory-mapping.md) + +
      + +## The same run from Python + +The CLI is a thin layer over `CloudGEngine`, so anything a command does you can do from Python, one phase at a time if you like. + +=== "CLI" + + ```bash + cloudg ingest --prowler ./prowler-output/ --trivy ./trivy.json -o ./reports + ``` + +=== "Python" + + ```python title="ingest.py" + from cloudg import CloudGConfig, CloudGEngine + + engine = CloudGEngine(CloudGConfig()) + result = engine.run_from_reports_sync( + {"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}, + output_dir="./reports", + ) + print(result.total_findings, result.severity_breakdown) + print(result.report_paths) # findings.json and report.html + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD:/app/input:ro" \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + ingest --prowler /app/input/prowler-output/ --trivy /app/input/trivy.json + ``` + +## What happens in a run + +Collectors read each provider concurrently and hand assets and edges to a NetworkX graph. Scanners run in parallel against the same accounts, and their output is normalised into one finding model, deduplicated and matched to compliance controls. The same data then leaves as an HTML report, JSON, GraphML, an RDF ontology, RAG chunks and, if you want it, an MCP server for your AI assistant. + +```mermaid +flowchart LR + P["AWS, Azure, GCP"] --> C[Collectors] + C --> G[(Graph)] + P --> S[Scanners] + S --> N[Normaliser] + G --> N + N --> R["report.html, JSON, SVG"] + G --> O["Ontology and RAG"] + N --> M["MCP server"] + O --> M +``` + +## Where to go next + +
      + +- :material-book-open-variant:{ .lg .middle } **Guides** + + --- + + Install, authenticate, map an account, run the scanners and read the report. + + [:octicons-arrow-right-24: Introduction](guides/introduction.md) + +- :material-console:{ .lg .middle } **CLI reference** + + --- + + Every command and every flag, generated from the click definitions so it matches `--help`. + + [:octicons-arrow-right-24: All commands](cli/index.md) + +- :material-language-python:{ .lg .middle } **Python API** + + --- + + `CloudGEngine` and the modules behind it, with signatures, parameters and runnable examples. + + [:octicons-arrow-right-24: Python API](api/index.md) + +- :material-code-tags:{ .lg .middle } **Examples** + + --- + + Complete scripts for mapping, ingesting, graph exports and error handling, each with its output. + + [:octicons-arrow-right-24: Examples](examples/index.md) + +- :material-robot-outline:{ .lg .middle } **MCP server** + + --- + + 74 tools for Claude Desktop, Claude Code, Cursor, VS Code or any other MCP client. + + [:octicons-arrow-right-24: MCP server](mcp/index.md) + +- :material-file-chart-outline:{ .lg .middle } **Sample report** + + --- + + The interactive report.html for a synthetic three-cloud estate, with no account needed. + + [:octicons-arrow-right-24: Open the sample report](sample-report.md) + +
      + + diff --git a/site/content/mcp/connect.md b/site/content/mcp/connect.md index f0d5e4b..751b8f7 100644 --- a/site/content/mcp/connect.md +++ b/site/content/mcp/connect.md @@ -1,17 +1,17 @@ --- title: Connect a client -lede: "`cloudg mcp config` writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools." -meta: - - [Command, "`cloudg mcp config`"] - - [Clients, "Claude Desktop, Claude Code, Cursor, VS Code"] - - [Transports, "stdio, streamable HTTP"] -source: docs/MCP.md -since: "0.6.0" +description: cloudg mcp config writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools. --- +`cloudg mcp config` writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools. +{ .lede } + +**Command** `cloudg mcp config` · **Clients** Claude Desktop, Claude Code, Cursor, VS Code · **Transports** stdio, streamable HTTP · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Most people run cloudg over stdio: the client starts `cloudg mcp serve` as a child process when it opens and stops it when it closes. You don't keep a server running, and nothing listens on a port. HTTP is for the other cases: a server shared by several people, a client on a different machine, or a gateway in front of it. -```mermaid caption="Which setup to use" +```mermaid flowchart TD A{"Does the client run on the machine that holds the reports?"} -->|yes| B{"One user, one client?"} A -->|no| E["HTTP: cloudg mcp serve --transport http, with --auth-token"] @@ -21,26 +21,33 @@ flowchart TD F -->|yes| G["Bridge with npx mcp-remote"] F -->|no| H["Client connects to the URL directly"] ``` +Which setup to use +{ .diagram-caption } ## Before you start You need cloudg 0.6.0 or newer on Python 3.11 or newer. The base install includes the MCP layer and a dependency-free server; the `mcp` extra adds the official SDK, which only matters if you want `--flavor sdk`. -```bash tab="pip" -pip install cloudg -``` +=== "pip" -```bash tab="uv" -uv pip install cloudg -``` + ```bash + pip install cloudg + ``` + +=== "uv" + + ```bash + uv pip install cloudg + ``` -You also need something to look at: an `inventory-map.json` from [cloudg map](/cli/map/), a `findings.json` from [cloudg run](/cli/run/), or native scanner output that [cloudg ingest](/cli/ingest/) understands. The examples below assume `./reports/inventory-map.json`. +You also need something to look at: an `inventory-map.json` from [cloudg map](../cli/map.md), a `findings.json` from [cloudg run](../cli/run.md), or native scanner output that [cloudg ingest](../cli/ingest.md) understands. The examples below assume `./reports/inventory-map.json`. If you installed cloudg in editable mode before 0.6.0, run `pip install -e .` again so the `cloudg-mcp` console script gets created. ## Set up a stdio client -::::steps +
      + ### Check that the server loads your data Run one tool from a terminal before touching any client config. `cloudg mcp call` goes through the same layer, policy and transforms the server uses, so if this works, the server will too. @@ -57,66 +64,74 @@ The result lists the dataset with its asset, edge and finding counts, and the al The output below is from an install where `cloudg-mcp` was found on `PATH`; your path will differ. -```json tab="Claude Desktop" title="claude_desktop_config.json" -{ - "mcpServers": { - "cloudg": { - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] +=== "Claude Desktop" + + ```json title="claude_desktop_config.json" + { + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` - -```json tab="Claude Code" title=".mcp.json" -{ - "mcpServers": { - "cloudg": { - "type": "stdio", - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ], - "env": {} + ``` + +=== "Claude Code" + + ```json title=".mcp.json" + { + "mcpServers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ], + "env": {} + } + } } - } -} -``` - -```json tab="Cursor" title="~/.cursor/mcp.json" -{ - "mcpServers": { - "cloudg": { - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] + ``` + +=== "Cursor" + + ```json title="~/.cursor/mcp.json" + { + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` - -```json tab="VS Code" title=".vscode/mcp.json" -{ - "servers": { - "cloudg": { - "type": "stdio", - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] + ``` + +=== "VS Code" + + ```json title=".vscode/mcp.json" + { + "servers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` + ``` The commands that produced them are `cloudg mcp config --client claude-desktop` (the default), `--client claude-code`, `--client cursor` and `--client vscode`. `--name` changes the server key from `cloudg`, and `--command "CMD ARGS"` replaces the detected launch command (it is split with `shlex`). -:::note Interpreter fallback -In a virtualenv where neither script is on `PATH`, the command is the interpreter and the arguments are `["-m", "cloudg.mcp", "serve"]`. That works just as well; it only looks different. -::: +!!! note "Interpreter fallback" + + In a virtualenv where neither script is on `PATH`, the command is the interpreter and the arguments are `["-m", "cloudg.mcp", "serve"]`. That works just as well; it only looks different. ### Put it where the client reads it @@ -138,7 +153,8 @@ claude mcp add-json cloudg '{"type": "stdio", "command": "/home/you/.venv/bin/cl ### Restart the client and start a conversation The server sends instructions on `initialize` that tell the model to begin with `workspace_status`. If you did not preload a dataset, ask the assistant to load one by absolute path, for example "Load /srv/cloudg/reports/inventory-map.json and show me the internet-exposed assets with HIGH findings." That path has to be inside an allowed root, which the next section explains. -:::: + +
      ## Load your reports with --dataset @@ -198,7 +214,7 @@ The separator is `os.pathsep`: `:` on macOS and Linux, `;` on Windows. Export to The counts are what `cloudg mcp tools --policy NAME` lists for the local user in 0.6.0. Run that command with your policy and, for HTTP, with `--role ROLE`, to see exactly what a client will get before you hand it out. -Under `strict`, pseudonyms come from a key that is random per process, so they change every time the client restarts the server and old ones stop resolving. Put `CLOUDG_MCP_VAULT_KEY` in the entry's `env` block if pseudonyms must stay stable across conversations, and keep that value away from the model. The [built-in profiles](/mcp/privacy/built-in-profiles/) page shows the same calls under every profile, and [loading and composing policies](/mcp/privacy/loading-and-composing-policies/) covers writing your own with `extends`. +Under `strict`, pseudonyms come from a key that is random per process, so they change every time the client restarts the server and old ones stop resolving. Put `CLOUDG_MCP_VAULT_KEY` in the entry's `env` block if pseudonyms must stay stable across conversations, and keep that value away from the model. The [built-in profiles](privacy/built-in-profiles.md) page shows the same calls under every profile, and [loading and composing policies](privacy/loading-and-composing-policies.md) covers writing your own with `extends`. ## Read-only mode @@ -222,67 +238,75 @@ It listens on `http://127.0.0.1:8765/mcp` (change with `--host`, `--port` and `- Then generate the client side with `--transport http` and `--token-env`, naming the variable that holds the token on the client's machine. `--policy`, `--dataset` and `--read-only` do nothing in this mode, since the server is already running with its own options. Use `--url` for anything other than the default address. -```json tab="Claude Desktop" title="claude_desktop_config.json" -{ - "mcpServers": { - "cloudg": { - "command": "npx", - "args": [ - "-y", - "mcp-remote", - "http://127.0.0.1:8765/mcp", - "--header", - "Authorization:Bearer ${CLOUDG_MCP_TOKEN}" - ] +=== "Claude Desktop" + + ```json title="claude_desktop_config.json" + { + "mcpServers": { + "cloudg": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "http://127.0.0.1:8765/mcp", + "--header", + "Authorization:Bearer ${CLOUDG_MCP_TOKEN}" + ] + } + } } - } -} -``` - -```json tab="Claude Code" title=".mcp.json" -{ - "mcpServers": { - "cloudg": { - "type": "http", - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + ``` + +=== "Claude Code" + + ```json title=".mcp.json" + { + "mcpServers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` - -```json tab="Cursor" title="~/.cursor/mcp.json" -{ - "mcpServers": { - "cloudg": { - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + ``` + +=== "Cursor" + + ```json title="~/.cursor/mcp.json" + { + "mcpServers": { + "cloudg": { + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` - -```json tab="VS Code" title=".vscode/mcp.json" -{ - "servers": { - "cloudg": { - "type": "http", - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + ``` + +=== "VS Code" + + ```json title=".vscode/mcp.json" + { + "servers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` + ``` Each client expands the variable in its own syntax: `${VAR}` in Claude Code, `${env:VAR}` in Cursor and VS Code. The token itself never appears in the file. Claude Desktop's config only launches local commands, so cloudg bridges to the URL with `npx mcp-remote` and passes the header on its command line. That entry has no `env` block, and whether `${CLOUDG_MCP_TOKEN}` gets a value depends on the environment Claude Desktop hands to `npx`, which the cloudg docs have not tested. If the server answers 401, add `"env": {"CLOUDG_MCP_TOKEN": "..."}` to the entry by hand. -To reach the server from another host, bind to a public address and keep the token: `--host 0.0.0.0` together with `--allowed-host 'mcp.internal.example:*'` (and `--allowed-origin` for browser clients). The [CLI reference](/mcp/server/cli-reference/) lists every serve option. +To reach the server from another host, bind to a public address and keep the token: `--host 0.0.0.0` together with `--allowed-host 'mcp.internal.example:*'` (and `--allowed-origin` for browser clients). The [CLI reference](server/cli-reference.md) lists every serve option. ## Troubleshooting @@ -317,13 +341,32 @@ The vault key is random per process unless `CLOUDG_MCP_VAULT_KEY` is set. Set it ### The client complains about a missing outputSchema -Under every profile except `open`, tools don't advertise `outputSchema`, because masking can change a field's type and a client must reject content that doesn't match an advertised schema. The structured content is still sent. This is deliberate; see [troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) for this and the other cases (timeouts, large results, "cooling down" from live tools). +Under every profile except `open`, tools don't advertise `outputSchema`, because masking can change a field's type and a client must reject content that doesn't match an advertised schema. The structured content is still sent. This is deliberate; see [troubleshooting and FAQ](server/troubleshooting-and-faq.md) for this and the other cases (timeouts, large results, "cooling down" from live tools). + +
      + +- [**Quick start**](server/quick-start-with-a-desktop-client.md) + + The same setup with captured output from the sample estate. + +- [**cloudg mcp config**](../cli/mcp-config.md) + + Every option of the config command. + +- [**cloudg mcp serve**](../cli/mcp-serve.md) + + Transports, flavors, tokens, Host and Origin checks. + +- [**Built-in profiles**](privacy/built-in-profiles.md) + + What each policy changes, call by call. + +- [**Principals and roles**](privacy/principals-and-roles.md) + + Map tokens to roles and give each role its own view. + +- [**Troubleshooting and FAQ**](server/troubleshooting-and-faq.md) + + The full list of known problems. -:::links -- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) The same setup with captured output from the sample estate. -- [cloudg mcp config](/cli/mcp-config/) Every option of the config command. -- [cloudg mcp serve](/cli/mcp-serve/) Transports, flavors, tokens, Host and Origin checks. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy changes, call by call. -- [Principals and roles](/mcp/privacy/principals-and-roles/) Map tokens to roles and give each role its own view. -- [Troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) The full list of known problems. -::: +
      diff --git a/site/content/mcp/index.md b/site/content/mcp/index.md index 495dbd0..663b06f 100644 --- a/site/content/mcp/index.md +++ b/site/content/mcp/index.md @@ -1,18 +1,17 @@ --- title: MCP server -lede: "cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request." -meta: - - [Command, "`cloudg mcp serve`"] - - [Python, "`cloudg.mcp.CloudGMCPLayer`"] - - [Catalog, "74 tools, 13 resources, 11 templates, 11 prompts"] - - [Default policy, "`standard`"] -source: docs/MCP.md -since: "0.6.0" +description: cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request. --- +cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request. +{ .lede } + +**Command** `cloudg mcp serve` · **Python** `cloudg.mcp.CloudGMCPLayer` · **Catalog** 74 tools, 13 resources, 11 templates, 11 prompts · **Default policy** `standard` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + The MCP layer lives in `cloudg/mcp/` and ships with the base `pip install cloudg`. It takes the files cloudg already writes (`inventory-map.json`, `findings.json`, native scanner output) and turns them into tools an assistant can call: "which internet-exposed assets have HIGH findings?", "what breaks if this IAM role goes away?", "which CIS controls fail in the shared-services account?". The model asks, cloudg answers from its own graph and data model, and a policy decides what the model may see and do on the way. -It is written for three kinds of reader. People who run Claude Desktop, Claude Code, Cursor or VS Code and want to point it at a cloud estate should read [Connect a client](/mcp/connect/) next. Operators who already run an MCP server or gateway can mount cloudg into it. And Python authors (an agent framework, a test suite, a batch job) can call the same tools in-process, with no MCP traffic at all. +It is written for three kinds of reader. People who run Claude Desktop, Claude Code, Cursor or VS Code and want to point it at a cloud estate should read [Connect a client](connect.md) next. Operators who already run an MCP server or gateway can mount cloudg into it. And Python authors (an agent framework, a test suite, a batch job) can call the same tools in-process, with no MCP traffic at all. ## One layer, three ways in @@ -24,7 +23,7 @@ The center of the package is one class, `CloudGMCPLayer`. It holds a registry of | Mounted | `layer.register_into(server)` | You already run an MCP server (official `mcp` SDK 1.x or 2.x, or `fastmcp` 2, 3 or 4) and want cloudg's tools inside it | | In-process | `await layer.call_tool(...)` | You are in Python and do not need MCP on the wire | -```mermaid caption="How the pieces fit together" +```mermaid flowchart LR A["Desktop or IDE client"] -->|"stdio or HTTP"| B["cloudg mcp serve (native, sdk or fastmcp)"] C["Your MCP server"] -->|register_into| L[CloudGMCPLayer] @@ -37,8 +36,10 @@ flowchart LR W --> F["inventory-map.json, findings.json, scanner output"] W --> X["Cloud APIs (live tools only)"] ``` +How the pieces fit together +{ .diagram-caption } -The [server section](/mcp/server/what-the-layer-is/) covers the layer in depth, [mounting](/mcp/server/mounting-into-an-existing-mcp-server/) covers the second mode and [in-process use](/mcp/server/in-process-use/) the third. +The [server section](server/what-the-layer-is.md) covers the layer in depth, [mounting](server/mounting-into-an-existing-mcp-server.md) covers the second mode and [in-process use](server/in-process-use.md) the third. ## What an assistant can do with it @@ -57,9 +58,9 @@ The default catalog has 74 tools in ten categories. The counts below come from r | `meta` | 4 | What this caller may use, and cloudg's vocabulary of asset, edge and relation types | | `privacy` | 5 | The active policy, a transform preview, the detector list, the decision log, and `reveal_token` | -The default `standard` policy hides `reveal_token`, so `cloudg mcp tools` lists 73. The stricter profiles hide more (see [policies](#policies-and-privacy) below). Every tool is documented with its arguments, return fields and a captured example in the [tool index](/mcp/tools/tool-index/) and the per-category pages such as [graph](/mcp/tools/graph/) and [findings](/mcp/tools/findings/). +The default `standard` policy hides `reveal_token`, so `cloudg mcp tools` lists 73. The stricter profiles hide more (see [policies](#policies-and-privacy) below). Every tool is documented with its arguments, return fields and a captured example in the [tool index](tools/tool-index.md) and the per-category pages such as [graph](tools/graph/index.md) and [findings](tools/findings.md). -The four collecting tools in `live` are the only ones that touch a cloud provider. They check credentials before the first API call, refuse to re-collect a scope that was collected a couple of minutes ago, and are hidden by every built-in profile except `open` and `standard`. The details are in [live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/). +The four collecting tools in `live` are the only ones that touch a cloud provider. They check credentials before the first API call, refuse to re-collect a scope that was collected a couple of minutes ago, and are hidden by every built-in profile except `open` and `standard`. The details are in [live tools and the credential preflight](server/live-tools-and-the-credential-preflight.md). ### Resources and prompts @@ -76,7 +77,7 @@ The 11 prompts are packaged analysis workflows that a user picks from the client | `incident_triage` | `finding_id` | | `drift_review` | `base` | -The full list with arguments and rendered output is in [resources and resource templates](/mcp/tools/resources-and-resource-templates/) and [prompts](/mcp/tools/prompts/). +The full list with arguments and rendered output is in [resources and resource templates](tools/resources-and-resource-templates.md) and [prompts](tools/prompts.md). ## Workspaces and datasets @@ -86,13 +87,13 @@ Datasets get into the workspace in three ways. The operator can preload them wit The graph, dependency graph, centrality scores and RDF ontology are built the first time a tool needs them and cached until the dataset changes. On a large estate the first `attack_paths` or `sparql_query` call is the slow one. The workspace holds 16 datasets; loading a 17th evicts the oldest inactive one. -Two safety rules matter in practice. First, the model can only read and write inside the allowed roots: by default the directory the server was started from and the configured report directory, with `/` and your home directory left out. Set `CLOUDG_MCP_ALLOWED_ROOTS` to change them. Second, paths given with `--dataset` skip that check, because the operator typed them. `snapshot_dataset` and `diff_datasets` let a model compare two versions of an estate, which is what the `drift_review` prompt does. More in [the workspace](/mcp/server/the-workspace/). +Two safety rules matter in practice. First, the model can only read and write inside the allowed roots: by default the directory the server was started from and the configured report directory, with `/` and your home directory left out. Set `CLOUDG_MCP_ALLOWED_ROOTS` to change them. Second, paths given with `--dataset` skip that check, because the operator typed them. `snapshot_dataset` and `diff_datasets` let a model compare two versions of an estate, which is what the `drift_review` prompt does. More in [the workspace](server/the-workspace.md). ## The request lifecycle Every tool call takes the same route, whatever transport it arrived on. -```mermaid caption="What happens to one tools/call" +```mermaid flowchart TD A[MCP client] --> B["Transport: stdio line or HTTP POST"] B --> C["Adapter resolves the principal (token, OAuth or local user)"] @@ -106,10 +107,12 @@ flowchart TD I --> J["Output pipeline: sanitize, redact, project, annotate"] J --> L["Result with cloudg/transforms in _meta"] ``` +What happens to one tools/call +{ .diagram-caption } A few details are easy to miss. A tool the policy hides gives exactly the same "Unknown tool" error as a tool that does not exist, so a model cannot probe for what it is missing. Arguments are validated against a pydantic model built from the handler's signature, and unknown keys are rejected. Synchronous handlers run in a worker thread so a slow graph build does not block other requests, and every call has a timeout (300 seconds by default, longer for the live tools). The result is sent twice, as `structuredContent` and as indented JSON text for clients that ignore structured output; the text copy is capped at 200,000 characters. Errors from the handler pass through the same output pipeline as results, because exception messages tend to quote ARNs. -The step-by-step version, including resources, prompts and completions, is in [the request lifecycle](/mcp/server/the-request-lifecycle/). Error codes are listed in [errors](/mcp/server/errors/) and the middleware hooks in [middleware](/mcp/server/middleware/). +The step-by-step version, including resources, prompts and completions, is in [the request lifecycle](server/the-request-lifecycle.md). Error codes are listed in [errors](server/errors.md) and the middleware hooks in [middleware](server/middleware.md). ## Policies and privacy @@ -131,7 +134,7 @@ The output pipeline in `standard` runs four transforms in order. `sanitize` stri Under `strict` the redactor also pseudonymises identifiers through the token vault. Pseudonyms keep their format, so a 12-digit account id stays 12 digits and an ARN stays a valid ARN, and they are deterministic for the life of the vault key. The model can pass `res-8de954a6d9` back as a `ref` and the input pipeline resolves the real asset before the handler runs. Set `CLOUDG_MCP_VAULT_KEY` if pseudonyms must survive a server restart; without it a random key is generated per process. -One side effect surprises people: under every profile except `open`, tools do not advertise an `outputSchema`, since masking can change a field's type and clients reject structured content that does not match a schema. The data is still sent. The [privacy section](/mcp/privacy/threat-model-and-goals/) covers the threat model, [what the policy touches](/mcp/privacy/what-the-policy-touches/), the [built-in profiles](/mcp/privacy/built-in-profiles/) file by file, and [the token vault](/mcp/privacy/the-token-vault/). +One side effect surprises people: under every profile except `open`, tools do not advertise an `outputSchema`, since masking can change a field's type and clients reject structured content that does not match a schema. The data is still sent. The [privacy section](privacy/threat-model-and-goals.md) covers the threat model, [what the policy touches](privacy/what-the-policy-touches.md), the [built-in profiles](privacy/built-in-profiles.md) file by file, and [the token vault](privacy/the-token-vault.md). ## Transports and flavors @@ -148,13 +151,13 @@ The flavor picks which implementation speaks MCP. The layer and results are iden | `sdk` | the official SDK's low-level `Server`, uvicorn for HTTP | `pip install "cloudg[mcp]"` | | `fastmcp` | `fastmcp.FastMCP` with cloudg components | `pip install fastmcp` | -The native flavor supports every serve option, including `--cors-origin`, `--page-size` and `/healthz`. See [transports and protocol versions](/mcp/server/transports-and-protocol-versions/) and [flavors](/mcp/server/flavors/). +The native flavor supports every serve option, including `--cors-origin`, `--page-size` and `/healthz`. See [transports and protocol versions](server/transports-and-protocol-versions.md) and [flavors](server/flavors.md). ## Security model in brief Every transport binds to 127.0.0.1 by default. On HTTP, `--auth-token TOKEN[:ROLES[:ID]]` maps static bearer tokens to principals with roles, which the policy's `roles:` section uses; tokens are kept only as SHA-256 digests and compared in constant time. Every flavor checks `Origin` and `Host` headers against allow-lists to block DNS rebinding, and the native server warns at start-up when bound to a non-loopback address without tokens. On stdio, tokens are ignored and the caller is the local user. -File access is confined to the allowed roots and writes to the output directory. Error messages never repeat the caller's arguments. The JSONL audit log (`--audit-log`) records every call with hashed argument values, and every pseudonym reversal is logged at WARNING. `--registry` imports and runs Python code, so its value must never come from untrusted input. The longer list is in [security model in brief](/mcp/server/security-model-in-brief/). +File access is confined to the allowed roots and writes to the output directory. Error messages never repeat the caller's arguments. The JSONL audit log (`--audit-log`) records every call with hashed argument values, and every pseudonym reversal is logged at WARNING. `--registry` imports and runs Python code, so its value must never come from untrusted input. The longer list is in [security model in brief](server/security-model-in-brief.md). ## Examples @@ -204,7 +207,6 @@ from pathlib import Path from cloudg.config import CloudGConfig from cloudg.mcp import CloudGMCPLayer, NotFoundError, Principal, Workspace - async def main() -> None: reports = Path("reports").resolve() ws = Workspace(CloudGConfig(), allowed_roots=[reports], output_dir=reports / "mcp-out") @@ -232,7 +234,6 @@ async def main() -> None: except NotFoundError as exc: print("refused:", exc) - asyncio.run(main()) ``` @@ -250,15 +251,46 @@ The pseudonyms change between runs unless `CLOUDG_MCP_VAULT_KEY` is set. Note th ## Where to go next -:::links -- [Connect a client](/mcp/connect/) Claude Desktop, Claude Code, Cursor and VS Code, step by step. -- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) Check your data in a terminal before wiring a client. -- [CLI reference](/mcp/server/cli-reference/) Every option of serve, tools, call, read and config. -- [Tool index](/mcp/tools/tool-index/) All 74 tools with category, sensitivity and a one-line summary. -- [Conventions](/mcp/tools/conventions-shared-by-every-tool/) Paging, field selection, asset references and the result envelope. -- [Recipes](/mcp/tools/recipes/) Multi-step investigations as tool-call sequences. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) The seven policies side by side, with the same calls under each. -- [Policy schema](/mcp/privacy/policy-schema-reference/) Write your own policy, roles and rules. -- [In-process use](/mcp/server/in-process-use/) CloudGMCPLayer, Principal and ToolContext from Python. -- [Internals](/mcp/internals/package-map/) How the package is built, for contributors. -::: +
      + +- [**Connect a client**](connect.md) + + Claude Desktop, Claude Code, Cursor and VS Code, step by step. + +- [**Quick start**](server/quick-start-with-a-desktop-client.md) + + Check your data in a terminal before wiring a client. + +- [**CLI reference**](server/cli-reference.md) + + Every option of serve, tools, call, read and config. + +- [**Tool index**](tools/tool-index.md) + + All 74 tools with category, sensitivity and a one-line summary. + +- [**Conventions**](tools/conventions-shared-by-every-tool.md) + + Paging, field selection, asset references and the result envelope. + +- [**Recipes**](tools/recipes.md) + + Multi-step investigations as tool-call sequences. + +- [**Built-in profiles**](privacy/built-in-profiles.md) + + The seven policies side by side, with the same calls under each. + +- [**Policy schema**](privacy/policy-schema-reference.md) + + Write your own policy, roles and rules. + +- [**In-process use**](server/in-process-use.md) + + CloudGMCPLayer, Principal and ToolContext from Python. + +- [**Internals**](internals/package-map.md) + + How the package is built, for contributors. + +
      diff --git a/site/content/reference/output-files.md b/site/content/reference/output-files.md index 3eff57a..3063cbe 100644 --- a/site/content/reference/output-files.md +++ b/site/content/reference/output-files.md @@ -1,15 +1,19 @@ --- title: Output files -lede: Every file cloudg writes, which command writes it, and what is inside. -source: docs/DOCUMENTATION.md -since: "0.6.0" +description: Every file cloudg writes, which command writes it, and what is inside. --- +Every file cloudg writes, which command writes it, and what is inside. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every command writes into one output directory, `-o` / `--output`, which defaults to `./reports` and is created when missing. Files with the same name are overwritten without a prompt, so give each run its own directory when you want to keep history. The exceptions to "everything goes into `-o`" are the Cloud Control type cache, the MCP server's audit log and vault, and a Terraform export sent elsewhere with `terraform.output_dir`, listed in their sections below. ## cloudg run -```mermaid caption="Where the files of cloudg run come from, phase by phase" +```mermaid flowchart LR A["Collect"] --> B["Graph"] B --> C["RAG export"] @@ -24,17 +28,19 @@ flowchart LR F -.-> F1["ontology.ttl, ontology.jsonld"] H -.-> H1["findings.json, topology.svg, report.html"] ``` +Where the files of cloudg run come from, phase by phase +{ .diagram-caption } | File | Written by | Contents | |---|---|---| -| `topology.graphml` | graph phase, unless `graph.persist_graphml: false` | The asset graph from [GraphBuilder](/api/graphbuilder/), for Gephi, yEd or `networkx.read_graphml`. Saved before the reachability walk, so `is_internet_exposed` is the collector's flag | +| `topology.graphml` | graph phase, unless `graph.persist_graphml: false` | The asset graph from [GraphBuilder](../api/graphbuilder.md), for Gephi, yEd or `networkx.read_graphml`. Saved before the reachability walk, so `is_internet_exposed` is the collector's flag | | `topology-cytoscape.json` | graph phase, with `graph.export_cytoscape: true` (off by default) | The same graph as Cytoscape.js `elements` | -| `rag_chunks.jsonl` | RAG export, unless `--no-rag-export` or `rag.enabled: false` | One retrieval chunk per line: entity, community and relation-group chunks ([RAGExporter](/api/ragexporter/)). Written once after the graph phase and rewritten after the scanners with every finding | +| `rag_chunks.jsonl` | RAG export, unless `--no-rag-export` or `rag.enabled: false` | One retrieval chunk per line: entity, community and relation-group chunks ([RAGExporter](../api/ragexporter.md)). Written once after the graph phase and rewritten after the scanners with every finding | | `rag_metadata_index.json` | RAG export | Chunk counts per kind, every `chunk_id` in file order, the chunk types | | `provider.tf.json`, `variables.tf.json`, `main.tf.json`, `import_commands.sh` | Terraform phase, with `--terraform` or `terraform.enabled: true` | Terraform JSON that recreates the collected resources, and a script of `terraform import` commands. Written to `/terraform`, so it follows `-o`. A `terraform.output_dir` set in `config.yaml` replaces that directory, except the old default `./reports/terraform`, which counts as unset | | `prowler//` | Prowler, when it runs | Prowler's own JSON-ASFF output files, which cloudg parses | | `scoutsuite//` | ScoutSuite, when it runs | ScoutSuite's report directory | -| `ontology.ttl`, `ontology.jsonld` | ontology phase, unless `--no-ontology` or `ontology.enabled: false` | The RDF graph with 64 relation types and the findings ([CloudOntology](/api/cloudontology/)). One file per `ontology.export_formats` entry; `xml` adds `ontology.rdf`, `nt` adds `ontology.nt` | +| `ontology.ttl`, `ontology.jsonld` | ontology phase, unless `--no-ontology` or `ontology.enabled: false` | The RDF graph with 64 relation types and the findings ([CloudOntology](../api/cloudontology.md)). One file per `ontology.export_formats` entry; `xml` adds `ontology.rdf`, `nt` adds `ontology.nt` | | `findings.json` | render phase | The normalised result: metadata, summary, assets, findings, compliance mappings, edges, D3 graph. Shape below | | `topology.svg` | render phase | Static topology diagram | | `report.html` | render phase | Interactive report: topology, findings table, compliance matrix. The data, Chart.js 4.4.0 and D3 7.9.0 are embedded, so it works offline. With `report.inline_js: false` the page loads the two libraries from `cdn.jsdelivr.net` instead | @@ -61,9 +67,9 @@ Checkov and Trivy write nothing to disk; cloudg reads their stdout. `cloudg run` | File | Written by | Contents | |---|---|---| -| `inventory-map.json` | every map | The self-contained map: `summary`, `providers`, `regions`, `assets`, `edges`, `coverage`, `unresolved_references`, and `throttling` when a cloud API pushed back. Read by `cloudg deps`, `InventoryResult.load()` and the MCP workspace. Field reference: [inventory-map.json](/reference/inventory/inventory-map-json/) | -| `inventory-map.graphml` | every map | The map's graph, with parallel rule edges merged ([field list](/reference/inventory/inventory-map-graphml/)) | -| `inventory-graph.json` | every map | D3 `nodes` and `links` for `docs/viewer.html` and other front ends ([field list](/reference/inventory/inventory-graph-json/)) | +| `inventory-map.json` | every map | The self-contained map: `summary`, `providers`, `regions`, `assets`, `edges`, `coverage`, `unresolved_references`, and `throttling` when a cloud API pushed back. Read by `cloudg deps`, `InventoryResult.load()` and the MCP workspace. Field reference: [inventory-map.json](inventory/inventory-map-json.md) | +| `inventory-map.graphml` | every map | The map's graph, with parallel rule edges merged ([field list](inventory/inventory-map-graphml.md)) | +| `inventory-graph.json` | every map | D3 `nodes` and `links` for `docs/viewer.html` and other front ends ([field list](inventory/inventory-graph-json.md)) | | `inventory-dependencies.json` | every map | Shared dependencies and largest blast radius (top 25 each), every cross-account edge, security service coverage, unresolved references | | `inventory-organization.json` | `--org`, or `aws.organization.enabled` | The AWS Organization and Control Tower topology: accounts, OU tree, SCPs, landing zone | | `asset-map.json` | `--findings`, when at least one finding loads | Every asset with its matched findings, sorted by finding count | @@ -87,7 +93,7 @@ cloudg deps arn:aws:iam::123456789012:role/orders-worker --json > role-deps.json cloudg deps --map ./reports --top 50 --json > overview.json ``` -The key-by-key shapes are in [cloudg deps --json](/reference/inventory/cloudg-deps-json/). +The key-by-key shapes are in [cloudg deps --json](inventory/cloudg-deps-json.md). ## cloudg ingest @@ -213,7 +219,7 @@ The global option `cloudg --log-file PATH ` (or `log_file` in `config.y |---|---| | `metadata` | Run identity. The timestamps are ISO 8601 strings, like the ones on assets and findings, and an unset `completed_at` is `null`. The CLI and the engine leave `provider`, `account_id` and `region` at `null` | | `summary` | Counts. `total_edges` counts the entries in `edges` | -| `assets` | `CloudAsset` in JSON mode, without `raw_data`, with the computed `display_id` (ARN, else id). Fields: [CloudAsset](/reference/inventory/cloudasset/) | +| `assets` | `CloudAsset` in JSON mode, without `raw_data`, with the computed `display_id` (ARN, else id). Fields: [CloudAsset](inventory/cloudasset.md) | | `findings` | `Finding` in JSON mode, sorted by severity, with the computed `risk_score` (severity base, averaged with `cvss_score` when present). Reachability findings have deterministic UUID5 ids; most scanner findings get a fresh UUID on every run. The normaliser can add frameworks the scanner did not set | | `compliance` | One `ComplianceResult` per framework control, with the ids of the findings that failed it. `-aggregate` collects findings mapped to a framework but to no specific control | | `edges` | Every collected `NetworkEdge` in JSON mode, parallel rules kept. `cloudg report` draws `topology.svg` from them | @@ -272,21 +278,37 @@ The map `cloudg map` writes, and `cloudg deps` and the MCP workspace read. The s | Key | Notes | |---|---| -| `summary` | Counts by type, service, region and account, edge counts by type and relationship, and health counters. `organization` and `throttling` blocks appear only when an organization was mapped or an API throttled. See [summary](/reference/inventory/summary/) | +| `summary` | Counts by type, service, region and account, edge counts by type and relationship, and health counters. `organization` and `throttling` blocks appear only when an organization was mapped or an API throttled. See [summary](inventory/summary.md) | | `assets` | Every asset, linked and deduplicated. Real collector assets carry rich `metadata`, including the `relations` the linker resolved | | `edges` | Every edge, parallel rules kept. An endpoint can be a CIDR or another non-asset string, so look endpoints up with `.get()` | | `unresolved_references` | Declared relations the linker could not resolve, with `source`, `source_name`, `target` and `edge_type` | | `coverage` | One record per provider, region and account collected: per service, `status` (`SUCCESS`, `PARTIAL`, `FAILED` or `SKIPPED`), `asset_count`, `error` and `duration_ms`. Trimmed to one service above | | `throttling` | Only when a cloud API throttled the run: the full throttling report of the run | -`edges` here and `links` in `inventory-graph.json` differ in length whenever parallel rules were merged for the graph. Count edges in this file. For every field of every structure, see the [inventory reference](/reference/inventory/inventory-map-json/). +`edges` here and `links` in `inventory-graph.json` differ in length whenever parallel rules were merged for the graph. Count edges in this file. For every field of every structure, see the [inventory reference](inventory/inventory-map-json.md). ## Related -:::links -- [cloudg run](/cli/run/) The flags that switch outputs on and off. -- [cloudg map](/cli/map/) The inventory map and its companions. -- [Reports](/guides/reports/) Reading `report.html` and `findings.json`. -- [Data models](/api/models/) The Pydantic models behind every JSON file. -- [CloudAsset reference](/reference/inventory/cloudasset/) Every asset field and identifier format. -::: +
      + +- [**cloudg run**](../cli/run.md) + + The flags that switch outputs on and off. + +- [**cloudg map**](../cli/map.md) + + The inventory map and its companions. + +- [**Reports**](../guides/reports.md) + + Reading `report.html` and `findings.json`. + +- [**Data models**](../api/models.md) + + The Pydantic models behind every JSON file. + +- [**CloudAsset reference**](inventory/cloudasset.md) + + Every asset field and identifier format. + +
      diff --git a/site/content/sample-report.md b/site/content/sample-report.md new file mode 100644 index 0000000..c4793e8 --- /dev/null +++ b/site/content/sample-report.md @@ -0,0 +1,53 @@ +--- +title: Sample report +description: The interactive report.html that cloudg writes, built from a synthetic three-cloud estate, so you can see it without an account. +hide: + - toc +--- + +This is the `report.html` cloudg writes at the end of `cloudg run`, `cloudg ingest` and `cloudg report`, built from a synthetic estate that spans AWS, Azure and GCP. Every account ID, ARN and finding in it is made up, so nothing here describes a real environment. +{ .lede } + +[Open full screen](sample-report.html){ .md-button .md-button--primary target="_blank" rel="noopener" } +[How to read a report](guides/reports.md){ .md-button } + + + +## Where it comes from + +The direct address of the file is [morpheuslord.github.io/cloudg/sample-report.html](https://morpheuslord.github.io/cloudg/sample-report.html). It is rebuilt from the fixtures the MCP test suite uses, with no cloud access, by a script in the repository: + +=== "Rebuild it" + + ```bash + python scripts/build_sample_report.py # writes site/static/sample-report.html + python scripts/build_sample_report.py --out ./tmp # or anywhere else + ``` + +=== "Python" + + The script is a few lines around the same classes you would use yourself. Run it from a clone of the repository, because the fixtures live in `tests/`: + + ```python title="build_sample_report.py" + from cloudg.graph.builder import GraphBuilder + from cloudg.renderers.html_report import HTMLReportGenerator + from cloudg.schema.models import ScanResult + from tests.mcp.fixtures import sample_estate + + inv = sample_estate.build_inventory() + result = ScanResult( + assets=inv.assets, + edges=inv.edges, + findings=sample_estate.findings(), + compliance=sample_estate.compliance(), + ) + graph = GraphBuilder() + graph.build(inv.assets, inv.edges) + + path = HTMLReportGenerator(output_dir="./tmp").generate( + result, graph_json=graph.to_d3_json(), filename="sample-report.html" + ) + print(path) + ``` + +The page is a single self-contained file: Chart.js and D3 are inlined (`report.inline_js`, on by default), so it works offline and can be mailed or attached to a ticket as it is. [Reports](guides/reports.md) explains each tab, and [Output files](reference/output-files.md) lists what else a run writes next to it. diff --git a/site/data/nav.yaml b/site/data/nav.yaml deleted file mode 100644 index ce90d23..0000000 --- a/site/data/nav.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Site map. Each tab has a sidebar made of groups; each group lists pages. -# -# page: hand-written page, read from site/content/.md -# split: docs/FILE.md one page per "##" section of a long reference document -# (sections over split_h3_kb are split again at "###") -# generated: built from code: cli-index, api-index, config, mcp-tools-index -# -# Paths are URLs under the site base, without slashes at either end. - -tabs: - - id: guides - label: Guides - root: guides/introduction - groups: - - title: Start - items: - - {title: Introduction, page: guides/introduction} - - {title: Installation, page: guides/installation} - - {title: Quickstart, page: guides/quickstart} - - {title: The pipeline, page: guides/pipeline} - - title: Mapping - items: - - {title: Inventory mapping, page: guides/inventory-mapping} - - {title: Dependencies & blast radius, page: guides/dependencies} - - {title: AWS Organizations, page: guides/aws-organizations} - - title: Scanning - items: - - {title: Running scanners, page: guides/running-scanners} - - {title: Ingesting existing output, page: guides/ingesting} - - {title: Compliance frameworks, page: guides/compliance} - - title: Analyse - items: - - {title: "Graph, ontology & RAG", page: guides/graph-ontology-rag} - - {title: Reports, page: guides/reports} - - title: Operate - items: - - {title: Authentication, page: guides/authentication} - - {title: Rate limits & throttling, page: guides/rate-limits} - - {title: Docker, page: guides/docker} - - {title: CI and automation, page: guides/ci-automation} - - {title: Python recipes, page: guides/python-recipes} - - - id: cli - label: CLI - root: cli - groups: - - title: Overview - items: - - {title: All commands, generated: cli-index, path: cli} - - title: Pipeline - items: - - {title: cloudg run, page: cli/run} - - {title: cloudg collect, page: cli/collect} - - {title: cloudg scan, page: cli/scan} - - {title: cloudg report, page: cli/report} - - title: Inventory - items: - - {title: cloudg map, page: cli/map} - - {title: cloudg deps, page: cli/deps} - - {title: cloudg ingest, page: cli/ingest} - - title: MCP - items: - - {title: cloudg mcp serve, page: cli/mcp-serve} - - {title: cloudg mcp config, page: cli/mcp-config} - - {title: cloudg mcp tools, page: cli/mcp-tools} - - {title: cloudg mcp resources, page: cli/mcp-resources} - - {title: cloudg mcp prompts, page: cli/mcp-prompts} - - {title: cloudg mcp call, page: cli/mcp-call} - - {title: cloudg mcp read, page: cli/mcp-read} - - title: Shared - items: - - {title: Provider & region flags, page: cli/provider-flags} - - {title: Auth flags, page: cli/auth-flags} - - {title: Output flags, page: cli/output-flags} - - - id: api - label: Python API - root: api - groups: - - title: Overview - items: - - {title: Python API, generated: api-index, path: api} - - title: Engine - items: - - {title: CloudGEngine, page: api/cloudgengine} - - {title: CloudGConfig, page: api/cloudgconfig} - - {title: Result objects, page: api/results} - - {title: Event hooks, page: api/event-hooks} - - title: Inventory - items: - - {title: InventoryMapper, page: api/inventorymapper} - - {title: InventoryResult, page: api/inventoryresult} - - {title: RelationshipLinker, page: api/relationshiplinker} - - {title: DependencyGraph, page: api/dependencygraph} - - title: Graph - items: - - {title: GraphBuilder, page: api/graphbuilder} - - {title: ReachabilityAnalyzer, page: api/reachabilityanalyzer} - - {title: CloudOntology, page: api/cloudontology} - - {title: RAGExporter, page: api/ragexporter} - - title: Findings - items: - - {title: FindingsNormaliser, page: api/findingsnormaliser} - - {title: parse_report, page: api/parse-report} - - {title: Data models, page: api/models} - - title: Extend - items: - - {title: Entry points, page: api/entry-points} - - {title: cloudg.resilience, page: api/resilience} - - {title: CloudGMCPLayer, page: api/mcp-layer} - - - id: mcp - label: MCP server - root: mcp - groups: - - title: Start - items: - - {title: Overview, page: mcp/index, path: mcp} - - {title: Connect a client, page: mcp/connect} - - title: Server - split: docs/MCP.md - prefix: mcp/server - skip: [Contents] - - title: Tools - split: docs/MCP_TOOLS.md - prefix: mcp/tools - skip: [Tool reference] - - title: Privacy & policy - split: docs/MCP_PRIVACY.md - prefix: mcp/privacy - skip: [Contents] - - title: Internals - split: docs/MCP_INTERNALS.md - prefix: mcp/internals - skip: [Contents] - - - id: reference - label: Reference - root: reference/config/providers - groups: - - title: config.yaml - generated: config - prefix: reference/config - - title: Outputs - items: - - {title: Output files, page: reference/output-files} - - title: Inventory structures - split: docs/INVENTORY_REFERENCE.md - prefix: reference/inventory - skip: [Contents] - - title: Catalogs - split: docs/INVENTORY_CATALOG.md - prefix: reference/catalog - skip: [Contents] - - title: Inventory internals - split: docs/INVENTORY_INTERNALS.md - prefix: reference/internals - skip: [Contents] - - title: Resilience internals - split: docs/RESILIENCE.md - prefix: reference/resilience - skip: [Contents] - - - id: changelog - label: Changelog - root: changelog - -# Links from the old one-page handbook (index.html#anchor) to their new homes. -redirects: - install: guides/installation - pipeline: guides/pipeline - run: cli/run - collect-scan: cli/collect - map: guides/inventory-mapping - deps: guides/dependencies - ingest: guides/ingesting - report: cli/report - inputs: guides/ingesting#input-requirements - outputs: reference/output-files - models: api/models - inventory-docs: reference/inventory - config: reference/config/providers - ratelimit: guides/rate-limits - api: api - recipes: guides/python-recipes - plugins: api/entry-points - auth: guides/authentication - docker: guides/docker - mcp: mcp - mcp-serve: cli/mcp-serve - mcp-embed: mcp/server/in-process-use - mcp-policies: mcp/privacy/built-in-profiles - mcp-catalog: mcp/tools/tool-index - mcp-cli: cli/mcp-serve - changelog: changelog diff --git a/site/data/stats.yaml b/site/data/stats.yaml deleted file mode 100644 index f76aabc..0000000 --- a/site/data/stats.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Figures on the docs home page. Entries with `compute` are counted from the -# code at build time; `value` is the fallback (and the only source for the -# framework count, which follows the README's way of counting). -stats: - - label: compliance frameworks - value: 28 - href: guides/compliance - - label: controls mapped - compute: controls - value: 4166 - href: guides/compliance - - label: asset types catalogued - compute: asset_types - value: 156 - href: reference/catalog - - label: MCP tools - compute: mcp_tools - value: 74 - href: mcp diff --git a/site/docsgen/__init__.py b/site/docsgen/__init__.py deleted file mode 100644 index 450dc9f..0000000 --- a/site/docsgen/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Static site generator for the cloudg documentation (see site/AUTHORING.md).""" diff --git a/site/docsgen/discover.py b/site/docsgen/discover.py deleted file mode 100644 index 0060357..0000000 --- a/site/docsgen/discover.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Page discovery: walks data/nav.yaml and creates every page before anything renders.""" - -from __future__ import annotations - -import yaml - -from docsgen import gen_config -from docsgen.page import REPO, SITE, Page -from docsgen.split import SplitDoc - -_GENERATED_SEARCH_TYPES = {"cli-index": "CLI", "api-index": "Python API"} -_TAB_KINDS = {"cli": "cli", "api": "api", "mcp": "guide", "reference": "guide"} -_TAB_SEARCH_TYPES = { - "guides": "Guides", - "cli": "CLI", - "api": "Python API", - "mcp": "MCP", - "reference": "Reference", -} - - -class DiscoverMixin: - """Nav walking for :class:`docsgen.site.Site`.""" - - nav: dict - tabs: list[dict] - sidebars: dict[str, list[dict]] - - def discover(self) -> None: - home = self.add( - Page("", "cloudg documentation", kind="home", search_type="Guides", in_sidebar=False) - ) - home.source = "site/templates/home.html" - for tab in self.nav["tabs"]: - self.tabs.append({"id": tab["id"], "label": tab["label"], "root": tab["root"]}) - self.sidebars[tab["id"]] = [ - { - "title": group["title"], - "num": f"{gi:02d}", - "pages": self._discover_group(tab["id"], group), - } - for gi, group in enumerate(tab.get("groups", []), start=1) - ] - self.add( - Page( - "changelog", - "Changelog", - tab="changelog", - kind="changelog", - search_type="Changelog", - source="CHANGELOG.md", - in_sidebar=False, - ) - ) - - def _discover_group(self, tab: str, group: dict) -> list[Page]: - if "split" in group: - return SplitDoc(self, tab, group).discover() - if group.get("generated") == "config": - return self._discover_config(tab, group) - return [self._discover_item(tab, group["title"], item) for item in group["items"]] - - def _discover_item(self, tab: str, group: str, item: dict) -> Page: - title = item["title"] - if "generated" in item: - page = Page(item["path"], title, tab=tab, group=group, kind=item["generated"]) - page.search_type = _GENERATED_SEARCH_TYPES.get(item["generated"], "Guides") - return self.add(page) - rel = item["page"] - file = SITE / "content" / f"{rel}.md" - page = Page( - item.get("path", rel), - title, - tab=tab, - group=group, - kind=_TAB_KINDS.get(tab, "guide"), - search_type=_TAB_SEARCH_TYPES[tab], - source=str(file.relative_to(REPO)), - nav_title=title, - ) - page.loader = file - return self.add(page) - - def _discover_config(self, tab: str, group: dict) -> list[Page]: - sections = gen_config.parse(REPO / "config.yaml") - gen_config.merge_model_descriptions(sections) - notes_file = SITE / "data" / "config-notes.yaml" - overrides = yaml.safe_load(notes_file.read_text()) or {} - pages = [] - for s in sections: - for k in s.keys: - note = overrides.get(f"{s.name}.{k.path}") - if note: - k.description = note - page = Page( - f"{group['prefix']}/{s.name}", - s.name, - tab=tab, - group=group["title"], - kind="config", - search_type="Config", - source="config.yaml", - mono_title=True, - ) - page.extra = {"section": s, "intro": overrides.get(s.name, "")} - pages.append(self.add(page)) - return pages diff --git a/site/docsgen/fences.py b/site/docsgen/fences.py deleted file mode 100644 index 782c50b..0000000 --- a/site/docsgen/fences.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Line-by-line tracking of fenced code blocks in markdown sources.""" - -from __future__ import annotations - -import re - -FENCE_RE = re.compile(r"^\s{0,3}(`{3,}|~{3,})(.*)$") - - -class FenceTracker: - """Follows ``` and ~~~ fences so callers can skip lines inside code. - - Feed every line in order; :meth:`feed` says whether that line belongs to a - fenced block (the opening and closing fence lines included). - """ - - def __init__(self) -> None: - self.fence: str | None = None - - def feed(self, line: str) -> bool: - m = FENCE_RE.match(line) - if self.fence is None: - if m: - self.fence = m.group(1) - return m is not None - if m and self._closes(m): - self.fence = None - return True - - def _closes(self, m: re.Match) -> bool: - mark = m.group(1) - return mark[0] == self.fence[0] and len(mark) >= len(self.fence) and not m.group(2).strip() diff --git a/site/docsgen/gen_api.py b/site/docsgen/gen_api.py deleted file mode 100644 index a3cb159..0000000 --- a/site/docsgen/gen_api.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Python API reference data, read from the live objects with ``inspect``.""" - -from __future__ import annotations - -import dataclasses -import enum -import inspect -import pkgutil -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -_SECTION_RE = re.compile( - r"^(Args|Arguments|Parameters|Params|Returns|Return|Yields|Raises|Example|Examples" - r"|Note|Notes|Attributes|Usage)\s*:\s*$" -) -_ITEM_RE = re.compile(r"^(\*{0,2}[\w.\[\], |]+?)\s*(\([^)]*\))?\s*:\s*(.*)$") -# objects documented on the API pages: dotted identifiers inside the cloudg package -_API_PATH_RE = re.compile(r"^cloudg(?:\.[A-Za-z_][A-Za-z0-9_]*)+$") - - -@dataclass -class Param: - name: str - type: str - default: str - description: str = "" - kind: str = "" - - -@dataclass -class Doc: - summary: str = "" - body: str = "" - params: dict[str, str] = field(default_factory=dict) - returns: str = "" - raises: list[tuple[str, str]] = field(default_factory=list) - examples: str = "" - attributes: dict[str, str] = field(default_factory=dict) - notes: str = "" - - -@dataclass -class Member: - name: str - kind: str # method, async, property, classmethod, staticmethod, attribute - signature: str - returns: str - doc: Doc - params: list[Param] - source_line: int = 0 - - -@dataclass -class ApiObject: - path: str - name: str - module: str - kind: str # class, function, module, enum, model, dataclass - signature: str - doc: Doc - params: list[Param] - returns: str - members: list[Member] = field(default_factory=list) - fields: list[Param] = field(default_factory=list) - enum_values: list[tuple[str, str]] = field(default_factory=list) - bases: list[str] = field(default_factory=list) - source_file: str = "" - source_line: int = 0 - - -def rst_to_md(text: str) -> str: - """Turn the reST bits docstrings use (``:func:`x```, double backticks) into markdown.""" - text = re.sub(r":[\w]+(?::[\w]+)?:`~?\.?([^`]+)`", r"`\1`", text) - return re.sub(r"``([^`]+)``", r"`\1`", text) - - -def _split_sections(text: str) -> dict[str, list[str]]: - """Group docstring lines under their Google-style section header ("" for the top).""" - sections: dict[str, list[str]] = {"": []} - current = "" - for line in text.split("\n"): - m = _SECTION_RE.match(line.strip()) if not line.startswith(" ") else None - if m: - current = m.group(1).lower() - sections.setdefault(current, []) - continue - sections.setdefault(current, []).append(line) - return sections - - -def _items(lines: list[str]) -> list[tuple[str, str]]: - """``name (type): text`` entries, with continuation lines folded in.""" - out: list[tuple[str, str]] = [] - lines = inspect.cleandoc("\n".join(lines)).split("\n") if lines else [] - for line in lines: - if not line.strip(): - continue - m = _ITEM_RE.match(line) if not line.startswith(" ") else None - if m: - out.append((m.group(1).strip("* "), m.group(3).strip())) - elif out: - out[-1] = (out[-1][0], (out[-1][1] + " " + line.strip()).strip()) - return out - - -def _take(sections: dict[str, list[str]], keys: tuple[str, ...]) -> list[list[str]]: - """Remove and return the sections named in ``keys``, in that order.""" - return [sections.pop(key) for key in keys if key in sections] - - -def _block(lines: list[str]) -> str: - return inspect.cleandoc("\n".join(lines)) - - -def parse_docstring(text: str | None) -> Doc: - doc = Doc() - if not text: - return doc - sections = _split_sections(rst_to_md(inspect.cleandoc(text))) - main = "\n".join(sections.pop("", [])).strip() - paras = re.split(r"\n\s*\n", main, maxsplit=1) - doc.summary = " ".join(paras[0].split()) - doc.body = paras[1].strip() if len(paras) > 1 else "" - for lines in _take(sections, ("args", "arguments", "parameters", "params")): - doc.params.update(dict(_items(lines))) - for lines in _take(sections, ("returns", "return", "yields")): - doc.returns = " ".join(" ".join(lines).split()) - for lines in _take(sections, ("raises",)): - doc.raises = _items(lines) - for lines in _take(sections, ("attributes",)): - doc.attributes = dict(_items(lines)) - for lines in _take(sections, ("example", "examples", "usage")): - doc.examples = _block(lines) - doc.notes = "\n\n".join(_block(lines) for lines in _take(sections, ("note", "notes"))) - return doc - - -def _ann(a: Any) -> str: - if a is inspect.Parameter.empty or a is inspect.Signature.empty: - return "" - if isinstance(a, str): - return a.strip("'\"") - if isinstance(a, type): - return a.__name__ if a.__module__ == "builtins" else a.__name__ - s = repr(a) - return re.sub(r"\b(?:[a-z_][\w]*\.)+([A-Z]\w*)", r"\1", s).replace("typing.", "") - - -def _default(d: Any) -> str: - if d is inspect.Parameter.empty: - return "" - if isinstance(d, enum.Enum): - return f"{type(d).__name__}.{d.name}" - r = repr(d) - return r if len(r) <= 60 else r[:57] + "..." - - -def _params(sig: inspect.Signature | None, doc: Doc, skip_self: bool) -> list[Param]: - if sig is None: - return [] - out = [] - for i, (name, p) in enumerate(sig.parameters.items()): - if skip_self and i == 0 and name in ("self", "cls"): - continue - prefix = "*" if p.kind is p.VAR_POSITIONAL else "**" if p.kind is p.VAR_KEYWORD else "" - desc = doc.params.get(name) or doc.params.get(prefix + name) or "" - out.append(Param(prefix + name, _ann(p.annotation), _default(p.default), desc, p.kind.name)) - return out - - -def _param_text(p: Param) -> str: - s = p.name - if p.type: - s += f": {p.type}" - if p.default: - s += f" = {p.default}" if p.type else f"={p.default}" - return s - - -def _has_var_positional(params: list[Param]) -> bool: - return any(q.name.startswith("*") and not q.name.startswith("**") for q in params) - - -def format_signature(name: str, params: list[Param], returns: str, width: int = 88) -> str: - pieces = [] - # a *args parameter already marks where the keyword-only ones start - star_written = _has_var_positional(params) - for p in params: - if p.kind == "KEYWORD_ONLY" and not star_written: - pieces.append("*") - star_written = True - pieces.append(_param_text(p)) - ret = f" -> {returns}" if returns else "" - one = f"{name}({', '.join(pieces)}){ret}" - if len(one) <= width: - return one - return f"{name}(\n" + "".join(f" {p},\n" for p in pieces) + f"){ret}" - - -def _signature(obj: Any) -> inspect.Signature | None: - try: - return inspect.signature(obj) - except (TypeError, ValueError): - return None - - -def _source(obj: Any, repo: Path) -> tuple[str, int]: - try: - target = inspect.unwrap(obj) - file = inspect.getsourcefile(target) or "" - line = inspect.getsourcelines(target)[1] - return str(Path(file).resolve().relative_to(repo.resolve())), line - except (TypeError, OSError, ValueError): - return "", 0 - - -def _is_pydantic(cls: type) -> bool: - return hasattr(cls, "model_fields") and hasattr(cls, "model_validate") - - -def _member(cls: type, name: str, repo: Path) -> Member | None: - raw = inspect.getattr_static(cls, name, None) - if raw is None: - return None - kind = "method" - target: Any = raw - if isinstance(raw, property): - kind, target = "property", raw.fget - elif isinstance(raw, classmethod): - kind, target = "classmethod", raw.__func__ - elif isinstance(raw, staticmethod): - kind, target = "staticmethod", raw.__func__ - elif inspect.iscoroutinefunction(raw): - kind = "async" - elif not callable(raw): - return Member(name, "attribute", name, _ann(type(raw)), Doc(), []) - doc = parse_docstring(inspect.getdoc(target)) - sig = _signature(target) - params = _params(sig, doc, skip_self=kind in ("method", "async", "classmethod", "property")) - returns = _ann(sig.return_annotation) if sig else "" - if kind == "property": - signature, params = name, [] - else: - signature = format_signature(name, params, returns) - _, line = _source(target, repo) - return Member(name, kind, signature, returns, doc, params, line) - - -def _public_members(cls: type) -> list[str]: - names = [] - base_names: set[str] = set() - for base in cls.__mro__[1:]: - if base.__module__.startswith(("pydantic", "builtins", "enum", "typing", "abc")): - base_names.update(vars(base)) - for name, value in vars(cls).items(): - if name.startswith("_") or name in base_names: - continue - if callable(value) or isinstance(value, (property, classmethod, staticmethod)): - names.append(name) - return names - - -def _resolve(path: str) -> tuple[Any, str, str]: - """Return (object, module name, attribute name) for a dotted ``cloudg.`` path. - - Only paths inside the cloudg package are accepted: the names come from the - front matter of the content pages. - """ - if not _API_PATH_RE.match(path): - raise ValueError(f"not a cloudg object path: {path!r}") - obj = pkgutil.resolve_name(path) - module_name, _, attr = path.rpartition(".") - if inspect.ismodule(obj): - module_name = path - return obj, module_name, attr - - -def _class_kind(cls: type) -> str: - if issubclass(cls, enum.Enum): - return "enum" - if _is_pydantic(cls): - return "model" - if dataclasses.is_dataclass(cls): - return "dataclass" - return "class" - - -def _factory_default(factory: Any) -> str: - name = getattr(factory, "__name__", "") - return f"{name}()" if name and name != "" else "generated" - - -def _model_field_default(finfo: Any) -> str: - if finfo.is_required(): - return "required" - if finfo.default_factory is not None: - return _factory_default(finfo.default_factory) - return _default(finfo.default) - - -def _dataclass_field_default(f: dataclasses.Field) -> str: - if f.default is not dataclasses.MISSING: - return _default(f.default) - if f.default_factory is not dataclasses.MISSING: # type: ignore[misc] - return _factory_default(f.default_factory) - return "required" - - -def _model_fields(cls: type, doc: Doc) -> list[Param]: - return [ - Param( - fname, - _ann(finfo.annotation), - _model_field_default(finfo), - finfo.description or doc.attributes.get(fname, ""), - ) - for fname, finfo in cls.model_fields.items() - ] - - -def _dataclass_fields(cls: type, doc: Doc) -> list[Param]: - return [ - Param(f.name, _ann(f.type), _dataclass_field_default(f), doc.attributes.get(f.name, "")) - for f in dataclasses.fields(cls) - ] - - -def _enum_values(cls: type) -> list[tuple[str, str]]: - return [(m.name, repr(m.value)) for m in cls] - - -def _member_names(cls: type, kind: str, members: list[str] | None) -> list[str]: - if members is not None: - return members - return [] if kind == "enum" else _public_members(cls) - - -def _fill_class(api: ApiObject, cls: type, repo: Path, members: list[str] | None) -> None: - kind = _class_kind(cls) - init_doc = parse_docstring(inspect.getdoc(cls.__init__)) if "__init__" in vars(cls) else Doc() - doc = api.doc - api.doc = Doc(**{**doc.__dict__, "params": {**init_doc.params, **doc.params}}) - api.name, api.kind, api.signature = cls.__name__, kind, cls.__name__ - if kind in ("class", "dataclass"): - init_sig = _signature(cls) - api.params = _params(init_sig, api.doc, skip_self=False) if init_sig else [] - api.signature = format_signature(cls.__name__, api.params, "") - api.bases = [b.__name__ for b in cls.__bases__ if b is not object] - if kind == "enum": - api.enum_values = _enum_values(cls) - fields_of = {"model": _model_fields, "dataclass": _dataclass_fields}.get(kind) - if fields_of: - api.fields = fields_of(cls, doc) - for name in _member_names(cls, kind, members): - m = _member(cls, name, repo) - if m is not None: - api.members.append(m) - - -def _fill_function(api: ApiObject, fn: Any) -> None: - sig = _signature(fn) - api.params = _params(sig, api.doc, skip_self=False) - api.returns = _ann(sig.return_annotation) if sig else "" - api.kind = "async" if inspect.iscoroutinefunction(fn) else "function" - api.signature = format_signature(api.name, api.params, api.returns) - - -def load(path: str, repo: Path, members: list[str] | None = None) -> ApiObject: - obj, module_name, attr = _resolve(path) - file, line = _source(obj, repo) - if inspect.ismodule(obj): - doc = parse_docstring(obj.__doc__) - return ApiObject( - path, attr, module_name, "module", path, doc, [], "", source_file=file, source_line=1 - ) - real_module = getattr(obj, "__module__", module_name) or module_name - doc = parse_docstring(inspect.getdoc(obj)) - api = ApiObject( - path, attr, real_module, "function", attr, doc, [], "", source_file=file, source_line=line - ) - if inspect.isclass(obj): - _fill_class(api, obj, repo, members) - else: - _fill_function(api, obj) - return api diff --git a/site/docsgen/gen_changelog.py b/site/docsgen/gen_changelog.py deleted file mode 100644 index 16de0a5..0000000 --- a/site/docsgen/gen_changelog.py +++ /dev/null @@ -1,54 +0,0 @@ -"""CHANGELOG.md split into releases and labelled blocks (Added, Changed, Fixed...).""" - -from __future__ import annotations - -import re -from dataclasses import dataclass, field -from pathlib import Path - -_RELEASE_RE = re.compile(r"^##\s+(.+?)\s*$") -_VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)\s*(?:\((\d{4}-\d{2}-\d{2})\))?") -_LABEL_RE = re.compile(r"^([A-Z][A-Za-z ]{2,30}):\s*$") - - -@dataclass -class Release: - title: str - version: str - date: str - anchor: str - intro: str - blocks: list[tuple[str, str]] = field(default_factory=list) - - -def _split_blocks(lines: list[str]) -> tuple[str, list[tuple[str, str]]]: - """Split a release body at ``Label:`` lines that sit outside code fences.""" - intro: list[str] = [] - blocks: list[tuple[str, list[str]]] = [] - fence = False - for line in lines: - if line.lstrip().startswith("```"): - fence = not fence - lm = _LABEL_RE.match(line) if not fence else None - if lm: - blocks.append((lm.group(1), [])) - continue - (blocks[-1][1] if blocks else intro).append(line) - return "\n".join(intro).strip(), [(k, "\n".join(v).strip()) for k, v in blocks] - - -def _release(part: str) -> Release: - lines = part.rstrip().split("\n") - title = _RELEASE_RE.match(lines[0]).group(1) - intro, blocks = _split_blocks(lines[1:]) - m = _VERSION_RE.match(title) - if m: - return Release(title, m.group(1), m.group(2) or "", m.group(1), intro, blocks) - anchor = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") - return Release(title, title, "", anchor, intro, blocks) - - -def parse(path: Path) -> tuple[str, list[Release]]: - parts = re.split(r"(?m)^(?=## )", path.read_text()) - head = re.sub(r"(?m)^#\s+.*$", "", parts[0]).strip() - return head, [_release(part) for part in parts[1:]] diff --git a/site/docsgen/highlight.py b/site/docsgen/highlight.py deleted file mode 100644 index ddc1dc4..0000000 --- a/site/docsgen/highlight.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Code blocks rendered as editor or terminal windows. - -Pygments does the tokenising; the HTML is built here, one ```` -per source line, so line numbers, highlighted lines and terminal prompts can be -drawn with CSS and stay out of the copied text. -""" - -from __future__ import annotations - -import re -from html import escape - -from pygments.lexers import get_lexer_by_name -from pygments.token import STANDARD_TYPES, Token -from pygments.util import ClassNotFound - -from docsgen.icons import icon - -TERMINAL_LANGS = {"bash", "sh", "shell", "zsh", "shell-session-commands"} -SESSION_LANGS = {"console", "shell-session"} -LANG_LABELS = { - "python": "Python", - "py": "Python", - "yaml": "YAML", - "yml": "YAML", - "json": "JSON", - "jsonc": "JSON", - "toml": "TOML", - "ini": "INI", - "dockerfile": "Dockerfile", - "docker": "Dockerfile", - "text": "Text", - "txt": "Text", - "sparql": "SPARQL", - "turtle": "Turtle", - "ttl": "Turtle", - "xml": "XML", - "html": "HTML", - "javascript": "JavaScript", - "js": "JavaScript", - "typescript": "TypeScript", - "hcl": "HCL", - "terraform": "HCL", - "sql": "SQL", - "powershell": "PowerShell", - "ps1": "PowerShell", - "diff": "Diff", - "jsonl": "JSON Lines", - "csv": "CSV", -} -_ATTR_RE = re.compile(r'(\w+)=("([^"]*)"|\'([^\']*)\'|(\S+))') - - -def parse_info(info: str) -> tuple[str, dict[str, str]]: - """Split a fence info string into language and attributes. - - ``python title="a.py" hl="2-4"`` gives ``("python", {"title": "a.py", "hl": "2-4"})``. - A bare ``{2-4}`` is accepted as ``hl`` too. - """ - info = info.strip() - if not info: - return "", {} - lang, _, rest = info.partition(" ") - attrs: dict[str, str] = {} - for m in _ATTR_RE.finditer(rest): - attrs[m.group(1)] = next(g for g in (m.group(3), m.group(4), m.group(5)) if g is not None) - brace = re.search(r"\{([\d,\-\s]+)\}", rest) - if brace and "hl" not in attrs: - attrs["hl"] = brace.group(1) - return lang.lower(), attrs - - -def parse_ranges(spec: str) -> set[int]: - out: set[int] = set() - for part in spec.replace(" ", "").split(","): - if not part: - continue - if "-" in part: - a, b = part.split("-", 1) - out.update(range(int(a), int(b) + 1)) - else: - out.add(int(part)) - return out - - -def _css_class(ttype) -> str: - while ttype not in STANDARD_TYPES and ttype is not Token: - ttype = ttype.parent - return STANDARD_TYPES.get(ttype, "") - - -def _lexer(lang: str): - alias = { - "jsonl": "json", - "yml": "yaml", - "docker": "dockerfile", - "ttl": "turtle", - "terraform": "hcl", - } - try: - return get_lexer_by_name(alias.get(lang, lang) or "text", stripnl=False, ensurenl=False) - except ClassNotFound: - return get_lexer_by_name("text", stripnl=False, ensurenl=False) - - -def tokenize_lines(code: str, lang: str) -> list[str]: - """Return one HTML string per source line.""" - lines: list[str] = [""] - for ttype, value in _lexer(lang).get_tokens(code): - cls = _css_class(ttype) - parts = value.split("\n") - for i, part in enumerate(parts): - if i: - lines.append("") - if part: - text = escape(part, quote=False) - lines[-1] += f'{text}' if cls else text - if lines and lines[-1] == "" and code.endswith("\n"): - lines.pop() - return lines - - -def _terminal_roles(raw_lines: list[str]) -> list[str]: - roles = [] - continued = False - in_heredoc: str | None = None - for line in raw_lines: - stripped = line.strip() - if in_heredoc is not None: - roles.append("out") - if stripped == in_heredoc: - in_heredoc = None - continue - if continued: - roles.append("cont") - elif not stripped: - roles.append("blank") - elif stripped.startswith("#"): - roles.append("cmt") - else: - roles.append("cmd") - m = re.search(r"<<-?\s*'?\"?(\w+)'?\"?\s*$", stripped) - if m and roles[-1] in ("cmd", "cont"): - in_heredoc = m.group(1) - continued = stripped.endswith("\\") and roles[-1] != "cmt" - return roles - - -def _session_roles(raw_lines: list[str]) -> list[tuple[str, str]]: - """For console blocks: (role, line without prompt).""" - out = [] - continued = False - for line in raw_lines: - if continued: - out.append(("cont", line)) - elif line.startswith("$ "): - out.append(("cmd", line[2:])) - elif line.startswith("# ") and not out: - out.append(("cmt", line)) - else: - out.append(("out", line)) - continued = out[-1][0] in ("cmd", "cont") and line.rstrip().endswith("\\") - return out - - -def code_bar_label(lang: str, attrs: dict[str, str]) -> tuple[str, str]: - """Icon name and label shown in the editor bar.""" - if lang in TERMINAL_LANGS or lang in SESSION_LANGS: - return "terminal", attrs.get("title", "Terminal") - return "file-code", attrs.get("title", LANG_LABELS.get(lang, lang.upper() if lang else "Text")) - - -def _session_lines(code: str, lang: str) -> list[tuple[str, str]]: # noqa: ARG001 - """Console transcript: commands highlighted as bash, output left plain.""" - lines = [] - for role, text in _session_roles(code.rstrip("\n").split("\n")): - if role in ("cmd", "cont"): - inner = tokenize_lines(text + "\n", "bash")[0] - else: - inner = escape(text, quote=False) - lines.append((role, inner)) - return lines - - -def _terminal_lines(code: str, lang: str) -> list[tuple[str, str]]: # noqa: ARG001 - raw = code.rstrip("\n").split("\n") - toks = tokenize_lines(code, "bash") - toks += [""] * (len(raw) - len(toks)) - return list(zip(_terminal_roles(raw), toks)) - - -def _editor_lines(code: str, lang: str) -> list[tuple[str, str]]: - return [("", t) for t in tokenize_lines(code, lang)] - - -def _line_span(n: int, role: str, inner: str, hl: set[int]) -> str: - classes = ["line"] - if role: - classes.append(role) - if n in hl: - classes.append("hl") - return f'{inner or " "}' - - -def render_code_body(code: str, lang: str, attrs: dict[str, str]) -> tuple[str, str]: - """Render the ``
      `` for a block. Returns (variant, html)."""
-    code = code.rstrip("\n") + "\n"
-    hl = parse_ranges(attrs["hl"]) if attrs.get("hl") else set()
-    if lang in SESSION_LANGS:
-        variant, lines_of = "terminal", _session_lines
-    elif lang in TERMINAL_LANGS:
-        variant, lines_of = "terminal", _terminal_lines
-    else:
-        variant, lines_of = "editor", _editor_lines
-    out = [
-        _line_span(n, role, inner, hl)
-        for n, (role, inner) in enumerate(lines_of(code, lang), start=1)
-    ]
-    lang_class = escape(lang or "text")
-    pre = (
-        f'
      ' + "\n".join(out) + "
      "
-    )
-    return variant, pre
-
-
-def render_code_block(code: str, info: str) -> str:
-    lang, attrs = parse_info(info)
-    variant, pre = render_code_body(code, lang, attrs)
-    ico, label = code_bar_label(lang, attrs)
-    lang_tag = (
-        LANG_LABELS.get(lang, lang) if lang and "title" in attrs and variant == "editor" else ""
-    )
-    bar = (
-        f'
      {icon(ico, 14)}{escape(label)}'
-        + (f'{escape(lang_tag)}' if lang_tag else "")
-        + copy_button()
-        + "
      "
-    )
-    return f'
      {bar}{pre}
      '
-
-
-def copy_button() -> str:
-    return (
-        f''
-    )
-
-
-def render_tab_group(blocks: list[tuple[str, str]]) -> str:
-    """Blocks are (code, info) pairs that all carry a ``tab`` attribute."""
-    tabs, panes = [], []
-    for i, (code, info) in enumerate(blocks):
-        lang, attrs = parse_info(info)
-        label = attrs.get("tab", f"Tab {i + 1}")
-        variant, pre = render_code_body(code, lang, attrs)
-        selected = "true" if i == 0 else "false"
-        ico = icon("terminal", 14) if variant == "terminal" else ""
-        tabs.append(
-            f''
-        )
-        sub = ""
-        if "title" in attrs:
-            sub_ico = "terminal" if variant == "terminal" else "file-code"
-            sub = f'
      {icon(sub_ico, 13)}{escape(attrs["title"])}
      '
-        hidden = "" if i == 0 else " hidden"
-        panes.append(
-            f'
      {sub}{pre}
      '
-        )
-    return (
-        '
      '
-        f'
      {"".join(tabs)}{copy_button()}
      '
-        f"{''.join(panes)}
      "
-    )
diff --git a/site/docsgen/icons.py b/site/docsgen/icons.py
deleted file mode 100644
index 44ef0c0..0000000
--- a/site/docsgen/icons.py
+++ /dev/null
@@ -1,63 +0,0 @@
-"""Lucide icons used by the site, inlined as SVG (stroke 2, currentColor)."""
-
-_PATHS = {
-    "search": '',
-    "chevron-down": '',
-    "github": (
-        ''
-    ),
-    "moon": '',
-    "sun": (
-        ''
-        ''
-        ''
-    ),
-    "pencil": (
-        ''
-    ),
-    "info": '',
-    "lightbulb": (
-        ''
-    ),
-    "triangle-alert": (
-        ''
-        ''
-    ),
-    "octagon-alert": (
-        ''
-    ),
-    "copy": '',
-    "check": '',
-    "arrow-left": '',
-    "arrow-right": '',
-    "terminal": '',
-    "corner-down-left": '',
-    "file-code": (
-        ''
-    ),
-    "menu": '',
-    "x": '',
-    "external": (
-        ''
-    ),
-    "list": (
-        ''
-        ''
-    ),
-}
-
-
-def icon(name: str, size: int = 16, cls: str = "") -> str:
-    attrs = f' class="{cls}"' if cls else ""
-    return (
-        f'