diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d796f4d..3221d8f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,8 +1,9 @@ name: Docs -# Builds the documentation site (site/build.py) and deploys it to GitHub Pages -# on every push to main that touches the docs, the content they are generated -# from (click commands, docstrings, config.yaml, CHANGELOG.md) or the site itself. +# Builds the documentation site (MkDocs Material, mkdocs.yml) and deploys it to +# GitHub Pages on every push to main that touches the docs, the content they are +# generated from (click commands, docstrings, config.yaml, CHANGELOG.md) or the +# site itself. The site is served at https://morpheuslord.github.io/cloudg/. # # One-time setup: under Settings -> Pages, set "Build and deployment" # source to "GitHub Actions". The GITHUB_TOKEN cannot create the Pages @@ -18,12 +19,15 @@ on: - "cloudg/**" - "config.yaml" - "CHANGELOG.md" + - "README.md" - "pyproject.toml" + - "mkdocs.yml" - ".github/workflows/pages.yml" pull_request: paths: - "docs/**" - "site/**" + - "mkdocs.yml" - ".github/workflows/pages.yml" workflow_dispatch: @@ -46,14 +50,19 @@ jobs: with: python-version: "3.12" - # The CLI and API reference pages are generated by importing cloudg, - # so the package is installed; its core dependencies already include - # everything the site generator needs (markdown-it-py, Pygments, Jinja2). - - name: Install cloudg - run: pip install . + # The CLI, config and API reference pages are generated from the code, + # so cloudg is installed next to the docs dependency group. pip reads + # dependency groups from 25.1 on. + - name: Install cloudg and the docs tools + run: | + python -m pip install --upgrade "pip>=25.1" + pip install . --group docs - name: Build the site - run: python site/build.py --strict + env: + NO_MKDOCS_2_WARNING: "1" + DISABLE_MKDOCS_2_WARNING: "true" + run: mkdocs build --strict - uses: actions/upload-pages-artifact@v3 if: github.event_name != 'pull_request' diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4fc7e..eb01592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Notable changes per release. Patch releases are folded into the major entry they belong to. +## 0.7.0 (2026-10-25) + +Azure and GCP are mapped as deeply as AWS, scanner and collector failures raise exceptions you can import and catch, report.html opens on an animated isometric map of the estate, and the docs site moves to MkDocs Material. + +Added: + +- Azure depth. Besides the Resource Graph breadth query, cloudg reads the `policyresources`, `securityresources`, `recoveryservicesresources`, `networkresources`, `maintenanceresources`, `insightsresources` and `kubernetesconfigurationresources` tables, management-group and tenant-scope role assignments, ARM child resources (APIM, Service Bus, Event Hubs, DNS record sets, Front Door, Virtual WAN, Data Factory and Synapse pipelines, AI Search, Azure OpenAI deployments and more), diagnostic settings, Entra ID through Microsoft Graph (users, groups, app registrations, service principals, app-role assignments, OAuth2 grants, directory roles, PIM eligibility) and the objects inside AKS clusters. The ARM type catalog grows from 125 to 278 types. `azure.map_entra`, `azure.child_resources`, `azure.diagnostic_settings` and `azure.data_plane` turn the extra sources off. +- GCP depth. 350 Cloud Asset Inventory types now have extractors (123 before), and API sources fill what the inventory does not return: Cloud DNS records, Security Command Center, Cloud Identity groups, IAM deny policies and principal access boundaries, PAM, audit configs, Cloud Monitoring, OS Config, Artifact Analysis, Backup and DR, Apigee, Cloud Scheduler and others. `gcp.deep_sources` picks which ones run. +- Azure and GCP record coverage per task family, as AWS does, show "not enabled" placeholders for Defender plans, Sentinel and Security Command Center services, and follow `--services` / `inventory.exclude_services`. +- `cloudg.exceptions`: `CloudGError` and its subclasses (`ScannerError`, `ScannerTimeoutError`, `ScannerNotInstalledError`, `ScannerExecutionError`, `ScannerOutputError`, `ReportParseError`, `ScannerFailed`, `CollectionError`, `CredentialError`, `ProviderSDKMissingError`, `ConfigurationError`), also exported from `cloudg`. Every scanner wrapper takes `strict=True` to raise on the first failure, records each failure in `.failures`, and has `raise_for_errors()`. `parse_report(path, strict=True)` raises `ReportParseError` for an unreadable report. +- report.html: an estate map in 3D (provider plates, account and region platforms, network slabs, subnet tiles, a control deck for identity and governance), with every relation drawn as a trace line on a lane of its own, asset selection, tracing from the internet, turning, zoom and pan. The D3 tree and force views stay one toggle away. The page also gains a posture overview, a top-risks list, an Assets tab, CSV export, light and dark themes and print styles. The Anatomy iso-kit and the Hairline kernel (both MIT) ship in `cloudg/templates/vendor/` and are always embedded. +- The docs site is built with MkDocs Material, with API pages generated from the docstrings, tabbed examples, an Examples section, an Exceptions page, a Credits page and a sample report at `/sample-report.html`. + +Changed: + +- Credential, SDK and collection failures raise `CredentialError`, `ProviderSDKMissingError` and `CollectionError`. They keep their old bases (`RuntimeError`, `ImportError`), so existing `except` clauses still match. `ResilienceError` and `LiveOperationRejected` now derive from `CloudGError`. +- The PyPI project page is a short overview (`docs/PYPI.md`) that links into the docs site. `docs/DOCUMENTATION.md` stays the full reference that the MCP server serves. + ## 0.6.1 (2026-10-10) A security fix for report.html, a round of fixes found while documenting every command and API, and the new docs site. Upgrade if you open reports built from scanner output you did not produce yourself. Several commands now behave the way their help text always said, so check the Changed list if you script `cloudg report`, `cloudg scan` or the exit codes. diff --git a/README.md b/README.md index 1b2c8f8..42bf89b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Want an AI agent to work with all of it? `cloudg mcp serve` turns cloudg into an **Full documentation:** the rendered handbook lives at [morpheuslord.github.io/cloudg](https://morpheuslord.github.io/cloudg/), with the same content as markdown in the [feature reference](https://github.com/morpheuslord/cloudg/blob/main/docs/DOCUMENTATION.md) and release notes in the [changelog](https://github.com/morpheuslord/cloudg/blob/main/CHANGELOG.md). The inventory mapper has its own deep-dive documents: the [inventory reference](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_REFERENCE.md) (every return structure and output file, field by field), the [inventory catalog](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_CATALOG.md) (all 156 asset types, their metadata and relationships) and the [inventory internals](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_INTERNALS.md) (how it works and how to extend it). The MCP server is covered by the [MCP guide](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP.md), the [tool reference](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP_TOOLS.md) and the [privacy reference](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP_PRIVACY.md), and throttling by [rate limits and throttling](https://github.com/morpheuslord/cloudg/blob/main/docs/RESILIENCE.md). +**Sample report:** to see what cloudg produces before running anything, [open the sample report](https://morpheuslord.github.io/cloudg/sample-report.html). It is the `report.html` of a synthetic AWS, Azure and GCP estate, so every ID in it is made up, and `scripts/build_sample_report.py` rebuilds it from the test fixtures. + --- ## Quick start @@ -375,7 +377,7 @@ Python 3.11 or newer. The moto/aiobotocore incompatibility around async response ## Credits -cloudg builds on other people's work, and some of it ships inside the package. +cloudg builds on other people's work, and some of it ships inside the package. The documentation site has the same list on its [credits page](https://morpheuslord.github.io/cloudg/credits/). [Prowler](https://github.com/prowler-cloud/prowler) (Apache-2.0). The 22 framework files in `cloudg/rules/frameworks/` are generated from Prowler's compliance JSON by `scripts/import_prowler_compliance.py`. That covers the control IDs, the control titles and descriptions, and the mapping from each control to Prowler check names. Every generated file says so in its header and in its `source` field. Prowler check names also appear in `cloudg/rules/check_equivalence.yaml`, and cloudg runs Prowler and parses its ASFF and OCSF output. @@ -385,6 +387,8 @@ Scanners. [Checkov](https://github.com/bridgecrewio/checkov) (Apache-2.0), [Triv [parliament](https://github.com/duo-labs/parliament) (BSD-3-Clause) from Duo Labs does the IAM policy linting. +Drawing engines. The estate map in report.html is drawn with the iso-kit from [Anatomy](https://github.com/wheresryan22/anatomy) (MIT, by Ryan), and its empty states use the kernel of [Hairline](https://hairline.lucasmarkes.com) 0.5.1 (MIT, by Lucas Marques). Both are copied into `cloudg/templates/vendor/` and embedded in every report, so the map works offline. The iso-kit is wrapped so it can sit inside a script element, and `cloudg/templates/vendor/NOTICE` lists exactly what changed. + [MITRE ATT&CK®](https://attack.mitre.org/). The `mitre_attack_*.yaml` files contain ATT&CK technique IDs and descriptions. © The MITRE Corporation. This work is reproduced and distributed with the permission of The MITRE Corporation. ATT&CK is a registered trademark of The MITRE Corporation. Standards. The hand-written rulesets in `cloudg/rules/` reference public control numbering from the CIS Benchmarks for AWS, Azure and GCP, NIST SP 800-53, NIST CSF, PCI DSS, the AICPA SOC 2 Trust Services Criteria, the HIPAA Security Rule, GDPR, ISO/IEC 27001:2022 and the AWS Foundational Security Best Practices. The standards belong to their publishers. CIS Benchmarks is a trademark of the Center for Internet Security, and cloudg is not affiliated with or endorsed by CIS or any other publisher named here. diff --git a/cloudg/__init__.py b/cloudg/__init__.py index 22da0bb..88bec05 100644 --- a/cloudg/__init__.py +++ b/cloudg/__init__.py @@ -1,6 +1,6 @@ """cloudg (cloud graphing): multi-cloud mapping and security intelligence.""" -__version__ = "0.6.0" +__version__ = "0.7.0" from cloudg.api import ( AnalysisResult, @@ -9,6 +9,20 @@ PipelineResult, ) from cloudg.config import CloudGConfig, load_config +from cloudg.exceptions import ( + CloudGError, + CollectionError, + ConfigurationError, + CredentialError, + ProviderSDKMissingError, + ReportParseError, + ScannerError, + ScannerExecutionError, + ScannerFailed, + ScannerNotInstalledError, + ScannerOutputError, + ScannerTimeoutError, +) from cloudg.inventory import InventoryMapper, InventoryResult __all__ = [ @@ -16,6 +30,18 @@ "AnalysisResult", "CloudGEngine", "CloudGConfig", + "CloudGError", + "CollectionError", + "ConfigurationError", + "CredentialError", + "ProviderSDKMissingError", + "ReportParseError", + "ScannerError", + "ScannerExecutionError", + "ScannerFailed", + "ScannerNotInstalledError", + "ScannerOutputError", + "ScannerTimeoutError", "CollectionResult", "InventoryMapper", "InventoryResult", diff --git a/cloudg/api_scanners.py b/cloudg/api_scanners.py index ed26b30..39f9c3c 100644 --- a/cloudg/api_scanners.py +++ b/cloudg/api_scanners.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Callable +from cloudg.exceptions import ScannerFailed from cloudg.schema.models import CloudAsset, Finding, NetworkEdge __all__ = [ @@ -64,24 +65,12 @@ def resolve_iac_dirs( # --------------------------------------------------------------------------- -class ScannerFailed(RuntimeError): - """A scanner started but did not finish (timeout, launch failure). - - ``findings`` holds whatever it produced before that (for example the - images Trivy did scan), so callers can keep partial results. - """ - - def __init__(self, scanner: str, errors: list[str], findings: list[Finding]) -> None: - super().__init__(f"{scanner} " + "; ".join(errors)) - self.errors = list(errors) - self.findings = findings - - def _finish(label: str, scanner: Any, findings: list[Finding]) -> list[Finding]: """Return ``findings``, or raise ScannerFailed when the wrapper recorded errors.""" errors = list(getattr(scanner, "errors", None) or []) if errors: - raise ScannerFailed(label, errors, findings) + failures = [f for f in getattr(scanner, "failures", None) or [] if f.detail in errors] + raise ScannerFailed(label, errors, findings, failures=failures) return findings diff --git a/cloudg/cli_inventory.py b/cloudg/cli_inventory.py index ed12678..a1e862d 100644 --- a/cloudg/cli_inventory.py +++ b/cloudg/cli_inventory.py @@ -292,7 +292,8 @@ def _merge_findings( @click.option( "--services", default=None, - help="Service families/collectors to map, comma-separated (e.g. containers,serverless,security)", + help="Service families or task names to map, comma-separated, for AWS and Azure " + "(e.g. containers,serverless,security)", ) @click.option( "--exclude-services", default=None, help="Service families/collectors to skip, comma-separated" @@ -300,7 +301,7 @@ def _merge_findings( @click.option( "--kubernetes/--no-kubernetes", default=None, - help="Map workloads inside EKS clusters through the Kubernetes API", + help="Map workloads inside EKS and AKS clusters through the Kubernetes API", ) @click.option( "--cloud-control/--no-cloud-control", diff --git a/cloudg/collectors/gcp.py b/cloudg/collectors/gcp.py index e891ee6..8ecb4b1 100644 --- a/cloudg/collectors/gcp.py +++ b/cloudg/collectors/gcp.py @@ -43,6 +43,7 @@ ) from cloudg.config import GCP_DEFAULT_SKIP_ASSET_TYPES from cloudg.coverage import CollectionCoverage, ServiceStatus +from cloudg.exceptions import CollectionError, ProviderSDKMissingError from cloudg.resilience.errors import describe_error from cloudg.resilience.gcp import RetryCounter, gcp_retry, gcp_scope, paced from cloudg.resilience.governor import get_governor @@ -186,9 +187,7 @@ def _get_client(self) -> Any: try: from google.cloud import asset_v1 except ImportError as exc: - raise RuntimeError( - "google-cloud-asset is not installed; install cloudg[gcp] to collect GCP" - ) from exc + raise ProviderSDKMissingError("gcp", missing="google-cloud-asset") from exc self._client = asset_v1.AssetServiceClient(credentials=self._credentials) return self._client @@ -301,7 +300,7 @@ async def _collect_from_search(self, error: Exception, start: float) -> list[dic error=describe_error(search_error), start=search_start, ) - raise RuntimeError( + raise CollectionError( f"GCP asset enumeration failed for {self._scope}: " f"ListAssets: {describe_error(error)}; " f"SearchAllResources: {describe_error(search_error)}" diff --git a/cloudg/collectors/multi.py b/cloudg/collectors/multi.py index 9bbedca..5e07e30 100644 --- a/cloudg/collectors/multi.py +++ b/cloudg/collectors/multi.py @@ -563,7 +563,10 @@ async def _azure_edges_and_coverage( """Collect one Azure subscription's edges and record its coverage. An edge failure is recorded as ``azure_edges`` and never discards the - assets; per-service collector errors become ``azure_`` entries. + assets; per-service collector errors become ``azure_`` entries, + and the per-task records of collectors that keep a ``coverage`` + (:class:`~cloudg.inventory.azure_deep.AzureDeepInventoryCollector`) are + copied in. ``azure_full`` is PARTIAL when any of them failed. """ edges: list[NetworkEdge] = [] try: @@ -573,12 +576,15 @@ async def _azure_edges_and_coverage( coverage.record("azure_edges", ServiceStatus.FAILED, error=describe_error(exc)) duration_ms = int((time.time() - start) * 1000) + task_records = list(getattr(getattr(collector, "coverage", None), "services", None) or []) + coverage.services.extend(task_records) + degraded = any(s.status in (ServiceStatus.FAILED, ServiceStatus.PARTIAL) for s in task_records) service_errors = dict(getattr(collector, "service_errors", None) or {}) for service, error in service_errors.items(): coverage.record(f"azure_{service}", ServiceStatus.FAILED, error=error) coverage.record( "azure_full", - ServiceStatus.PARTIAL if service_errors else ServiceStatus.SUCCESS, + ServiceStatus.PARTIAL if service_errors or degraded else ServiceStatus.SUCCESS, asset_count=asset_count, duration_ms=duration_ms, ) diff --git a/cloudg/config.py b/cloudg/config.py index 00e6c7c..bd86c03 100644 --- a/cloudg/config.py +++ b/cloudg/config.py @@ -166,6 +166,49 @@ class AzureConfig(BaseModel): description="Map the management group / subscription / Azure Policy hierarchy (inventory)", ) + # Azure parity + map_entra: bool = Field( + default=True, + description="Inventory: map Entra ID (users, groups, apps, service principals, " + "directory roles, PIM) through Microsoft Graph; needs Graph read permissions", + ) + entra_max_objects: int = Field( + default=10000, ge=0, description="Inventory: most objects read per Microsoft Graph listing" + ) + entra_max_expansions: int = Field( + default=1000, + ge=0, + description="Inventory: most groups / apps / service principals whose members, owners, " + "federated credentials or app-role assignments are read one by one", + ) + child_resources: bool = Field( + default=True, + description="Inventory: read child resources Resource Graph does not return (APIM APIs, " + "Service Bus queues, DNS records, functions, ...) through ARM REST", + ) + max_child_parents: int = Field( + default=500, + ge=0, + description="Inventory: most parent resources queried per child-resource kind", + ) + diagnostic_settings: bool = Field( + default=True, description="Inventory: read diagnostic settings of logged resource types" + ) + max_diagnostic_settings: int = Field( + default=1000, + ge=0, + description="Inventory: most resources whose diagnostic settings are read", + ) + data_plane: bool = Field( + default=True, + description="Inventory: read AI Search and Synapse data-plane metadata (data sources, " + "indexers, pipelines, linked services); needs data-plane reader roles", + ) + rest_concurrency: int = Field( + default=8, ge=1, le=32, description="Inventory: concurrent ARM / Graph REST calls" + ) + # end Azure parity + # High-churn Cloud Asset Inventory types the GCP collector skips by default # (GCPConfig.skip_asset_types; cloudg.collectors.gcp.DEFAULT_SKIP_ASSET_TYPES) @@ -231,6 +274,33 @@ class GCPConfig(BaseModel): vpc_service_controls: bool = Field( default=True, description="Map VPC Service Controls perimeters" ) + # GCP parity: API-backed sources of the deep inventory collector + deep_sources: bool = Field( + default=True, + description="Inventory mapping: also read what Cloud Asset Inventory does not return " + "(Cloud DNS records, Security Command Center, Cloud Identity groups, deny and access " + "boundary policies, PAM, audit-log configs, Monitoring, OS Config, Artifact Analysis, " + "Backup and DR, Cloud Scheduler, Apigee, Infrastructure / Deployment Manager)", + ) + rest_timeout_seconds: float = Field(default=60.0, ge=5.0) + cloud_identity_customer_id: str | None = Field( + default=None, + description="Cloud Identity customer ID (Cxxxxxxx) for group membership; " + "found from the organization when unset", + ) + max_dns_records_per_zone: int = Field(default=5000, ge=0) + max_vulnerability_occurrences: int = Field( + default=5000, ge=0, description="Artifact Analysis occurrences read per project" + ) + max_groups: int = Field(default=2000, ge=0) + max_group_members: int = Field(default=5000, ge=0) + expand_predefined_roles: bool = Field( + default=True, description="Read the permissions of bound predefined roles (IAM Roles API)" + ) + max_role_expansions: int = Field(default=500, ge=0) + max_scheduler_locations: int = Field(default=25, ge=0) + source_concurrency: int = Field(default=4, ge=1, le=32) + # end GCP parity class ScannerConfig(BaseModel): diff --git a/cloudg/credentials.py b/cloudg/credentials.py index ed51aa6..4454eb8 100644 --- a/cloudg/credentials.py +++ b/cloudg/credentials.py @@ -39,6 +39,8 @@ from dataclasses import dataclass from typing import Any +from cloudg.exceptions import CredentialError, ProviderSDKMissingError + logger = logging.getLogger(__name__) _GCP_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] @@ -94,7 +96,9 @@ def _aws_session_from_web_identity( WebIdentityToken=token, ) except Exception as exc: - raise RuntimeError(f"AssumeRoleWithWebIdentity failed for {role_arn}: {exc}") from exc + raise CredentialError( + "aws", f"AssumeRoleWithWebIdentity failed for {role_arn}: {exc}" + ) from exc creds = resp["Credentials"] return boto3.Session( aws_access_key_id=creds["AccessKeyId"], @@ -162,7 +166,7 @@ def _aws_assume_role( try: assumed = session.client("sts").assume_role(**assume_kwargs) except Exception as exc: - raise RuntimeError(f"AssumeRole failed for {target_role}: {exc}") from exc + raise CredentialError("aws", f"AssumeRole failed for {target_role}: {exc}") from exc creds = assumed["Credentials"] return boto3.Session( aws_access_key_id=creds["AccessKeyId"], @@ -191,16 +195,14 @@ def build_aws_session( boto3.Session ready for API calls. Raises: - ImportError: If boto3 is not installed. - RuntimeError: If a requested role assumption fails, or account_id + ProviderSDKMissingError: If boto3 is not installed (an ImportError). + CredentialError: If a requested role assumption fails, or account_id is given without cfg.role_name (there is no role to reach it). """ try: import boto3 except ImportError as exc: - raise ImportError( - "boto3 is required for AWS. Install with: pip install cloudg[aws]" - ) from exc + raise ProviderSDKMissingError("aws", missing="boto3") from exc session_name = getattr(cfg, "role_session_name", None) or "cloudg-scan" role_arn = getattr(cfg, "role_arn", None) @@ -237,9 +239,10 @@ def _aws_target_roles( if account_id: role_name = getattr(cfg, "role_name", None) if not role_name: - raise RuntimeError( + raise CredentialError( + "aws", f"Cannot reach account {account_id}: no role to assume there. " - "Set aws.role_name (--role-name) to the role that exists in each account." + "Set aws.role_name (--role-name) to the role that exists in each account.", ) member_role = f"arn:aws:iam::{account_id}:role/{role_name}" if member_role not in roles: @@ -295,7 +298,7 @@ def aws_scanner_auth(cfg: Any, region: str | None = None) -> tuple[str | None, d them for (one hour). Raises: - RuntimeError: assuming role_arn failed. + CredentialError: assuming role_arn failed (a RuntimeError). """ if not region or region.upper() == "ALL": region = "us-east-1" @@ -307,7 +310,7 @@ def aws_scanner_auth(cfg: Any, region: str | None = None) -> tuple[str | None, d session = build_aws_session(cfg, region) credentials = session.get_credentials() if credentials is None: - raise RuntimeError(f"No credentials resolved for role {role_arn}") + raise CredentialError("aws", f"No credentials resolved for role {role_arn}") frozen = credentials.get_frozen_credentials() return None, _aws_key_env(frozen.access_key, frozen.secret_key, frozen.token) if access_key and secret_key: @@ -353,14 +356,12 @@ def build_azure_credential(cfg: Any) -> Any: A TokenCredential usable with all Azure mgmt SDK clients. Raises: - ImportError: If azure-identity is not installed. + ProviderSDKMissingError: If azure-identity is not installed (an ImportError). """ try: from azure import identity except ImportError as exc: - raise ImportError( - "azure-identity is required for Azure. Install with: pip install cloudg[azure]" - ) from exc + raise ProviderSDKMissingError("azure", missing="azure-identity") from exc explicit = _azure_service_principal(identity, cfg) if explicit is not None: @@ -443,14 +444,12 @@ def build_gcp_credentials(cfg: Any) -> tuple[Any, str | None]: auth method carries no project (e.g. impersonation). Raises: - ImportError: If google-auth is not installed. + ProviderSDKMissingError: If google-auth is not installed (an ImportError). """ try: import google.auth except ImportError as exc: - raise ImportError( - "google-auth is required for GCP. Install with: pip install cloudg[gcp]" - ) from exc + raise ProviderSDKMissingError("gcp", missing="google-auth") from exc credentials_file = getattr(cfg, "credentials_file", None) or os.environ.get( "GOOGLE_APPLICATION_CREDENTIALS" diff --git a/cloudg/exceptions.py b/cloudg/exceptions.py new file mode 100644 index 0000000..2ce20cb --- /dev/null +++ b/cloudg/exceptions.py @@ -0,0 +1,212 @@ +"""Every exception cloudg raises on purpose, in one importable place. + +All of them derive from :class:`CloudGError`, so one ``except CloudGError`` +catches anything cloudg itself raised, while SDK and OS errors keep passing +through untouched. The narrower classes also keep the built-in base they +always had (``ValueError``, ``RuntimeError``, ``ImportError``), so existing +``except ValueError`` code goes on working. + +:: + + CloudGError + ├── ConfigurationError (ValueError) + ├── ProviderSDKMissingError (ImportError) + ├── CollectionError (RuntimeError) + │ ├── CredentialError + │ └── OrganizationScopeError (ValueError) + ├── ScannerError + │ ├── ScannerNotInstalledError + │ ├── ScannerTimeoutError + │ ├── ScannerExecutionError + │ ├── ScannerOutputError + │ │ └── ReportParseError + │ └── ScannerFailed (RuntimeError) + ├── ResilienceError (cloudg.resilience) + └── LiveOperationRejected (cloudg.resilience) + +Scanner wrappers record failures instead of raising, so one broken scanner +never stops a pipeline. To get exceptions instead, pass ``strict=True`` to +the wrapper, or call :meth:`raise_for_errors` after a run:: + + from cloudg.exceptions import ScannerError, ScannerTimeoutError + from cloudg.scanners.checkov import CheckovScanner + + scanner = CheckovScanner(target_dir="infra/", timeout_seconds=600) + findings = scanner.run() + try: + scanner.raise_for_errors() + except ScannerTimeoutError as exc: + print(f"{exc.scanner} gave up after {exc.timeout_seconds}s") + except ScannerError as exc: + print(f"{exc.scanner} failed: {exc.detail}") + +This module imports nothing from the rest of cloudg, so any module can +import it without a cycle. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "CloudGError", + "CollectionError", + "ConfigurationError", + "CredentialError", + "OrganizationScopeError", + "ProviderSDKMissingError", + "ReportParseError", + "ScannerError", + "ScannerExecutionError", + "ScannerFailed", + "ScannerNotInstalledError", + "ScannerOutputError", + "ScannerTimeoutError", +] + + +class CloudGError(Exception): + """Base of every exception cloudg raises on purpose.""" + + +class ConfigurationError(CloudGError, ValueError): + """A configuration value is missing, malformed or contradicts another.""" + + +class ProviderSDKMissingError(CloudGError, ImportError): + """A provider's SDK is not installed. + + ``provider`` is ``"aws"``, ``"azure"`` or ``"gcp"`` and ``extra`` the pip + extra that installs it, so the message can say ``pip install cloudg[gcp]``. + """ + + def __init__(self, provider: str, extra: str | None = None, missing: str | None = None) -> None: + self.provider = provider + self.extra = extra or provider + self.missing = missing + what = f" ({missing})" if missing else "" + super().__init__( + f"the {provider} SDK is not installed{what}; run: pip install 'cloudg[{self.extra}]'" + ) + + +class CollectionError(CloudGError, RuntimeError): + """Collecting inventory from a cloud provider failed.""" + + +class CredentialError(CollectionError): + """No usable credentials were found for a provider, or they were refused.""" + + def __init__(self, provider: str, detail: str) -> None: + self.provider = provider + self.detail = detail + super().__init__(f"{provider}: {detail}") + + +class OrganizationScopeError(CollectionError, ValueError): + """The requested organization scope (an OU filter) names nothing in the org. + + Raised instead of quietly mapping only the caller's account. + """ + + +# --------------------------------------------------------------------------- +# Scanners +# --------------------------------------------------------------------------- + + +class ScannerError(CloudGError): + """A security scanner could not produce a complete result. + + ``scanner`` is the wrapper's short name (``"prowler"``, ``"checkov"``, + ...), ``detail`` the reason in a few words and ``target`` what was being + scanned (an image, a directory, a report file), when there is one. + """ + + def __init__(self, scanner: str, detail: str, *, target: str | None = None) -> None: + self.scanner = scanner + self.detail = detail + self.target = target + super().__init__(f"{scanner}: {detail}") + + +class ScannerNotInstalledError(ScannerError): + """The scanner's executable is not on ``PATH``.""" + + def __init__(self, scanner: str, executable: str, install_hint: str = "") -> None: + self.executable = executable + self.install_hint = install_hint + detail = f"{executable!r} is not on PATH" + if install_hint: + detail += f"; install with: {install_hint}" + super().__init__(scanner, detail) + + +class ScannerTimeoutError(ScannerError): + """The scanner ran longer than its ``timeout_seconds`` and was killed.""" + + def __init__(self, scanner: str, timeout_seconds: float, *, target: str | None = None) -> None: + self.timeout_seconds = timeout_seconds + detail = f"timed out after {timeout_seconds:g}s" + if target: + detail += f" scanning {target}" + super().__init__(scanner, detail, target=target) + + +class ScannerExecutionError(ScannerError): + """The scanner process could not be started, or crashed. + + ``returncode`` is the exit status when the process ran; ``output`` the + tail of what it printed, when that was captured. + """ + + def __init__( + self, + scanner: str, + detail: str, + *, + target: str | None = None, + returncode: int | None = None, + output: str = "", + ) -> None: + self.returncode = returncode + self.output = output + super().__init__(scanner, detail, target=target) + + +class ScannerOutputError(ScannerError): + """The scanner ran, but its output could not be read or parsed.""" + + def __init__( + self, scanner: str, detail: str, *, target: str | None = None, path: str | None = None + ) -> None: + self.path = path + super().__init__(scanner, detail, target=target or path) + + +class ReportParseError(ScannerOutputError): + """An existing report handed to ``parse_report`` or ``cloudg ingest`` is unreadable.""" + + +class ScannerFailed(ScannerError, RuntimeError): + """A scanner started but did not finish (timeout, launch failure). + + ``findings`` holds whatever it produced before that (for example the + images Trivy did scan), so callers can keep partial results. ``errors`` + are the recorded reasons as text and ``failures`` the matching + :class:`ScannerError` objects, when the wrapper recorded them. + """ + + def __init__( + self, + scanner: str, + errors: list[str], + findings: list[Any], + failures: list[ScannerError] | None = None, + ) -> None: + self.errors = list(errors) + self.findings = findings + self.failures = list(failures or []) + ScannerError.__init__(self, scanner, "; ".join(self.errors)) + # Keep the message format callers already match on: " " + self.args = (f"{scanner} " + "; ".join(self.errors),) diff --git a/cloudg/ingest.py b/cloudg/ingest.py index b61b30a..3f00a77 100644 --- a/cloudg/ingest.py +++ b/cloudg/ingest.py @@ -20,6 +20,7 @@ import logging from pathlib import Path +from cloudg.exceptions import ConfigurationError from cloudg.normaliser import FindingList from cloudg.schema.models import Finding @@ -70,7 +71,7 @@ def parse_report(tool: str, path: str | Path) -> list[Finding]: return TrivyScanner.parse_report(str(p)) - raise ValueError(f"Unsupported tool '{tool}'. Supported: {', '.join(SUPPORTED_TOOLS)}") + raise ConfigurationError(f"Unsupported tool '{tool}'. Supported: {', '.join(SUPPORTED_TOOLS)}") def ingest_reports(reports: dict[str, list[str | Path]]) -> list[Finding]: diff --git a/cloudg/inventory/azure_deep.py b/cloudg/inventory/azure_deep.py index 604abd1..1aed212 100644 --- a/cloudg/inventory/azure_deep.py +++ b/cloudg/inventory/azure_deep.py @@ -13,15 +13,34 @@ encryption sets), resource groups, and the ARM ``resources.list()`` catch-all sweep for everything else. Both paths feed the same row pipeline, so assets carry the same metadata and relations. + +Depth beyond the breadth query (both paths, all best-effort, one coverage +record per task in :attr:`AzureDeepInventoryCollector.coverage`): + +- more Resource Graph tables (policy, policy states, Defender assessments, + alerts and secure score, backup items, Kubernetes configuration, + Virtual Network Manager, guest configuration, maintenance, data + collection rule associations), management group role assignments and + role definitions (Resource Graph path only), +- child resources and subscription-level listings read through ARM REST + (:mod:`cloudg.inventory.azure_graph.children`), +- AI Search and Synapse data-plane metadata, +- in-cluster AKS mapping (:mod:`cloudg.inventory.azure_kubernetes`). + +``services`` / ``exclude_services`` select tasks by task or family name, +like the AWS deep collector (:mod:`cloudg.inventory.azure_graph.families`). """ from __future__ import annotations import asyncio import logging +import time +from dataclasses import dataclass from typing import Any, Callable from cloudg.collectors.azure import AzureCollector, subscription_client +from cloudg.coverage import CollectionCoverage from cloudg.inventory.azure_graph import ( _ARM_TYPE_MAP, AzureAssetBuilder, @@ -30,11 +49,67 @@ default_graph_client_factory, to_rest, ) +from cloudg.inventory.azure_graph.children import CHILD_SPECS, fetch_children +from cloudg.inventory.azure_graph.dataplane import fetch_data_planes +from cloudg.inventory.azure_graph.families import ( + AZURE_TASK_FAMILIES, + TaskRecorder, + row_filter, + select_azure_tasks, +) +from cloudg.inventory.azure_graph.rest import ( + default_arm_client_factory, + default_dataplane_client_factory, +) from cloudg.schema.models import CloudAsset logger = logging.getLogger(__name__) -__all__ = ["AzureDeepInventoryCollector", "asset_type_from_arm", "_ARM_TYPE_MAP"] +__all__ = [ + "AzureDeepInventoryCollector", + "AzureDeepOptions", + "asset_type_from_arm", + "_ARM_TYPE_MAP", +] + +# Tasks that need Resource Graph (skipped on the SDK fallback path) +_GRAPH_TASKS = ( + "role_assignments", + "management_group_role_assignments", + "role_definitions", + "defender_pricings", + "policy", + "policy_states", + "backup_items", + "kubernetes_configuration", + "network_resources", + "guest_configuration", + "maintenance", + "insights", + "security_assessments", + "secure_score", + "security_alerts", +) +_REST_TASKS = tuple( + dict.fromkeys([s.task for s in CHILD_SPECS] + ["sentinel", "diagnostic_settings"]) +) +_PLANE_TASKS = ("search_data_plane", "synapse_data_plane") + + +@dataclass +class AzureDeepOptions: + """Depth options of :class:`AzureDeepInventoryCollector` (all on by default).""" + + services: list[str] | None = None + exclude_services: list[str] | None = None + child_resources: bool = True + max_child_parents: int = 500 + diagnostic_settings: bool = True + max_diagnostic_settings: int = 1000 + data_plane: bool = True + kubernetes: bool = True + kubernetes_timeout: int = 10 + rest_concurrency: int = 8 class AzureDeepInventoryCollector(AzureCollector): @@ -47,12 +122,23 @@ class AzureDeepInventoryCollector(AzureCollector): with ``resources(QueryRequest)`` (injectable for tests). Defaults to ``azure.mgmt.resourcegraph.ResourceGraphClient``. use_resource_graph: Set False to force the per-service SDK path. + arm_client_factory: ``credential -> client`` with ``list(path, + api_version, **params)``, ``get(...)`` and ``post(...)`` for ARM + REST (injectable for tests). Defaults to + :func:`~cloudg.inventory.azure_graph.rest.default_arm_client_factory`. + dataplane_client_factory: ``(credential, base_url, scope) -> client`` + for the AI Search and Synapse data planes. + options: :class:`AzureDeepOptions` (or keyword arguments of it). Attributes: collection_mode: ``"resource-graph"`` or ``"sdk"`` after :meth:`collect`. + coverage: Per-task coverage records of the last :meth:`collect`. """ graph_client_factory: Callable[[Any], Any] | None = None + arm_client_factory: Callable[[Any], Any] | None = None + dataplane_client_factory: Callable[[Any, str, str], Any] | None = None + aks_reader_factory: Callable[..., Any] | None = None def __init__( self, @@ -60,12 +146,29 @@ def __init__( subscription_id: str, graph_client_factory: Callable[[Any], Any] | None = None, use_resource_graph: bool = True, + *, + arm_client_factory: Callable[[Any], Any] | None = None, + dataplane_client_factory: Callable[[Any, str, str], Any] | None = None, + options: AzureDeepOptions | None = None, + **option_kwargs: Any, ) -> None: super().__init__(credential, subscription_id) if graph_client_factory is not None: self.graph_client_factory = graph_client_factory + if arm_client_factory is not None: + self.arm_client_factory = arm_client_factory + if dataplane_client_factory is not None: + self.dataplane_client_factory = dataplane_client_factory self._use_resource_graph = use_resource_graph + self.options = options or AzureDeepOptions(**option_kwargs) self.collection_mode: str | None = None + self.coverage = CollectionCoverage(provider="azure", account_id=subscription_id) + self._tasks = self._selected_tasks() + + def _selected_tasks(self) -> set[str]: + opts = self.options + names = [t for t in AZURE_TASK_FAMILIES if t != "entra"] + return set(select_azure_tasks(names, opts.services, opts.exclude_services)) # ------------------------------------------------------------------ # Resource Graph path @@ -74,12 +177,112 @@ def __init__( def _graph_builder(self) -> AzureAssetBuilder: factory = self.graph_client_factory or default_graph_client_factory client = factory(self._credential) # ImportError when the SDK is absent - return collect_subscription_graph(client, self._subscription_id, errors=self.service_errors) + return collect_subscription_graph( + client, + self._subscription_id, + errors=self.service_errors, + tasks=self._tasks, + recorder=TaskRecorder(self.coverage), + keep_row=row_filter(self.options.services, self.options.exclude_services), + ) + + def _graph_and_depth(self) -> AzureAssetBuilder: + builder = self._graph_builder() + self._safe_depth(builder) + return builder async def _collect_via_graph(self) -> list[CloudAsset]: - builder = await asyncio.to_thread(self._graph_builder) + builder = await asyncio.to_thread(self._graph_and_depth) return builder.build() + # ------------------------------------------------------------------ + # Depth: ARM REST children, data planes, in-cluster AKS + # ------------------------------------------------------------------ + + def _depth(self, builder: AzureAssetBuilder) -> None: + """Every selected REST / data-plane / Kubernetes task (best-effort).""" + rec = TaskRecorder(self.coverage) + opts = self.options + selected = self._tasks + for task in (*_REST_TASKS, *_PLANE_TASKS, "aks_kubernetes"): + if task not in selected: + rec.skip(task, "not selected (inventory.services)") + rest_tasks = {t for t in _REST_TASKS if t in selected} + if not opts.child_resources: + for task in sorted(rest_tasks): + rec.skip(task, "disabled (azure.child_resources)") + rest_tasks = set() + if not opts.diagnostic_settings and "diagnostic_settings" in rest_tasks: + rec.skip("diagnostic_settings", "disabled (azure.diagnostic_settings)") + arm = None + if rest_tasks or "aks_kubernetes" in selected: + try: + arm = (self.arm_client_factory or default_arm_client_factory)(self._credential) + except Exception as exc: + for task in sorted(rest_tasks): + rec.record(task, "FAILED", error=f"ARM REST client: {exc}"[:300]) + rest_tasks = set() + if arm is not None and rest_tasks: + fetch_children( + arm, + builder, + rest_tasks, + rec, + max_parents=opts.max_child_parents, + concurrency=opts.rest_concurrency, + diagnostic_settings=opts.diagnostic_settings, + max_diagnostic_settings=opts.max_diagnostic_settings, + ) + plane_tasks = {t for t in _PLANE_TASKS if t in selected} + if plane_tasks and not opts.data_plane: + for task in sorted(plane_tasks): + rec.skip(task, "disabled (azure.data_plane)") + elif plane_tasks: + fetch_data_planes( + self._credential, + self.dataplane_client_factory or default_dataplane_client_factory, + builder, + plane_tasks, + rec, + max_parents=opts.max_child_parents, + ) + if "aks_kubernetes" in selected: + self._aks_workloads(builder, arm, rec) + + def _aks_workloads(self, builder: AzureAssetBuilder, arm: Any, rec: TaskRecorder) -> None: + """In-cluster objects of every Entra-integrated AKS cluster.""" + from cloudg.inventory.azure_kubernetes import aks_mappable, collect_aks_workloads + + clusters = builder.drafts_of("microsoft.containerservice/managedclusters") + if not self.options.kubernetes: + rec.skip("aks_kubernetes", "disabled (inventory.kubernetes)") + return + if arm is None: + rec.skip("aks_kubernetes", "no ARM REST client") + return + for d in clusters: + task = f"kubernetes:{d.name}" + reason = aks_mappable(d.row) + if reason: + rec.skip(task, reason) + continue + started = time.time() + try: + found = collect_aks_workloads( + self._credential, + arm, + {**d.row, "properties": d.props}, + timeout=self.options.kubernetes_timeout, + reader_factory=self.aks_reader_factory, + ) + except Exception as exc: + logger.warning("Kubernetes API mapping skipped for %s: %s", d.name, exc) + rec.record(task, "FAILED", error=str(exc)[:300], started=started) + continue + for asset in found: + builder.add_extra(asset) + rec.record(task, "SUCCESS", len(found), started=started) + # ------------------------------------------------------------------ # SDK fallback: network fabric, containers and the ARM sweep # ------------------------------------------------------------------ @@ -189,13 +392,29 @@ async def _collect_via_sdk(self) -> list[CloudAsset]: self._run_service("resource_groups", self._fetch_resource_groups), self._run_service("subscription", self._fetch_subscription), ) + keep = row_filter(self.options.services, self.options.exclude_services) + if keep is not None: + detailed = [r for r in detailed if keep(r)] + sweep = [r for r in sweep if keep(r)] builder = AzureAssetBuilder(self._subscription_id, "sdk") builder.add_rows(detailed) # detailed rows first: they win over sweep hits builder.add_rows(sweep, discovered_via="arm-sweep") builder.add_containers(groups) builder.include_subscription(subscription[0] if subscription else None) + rec = TaskRecorder(self.coverage) + rec.record("resources", "SUCCESS", len(detailed) + len(sweep), "per-service SDK fallback") + for task in _GRAPH_TASKS: + rec.skip(task, "Resource Graph unavailable (SDK fallback)") + await asyncio.to_thread(self._safe_depth, builder) return builder.build() + def _safe_depth(self, builder: AzureAssetBuilder) -> None: + try: + self._depth(builder) + except Exception as exc: # depth never costs the breadth inventory + logger.warning("Azure depth collection failed for %s: %s", self._subscription_id, exc) + TaskRecorder(self.coverage).record("depth", "FAILED", error=str(exc)[:300]) + # ------------------------------------------------------------------ # Main interface # ------------------------------------------------------------------ @@ -204,6 +423,7 @@ async def collect(self) -> list[CloudAsset]: """Inventory the subscription via Resource Graph, else via the SDKs.""" logger.info("Starting deep Azure inventory for subscription %s", self._subscription_id) self.service_errors = {} + self.coverage = CollectionCoverage(provider="azure", account_id=self._subscription_id) assets: list[CloudAsset] | None = None if self._use_resource_graph: try: @@ -221,6 +441,7 @@ async def collect(self) -> list[CloudAsset]: ) self.service_errors["resource_graph"] = str(exc) if assets is None: + self.coverage = CollectionCoverage(provider="azure", account_id=self._subscription_id) assets = await self._collect_via_sdk() self.collection_mode = "sdk" diff --git a/cloudg/inventory/azure_entra.py b/cloudg/inventory/azure_entra.py new file mode 100644 index 0000000..5831363 --- /dev/null +++ b/cloudg/inventory/azure_entra.py @@ -0,0 +1,721 @@ +"""Microsoft Entra ID through Microsoft Graph: the identity layer of an Azure map. + +Read once per tenant (not per subscription) with the azure-identity +credential cloudg already holds (``get_token("https://graph.microsoft.com/.default")``), +using only ``GET`` on Microsoft Graph v1.0. Every listing is best-effort: +a listing the credential may not read (missing Graph application +permissions such as ``Directory.Read.All``, ``Application.Read.All``, +``RoleManagement.Read.Directory``, ``RoleEligibilitySchedule.Read.Directory``) +becomes a FAILED coverage record and the rest is still mapped. + +Mapped objects and relations: + +- tenant (``ORGANIZATION``, ``entra:tenant/``), +- users (``IDENTITY_USER``) and groups (``IDENTITY_GROUP``) with direct + members (group ``CONTAINS`` member), +- app registrations (``APP_REGISTRATION``): credential counts and expiry + (never values), owners (``MANAGES / OWNED_BY``), federated identity + credentials (``IAM_TRUST`` from the issuer: ``CROSS_ACCOUNT_TRUST`` for + external issuers), the API permissions they request; the app ``MANAGES / + OWNED_BY`` its service principal, +- service principals (``SERVICE_PRINCIPAL``; managed identities are left + to the resources that own them), owners, app-role assignments + (principal ``GRANTS_ACCESS`` the resource service principal, with the + role value, e.g. Microsoft Graph ``RoleManagement.ReadWrite.Directory``, + and ``privileged``), delegated OAuth2 permission grants, +- directory roles (``IAM_POLICY``): active assignments and PIM eligible + assignments (``IAM_POLICY_ATTACHMENT`` with ``USER_HAS_POLICY`` / + ``GROUP_HAS_POLICY`` / ``ROLE_HAS_POLICY``, plus ``GRANTS_ACCESS`` the + tenant with ``eligible`` set for PIM). + +Principals use the identifier the subscription collectors use for role +assignment placeholders (``entra:principal/``), so the two merge +and every Azure RBAC grant lands on the real Entra object. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Iterable + +from cloudg.coverage import CollectionCoverage +from cloudg.exceptions import CredentialError +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.families import TaskRecorder +from cloudg.inventory.azure_graph.helpers import issuer_kind, principal_ref +from cloudg.inventory.azure_graph.rest import AzureRestError, default_msgraph_client_factory +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType + +logger = logging.getLogger(__name__) + +_V1 = "/v1.0" +MICROSOFT_GRAPH_APP_ID = "00000003-0000-0000-c000-000000000000" + +# Directory role template IDs that hand over the tenant (or most of it) +PRIVILEGED_DIRECTORY_ROLES = { + "62e90394-69f5-4237-9190-012177145e10", # Global Administrator + "e8611ab8-c189-46e8-94e1-60213ab1f814", # Privileged Role Administrator + "7be44c8a-adaf-4e2a-84d6-ab2649e08a13", # Privileged Authentication Administrator + "9b895d92-2cd3-44c7-9d02-a6ac2d5ea5c3", # Application Administrator + "158c047a-c907-4556-b7ef-446551a6b5f7", # Cloud Application Administrator + "fe930be7-5e62-47db-91af-98c3a49a38b1", # User Administrator + "29232cdf-9323-42fd-ade2-1d097af3e4de", # Exchange Administrator + "194ae4cb-b126-40b2-bd5b-6091b380977d", # Security Administrator + "b1be1c3e-b65d-4f19-8427-f6fa0d97feb9", # Conditional Access Administrator + "c4e39bd9-1100-46d3-8c65-fb160da0071f", # Authentication Administrator + "729827e3-9c14-49f7-bb1b-9608f156bbb8", # Helpdesk Administrator + "966707d0-3269-4727-9be2-8c3a10f19b9d", # Password Administrator + "7698a772-787b-4ac8-901f-60d6b08affd2", # Cloud Device Administrator + "8ac3fc64-6eca-42ea-9e69-59f4c7b60eb2", # Hybrid Identity Administrator +} + +# Microsoft Graph application permissions that amount to tenant takeover +PRIVILEGED_GRAPH_ROLES = { + "roledefinition.readwrite.directory", + "rolemanagement.readwrite.directory", + "approleassignment.readwrite.all", + "application.readwrite.all", + "directory.readwrite.all", + "user.readwrite.all", + "group.readwrite.all", + "groupmember.readwrite.all", + "serviceprincipalendpoint.readwrite.all", + "policy.readwrite.conditionalaccess", + "privilegedaccess.readwrite.azuread", + "user.manageidentities.all", +} + +_PRIVILEGED_SCOPES = { + "directory.readwrite.all", + "rolemanagement.readwrite.directory", + "application.readwrite.all", + "approleassignment.readwrite.all", + "user.readwrite.all", + "mail.readwrite", + "files.readwrite.all", + "sites.fullcontrol.all", + "full_access_as_app", +} + + +def _asset( + arn: str, + name: str, + asset_type: AssetType, + tenant: str | None, + metadata: dict[str, Any], + relations: Iterable[dict[str, Any] | None] = (), + aliases: Iterable[str | None] = (), +) -> CloudAsset: + md = {k: v for k, v in metadata.items() if v not in (None, "", [], {})} + md["collected_via"] = "microsoft-graph" + md["tenant_id"] = tenant + rels = [r for r in relations if r] + if rels: + md["relations"] = rels + alias_list = list(dict.fromkeys(a for a in aliases if a and a != arn)) + if alias_list: + md["aliases"] = alias_list + return CloudAsset( + arn=arn, + name=name or arn, + asset_type=asset_type, + provider=CloudProvider.AZURE, + region="global", + account_id=None, + metadata=md, + ) + + +def _credential_summary(creds: list[dict[str, Any]]) -> dict[str, Any]: + ends = sorted(str(c.get("endDateTime") or "") for c in creds if c.get("endDateTime")) + return {"count": len(creds), "latest_expiry": ends[-1] if ends else None} + + +class EntraCollector: + """Maps one Entra tenant through Microsoft Graph. + + Args: + client: Microsoft Graph REST client (``list(path, **params)``). + recorder: Task coverage recorder. + max_objects: Most objects read per listing. + max_expansions: Most objects whose members / owners / credentials / + assignments are read one by one. + """ + + def __init__( + self, + client: Any, + recorder: TaskRecorder, + *, + max_objects: int = 10000, + max_expansions: int = 1000, + ) -> None: + self.client = client + self.rec = recorder + self.max_objects = max_objects + self.max_expansions = max_expansions + self.tenant: str | None = None + self.assets: dict[str, CloudAsset] = {} + self.credential_error: str | None = None + self._sp_by_app: dict[str, str] = {} + self._app_roles: dict[str, dict[str, str]] = {} + self._sp_names: dict[str, str] = {} + + # ── plumbing ── + + def _list(self, path: str, **params: Any) -> list[dict[str, Any]]: + return self.client.list(f"{_V1}{path}", max_items=self.max_objects, **params) + + def _task(self, name: str, fn: Callable[[], int]) -> None: + if self.credential_error: + self.rec.skip(name, self.credential_error[:300]) + return + started = time.time() + try: + count = fn() + except CredentialError as exc: + self.credential_error = str(exc) + self.rec.skip(name, self.credential_error[:300]) + return + except AzureRestError as exc: + reason = "Graph permission missing: " if exc.forbidden else "" + self.rec.record(name, "FAILED", error=f"{reason}{exc}"[:300], started=started) + return + except Exception as exc: + self.rec.record( + name, "FAILED", error=f"{type(exc).__name__}: {exc}"[:300], started=started + ) + return + self.rec.record(name, "SUCCESS", count, started=started) + + def _expand(self, name: str, ids: list[str], fn: Callable[[str], None]) -> None: + """Per-object follow-up calls; some may fail (PARTIAL).""" + if self.credential_error: + self.rec.skip(name, self.credential_error[:300]) + return + started = time.time() + failures: list[str] = [] + done = 0 + for oid in ids[: self.max_expansions]: + try: + fn(oid) + done += 1 + except CredentialError as exc: + self.credential_error = str(exc) + failures.append(str(exc)) + break + except Exception as exc: + failures.append(str(exc)[:200]) + self.rec.outcome(name, done, failures, min(len(ids), self.max_expansions), started) + + def _add(self, asset: CloudAsset) -> CloudAsset: + existing = self.assets.get(asset.arn) + if existing is None: + self.assets[asset.arn] = asset + return asset + existing.metadata.setdefault("relations", []).extend(asset.metadata.get("relations", [])) + return existing + + def _relate(self, arn: str, *relations: dict[str, Any] | None) -> None: + asset = self.assets.get(arn) + if asset is not None: + asset.metadata.setdefault("relations", []).extend(r for r in relations if r) + + def _tenant_ref(self) -> str | None: + return f"entra:tenant/{self.tenant}" if self.tenant else None + + # ── listings ── + + def organization(self) -> int: + orgs = self._list( + "/organization", **{"$select": "id,displayName,tenantType,verifiedDomains"} + ) + for org in orgs: + self.tenant = org.get("id") + domains = [d.get("name") for d in org.get("verifiedDomains") or [] if d.get("name")] + self._add( + _asset( + f"entra:tenant/{self.tenant}", + org.get("displayName") or f"tenant {self.tenant}", + AssetType.ORGANIZATION, + self.tenant, + {"tenant_type": org.get("tenantType"), "verified_domains": domains[:50]}, + ) + ) + return len(orgs) + + def users(self) -> int: + users = self._list( + "/users", + **{ + "$select": "id,displayName,userPrincipalName,userType,accountEnabled," + "createdDateTime,externalUserState,onPremisesSyncEnabled", + "$top": "999", + }, + ) + for u in users: + oid = str(u.get("id") or "").lower() + if not oid: + continue + self._add( + _asset( + principal_ref(oid) or oid, + u.get("displayName") or u.get("userPrincipalName") or oid, + AssetType.IDENTITY_USER, + self.tenant, + { + "principal_id": oid, + "principal_type": "User", + "user_principal_name": u.get("userPrincipalName"), + "user_type": u.get("userType"), + "guest": str(u.get("userType") or "").lower() == "guest", + "account_enabled": u.get("accountEnabled"), + "synced_from_on_premises": u.get("onPremisesSyncEnabled"), + "external_state": u.get("externalUserState"), + }, + aliases=[ + f"entra:upn/{str(u.get('userPrincipalName') or '').lower()}" + if u.get("userPrincipalName") + else None + ], + ) + ) + return len(users) + + def groups(self) -> int: + groups = self._list( + "/groups", + **{ + "$select": "id,displayName,securityEnabled,mailEnabled,groupTypes," + "isAssignableToRole,membershipRule,onPremisesSyncEnabled", + "$top": "999", + }, + ) + for g in groups: + oid = str(g.get("id") or "").lower() + if not oid: + continue + self._add( + _asset( + principal_ref(oid) or oid, + g.get("displayName") or oid, + AssetType.IDENTITY_GROUP, + self.tenant, + { + "principal_id": oid, + "principal_type": "Group", + "security_enabled": g.get("securityEnabled"), + "role_assignable": g.get("isAssignableToRole"), + "dynamic": "DynamicMembership" in (g.get("groupTypes") or []), + "membership_rule": g.get("membershipRule"), + "synced_from_on_premises": g.get("onPremisesSyncEnabled"), + }, + ) + ) + return len(groups) + + def group_members(self, gid: str) -> None: + members = self._list(f"/groups/{gid}/members", **{"$select": "id", "$top": "999"}) + self._relate( + principal_ref(gid) or gid, + *[ + rel(principal_ref(m.get("id")), EdgeType.CONTAINS, description="group member") + for m in members + ], + ) + + def applications(self) -> int: + apps = self._list( + "/applications", + **{ + "$select": "id,appId,displayName,signInAudience,requiredResourceAccess," + "passwordCredentials,keyCredentials,publisherDomain,createdDateTime,web", + "$top": "999", + }, + ) + for app in apps: + oid = str(app.get("id") or "").lower() + app_id = str(app.get("appId") or "").lower() + if not oid: + continue + requested = [ + { + "resource_app_id": r.get("resourceAppId"), + "count": len(r.get("resourceAccess") or []), + } + for r in app.get("requiredResourceAccess") or [] + ] + redirect = ((app.get("web") or {}).get("redirectUris") or [])[:20] + self._add( + _asset( + f"entra:application/{oid}", + app.get("displayName") or app_id, + AssetType.APP_REGISTRATION, + self.tenant, + { + "object_id": oid, + "app_id": app_id, + "sign_in_audience": app.get("signInAudience"), + "multi_tenant": "multiple" in str(app.get("signInAudience") or "").lower() + or "personal" in str(app.get("signInAudience") or "").lower(), + "secrets": _credential_summary(app.get("passwordCredentials") or []), + "certificates": _credential_summary(app.get("keyCredentials") or []), + "requested_permissions": requested, + "redirect_uris": redirect, + "publisher_domain": app.get("publisherDomain"), + }, + aliases=[f"entra:app/{app_id}" if app_id else None], + ) + ) + return len(apps) + + def application_owners(self, oid: str) -> None: + owners = self._list(f"/applications/{oid}/owners", **{"$select": "id"}) + self._relate( + f"entra:application/{oid}", + *[ + rel( + principal_ref(o.get("id")), + EdgeType.MANAGES, + "OWNED_BY", + reverse=True, + description="application owner", + ) + for o in owners + ], + ) + + def federated_credentials(self, oid: str) -> None: + creds = self._list(f"/applications/{oid}/federatedIdentityCredentials") + relations = [] + for c in creds: + issuer = str(c.get("issuer") or "").rstrip("/") + aks = issuer_kind(issuer) == "aks" + relations.append( + rel( + issuer, + EdgeType.IAM_TRUST, + "ROLE_ASSUMES_ROLE" if aks else "CROSS_ACCOUNT_TRUST", + reverse=True, + description=f"federated credential {c.get('name')}", + subject=c.get("subject"), + audiences=c.get("audiences"), + ) + ) + asset = self.assets.get(f"entra:application/{oid}") + if asset is not None and creds: + asset.metadata["federated_credentials"] = [ + {"name": c.get("name"), "issuer": c.get("issuer"), "subject": c.get("subject")} + for c in creds + ] + self._relate(f"entra:application/{oid}", *relations) + + def service_principals(self) -> int: + sps = self._list( + "/servicePrincipals", + **{ + "$select": "id,appId,displayName,servicePrincipalType,appOwnerOrganizationId," + "accountEnabled,appRoleAssignmentRequired,appRoles,alternativeNames,publisherName", + "$top": "999", + }, + ) + count = 0 + for sp in sps: + oid = str(sp.get("id") or "").lower() + app_id = str(sp.get("appId") or "").lower() + if not oid: + continue + self._sp_by_app[app_id] = oid + self._sp_names[oid] = sp.get("displayName") or app_id + self._app_roles[oid] = { + str(r.get("id")).lower(): str(r.get("value") or r.get("displayName") or "") + for r in sp.get("appRoles") or [] + } + if str(sp.get("servicePrincipalType") or "").lower() == "managedidentity": + continue # represented by the resource / user-assigned identity that owns it + owner_org = str(sp.get("appOwnerOrganizationId") or "").lower() + external = bool(owner_org and self.tenant and owner_org != str(self.tenant).lower()) + count += 1 + self._add( + _asset( + principal_ref(oid) or oid, + sp.get("displayName") or app_id, + AssetType.SERVICE_PRINCIPAL, + self.tenant, + { + "principal_id": oid, + "principal_type": "ServicePrincipal", + "app_id": app_id, + "service_principal_type": sp.get("servicePrincipalType"), + "app_owner_tenant_id": owner_org or None, + "external_application": external, + "microsoft_first_party": owner_org + == "f8cdef31-a31e-4b4a-93e4-5f571e91255a", + "enabled": sp.get("accountEnabled"), + "assignment_required": sp.get("appRoleAssignmentRequired"), + "publisher": sp.get("publisherName"), + }, + aliases=[f"azure-client-id:{app_id}" if app_id else None], + ) + ) + for asset in list(self.assets.values()): + if asset.asset_type != AssetType.APP_REGISTRATION: + continue + sp = self._sp_by_app.get(str(asset.metadata.get("app_id") or "")) + if sp: + asset.metadata.setdefault("relations", []).append( + rel( + principal_ref(sp), + EdgeType.MANAGES, + "OWNED_BY", + description="service principal of this app", + ) + ) + return count + + def sp_owners(self, oid: str) -> None: + owners = self._list(f"/servicePrincipals/{oid}/owners", **{"$select": "id"}) + self._relate( + principal_ref(oid) or oid, + *[ + rel( + principal_ref(o.get("id")), + EdgeType.MANAGES, + "OWNED_BY", + reverse=True, + description="service principal owner", + ) + for o in owners + ], + ) + + def app_role_assignments(self, resource_oid: str) -> None: + assignments = self._list( + f"/servicePrincipals/{resource_oid}/appRoleAssignedTo", **{"$top": "999"} + ) + roles = self._app_roles.get(resource_oid, {}) + resource_name = self._sp_names.get(resource_oid, resource_oid) + relations = [] + for a in assignments: + value = roles.get(str(a.get("appRoleId") or "").lower()) or "default access" + relations.append( + rel( + principal_ref(a.get("principalId")), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description=f"app role {value} on {resource_name}", + role=value, + principal_type=a.get("principalType"), + privileged=value.lower() in PRIVILEGED_GRAPH_ROLES or None, + application_permission=str(a.get("principalType") or "") == "ServicePrincipal" + or None, + ) + ) + self._relate(principal_ref(resource_oid) or resource_oid, *relations) + + def oauth2_grants(self) -> int: + grants = self._list("/oauth2PermissionGrants") + for g in grants: + client = str(g.get("clientId") or "").lower() + scopes = [s for s in str(g.get("scope") or "").split() if s] + all_users = str(g.get("consentType") or "") == "AllPrincipals" + self._relate( + principal_ref(client) or client, + rel( + principal_ref(g.get("resourceId")), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + description=f"delegated {', '.join(scopes[:5])}", + delegated=True, + scopes=scopes[:50], + consent_type=g.get("consentType"), + on_behalf_of=None if all_users else g.get("principalId"), + privileged=any(s.lower() in _PRIVILEGED_SCOPES for s in scopes) + and all_users + or None, + ), + ) + return len(grants) + + def directory_roles(self) -> int: + definitions = self._list( + "/roleManagement/directory/roleDefinitions", + **{"$select": "id,displayName,isBuiltIn,templateId,description,rolePermissions"}, + ) + by_id = {str(d.get("id")).lower(): d for d in definitions} + assignments = self._list( + "/roleManagement/directory/roleAssignments", + **{"$select": "id,principalId,roleDefinitionId,directoryScopeId"}, + ) + self._role_assignments(by_id, assignments, eligible=False) + self._role_defs = by_id + return len(assignments) + + def pim_eligibility(self) -> int: + by_id = getattr(self, "_role_defs", {}) + schedules = self._list( + "/roleManagement/directory/roleEligibilitySchedules", + **{"$select": "id,principalId,roleDefinitionId,directoryScopeId,memberType"}, + ) + self._role_assignments(by_id, schedules, eligible=True) + return len(schedules) + + def _role_asset(self, rid: str, definition: dict[str, Any]) -> str: + arn = f"entra:directoryRole/{rid}" + template = str(definition.get("templateId") or rid).lower() + actions = [ + a + for p in definition.get("rolePermissions") or [] + for a in p.get("allowedResourceActions") or [] + ] + self._add( + _asset( + arn, + definition.get("displayName") or rid, + AssetType.IAM_POLICY, + self.tenant, + { + "directory_role": True, + "role_template_id": template, + "built_in": definition.get("isBuiltIn"), + "privileged": template in PRIVILEGED_DIRECTORY_ROLES, + "description": definition.get("description"), + "actions": actions[:200], + }, + ) + ) + return arn + + def _role_assignments( + self, by_id: dict[str, Any], rows: list[dict[str, Any]], *, eligible: bool + ) -> None: + for row in rows: + rid = str(row.get("roleDefinitionId") or "").lower() + pid = str(row.get("principalId") or "").lower() + if not rid or not pid: + continue + definition = by_id.get(rid) or {"displayName": rid} + role_arn = self._role_asset(rid, definition) + principal = self.assets.get(principal_ref(pid) or "") + kind = principal.asset_type if principal is not None else None + relationship = { + AssetType.IDENTITY_USER: "USER_HAS_POLICY", + AssetType.IDENTITY_GROUP: "GROUP_HAS_POLICY", + }.get(kind, "ROLE_HAS_POLICY") + privileged = ( + str(definition.get("templateId") or rid).lower() in PRIVILEGED_DIRECTORY_ROLES + ) + relations = [ + rel( + role_arn, + EdgeType.IAM_POLICY_ATTACHMENT, + relationship, + description=f"directory role {definition.get('displayName')}", + eligible=eligible or None, + directory_scope=row.get("directoryScopeId"), + ), + rel( + self._tenant_ref(), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + description=f"{definition.get('displayName')} (directory role)", + role=definition.get("displayName"), + privileged=privileged or None, + eligible=eligible or None, + directory_scope=row.get("directoryScopeId"), + ), + ] + if principal is not None: + principal.metadata.setdefault("relations", []).extend(r for r in relations if r) + else: + # principal not listed (managed identity, or listing not permitted) + self._add( + _asset( + principal_ref(pid) or pid, + f"Entra principal {pid}", + AssetType.SERVICE_PRINCIPAL, + self.tenant, + { + "principal_id": pid, + "placeholder": True, + "discovered_via": "entra-principal-reference", + }, + relations, + ) + ) + + # ── run ── + + def collect(self) -> list[CloudAsset]: + self._task("entra_tenant", self.organization) + self._task("entra_users", self.users) + self._task("entra_groups", self.groups) + self._task("entra_applications", self.applications) + self._task("entra_service_principals", self.service_principals) + groups = [ + a.metadata["principal_id"] + for a in self.assets.values() + if a.asset_type == AssetType.IDENTITY_GROUP + ] + apps = [ + a.metadata["object_id"] + for a in self.assets.values() + if a.asset_type == AssetType.APP_REGISTRATION + ] + sps = [ + a.metadata["principal_id"] + for a in self.assets.values() + if a.asset_type == AssetType.SERVICE_PRINCIPAL + ] + self._expand("entra_group_members", groups, self.group_members) + self._expand("entra_application_owners", apps, self.application_owners) + self._expand("entra_federated_credentials", apps, self.federated_credentials) + self._expand( + "entra_service_principal_owners", + [s for s in sps if not self._is_first_party(s)], + self.sp_owners, + ) + resources = [oid for oid, roles in self._app_roles.items() if roles] + resources.sort(key=lambda oid: oid != self._sp_by_app.get(MICROSOFT_GRAPH_APP_ID)) + self._expand("entra_app_role_assignments", resources, self.app_role_assignments) + self._task("entra_oauth2_grants", self.oauth2_grants) + self._task("entra_directory_roles", self.directory_roles) + self._task("entra_pim_eligibility", self.pim_eligibility) + return list(self.assets.values()) + + def _is_first_party(self, oid: str) -> bool: + asset = self.assets.get(principal_ref(oid) or "") + return bool(asset and asset.metadata.get("microsoft_first_party")) + + +def discover_entra( + credential: Any, + client_factory: Callable[[Any], Any] | None = None, + *, + coverage: CollectionCoverage | None = None, + max_objects: int = 10000, + max_expansions: int = 1000, +) -> list[CloudAsset]: + """Map the Entra tenant of ``credential`` (blocking; call via ``asyncio.to_thread``). + + Raises: + CredentialError: No Microsoft Graph token could be obtained at all. + """ + client = (client_factory or default_msgraph_client_factory)(credential) + collector = EntraCollector( + client, + TaskRecorder(coverage), + max_objects=max_objects, + max_expansions=max_expansions, + ) + assets = collector.collect() + if collector.credential_error and not assets: + raise CredentialError("azure", f"Microsoft Graph: {collector.credential_error}") + logger.info("Entra ID: %d objects mapped", len(assets)) + return assets + + +__all__ = ["EntraCollector", "discover_entra"] diff --git a/cloudg/inventory/azure_graph/__init__.py b/cloudg/inventory/azure_graph/__init__.py index 1b701e4..12357ba 100644 --- a/cloudg/inventory/azure_graph/__init__.py +++ b/cloudg/inventory/azure_graph/__init__.py @@ -17,6 +17,10 @@ parent / resource-group containment, private endpoint connections, PaaS exposure) and then the type-specific ones, which live in one module per service area and register themselves on import below. +- Depth beyond ``resources``: more Resource Graph tables (``tables``), + ARM REST child resources and subscription listings (``children``), data + planes (``dataplane``) and the REST client they share (``rest``); task + names, families and ``--services`` filtering live in ``families``. Identifier conventions are documented in :mod:`.helpers`. """ @@ -26,9 +30,15 @@ # Importing the extractor modules registers their extractors. from cloudg.inventory.azure_graph import ( # noqa: F401 compute, + data_ml, data_security, edge, + governance, + hybrid, + integration, network, + ops, + security, web_containers, ) from cloudg.inventory.azure_graph.arm_types import _ARM_TYPE_MAP, asset_type_from_arm diff --git a/cloudg/inventory/azure_graph/builder.py b/cloudg/inventory/azure_graph/builder.py index eadb22b..b0eeb39 100644 --- a/cloudg/inventory/azure_graph/builder.py +++ b/cloudg/inventory/azure_graph/builder.py @@ -3,7 +3,8 @@ from __future__ import annotations import logging -from typing import Any, Iterable +import time +from typing import Any, Callable, Iterable from cloudg.inventory.aws_services._base import rel from cloudg.inventory.azure_graph.arm_types import asset_type_from_arm @@ -29,7 +30,7 @@ role_assignments_query, run_query, ) -from cloudg.inventory.azure_graph.registry import _COMMON, _EXTRACTORS, _Draft +from cloudg.inventory.azure_graph.registry import _COMMON, _EXTRACTORS, _ROW_HANDLERS, _Draft from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType logger = logging.getLogger("cloudg.inventory.azure_graph") @@ -45,21 +46,34 @@ def _role_relation( role_name = ra.get("roleName") or BUILTIN_ROLES.get(guid) or guid or "unknown role" rl = role_name.lower() privileged = rl in PRIVILEGED_ROLES or rl.endswith("administrator") or rl.endswith("data owner") + eligible = bool(ra.get("_eligible")) relation = rel( scope, EdgeType.GRANTS_ACCESS, "POLICY_ALLOWS_ACTION", - description=f"{role_name} on {scope}", + description=f"{role_name} on {scope}" + (" (PIM eligible)" if eligible else ""), role=role_name, role_definition_id=rd, privileged=privileged, principal_type=ptype, assignment_id=ra.get("id"), condition=_get(props, "condition"), + eligible=eligible or None, + inherited_from=ra.get("_inherited_from"), ) return guid, relation +def _pricing_resource(pricing_id: Any) -> str | None: + """The resource a resource-level Defender pricing belongs to (None at subscription level).""" + rid = str(pricing_id or "") + cut = rid.lower().find("/providers/microsoft.security/pricings/") + if cut <= 0: + return None + owner = rid[:cut] + return owner if "/providers/" in owner.lower() else None + + def _unique_relations(relations: list[dict[str, Any]]) -> list[dict[str, Any]]: """Drop repeated (target, edge, direction, relationship) relations, keeping order.""" rels: list[dict[str, Any]] = [] @@ -119,6 +133,11 @@ def __init__(self, subscription_id: str, discovered_via: str = "resource-graph") self._extra: list[CloudAsset] = [] self._subscription_row: dict[str, Any] | None = None self._with_subscription = False + self._finalizers: list[Callable[["AzureAssetBuilder"], None]] = [] + self._role_definitions: dict[str, dict[str, Any]] = {} + self._pricing_overrides: dict[str, bool] = {} + self._pricings_seen = False + self.state: dict[str, Any] = {} # scratch space shared by extractors and handlers # ── registration hooks used by extractors ── @@ -139,6 +158,32 @@ def defer(self, owner_id: str | None, relation: dict[str, Any] | None) -> None: if owner_id and relation: self._deferred.append((owner_id.lower(), relation)) + def at_build(self, fn: Callable[["AzureAssetBuilder"], None]) -> None: + """Run ``fn(builder)`` once at the start of :meth:`build` (cross-row work).""" + if fn not in self._finalizers: + self._finalizers.append(fn) + + def add_extra(self, asset: CloudAsset | None) -> None: + """Add a fully built asset (Defender, Sentinel, Kubernetes objects, ...).""" + if asset is not None: + self._extra.append(asset) + + def draft(self, resource_id: str | None) -> _Draft | None: + return self._drafts.get(resource_id.lower()) if resource_id else None + + def drafts_of(self, *types: str) -> list[_Draft]: + """Drafts of the given lowercase ARM types, in insertion order.""" + wanted = {t.lower() for t in types} + return [d for d in self._drafts.values() if d.type in wanted] + + @property + def drafts(self) -> list[_Draft]: + return list(self._drafts.values()) + + @property + def extras(self) -> list[CloudAsset]: + return list(self._extra) + # ── input ── def add_rows(self, rows: Iterable[dict[str, Any]], discovered_via: str | None = None) -> None: @@ -168,6 +213,13 @@ def add_row( if rtype == "microsoft.resources/subscriptions": self._subscription_row = row return None + handler = _ROW_HANDLERS.get(rtype) + if handler is not None: + try: + handler(row, self) + except Exception: + logger.debug("Azure row handler failed for %s", rid, exc_info=True) + return None key = rid.lower() existing = self._drafts.get(key) has_props = isinstance(row.get("properties"), dict) @@ -253,18 +305,59 @@ def include_subscription(self, row: dict[str, Any] | None = None) -> None: if row: self._subscription_row = row - def add_role_assignments(self, rows: Iterable[dict[str, Any]]) -> None: - self._role_rows.extend(r for r in rows if isinstance(r, dict)) + def add_role_assignments( + self, + rows: Iterable[dict[str, Any]], + *, + eligible: bool = False, + inherited_from: str | None = None, + ) -> None: + """RBAC role assignment rows (ARG ``authorizationresources`` or ARM REST). + + Args: + eligible: The rows are PIM eligibility schedules, not active grants. + inherited_from: Label for assignments made above the subscription + (management group or tenant root). + """ + for r in rows: + if not isinstance(r, dict): + continue + if eligible or inherited_from: + r = dict(r) + r["_eligible"] = eligible + r["_inherited_from"] = inherited_from + self._role_rows.append(r) + + def add_role_definitions(self, rows: Iterable[dict[str, Any]]) -> None: + """Role definition rows; assigned and custom ones become IAM_POLICY assets.""" + for row in rows: + if not isinstance(row, dict): + continue + if _lower(row.get("type")) != "microsoft.authorization/roledefinitions": + continue + m = _GUID_TAIL_RE.search(str(row.get("id") or "")) + guid = str(row.get("roleGuid") or (m.group(1) if m else "") or "").lower() + if guid: + self._role_definitions.setdefault(guid, row) def add_defender_pricings(self, rows: Iterable[dict[str, Any]]) -> None: sub_ref = subscription_ref(self.subscription_id) + self._pricings_seen = True for row in rows: if not isinstance(row, dict): continue + if _lower(row.get("type")) not in ("", "microsoft.security/pricings"): + continue props = row.get("properties") or {} plan = row.get("name") or str(row.get("id", "")).rsplit("/", 1)[-1] if not plan: continue + resource = _pricing_resource(row.get("id")) + if resource: # a resource-level override of a subscription plan + self._pricing_overrides[f"{resource.lower()}|{plan.lower()}"] = ( + _lower(_get(props, "pricingTier")) == "standard" + ) + continue tier = _get(props, "pricingTier") enabled = _lower(tier) == "standard" service = f"defender-{plan.lower()}" @@ -285,6 +378,12 @@ def add_defender_pricings(self, rows: Iterable[dict[str, Any]]) -> None: description=f"Defender for Cloud plan {plan}", ) ] + extensions = [e for e in _get(props, "extensions") or [] if isinstance(e, dict)] + if extensions: + md["extensions"] = [ + {"name": _get(e, "name"), "enabled": _get(e, "isEnabled")} + for e in extensions + ] else: md["status"] = "not enabled" self._extra.append( @@ -319,6 +418,9 @@ def _role_assignments(self) -> None: # AKS kubelet identity with AcrPull -> the cluster pulls from the registry if guid == ACR_PULL_ROLE_ID and pid.lower() in self._kubelets and scope: self._kubelet_acr_pull(pid, scope, registries) + # role definition attachments (ROLE/USER/GROUP_HAS_POLICY), when definitions were read + for pid, ptype, relation in self.state.get("role_attachments", []): + self._attach_principal_relation(pid, ptype, relation, placeholders) self._extra.extend(placeholders.values()) # principals referenced elsewhere (Key Vault policies, SQL admins) without an owner for key, ptype in self._principal_types.items(): @@ -415,7 +517,18 @@ def _subscription_asset(self) -> CloudAsset: ) def build(self) -> list[CloudAsset]: - """Finalize: role assignments, deferred relations, then CloudAssets.""" + """Finalize: build hooks, role assignments, deferred relations, then CloudAssets.""" + from cloudg.inventory.azure_graph.governance import role_definition_assets + from cloudg.inventory.azure_graph.security import defender_coverage + + for fn in list(self._finalizers): + try: + fn(self) + except Exception: + logger.debug("Azure build hook %s failed", fn, exc_info=True) + defender_coverage(self, self._pricing_overrides, self._pricings_seen) + if self._role_definitions: + role_definition_assets(self, self._role_definitions, self._role_rows) self._role_assignments() for owner, relation in self._deferred: draft = self._drafts.get(owner) @@ -447,37 +560,79 @@ def collect_subscription_graph( role_assignments: bool = True, defender: bool = True, errors: dict[str, str] | None = None, + tasks: Iterable[str] | None = None, + recorder: Any = None, + keep_row: Callable[[dict[str, Any]], bool] | None = None, ) -> AzureAssetBuilder: """Query ARG for one subscription and return a populated builder. The ``resources`` query must succeed (its exception propagates so the caller can fall back to the SDK path); the container, role-assignment and Defender queries are best-effort and record failures in ``errors``. + The other Resource Graph tables (policy, security, backup, Kubernetes + configuration, network manager, guest configuration, maintenance, + insights) and the management-group role assignments and role + definitions are best-effort too: their failures become FAILED coverage + records on ``recorder``. + + Args: + tasks: Task names to run (:mod:`.families`); None runs every task. + recorder: :class:`~.families.TaskRecorder` for per-task coverage. + keep_row: Predicate filtering the ``resources`` rows (``--services``). """ + from cloudg.inventory.azure_graph.families import TaskRecorder + from cloudg.inventory.azure_graph.tables import collect_tables + errors = errors if errors is not None else {} + rec = recorder if recorder is not None else TaskRecorder() + selected = set(tasks) if tasks is not None else None + + def on(task: str) -> bool: + if selected is None or task in selected: + return True + rec.skip(task, "not selected (inventory.services)") + return False + subs = [subscription_id] builder = AzureAssetBuilder(subscription_id, "resource-graph") + started = time.time() resource_rows = run_query(client, resources_query(subscription_id), subscriptions=subs) + if keep_row is not None: + resource_rows = [r for r in resource_rows if keep_row(r)] builder.add_rows(resource_rows) + rec.record("resources", "SUCCESS", len(resource_rows), started=started) builder.include_subscription() + started = time.time() + container_rows: list[dict[str, Any]] = [] try: - builder.add_containers( - run_query(client, containers_query(subscription_id), subscriptions=subs) - ) + container_rows = run_query(client, containers_query(subscription_id), subscriptions=subs) + builder.add_containers(container_rows) + rec.record("resource_containers", "SUCCESS", len(container_rows), started=started) except Exception as exc: errors["resource_graph_containers"] = str(exc) - if role_assignments: + if role_assignments and on("role_assignments"): + started = time.time() try: - builder.add_role_assignments( - run_query(client, role_assignments_query(subscription_id), subscriptions=subs) - ) + rows = run_query(client, role_assignments_query(subscription_id), subscriptions=subs) + builder.add_role_assignments(rows) + rec.record("role_assignments", "SUCCESS", len(rows), started=started) except Exception as exc: errors["resource_graph_role_assignments"] = str(exc) - if defender: + if defender and on("defender_pricings"): + started = time.time() try: - builder.add_defender_pricings( - run_query(client, defender_query(subscription_id), subscriptions=subs) - ) + rows = run_query(client, defender_query(subscription_id), subscriptions=subs) + builder.add_defender_pricings(rows) + rec.record("defender_pricings", "SUCCESS", len(rows), started=started) except Exception as exc: errors["resource_graph_defender"] = str(exc) + collect_tables( + client, + builder, + subscription_id, + on=on, + recorder=rec, + container_rows=container_rows, + role_assignments=role_assignments, + ) return builder diff --git a/cloudg/inventory/azure_graph/children.py b/cloudg/inventory/azure_graph/children.py new file mode 100644 index 0000000..c0e1128 --- /dev/null +++ b/cloudg/inventory/azure_graph/children.py @@ -0,0 +1,662 @@ +"""Child resources and subscription-level listings read through ARM REST. + +Resource Graph returns tracked resources; most proxy resources (APIM APIs, +Service Bus queues, DNS record sets, function bindings, Data Factory +pipelines, diagnostic settings, ...) and several subscription-level +collections (deny assignments, locks, deployments, Lighthouse, activity +log export, PIM eligibility) only exist in ARM. ARM answers them with the +same ARM-shaped JSON Resource Graph uses, so returned rows go straight +into :class:`~.builder.AzureAssetBuilder` and through the extractors +registered for their types. + +:data:`CHILD_SPECS` is the plan: each :class:`ChildSpec` names its task +(:mod:`.families`), the parent types it hangs off, the relative path and +API version, and optional nested specs applied to every returned child. +:func:`fetch_children` runs the selected specs with bounded concurrency +(the HTTP calls run in a thread pool; rows are applied to the builder on +the calling thread), and records one coverage record per task: +SUCCESS, PARTIAL when some calls were refused or throttled, FAILED when +every call failed, SKIPPED when the credential cannot issue ARM tokens. +""" + +from __future__ import annotations + +import logging +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable + +from cloudg.exceptions import CredentialError +from cloudg.inventory.azure_graph.helpers import _get, _lower, subscription_ref +from cloudg.inventory.azure_graph.rest import AzureRestError + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + from cloudg.inventory.azure_graph.registry import _Draft + +logger = logging.getLogger("cloudg.inventory.azure_graph") + +SUBSCRIPTION = "@subscription" +RESOURCE_GROUPS = "microsoft.resources/subscriptions/resourcegroups" + + +@dataclass(frozen=True) +class ChildSpec: + """One kind of child resource (or subscription-level listing). + + Args: + task: Coverage task name. + parents: Lowercase ARM types of the parent drafts, or + :data:`SUBSCRIPTION` for ``/subscriptions/``. + path: Path appended to the parent ID (``apis``, + ``providers/Microsoft.Insights/diagnosticSettings``). + api_version: ARM ``api-version``. + child_type: Type set on rows that lack one. + single: The path returns one object, not a ``value`` list. + params: Extra query parameters. + nested: Specs applied to every returned child. + when: Predicate on the parent draft (SKU, kind, feature flag). + force_type: Always set ``child_type``, even when rows carry a type. + """ + + task: str + parents: tuple[str, ...] + path: str + api_version: str + child_type: str | None = None + single: bool = False + params: tuple[tuple[str, str], ...] = () + nested: tuple["ChildSpec", ...] = () + when: Callable[["_Draft"], bool] | None = None + force_type: bool = False + + +def _kind_has(word: str) -> Callable[["_Draft"], bool]: + return lambda d: word in _lower(d.row.get("kind")) + + +def _sku_in(*names: str) -> Callable[["_Draft"], bool]: + def check(d: "_Draft") -> bool: + sku = d.row.get("sku") + name = _lower(_get(sku, "name") if isinstance(sku, dict) else sku) + return any(n in name for n in names) + + return check + + +def _prop_true(key: str) -> Callable[["_Draft"], bool]: + return lambda d: bool(_get(d.props, key)) + + +def _not_function(d: "_Draft") -> bool: + return "functionapp" not in _lower(d.row.get("kind")) + + +_APIM = "microsoft.apimanagement/service" +_SITES = "microsoft.web/sites" +_ADF = "microsoft.datafactory/factories" +_AA = "microsoft.automation/automationaccounts" +_CDN = "microsoft.cdn/profiles" + +CHILD_SPECS: tuple[ChildSpec, ...] = ( + # ── subscription-level listings ── + ChildSpec( + "deny_assignments", + (SUBSCRIPTION,), + "providers/Microsoft.Authorization/denyAssignments", + "2022-04-01", + ), + ChildSpec("locks", (SUBSCRIPTION,), "providers/Microsoft.Authorization/locks", "2020-05-01"), + ChildSpec( + "pim_eligibility", + (SUBSCRIPTION,), + "providers/Microsoft.Authorization/roleEligibilityScheduleInstances", + "2020-10-01", + params=(("$filter", "atScope()"),), + ), + ChildSpec( + "lighthouse", + (SUBSCRIPTION,), + "providers/Microsoft.ManagedServices/registrationAssignments", + "2022-10-01", + params=(("$expandRegistrationDefinition", "true"),), + ), + ChildSpec( + "activity_log", + (SUBSCRIPTION,), + "providers/Microsoft.Insights/diagnosticSettings", + "2021-05-01-preview", + ), + ChildSpec( + "deployments", + (SUBSCRIPTION, RESOURCE_GROUPS), + "providers/Microsoft.Resources/deployments", + "2024-03-01", + params=(("$top", "50"),), + ), + ChildSpec( + "deployment_stacks", + (SUBSCRIPTION, RESOURCE_GROUPS), + "providers/Microsoft.Resources/deploymentStacks", + "2024-03-01", + ), + # ── identity ── + ChildSpec( + "managed_identity_credentials", + ("microsoft.managedidentity/userassignedidentities",), + "federatedIdentityCredentials", + "2023-01-31", + ), + ChildSpec("key_vault_items", ("microsoft.keyvault/vaults",), "keys", "2023-07-01"), + ChildSpec("key_vault_items", ("microsoft.keyvault/vaults",), "secrets", "2023-07-01"), + # ── API Management ── + ChildSpec( + "apim", + (_APIM,), + "apis", + "2022-08-01", + params=(("$filter", "isCurrent eq true"),), + nested=(ChildSpec("apim", (), "policies", "2022-08-01"),), + ), + ChildSpec("apim", (_APIM,), "backends", "2022-08-01"), + ChildSpec( + "apim", + (_APIM,), + "products", + "2022-08-01", + nested=( + ChildSpec( + "apim", + (), + "apis", + "2022-08-01", + child_type="microsoft.apimanagement/service/products/apis", + force_type=True, + ), + ), + ), + ChildSpec("apim", (_APIM,), "namedValues", "2022-08-01"), + ChildSpec("apim", (_APIM,), "gateways", "2022-08-01", when=_sku_in("premium", "developer")), + ChildSpec("apim", (_APIM,), "authorizationServers", "2022-08-01"), + ChildSpec("apim", (_APIM,), "openidConnectProviders", "2022-08-01"), + ChildSpec("apim", (_APIM,), "policies", "2022-08-01"), + # ── App Service, Functions, Static Web Apps ── + ChildSpec("functions", (_SITES,), "functions", "2023-01-01", when=_kind_has("functionapp")), + ChildSpec("web_apps", (_SITES,), "config/web", "2023-01-01", single=True), + ChildSpec( + "web_apps", (_SITES,), "sourcecontrols/web", "2023-01-01", single=True, when=_not_function + ), + ChildSpec("web_apps", (_SITES,), "basicPublishingCredentialsPolicies", "2023-01-01"), + ChildSpec("static_sites", ("microsoft.web/staticsites",), "linkedBackends", "2023-01-01"), + ChildSpec( + "static_sites", ("microsoft.web/staticsites",), "userProvidedFunctionApps", "2023-01-01" + ), + # ── messaging ── + ChildSpec("service_bus", ("microsoft.servicebus/namespaces",), "queues", "2021-11-01"), + ChildSpec( + "service_bus", + ("microsoft.servicebus/namespaces",), + "topics", + "2021-11-01", + nested=(ChildSpec("service_bus", (), "subscriptions", "2021-11-01"),), + ), + ChildSpec("event_hubs", ("microsoft.eventhub/namespaces",), "eventhubs", "2024-01-01"), + ChildSpec( + "stream_analytics", ("microsoft.streamanalytics/streamingjobs",), "inputs", "2020-03-01" + ), + ChildSpec( + "stream_analytics", ("microsoft.streamanalytics/streamingjobs",), "outputs", "2020-03-01" + ), + # ── analytics ── + ChildSpec("data_factory", (_ADF,), "linkedservices", "2018-06-01"), + ChildSpec("data_factory", (_ADF,), "datasets", "2018-06-01"), + ChildSpec("data_factory", (_ADF,), "integrationRuntimes", "2018-06-01"), + ChildSpec("data_factory", (_ADF,), "pipelines", "2018-06-01"), + ChildSpec("data_factory", (_ADF,), "triggers", "2018-06-01"), + ChildSpec("synapse", ("microsoft.synapse/workspaces",), "sqlPools", "2021-06-01"), + ChildSpec("synapse", ("microsoft.synapse/workspaces",), "bigDataPools", "2021-06-01"), + ChildSpec("synapse", ("microsoft.synapse/workspaces",), "integrationRuntimes", "2021-06-01"), + ChildSpec( + "kusto", + ("microsoft.kusto/clusters",), + "databases", + "2023-08-15", + nested=(ChildSpec("kusto", (), "dataConnections", "2023-08-15"),), + ), + ChildSpec("kusto", ("microsoft.kusto/clusters",), "principalAssignments", "2023-08-15"), + # ── data ── + ChildSpec("sql", ("microsoft.sql/servers",), "failoverGroups", "2021-11-01"), + ChildSpec("sql", ("microsoft.sql/servers",), "auditingSettings", "2021-11-01"), + ChildSpec( + "cosmos", + ("microsoft.documentdb/databaseaccounts",), + "sqlRoleAssignments", + "2023-04-15", + when=lambda d: _lower(d.row.get("kind")) in ("", "globaldocumentdb"), + ), + ChildSpec( + "storage", ("microsoft.storage/storageaccounts",), "objectReplicationPolicies", "2023-01-01" + ), + ChildSpec( + "storage", + ("microsoft.storage/storageaccounts",), + "localUsers", + "2023-01-01", + when=_prop_true("isSftpEnabled"), + ), + ChildSpec( + "storage", + ("microsoft.storage/storageaccounts",), + "blobServices/default", + "2023-01-01", + single=True, + child_type="microsoft.storage/storageaccounts/blobservices", + ), + # ── DNS ── + ChildSpec("dns_records", ("microsoft.network/dnszones",), "all", "2018-05-01"), + ChildSpec("dns_records", ("microsoft.network/privatednszones",), "ALL", "2020-06-01"), + ChildSpec( + "dns_resolver", ("microsoft.network/dnsresolvers",), "inboundEndpoints", "2022-07-01" + ), + ChildSpec( + "dns_resolver", ("microsoft.network/dnsresolvers",), "outboundEndpoints", "2022-07-01" + ), + ChildSpec( + "dns_resolver", + ("microsoft.network/dnsforwardingrulesets",), + "forwardingRules", + "2022-07-01", + ), + ChildSpec( + "dns_resolver", + ("microsoft.network/dnsforwardingrulesets",), + "virtualNetworkLinks", + "2022-07-01", + ), + # ── hybrid ── + ChildSpec( + "virtual_wan", + ("microsoft.network/virtualhubs",), + "hubVirtualNetworkConnections", + "2023-09-01", + ), + ChildSpec("virtual_wan", ("microsoft.network/virtualhubs",), "hubRouteTables", "2023-09-01"), + ChildSpec("virtual_wan", ("microsoft.network/virtualhubs",), "routingIntent", "2023-09-01"), + ChildSpec("virtual_wan", ("microsoft.network/virtualhubs",), "bgpConnections", "2023-09-01"), + ChildSpec("virtual_wan", ("microsoft.network/virtualhubs",), "ipConfigurations", "2023-09-01"), + ChildSpec( + "network_manager", + ("microsoft.network/networkmanagers",), + "connectivityConfigurations", + "2023-09-01", + ), + ChildSpec( + "network_manager", + ("microsoft.network/networkmanagers",), + "networkGroups", + "2023-09-01", + nested=(ChildSpec("network_manager", (), "staticMembers", "2023-09-01"),), + ), + ChildSpec( + "network_manager", + ("microsoft.network/networkmanagers",), + "securityAdminConfigurations", + "2023-09-01", + nested=( + ChildSpec( + "network_manager", + (), + "ruleCollections", + "2023-09-01", + nested=(ChildSpec("network_manager", (), "rules", "2023-09-01"),), + ), + ), + ), + # ── Front Door Standard / Premium ── + ChildSpec( + "front_door", + (_CDN,), + "afdEndpoints", + "2023-05-01", + when=_sku_in("azurefrontdoor"), + nested=(ChildSpec("front_door", (), "routes", "2023-05-01"),), + ), + ChildSpec( + "front_door", + (_CDN,), + "originGroups", + "2023-05-01", + when=_sku_in("azurefrontdoor"), + nested=(ChildSpec("front_door", (), "origins", "2023-05-01"),), + ), + ChildSpec( + "front_door", + (_CDN,), + "ruleSets", + "2023-05-01", + when=_sku_in("azurefrontdoor"), + nested=(ChildSpec("front_door", (), "rules", "2023-05-01"),), + ), + ChildSpec("front_door", (_CDN,), "customDomains", "2023-05-01", when=_sku_in("azurefrontdoor")), + ChildSpec( + "front_door", (_CDN,), "securityPolicies", "2023-05-01", when=_sku_in("azurefrontdoor") + ), + ChildSpec("front_door", (_CDN,), "secrets", "2023-05-01", when=_sku_in("azurefrontdoor")), + # ── containers ── + ChildSpec( + "acr", + ("microsoft.containerregistry/registries",), + "replications", + "2023-07-01", + when=_sku_in("premium"), + ), + ChildSpec("acr", ("microsoft.containerregistry/registries",), "webhooks", "2023-07-01"), + ChildSpec("acr", ("microsoft.containerregistry/registries",), "tasks", "2019-06-01-preview"), + ChildSpec("acr", ("microsoft.containerregistry/registries",), "scopeMaps", "2023-07-01"), + ChildSpec("acr", ("microsoft.containerregistry/registries",), "tokens", "2023-07-01"), + ChildSpec( + "aks", + ("microsoft.containerservice/managedclusters",), + "trustedAccessRoleBindings", + "2024-05-01", + ), + # ── AI and ML ── + ChildSpec( + "cognitive_services", ("microsoft.cognitiveservices/accounts",), "deployments", "2024-10-01" + ), + ChildSpec( + "cognitive_services", ("microsoft.cognitiveservices/accounts",), "raiPolicies", "2024-10-01" + ), + ChildSpec( + "machine_learning", + ("microsoft.machinelearningservices/workspaces",), + "datastores", + "2024-04-01", + ), + ChildSpec( + "machine_learning", + ("microsoft.machinelearningservices/workspaces",), + "connections", + "2024-04-01", + ), + ChildSpec( + "machine_learning", + ("microsoft.machinelearningservices/workspaces",), + "computes", + "2024-04-01", + ), + ChildSpec( + "machine_learning", + ("microsoft.machinelearningservices/workspaces",), + "onlineEndpoints", + "2024-04-01", + nested=(ChildSpec("machine_learning", (), "deployments", "2024-04-01"),), + ), + ChildSpec( + "search", ("microsoft.search/searchservices",), "sharedPrivateLinkResources", "2023-11-01" + ), + # ── operations ── + ChildSpec("automation", (_AA,), "schedules", "2023-11-01"), + ChildSpec("automation", (_AA,), "jobSchedules", "2023-11-01"), + ChildSpec( + "automation", + (_AA,), + "hybridRunbookWorkerGroups", + "2023-11-01", + nested=(ChildSpec("automation", (), "hybridRunbookWorkers", "2023-11-01"),), + ), + ChildSpec("automation", (_AA,), "softwareUpdateConfigurations", "2019-06-01"), + ChildSpec("automation", (_AA,), "webhooks", "2015-10-31"), + ChildSpec( + "monitor_private_link", + ("microsoft.insights/privatelinkscopes",), + "scopedResources", + "2021-07-01-preview", + ), + ChildSpec("batch", ("microsoft.batch/batchaccounts",), "pools", "2024-02-01"), + ChildSpec("spring_apps", ("microsoft.appplatform/spring",), "apps", "2023-12-01"), + ChildSpec( + "data_protection", + ("microsoft.dataprotection/backupvaults",), + "backupPolicies", + "2024-04-01", + ), + ChildSpec("dev_center", ("microsoft.devcenter/devcenters",), "catalogs", "2024-02-01"), + ChildSpec("dev_center", ("microsoft.devcenter/projects",), "environmentTypes", "2024-02-01"), + ChildSpec( + "app_configuration", + ("microsoft.appconfiguration/configurationstores",), + "replicas", + "2023-03-01", + ), +) + +# Resource types whose diagnostic settings are read (task diagnostic_settings) +DIAGNOSTIC_TYPES: tuple[str, ...] = ( + "microsoft.keyvault/vaults", + "microsoft.storage/storageaccounts", + "microsoft.sql/servers/databases", + "microsoft.network/networksecuritygroups", + "microsoft.network/applicationgateways", + "microsoft.network/azurefirewalls", + "microsoft.network/frontdoors", + "microsoft.cdn/profiles", + "microsoft.network/publicipaddresses", + "microsoft.network/virtualnetworkgateways", + "microsoft.network/bastionhosts", + "microsoft.apimanagement/service", + "microsoft.containerservice/managedclusters", + "microsoft.containerregistry/registries", + "microsoft.web/sites", + "microsoft.cognitiveservices/accounts", + "microsoft.documentdb/databaseaccounts", + "microsoft.eventhub/namespaces", + "microsoft.servicebus/namespaces", + "microsoft.logic/workflows", + "microsoft.recoveryservices/vaults", + "microsoft.datafactory/factories", + "microsoft.synapse/workspaces", + "microsoft.search/searchservices", + "microsoft.cache/redis", + "microsoft.dbforpostgresql/flexibleservers", + "microsoft.dbformysql/flexibleservers", + "microsoft.machinelearningservices/workspaces", + "microsoft.kusto/clusters", + "microsoft.app/managedenvironments", + "microsoft.automation/automationaccounts", +) + +# Sentinel children under a Log Analytics workspace +_SENTINEL = "providers/Microsoft.SecurityInsights" +SENTINEL_SPECS: tuple[ChildSpec, ...] = ( + ChildSpec("sentinel", (), f"{_SENTINEL}/dataConnectors", "2024-03-01"), + ChildSpec("sentinel", (), f"{_SENTINEL}/alertRules", "2024-03-01"), + ChildSpec("sentinel", (), f"{_SENTINEL}/automationRules", "2024-03-01"), +) + + +@dataclass +class _TaskStats: + calls: int = 0 + rows: int = 0 + failures: list[str] = field(default_factory=list) + started: float = field(default_factory=time.time) + + +class ChildFetcher: + """Runs child specs against one builder. + + Args: + client: ARM REST client (``list(path, api_version, **params)`` and + ``get(path, api_version)``), e.g. :class:`~.rest.AzureRestClient`. + builder: The subscription's builder. + max_parents: Most parents queried per spec. + concurrency: Concurrent HTTP calls. + """ + + def __init__( + self, + client: Any, + builder: "AzureAssetBuilder", + *, + max_parents: int = 500, + concurrency: int = 8, + ) -> None: + self.client = client + self.b = builder + self.max_parents = max_parents + self.concurrency = max(1, concurrency) + self.stats: dict[str, _TaskStats] = {} + self.credential_error: str | None = None + + # ── one call ── + + def _call(self, parent_id: str, spec: ChildSpec) -> tuple[list[dict[str, Any]], str | None]: + if self.credential_error: + return [], self.credential_error + path = f"{parent_id.rstrip('/')}/{spec.path}" + params = dict(spec.params) + try: + if spec.single: + row = self.client.get(path, spec.api_version, **params) + return ([row] if isinstance(row, dict) and row.get("id") else []), None + return self.client.list(path, spec.api_version, **params), None + except CredentialError as exc: + self.credential_error = str(exc) + return [], self.credential_error + except AzureRestError as exc: + if exc.not_applicable: + return [], None + return [], f"{spec.path}: {exc}" + except Exception as exc: # network errors, malformed JSON + return [], f"{spec.path}: {type(exc).__name__}: {str(exc)[:200]}" + + def _parents(self, spec: ChildSpec) -> list[tuple[str, "_Draft | None"]]: + out: list[tuple[str, "_Draft | None"]] = [] + for kind in spec.parents: + if kind == SUBSCRIPTION: + out.append((subscription_ref(self.b.subscription_id), None)) + continue + for d in self.b.drafts_of(kind): + if spec.when is None or spec.when(d): + out.append((d.id, d)) + return out[: self.max_parents] + + def _apply(self, rows: list[dict[str, Any]], parent: "_Draft | None", spec: ChildSpec) -> list: + """Feed rows to the builder; returns the drafts created (for nested specs).""" + created = [] + for row in rows: + if not isinstance(row, dict) or not row.get("id"): + continue + row = dict(row) + if spec.child_type and (spec.force_type or spec.single or not row.get("type")): + row["type"] = spec.child_type + if parent is not None: + row.setdefault("location", parent.region) + row.setdefault("subscriptionId", self.b.subscription_id) + draft = self.b.add_row(row, discovered_via="arm-rest") + created.append((row["id"], draft)) + return created + + def run_spec(self, spec: ChildSpec, parents: list[tuple[str, "_Draft | None"]]) -> None: + stats = self.stats.setdefault(spec.task, _TaskStats()) + if not parents: + return + with ThreadPoolExecutor(max_workers=min(self.concurrency, len(parents))) as pool: + results = list(pool.map(lambda p: self._call(p[0], spec), parents)) + children: list[tuple[str, Any]] = [] + for (pid, parent), (rows, error) in zip(parents, results): + stats.calls += 1 + if error: + stats.failures.append(error) + continue + stats.rows += len(rows) + created = self._apply(rows, parent, spec) + children.extend(created) + for nested in spec.nested: + nested_parents = [ + (cid, draft) + for cid, draft in children + if nested.when is None or (draft and nested.when(draft)) + ][: self.max_parents] + self.run_spec(nested, nested_parents) + + def run(self, specs: list[ChildSpec]) -> None: + for spec in specs: + self.run_spec(spec, self._parents(spec)) + + def record(self, recorder: Any) -> None: + for task, stats in self.stats.items(): + if self.credential_error and not stats.rows: + recorder.record( + task, "SKIPPED", error=self.credential_error[:300], started=stats.started + ) + continue + recorder.outcome(task, stats.rows, stats.failures, stats.calls, stats.started) + + +def fetch_children( + client: Any, + builder: "AzureAssetBuilder", + tasks: set[str], + recorder: Any, + *, + max_parents: int = 500, + concurrency: int = 8, + diagnostic_settings: bool = True, + max_diagnostic_settings: int = 1000, +) -> ChildFetcher: + """Run every selected child spec, diagnostic settings and Sentinel.""" + fetcher = ChildFetcher(client, builder, max_parents=max_parents, concurrency=concurrency) + fetcher.run([s for s in CHILD_SPECS if s.task in tasks]) + if "sentinel" in tasks: + _sentinel(fetcher) + if diagnostic_settings and "diagnostic_settings" in tasks: + spec = ChildSpec( + "diagnostic_settings", + DIAGNOSTIC_TYPES, + "providers/Microsoft.Insights/diagnosticSettings", + "2021-05-01-preview", + ) + parents = [] + for kind in DIAGNOSTIC_TYPES: + parents.extend((d.id, d) for d in builder.drafts_of(kind)) + fetcher.run_spec(spec, parents[:max_diagnostic_settings]) + fetcher.record(recorder) + return fetcher + + +def _sentinel(fetcher: ChildFetcher) -> None: + """Sentinel onboarding per workspace, then its connectors and rules.""" + from cloudg.inventory.azure_graph.security import sentinel_placeholder + + b = fetcher.b + workspaces = [(d.id, d) for d in b.drafts_of("microsoft.operationalinsights/workspaces")] + onboarding = ChildSpec( + "sentinel", + (), + f"{_SENTINEL}/onboardingStates", + "2024-03-01", + child_type="microsoft.securityinsights/onboardingstates", + ) + fetcher.run_spec(onboarding, workspaces[: fetcher.max_parents]) + onboarded = { + str(d.md.get("workspace_id") or "").lower() + for d in b.drafts_of("microsoft.securityinsights/onboardingstates") + } + enabled = [(wid, d) for wid, d in workspaces if wid.lower() in onboarded] + for spec in SENTINEL_SPECS: + fetcher.run_spec(spec, enabled) + fetcher.stats.setdefault("sentinel", _TaskStats()) + b.at_build(sentinel_placeholder) + + +__all__ = [ + "CHILD_SPECS", + "DIAGNOSTIC_TYPES", + "ChildFetcher", + "ChildSpec", + "SUBSCRIPTION", + "fetch_children", +] diff --git a/cloudg/inventory/azure_graph/data_ml.py b/cloudg/inventory/azure_graph/data_ml.py new file mode 100644 index 0000000..9dfeff4 --- /dev/null +++ b/cloudg/inventory/azure_graph/data_ml.py @@ -0,0 +1,1125 @@ +"""Extractors for data platforms and AI: pipelines, warehouses, replication, models. + +- Data Factory and Synapse: linked services (``DEPENDS_ON`` the stores + their endpoints name, and their integration runtime), datasets, + pipelines (``READS_FROM`` / ``WRITES_TO`` datasets, ``INVOKES`` linked + compute, other pipelines and HTTP hosts), triggers (``SCHEDULED_BY`` / + ``TRIGGERED_BY``), integration runtimes, Git configuration. +- Synapse SQL and Spark pools, Data Explorer (Kusto) clusters, databases, + data connections (``READS_FROM``) and principal assignments, HDInsight. +- SQL managed instances, elastic pools, geo-replicas and failover groups + (``REPLICATES_TO``), auditing (``LOGS_TO``). +- Cosmos DB regions as replica assets (``REPLICATES_TO``) and data-plane + role assignments (``GRANTS_ACCESS``). +- Storage object replication (``REPLICATES_TO``), SFTP local users, + blob data protection settings; Azure NetApp Files volumes (replication, + backup, snapshot policies). +- Azure OpenAI / AI Services deployments (``ML_MODEL``, protected by their + content filter ``AI_GUARDRAIL``), Foundry projects, Azure ML online + endpoints, deployments, compute and datastores, connections, AI Search + shared private links and data-plane objects (data sources, indexers, + indexes, skillsets), Purview. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, Iterable + +from cloudg.inventory._util import walk_strings +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + _get, + _list, + _lower, + _path, + _rid, + host_of, + parent_resource_id, + principal_ref, +) +from cloudg.inventory.azure_graph.registry import ( + _Draft, + _keyvault_key_ref, + _set_exposure, + extractor, + relation_row, +) +from cloudg.schema.models import EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +_SERVER_RE = re.compile( + r"(?:server|data source|host)\s*=\s*(?:tcp:)?([a-z0-9.-]+\.[a-z]{2,})", re.I +) +_ACCOUNT_RE = re.compile(r"accountname\s*=\s*([a-z0-9]+)", re.I) +_ENDPOINT_RE = re.compile(r"(?:endpoint|accountendpoint)\s*=\s*([a-z]+://[^;\s]+)", re.I) +_RESOURCE_ID_RE = re.compile(r"resourceid\s*=\s*(/subscriptions/[^;\s]+)", re.I) +_STORAGE_SUFFIX = { + "azureblobstorage": "blob", + "azureblobfs": "dfs", + "azuredatalakestore": "dfs", + "azuretablestorage": "table", + "azurefilestorage": "file", + "azureblob": "blob", + "adlsgen2": "dfs", + "azuretable": "table", +} + + +def endpoint_refs(value: Any, kind: str = "", limit: int = 10) -> list[str]: + """Hosts / resource IDs named by a configuration value, never its secrets.""" + refs: list[str] = [] + suffix = _STORAGE_SUFFIX.get(_lower(kind), "blob") + + def add(ref: str | None) -> None: + if ref and ref not in refs and len(refs) < limit: + refs.append(ref) + + for text in walk_strings(value): + if len(text) > 4000: + continue + for rid in _RESOURCE_ID_RE.findall(text): + add(rid.rstrip("/")) + for host in _SERVER_RE.findall(text): + add(host.lower()) + for acct in _ACCOUNT_RE.findall(text): + add(f"{acct.lower()}.{suffix}.core.windows.net") + for url in _ENDPOINT_RE.findall(text): + add(host_of(url)) + stripped = text.strip() + if stripped.lower().startswith(("https://", "http://")) and "@" not in stripped: + add(host_of(stripped)) + elif stripped.lower().startswith("/subscriptions/") and "/providers/" in stripped.lower(): + add(stripped.split("?", 1)[0]) + return refs + + +def _factory_of(child_id: str) -> str | None: + """Data Factory / Synapse workspace of a pipeline, dataset, ... ID.""" + return parent_resource_id(child_id) + + +def _ref(owner: str | None, kind: str, item: Any) -> str | None: + name = _get(item, "referenceName") if isinstance(item, dict) else item + if not owner or not isinstance(name, str) or not name or "@" in name: + return None + return f"{owner}/{kind}/{name}" + + +# ── Data Factory / Synapse ── + +_ADF = "microsoft.datafactory/factories" +_SYN = "microsoft.synapse/workspaces" + + +@extractor(f"{_ADF}/linkedservices", f"{_SYN}/linkedservices") +def _x_linked_service(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + owner = _factory_of(d.id) + kind = _get(p, "type") + tp = _get(p, "typeProperties") or {} + refs = endpoint_refs(tp, str(kind or "")) + d.md.update({"linked_service_type": kind, "endpoints": refs}) + for ref in refs: + d.add(rel(ref, EdgeType.REFERENCES, "DEPENDS_ON", description=f"{kind} endpoint")) + d.add( + rel( + _ref(owner, "integrationruntimes", _get(p, "connectVia")), + EdgeType.REFERENCES, + "RUNS_ON", + description="integration runtime", + ) + ) + for store in _key_vault_stores(tp): + d.add(rel(_ref(owner, "linkedservices", store), EdgeType.REFERENCES, "READS_FROM")) + d.props = {k: v for k, v in p.items() if _lower(k) != "typeproperties"} + + +def _key_vault_stores(value: Any, depth: int = 0) -> Iterable[Any]: + if depth > 8: + return + if isinstance(value, dict): + if _lower(_get(value, "type")) == "azurekeyvaultsecret": + yield _get(value, "store") + for v in value.values(): + yield from _key_vault_stores(v, depth + 1) + elif isinstance(value, list): + for v in value: + yield from _key_vault_stores(v, depth + 1) + + +@extractor(f"{_ADF}/datasets", f"{_SYN}/datasets") +def _x_dataset(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + owner = _factory_of(d.id) + location = _path(p, "typeProperties", "location") or {} + d.md.update( + { + "dataset_type": _get(p, "type"), + "container": _get(location, "container") or _get(location, "fileSystem"), + "folder": _get(location, "folderPath"), + "table": _path(p, "typeProperties", "tableName") or _path(p, "typeProperties", "table"), + } + ) + d.add( + rel( + _ref(owner, "linkedservices", _get(p, "linkedServiceName")), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + ) + + +def _activities(acts: Any) -> Iterable[dict[str, Any]]: + stack = list(_list(acts)) + while stack: + act = stack.pop() + if not isinstance(act, dict): + continue + yield act + tp = _get(act, "typeProperties") or {} + for key in ("activities", "ifTrueActivities", "ifFalseActivities", "defaultActivities"): + stack.extend(_list(_get(tp, key))) + for case in _list(_get(tp, "cases")): + stack.extend(_list(_get(case, "activities"))) + + +def _activity_relations(d: _Draft, owner: str | None, act: dict[str, Any]) -> None: + kind = _lower(_get(act, "type")) + tp = _get(act, "typeProperties") or {} + name = _get(act, "name") + for ds in _list(_get(act, "inputs")): + d.add( + rel( + _ref(owner, "datasets", ds), + EdgeType.REFERENCES, + "READS_FROM", + description=f"{name} input", + ) + ) + for ds in _list(_get(act, "outputs")): + d.add( + rel( + _ref(owner, "datasets", ds), + EdgeType.REFERENCES, + "WRITES_TO", + description=f"{name} output", + ) + ) + d.add(rel(_ref(owner, "datasets", _get(tp, "dataset")), EdgeType.REFERENCES, "READS_FROM")) + d.add( + rel( + _ref(owner, "linkedservices", _get(act, "linkedServiceName")), + EdgeType.INVOKES, + "INVOKES", + description=f"{kind} activity", + ) + ) + d.add( + rel( + _ref(owner, "pipelines", _get(tp, "pipeline")), + EdgeType.INVOKES, + "INVOKES", + description="execute pipeline", + ) + ) + url = _get(tp, "url") + if kind in ("webactivity", "webhook") and isinstance(url, str) and "@" not in url: + d.add(rel(host_of(url), EdgeType.INVOKES, "INVOKES", description=f"{name} call")) + + +@extractor(f"{_ADF}/pipelines", f"{_SYN}/pipelines") +def _x_pipeline(d: _Draft, b: "AzureAssetBuilder") -> None: + owner = _factory_of(d.id) + kinds = [] + for act in _activities(_get(d.props, "activities")): + kinds.append(_get(act, "type")) + _activity_relations(d, owner, act) + d.md["activity_types"] = sorted({k for k in kinds if k}) + d.md["activity_count"] = len(kinds) + d.props = {k: v for k, v in d.props.items() if _lower(k) != "activities"} + + +@extractor(f"{_ADF}/triggers", f"{_SYN}/triggers") +def _x_pipeline_trigger(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + owner = _factory_of(d.id) + kind = _lower(_get(p, "type")) + tp = _get(p, "typeProperties") or {} + targets = [_get(x, "pipelineReference") for x in _list(_get(p, "pipelines"))] + targets.append(_get(p, "pipeline") and _get(_get(p, "pipeline"), "pipelineReference")) + scope = _get(tp, "scope") + d.md.update( + { + "trigger_type": _get(p, "type"), + "runtime_state": _get(p, "runtimeState"), + "recurrence": _get(tp, "recurrence"), + "event_scope": scope, + "events": _list(_get(tp, "events")), + } + ) + for target in targets: + pipeline = _ref(owner, "pipelines", target) + if kind in ("scheduletrigger", "tumblingwindowtrigger"): + d.add(rel(pipeline, EdgeType.MANAGES, "SCHEDULED_BY", description=_get(p, "type"))) + else: + d.add(rel(pipeline, EdgeType.INVOKES, "INVOKES", description=_get(p, "type"))) + if pipeline and isinstance(scope, str): + b.defer( + pipeline, + rel( + scope, + EdgeType.INVOKES, + "TRIGGERED_BY", + reverse=True, + description="event trigger", + ), + ) + + +@extractor(f"{_ADF}/integrationruntimes", f"{_SYN}/integrationruntimes") +def _x_integration_runtime(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + tp = _get(p, "typeProperties") or {} + subnet = _path(tp, "computeProperties", "vNetProperties", "subnetId") or _path( + tp, "ssisProperties", "vNetProperties", "subnetId" + ) + d.md.update( + { + "runtime_type": _get(p, "type"), + "managed_vnet": _path(p, "managedVirtualNetwork", "referenceName"), + "self_hosted": _lower(_get(p, "type")) == "selfhosted", + "location": _path(tp, "computeProperties", "location"), + } + ) + d.add(rel(subnet, EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + + +@extractor(_ADF) +def _x_factory_git(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import repository_ref + + repo = _get(d.props, "repoConfiguration") or {} + kind = _lower(_get(repo, "type")) + account, name = _get(repo, "accountName"), _get(repo, "repositoryName") + url = None + if account and name and "github" in kind: + url = f"https://{_get(repo, 'hostName') or 'github.com'}/{account}/{name}".replace( + "https://https://", "https://" + ) + elif account and name: + url = f"https://dev.azure.com/{account}/{_get(repo, 'projectName')}/_git/{name}" + d.md["git_repository"] = url + d.add(rel(repository_ref(b, url), EdgeType.REFERENCES, "READS_FROM", description="factory Git")) + d.add( + rel( + _path(d.props, "purviewConfiguration", "purviewResourceId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + ) + + +@extractor(_SYN) +def _x_synapse_workspace(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + lake = _get(p, "defaultDataLakeStorage") or {} + d.add( + rel( + host_of(_get(lake, "accountUrl")), + EdgeType.REFERENCES, + "READS_FROM", + description="default data lake", + ) + ) + d.add( + rel( + _get(lake, "resourceId"), + EdgeType.REFERENCES, + "READS_FROM", + description="default data lake", + ) + ) + for endpoint in (_get(p, "connectivityEndpoints") or {}).values(): + d.alias(host_of(endpoint)) + d.md["managed_vnet"] = _get(p, "managedVirtualNetwork") + d.md["sql_admin"] = _get(p, "sqlAdministratorLogin") + d.add( + rel( + _path(p, "virtualNetworkProfile", "computeSubnetId"), + EdgeType.CONTAINS, + "SUBNET_CONTAINS_INSTANCE", + reverse=True, + ) + ) + _keyvault_key_ref(d, _path(p, "encryption", "cmk", "key", "keyVaultUrl")) + + +@extractor(f"{_SYN}/sqlpools", f"{_SYN}/bigdatapools") +def _x_synapse_pool(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "status": _get(p, "status"), + "node_size": _get(p, "nodeSize"), + "spark_version": _get(p, "sparkVersion"), + } + ) + + +# ── Data Explorer, HDInsight ── + + +@extractor("microsoft.kusto/clusters") +def _x_kusto(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for key in ("uri", "dataIngestionUri"): + d.alias(host_of(_get(p, key))) + d.add( + rel( + _path(p, "virtualNetworkConfiguration", "subnetId"), + EdgeType.CONTAINS, + "SUBNET_CONTAINS_INSTANCE", + reverse=True, + ) + ) + _keyvault_key_ref(d, _path(p, "keyVaultProperties", "keyVaultUri")) + tenants = [_get(t, "value") for t in _list(_get(p, "trustedExternalTenants"))] + d.md.update( + { + "trusted_external_tenants": tenants, + "any_tenant_trusted": "*" in tenants, + "restrict_outbound": _get(p, "restrictOutboundNetworkAccess"), + "allowed_fqdns": _list(_get(p, "allowedFqdnList")), + } + ) + _set_exposure(d) + + +@relation_row("microsoft.kusto/clusters/databases/dataconnections") +def _kusto_data_connection(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + db = parent_resource_id(row.get("id")) + for key in ( + "eventHubResourceId", + "iotHubResourceId", + "storageAccountResourceId", + "cosmosDbAccountResourceId", + "eventGridResourceId", + ): + b.defer( + db, + rel( + _get(p, key), + EdgeType.REFERENCES, + "READS_FROM", + description=f"{row.get('kind')} data connection", + ), + ) + + +@relation_row( + "microsoft.kusto/clusters/principalassignments", + "microsoft.kusto/clusters/databases/principalassignments", +) +def _kusto_principal(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + pid = _get(p, "principalId") + if not pid: + return + b.note_principal(pid, _get(p, "principalType")) + role = _get(p, "role") + b.defer( + parent_resource_id(row.get("id")), + rel( + principal_ref(pid), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description=f"Data Explorer {role}", + role=role, + privileged="admin" in _lower(role) or None, + tenant_id=_get(p, "tenantId"), + ), + ) + + +@extractor("microsoft.hdinsight/clusters") +def _x_hdinsight(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for role in _list(_path(p, "computeProfile", "roles")): + subnet = _path(role, "virtualNetworkProfile", "subnet") + d.add(rel(subnet, EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + for acct in _list(_path(p, "storageProfile", "storageaccounts")): + d.add( + rel( + _get(acct, "resourceId") or str(_get(acct, "name") or "").lower() or None, + EdgeType.REFERENCES, + "READS_FROM", + description="cluster storage", + ) + ) + d.add(rel(_get(acct, "msiResourceId"), EdgeType.ASSUMES_ROLE, "RUNS_ON")) + _keyvault_key_ref(d, _path(p, "diskEncryptionProperties", "vaultUri")) + endpoints = [_get(e, "location") for e in _list(_get(p, "connectivityEndpoints"))] + for host in endpoints: + d.alias(str(host).lower() if host else None) + d.md.update( + { + "cluster_kind": _path(p, "clusterDefinition", "kind"), + "cluster_version": _get(p, "clusterVersion"), + "esp_domain": _path(p, "securityProfile", "domain"), + "public_network": _path(p, "networkProperties", "publicNetworkAccess"), + } + ) + if _lower(_path(p, "networkProperties", "resourceProviderConnection")) != "outbound": + d.exposed = any(str(h or "").endswith(".azurehdinsight.net") for h in endpoints) + d.add(rel(_path(p, "securityProfile", "aaddsResourceId"), EdgeType.REFERENCES, "DEPENDS_ON")) + + +# ── SQL ── + + +@extractor("microsoft.sql/managedinstances") +def _x_sql_mi(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + fqdn = _get(p, "fullyQualifiedDomainName") + d.alias(fqdn.lower() if isinstance(fqdn, str) else None) + d.add(rel(_get(p, "subnetId"), EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + public = bool(_get(p, "publicDataEndpointEnabled")) + d.md.update( + { + "fqdn": fqdn, + "public_data_endpoint": public, + "proxy_override": _get(p, "proxyOverride"), + "minimal_tls_version": _get(p, "minimalTlsVersion"), + } + ) + if public: + d.exposed = True + admins = _get(p, "administrators") or {} + sid = _get(admins, "sid") + if sid: + b.note_principal(sid, _get(admins, "principalType")) + d.add( + rel( + principal_ref(sid), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="Entra administrator", + privileged=True, + ) + ) + _keyvault_key_ref(d, _get(p, "keyId")) + + +@extractor("microsoft.sql/servers/elasticpools") +def _x_elastic_pool(d: _Draft, b: "AzureAssetBuilder") -> None: + sku = d.row.get("sku") or {} + d.md.update( + { + "role": "elastic pool", + "edition": _get(sku, "tier"), + "zone_redundant": _get(d.props, "zoneRedundant"), + "max_size_bytes": _get(d.props, "maxSizeBytes"), + } + ) + + +@extractor("microsoft.sql/servers/databases") +def _x_sql_geo_replica(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + source = _get(p, "sourceDatabaseId") + secondary = _get(p, "secondaryType") + d.md["secondary_type"] = secondary + if source and (secondary or _lower(_get(p, "createMode")) in ("secondary", "onlinesecondary")): + d.add( + rel( + source, + EdgeType.REFERENCES, + "REPLICATES_TO", + reverse=True, + description=f"{secondary or 'geo'} replica", + ) + ) + + +@relation_row("microsoft.sql/servers/failovergroups") +def _sql_failover_group(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + server = parent_resource_id(row.get("id")) + for partner in _list(_get(p, "partnerServers")): + b.defer( + server, + rel( + _rid(partner), + EdgeType.REFERENCES, + "REPLICATES_TO", + description=f"failover group {row.get('name')}", + role=_get(partner, "replicationRole"), + ), + ) + for db in _list(_get(p, "databases")): + for partner in _list(_get(p, "partnerServers")): + b.defer( + db, + rel( + _rid(partner), + EdgeType.REFERENCES, + "REPLICATES_TO", + description="failover group database", + ), + ) + + +@relation_row("microsoft.sql/locations/instancefailovergroups") +def _mi_failover_group(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + for pair in _list(_get(row.get("properties") or {}, "managedInstancePairs")): + b.defer( + _get(pair, "primaryManagedInstanceId"), + rel( + _get(pair, "partnerManagedInstanceId"), + EdgeType.REFERENCES, + "REPLICATES_TO", + description="instance failover group", + ), + ) + + +@relation_row("microsoft.sql/servers/auditingsettings") +def _sql_auditing(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import _set_md_later + + p = row.get("properties") or {} + server = parent_resource_id(row.get("id")) + enabled = _lower(_get(p, "state")) == "enabled" + _set_md_later( + b, + server, + { + "auditing_enabled": enabled, + "audit_to_azure_monitor": _get(p, "isAzureMonitorTargetEnabled"), + }, + ) + if enabled: + b.defer( + server, + rel( + host_of(_get(p, "storageEndpoint")), + EdgeType.LOGS_TO, + "LOGS_TO", + description="SQL auditing", + ), + ) + + +# ── Cosmos DB ── + + +@extractor("microsoft.documentdb/databaseaccounts") +def _x_cosmos_regions(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + locations = [ + x + for x in _list(_get(p, "locations")) or _list(_get(p, "readLocations")) + if isinstance(x, dict) + ] + d.md.update( + { + "multi_region_writes": _get(p, "enableMultipleWriteLocations"), + "consistency": _path(p, "consistencyPolicy", "defaultConsistencyLevel"), + "backup_policy": _path(p, "backupPolicy", "type"), + "regions": [_get(x, "locationName") for x in locations], + } + ) + for loc in locations: + if not _get(loc, "failoverPriority"): + continue # the write region is the account itself + region = str(_get(loc, "locationName") or "").replace(" ", "").lower() + if not region: + continue + rid = f"{d.id}/regions/{region}" + b.add_row( + { + "id": rid, + "name": f"{d.name} ({_get(loc, 'locationName')})", + "type": "microsoft.documentdb/databaseaccounts/regions", + "location": region, + "subscriptionId": d.account_id, + "properties": { + "failoverPriority": _get(loc, "failoverPriority"), + "isZoneRedundant": _get(loc, "isZoneRedundant"), + }, + }, + synthetic=True, + ) + d.add( + rel( + rid, + EdgeType.REFERENCES, + "REPLICATES_TO", + description="Cosmos DB region replica", + failover_priority=_get(loc, "failoverPriority"), + ) + ) + + +@relation_row( + "microsoft.documentdb/databaseaccounts/sqlroleassignments", + "microsoft.documentdb/databaseaccounts/mongodbroleassignments", +) +def _cosmos_role_assignment(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + pid = _get(p, "principalId") + if not pid: + return + b.note_principal(pid, None) + b.defer( + parent_resource_id(row.get("id")), + rel( + principal_ref(pid), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="Cosmos DB data-plane role", + role_definition_id=_get(p, "roleDefinitionId"), + scope=_get(p, "scope"), + data_plane=True, + ), + ) + + +# ── Storage, NetApp ── + + +@relation_row("microsoft.storage/storageaccounts/objectreplicationpolicies") +def _object_replication(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + source = _get(p, "sourceAccount") or parent_resource_id(row.get("id")) + dest = _get(p, "destinationAccount") + owner = ( + source + if isinstance(source, str) and source.startswith("/") + else parent_resource_id(row.get("id")) + ) + if isinstance(dest, str) and dest and owner and dest.lower() != str(owner).lower(): + b.defer( + owner, + rel( + dest, + EdgeType.REFERENCES, + "REPLICATES_TO", + description="object replication", + containers=[_get(r, "sourceContainer") for r in _list(_get(p, "rules"))], + ), + ) + + +@extractor("microsoft.storage/storageaccounts/localusers") +def _x_storage_local_user(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + account = parent_resource_id(d.id) + scopes = [ + { + "service": _get(s, "service"), + "resource": _get(s, "resourceName"), + "permissions": _get(s, "permissions"), + } + for s in _list(_get(p, "permissionScopes")) + ] + d.md.update( + { + "sftp_user": True, + "has_ssh_key": _get(p, "hasSshKey"), + "has_ssh_password": _get(p, "hasSshPassword"), + "has_shared_key": _get(p, "hasSharedKey"), + "home_directory": _get(p, "homeDirectory"), + "permission_scopes": scopes, + } + ) + d.add( + rel( + account, + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + description="SFTP local user", + permissions=[s["permissions"] for s in scopes], + ) + ) + d.props = {k: v for k, v in p.items() if _lower(k) not in ("sshauthorizedkeys", "sshpassword")} + + +@relation_row("microsoft.storage/storageaccounts/blobservices") +def _blob_service(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import _set_md_later + + p = row.get("properties") or {} + _set_md_later( + b, + parent_resource_id(row.get("id")), + { + "blob_soft_delete": _path(p, "deleteRetentionPolicy", "enabled"), + "container_soft_delete": _path(p, "containerDeleteRetentionPolicy", "enabled"), + "versioning": _get(p, "isVersioningEnabled"), + "change_feed": _path(p, "changeFeed", "enabled"), + "point_in_time_restore": _path(p, "restorePolicy", "enabled"), + }, + ) + + +@extractor("microsoft.storage/storageaccounts") +def _x_storage_replication(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + sku = d.row.get("sku") + name = str(_get(sku, "name") if isinstance(sku, dict) else sku or "") + d.md.update( + { + "sftp_enabled": _get(p, "isSftpEnabled"), + "local_users_enabled": _get(p, "isLocalUserEnabled"), + "geo_replicated": "grs" in name.lower() or "gzrs" in name.lower(), + "secondary_location": _get(p, "secondaryLocation"), + "immutable_storage": _path(p, "immutableStorageWithVersioning", "enabled"), + } + ) + + +@extractor("microsoft.netapp/netappaccounts/capacitypools/volumes") +def _x_netapp_volume(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.add(rel(_get(p, "subnetId"), EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + for target in _list(_get(p, "mountTargets")): + d.alias(_get(target, "ipAddress")) + dp = _get(p, "dataProtection") or {} + replication = _get(dp, "replication") or {} + remote = _get(replication, "remoteVolumeResourceId") + is_destination = _lower(_get(replication, "endpointType")) == "dst" + d.add( + rel( + remote, + EdgeType.REFERENCES, + "REPLICATES_TO", + reverse=is_destination, + description="cross-region replication", + schedule=_get(replication, "replicationSchedule"), + ) + ) + backup = _get(dp, "backup") or {} + d.add( + rel( + _get(backup, "backupVaultId"), + EdgeType.REFERENCES, + "BACKUP_TO", + description="NetApp backup", + ) + ) + d.add( + rel( + _get(backup, "backupPolicyId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="backup policy", + ) + ) + snap_policy = _path(dp, "snapshot", "snapshotPolicyId") + if snap_policy: + b.defer( + snap_policy, rel(d.id, EdgeType.MANAGES, "SCHEDULED_BY", description="snapshot policy") + ) + rules = _list(_path(p, "exportPolicy", "rules")) + d.md.update( + { + "protocols": _list(_get(p, "protocolTypes")), + "export_clients": [_get(r, "allowedClients") for r in rules], + "open_export": any( + str(_get(r, "allowedClients") or "").strip() in ("0.0.0.0/0", "*") for r in rules + ), + } + ) + + +# ── AI and ML ── + + +@extractor("microsoft.cognitiveservices/accounts") +def _x_cognitive_account(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for endpoint in (_get(p, "endpoints") or {}).values(): + d.alias(host_of(endpoint)) + sub = _get(p, "customSubDomainName") + d.md.update( + { + "ai_kind": d.row.get("kind"), + "custom_subdomain": sub, + "local_auth_disabled": _get(p, "disableLocalAuth"), + "restrict_outbound": _get(p, "restrictOutboundNetworkAccess"), + "allowed_fqdns": _list(_get(p, "allowedFqdnList")), + } + ) + for store in _list(_get(p, "userOwnedStorage")): + d.add( + rel( + _get(store, "resourceId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="user-owned storage", + ) + ) + + +@extractor("microsoft.cognitiveservices/accounts/deployments") +def _x_ai_deployment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + model = _get(p, "model") or {} + account = parent_resource_id(d.id) + rai = _get(p, "raiPolicyName") + sku = d.row.get("sku") or {} + d.md.update( + { + "model_name": _get(model, "name"), + "model_version": _get(model, "version"), + "model_format": _get(model, "format"), + "content_filter": rai, + "capacity": _get(sku, "capacity") if isinstance(sku, dict) else None, + "deployment_type": _get(sku, "name") if isinstance(sku, dict) else None, + } + ) + if rai and account: + d.add( + rel( + f"{account}/raiPolicies/{rai}", + EdgeType.PROTECTS, + reverse=True, + description="content filter", + ) + ) + + +@extractor("microsoft.cognitiveservices/accounts/raipolicies") +def _x_rai_policy(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + filters = [ + { + "name": _get(f, "name"), + "source": _get(f, "source"), + "blocking": _get(f, "blocking"), + "enabled": _get(f, "enabled"), + "threshold": _get(f, "severityThreshold") or _get(f, "allowedContentLevel"), + } + for f in _list(_get(p, "contentFilters")) + ] + d.md.update( + { + "mode": _get(p, "mode"), + "base_policy": _get(p, "basePolicyName"), + "content_filters": filters, + "blocklists": [_get(x, "blocklistName") for x in _list(_get(p, "customBlocklists"))], + } + ) + + +@extractor( + "microsoft.cognitiveservices/accounts/projects", "microsoft.machinelearningservices/workspaces" +) +def _x_ai_project(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["workspace_kind"] = d.row.get("kind") + d.md["isolation_mode"] = _path(p, "managedNetwork", "isolationMode") + d.add(rel(_get(p, "hubResourceId"), EdgeType.CONTAINS, reverse=True, description="AI hub")) + + +@extractor( + "microsoft.machinelearningservices/workspaces/onlineendpoints", + "microsoft.machinelearningservices/workspaces/serverlessendpoints", +) +def _x_ml_endpoint(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for key in ("scoringUri", "swaggerUri", "inferenceEndpoint"): + value = _get(p, key) + d.alias(host_of(_get(value, "uri") if isinstance(value, dict) else value)) + d.md.update({"auth_mode": _get(p, "authMode"), "traffic": _get(p, "traffic")}) + if _lower(_get(p, "publicNetworkAccess") or "enabled") == "enabled": + d.exposed = True + + +@extractor("microsoft.machinelearningservices/workspaces/onlineendpoints/deployments") +def _x_ml_deployment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "model": _get(p, "model"), + "instance_type": _get(p, "instanceType"), + "egress_public": _get(p, "egressPublicNetworkAccess"), + } + ) + + +@extractor("microsoft.machinelearningservices/workspaces/computes") +def _x_ml_compute(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + inner = _get(p, "properties") or {} + d.md.update( + { + "compute_type": _get(p, "computeType"), + "vm_size": _get(inner, "vmSize"), + "node_public_ip": _get(inner, "enableNodePublicIp"), + } + ) + d.add( + rel( + _path(inner, "subnet", "id"), + EdgeType.CONTAINS, + "SUBNET_CONTAINS_INSTANCE", + reverse=True, + ) + ) + d.add( + rel(_get(p, "resourceId"), EdgeType.REFERENCES, "RUNS_ON", description="attached compute") + ) + d.exposed = bool(_get(inner, "enableNodePublicIp")) + + +@extractor("microsoft.machinelearningservices/workspaces/datastores") +def _x_ml_datastore(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + kind = _lower(_get(p, "datastoreType")) + account = _lower(_get(p, "accountName")) + suffix = {"azuredatalakegen2": "dfs", "azurefile": "file"}.get(kind, "blob") + target = f"{account}.{suffix}.{_get(p, 'endpoint') or 'core.windows.net'}" if account else None + if kind == "azuredatalakegen1": + target = f"{_lower(_get(p, 'storeName'))}.azuredatalakestore.net" + d.md.update( + { + "datastore_type": _get(p, "datastoreType"), + "container": _get(p, "containerName") or _get(p, "filesystem"), + "credential_type": _path(p, "credentials", "credentialsType"), + } + ) + d.add(rel(target, EdgeType.REFERENCES, "READS_FROM", description="datastore")) + ws = parent_resource_id(d.id) + if ws and target: + b.defer( + ws, rel(target, EdgeType.REFERENCES, "READS_FROM", description=f"datastore {d.name}") + ) + d.props = {k: v for k, v in p.items() if _lower(k) != "credentials"} + + +@extractor( + "microsoft.machinelearningservices/workspaces/connections", + "microsoft.cognitiveservices/accounts/connections", + "microsoft.cognitiveservices/accounts/projects/connections", +) +def _x_ai_connection(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + target = _get(p, "target") + d.md.update( + {"category": _get(p, "category"), "auth_type": _get(p, "authType"), "target": target} + ) + d.add( + rel( + host_of(target) if isinstance(target, str) and "://" in target else target, + EdgeType.REFERENCES, + "DEPENDS_ON", + description=f"{_get(p, 'category')} connection", + ) + ) + d.add(rel(_path(p, "metadata", "ResourceId"), EdgeType.REFERENCES, "DEPENDS_ON")) + d.props = {k: v for k, v in p.items() if _lower(k) not in ("credentials",)} + + +@relation_row("microsoft.search/searchservices/sharedprivatelinkresources") +def _search_shared_link(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + b.defer( + parent_resource_id(row.get("id")), + rel( + _get(p, "privateLinkResourceId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="shared private link", + status=_get(p, "status"), + ), + ) + + +@extractor("microsoft.search/searchservices") +def _x_search_service(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.alias(f"{d.name.lower()}.search.windows.net") + d.md.update( + { + "local_auth_disabled": _get(p, "disableLocalAuth"), + "auth_options": list((_get(p, "authOptions") or {}).keys()), + "semantic_search": _get(p, "semanticSearch"), + } + ) + + +@extractor("microsoft.search/searchservices/datasources") +def _x_search_datasource(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + kind = _get(p, "type") + refs = endpoint_refs(_get(p, "credentials") or {}, str(kind or "")) + d.md.update( + {"datasource_type": kind, "container": _path(p, "container", "name"), "endpoints": refs} + ) + for ref in refs: + d.add(rel(ref, EdgeType.REFERENCES, "READS_FROM", description=f"{kind} data source")) + d.props = {k: v for k, v in p.items() if _lower(k) != "credentials"} + + +@extractor("microsoft.search/searchservices/indexers") +def _x_search_indexer(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + svc = parent_resource_id(d.id) + d.md["schedule"] = _path(p, "schedule", "interval") + d.add( + rel(_ref(svc, "datasources", _get(p, "dataSourceName")), EdgeType.REFERENCES, "READS_FROM") + ) + d.add(rel(_ref(svc, "indexes", _get(p, "targetIndexName")), EdgeType.REFERENCES, "WRITES_TO")) + d.add(rel(_ref(svc, "skillsets", _get(p, "skillsetName")), EdgeType.REFERENCES, "DEPENDS_ON")) + + +@extractor("microsoft.search/searchservices/indexes") +def _x_search_index(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + vectorizers = _list(_path(p, "vectorSearch", "vectorizers")) + d.md.update( + { + "field_count": len(_list(_get(p, "fields"))), + "vector_search": bool(_get(p, "vectorSearch")), + } + ) + for v in vectorizers: + uri = _path(v, "azureOpenAIParameters", "resourceUri") or _path( + v, "customWebApiParameters", "uri" + ) + d.add(rel(host_of(uri), EdgeType.INVOKES, "INVOKES", description="vectorizer")) + d.props = {k: v for k, v in p.items() if _lower(k) not in ("fields",)} + + +@extractor("microsoft.search/searchservices/skillsets") +def _x_search_skillset(d: _Draft, b: "AzureAssetBuilder") -> None: + skills = _list(_get(d.props, "skills")) + d.md["skills"] = [_get(s, "@odata.type") for s in skills] + for s in skills: + uri = _get(s, "uri") or _get(s, "resourceUri") + d.add( + rel(host_of(uri), EdgeType.INVOKES, "INVOKES", description=str(_get(s, "@odata.type"))) + ) + d.props = {k: v for k, v in d.props.items() if _lower(k) not in ("cognitiveservices",)} + + +@extractor("microsoft.purview/accounts") +def _x_purview(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for endpoint in (_get(p, "endpoints") or {}).values(): + d.alias(host_of(endpoint)) + for key, target in (_get(p, "managedResources") or {}).items(): + d.add(rel(target, EdgeType.MANAGES, "OWNED_BY", description=f"managed {key}")) + _set_exposure(d) + + +__all__ = ["endpoint_refs"] diff --git a/cloudg/inventory/azure_graph/dataplane.py b/cloudg/inventory/azure_graph/dataplane.py new file mode 100644 index 0000000..69c02e5 --- /dev/null +++ b/cloudg/inventory/azure_graph/dataplane.py @@ -0,0 +1,131 @@ +"""Read-only data-plane metadata: AI Search and Synapse artifacts. + +AI Search data sources, indexers, indexes and skillsets, and Synapse +pipelines, linked services, datasets and triggers, live behind each +service's own endpoint, not ARM. They are read with an Entra token for the +service's data-plane audience (``https://search.azure.com/.default``, +``https://dev.azuresynapse.net/.default``), which needs a data-plane +reader role (Search Index Data Reader / Search Service Contributor, +Synapse Artifact User). Connection strings in the responses are scanned +for endpoints and then dropped; nothing secret is kept. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any, Callable + +from cloudg.exceptions import CredentialError +from cloudg.inventory.azure_graph.helpers import _get, host_of +from cloudg.inventory.azure_graph.rest import AzureRestError + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +SEARCH_SCOPE = "https://search.azure.com/.default" +SYNAPSE_SCOPE = "https://dev.azuresynapse.net/.default" +SEARCH_API = "2024-07-01" +SYNAPSE_API = "2020-12-01" +_SEARCH_KINDS = ("datasources", "indexers", "indexes", "skillsets") +_SYNAPSE_KINDS = ("linkedservices", "datasets", "pipelines", "triggers") + + +def _search_rows(svc_id: str, kind: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows = [] + for item in items: + name = item.get("name") + if not name: + continue + rows.append( + { + "id": f"{svc_id}/{kind}/{name}", + "name": name, + "type": f"microsoft.search/searchservices/{kind}", + "properties": {k: v for k, v in item.items() if not k.startswith("@odata.etag")}, + } + ) + return rows + + +def _synapse_rows(ws_id: str, kind: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows = [] + for item in items: + name = item.get("name") + if not name: + continue + row = dict(item) + row["id"] = item.get("id") or f"{ws_id}/{kind}/{name}" + row["type"] = f"microsoft.synapse/workspaces/{kind}" + rows.append(row) + return rows + + +class _PlaneRun: + def __init__(self, task: str) -> None: + self.task = task + self.calls = 0 + self.rows = 0 + self.failures: list[str] = [] + self.started = time.time() + self.skipped: str | None = None + + def call(self, fn: Callable[[], list[dict[str, Any]]]) -> list[dict[str, Any]]: + self.calls += 1 + try: + rows = fn() + except CredentialError as exc: + self.skipped = str(exc) + return [] + except AzureRestError as exc: + if not exc.not_applicable: + self.failures.append(str(exc)[:300]) + return [] + except Exception as exc: # unreachable endpoint (private), malformed JSON + self.failures.append(f"{type(exc).__name__}: {str(exc)[:200]}") + return [] + self.rows += len(rows) + return rows + + def record(self, recorder: Any) -> None: + if self.skipped and not self.rows: + recorder.record(self.task, "SKIPPED", error=self.skipped[:300], started=self.started) + else: + recorder.outcome(self.task, self.rows, self.failures, self.calls, self.started) + + +def fetch_data_planes( + credential: Any, + factory: Callable[[Any, str, str], Any], + builder: "AzureAssetBuilder", + tasks: set[str], + recorder: Any, + *, + max_parents: int = 500, +) -> None: + """Read AI Search and Synapse data-plane objects into ``builder``.""" + if "search_data_plane" in tasks: + run = _PlaneRun("search_data_plane") + for d in builder.drafts_of("microsoft.search/searchservices")[:max_parents]: + if run.skipped: + break + client = factory( + credential, f"https://{d.name.lower()}.search.windows.net", SEARCH_SCOPE + ) + for kind in _SEARCH_KINDS: + items = run.call(lambda k=kind: client.list(f"/{k}", SEARCH_API)) + builder.add_rows(_search_rows(d.id, kind, items), discovered_via="data-plane") + run.record(recorder) + if "synapse_data_plane" in tasks: + run = _PlaneRun("synapse_data_plane") + for d in builder.drafts_of("microsoft.synapse/workspaces")[:max_parents]: + dev = host_of(_get(_get(d.props, "connectivityEndpoints") or {}, "dev")) + if run.skipped or not dev: + continue + client = factory(credential, f"https://{dev}", SYNAPSE_SCOPE) + for kind in _SYNAPSE_KINDS: + items = run.call(lambda k=kind: client.list(f"/{k}", SYNAPSE_API)) + builder.add_rows(_synapse_rows(d.id, kind, items), discovered_via="data-plane") + run.record(recorder) + + +__all__ = ["fetch_data_planes"] diff --git a/cloudg/inventory/azure_graph/families.py b/cloudg/inventory/azure_graph/families.py new file mode 100644 index 0000000..2bc8ffb --- /dev/null +++ b/cloudg/inventory/azure_graph/families.py @@ -0,0 +1,338 @@ +"""Azure collection tasks, their service families, and ``--services`` filtering. + +The Azure inventory is split into named *tasks* the same way the AWS deep +collector is: one per Resource Graph table, per ARM REST child-resource +kind, per subscription-level REST listing, plus Entra ID and in-cluster +AKS mapping. Every task belongs to a service family (the AWS family names +where one fits: ``network``, ``compute``, ``containers``, ``identity``, +...), and ``inventory.services`` / ``inventory.exclude_services`` select +tasks by task or family name exactly like +:func:`cloudg.inventory.aws_deep_tasks.select_tasks`. + +The breadth query (``resources``) always runs, but when a filter is set its +rows are kept only for the selected families (resource groups and the +subscription are always kept), so ``--services network`` maps the network +and nothing else. +""" + +from __future__ import annotations + +import time +from typing import Any, Iterable + +# Task name -> service family. Order is the order tasks run in. +AZURE_TASK_FAMILIES: dict[str, str] = { + # Resource Graph tables + "resources": "sweep", + "resource_containers": "governance", + "role_assignments": "identity", + "management_group_role_assignments": "identity", + "role_definitions": "identity", + "defender_pricings": "security", + "policy": "governance", + "policy_states": "governance", + "security_assessments": "security", + "security_alerts": "security", + "secure_score": "security", + "backup_items": "backup", + "kubernetes_configuration": "containers", + "network_resources": "network", + "guest_configuration": "operations", + "maintenance": "operations", + "insights": "logging", + # Subscription-level ARM REST listings + "deny_assignments": "identity", + "pim_eligibility": "identity", + "lighthouse": "identity", + "locks": "governance", + "deployments": "iac", + "deployment_stacks": "iac", + "activity_log": "logging", + # ARM REST child resources + "managed_identity_credentials": "identity", + "key_vault_items": "identity", + "apim": "serverless", + "functions": "serverless", + "web_apps": "serverless", + "static_sites": "serverless", + "service_bus": "integration", + "event_hubs": "integration", + "stream_analytics": "analytics", + "data_factory": "analytics", + "synapse": "analytics", + "kusto": "analytics", + "sql": "data", + "cosmos": "data", + "storage": "storage", + "dns_records": "dns", + "dns_resolver": "dns", + "virtual_wan": "hybrid", + "network_manager": "network", + "front_door": "network", + "acr": "containers", + "aks": "containers", + "cognitive_services": "ml", + "machine_learning": "ml", + "search": "ml", + "automation": "operations", + "app_configuration": "operations", + "monitor_private_link": "logging", + "batch": "compute", + "spring_apps": "compute", + "data_protection": "backup", + "dev_center": "cicd", + "diagnostic_settings": "logging", + "sentinel": "security", + # Data planes + "search_data_plane": "ml", + "synapse_data_plane": "analytics", + # In-cluster + "aks_kubernetes": "kubernetes", + # Entra ID (Microsoft Graph), collected once per tenant + "entra": "identity", +} + +# Tasks that always run when their source is reachable: they carry the +# subscription, its resource groups and the relations every family uses. +CORE_TASKS = frozenset({"resources", "resource_containers"}) + +# ARM namespace (or full type) -> family of the rows of the breadth query. +_TYPE_FAMILIES: dict[str, str] = { + "microsoft.compute/disks": "storage", + "microsoft.compute/snapshots": "storage", + "microsoft.compute/diskencryptionsets": "identity", + "microsoft.compute/diskaccesses": "storage", + "microsoft.network/dnszones": "dns", + "microsoft.network/privatednszones": "dns", + "microsoft.network/dnsresolvers": "dns", + "microsoft.network/dnsforwardingrulesets": "dns", + "microsoft.network/trafficmanagerprofiles": "dns", + "microsoft.network/virtualwans": "hybrid", + "microsoft.network/virtualhubs": "hybrid", + "microsoft.network/vpngateways": "hybrid", + "microsoft.network/vpnsites": "hybrid", + "microsoft.network/p2svpngateways": "hybrid", + "microsoft.network/expressroutecircuits": "hybrid", + "microsoft.network/expressroutegateways": "hybrid", + "microsoft.network/expressrouteports": "hybrid", + "microsoft.network/localnetworkgateways": "hybrid", + "microsoft.network/connections": "hybrid", + "microsoft.network/virtualnetworkgateways": "hybrid", + "microsoft.network/azurefirewalls": "security", + "microsoft.network/firewallpolicies": "security", + "microsoft.network/ddosprotectionplans": "security", + "microsoft.network/frontdoorwebapplicationfirewallpolicies": "security", + "microsoft.network/applicationgatewaywebapplicationfirewallpolicies": "security", + "microsoft.cdn/cdnwebapplicationfirewallpolicies": "security", + "microsoft.network/networkwatchers": "logging", + "microsoft.network/networkwatchers/flowlogs": "logging", + "microsoft.web/connections": "integration", + "microsoft.web/certificates": "identity", + "microsoft.authorization/policyassignments": "governance", + "microsoft.authorization/policyexemptions": "governance", + "microsoft.authorization/policydefinitions": "governance", + "microsoft.authorization/policysetdefinitions": "governance", + "microsoft.authorization/locks": "governance", + "microsoft.authorization/roleassignments": "identity", + "microsoft.authorization/roledefinitions": "identity", + "microsoft.authorization/denyassignments": "identity", + "microsoft.insights/components": "logging", + "microsoft.operationsmanagement/solutions": "security", +} + +_NAMESPACE_FAMILIES: dict[str, str] = { + "microsoft.compute": "compute", + "microsoft.hybridcompute": "hybrid", + "microsoft.batch": "compute", + "microsoft.appplatform": "compute", + "microsoft.avs": "compute", + "microsoft.network": "network", + "microsoft.cdn": "network", + "microsoft.storage": "storage", + "microsoft.storagesync": "storage", + "microsoft.netapp": "storage", + "microsoft.elasticsan": "storage", + "microsoft.sql": "data", + "microsoft.dbforpostgresql": "data", + "microsoft.dbformysql": "data", + "microsoft.dbformariadb": "data", + "microsoft.documentdb": "data", + "microsoft.cache": "data", + "microsoft.kusto": "analytics", + "microsoft.synapse": "analytics", + "microsoft.databricks": "analytics", + "microsoft.hdinsight": "analytics", + "microsoft.datafactory": "analytics", + "microsoft.streamanalytics": "analytics", + "microsoft.purview": "analytics", + "microsoft.machinelearningservices": "ml", + "microsoft.cognitiveservices": "ml", + "microsoft.search": "ml", + "microsoft.botservice": "ml", + "microsoft.keyvault": "identity", + "microsoft.managedidentity": "identity", + "microsoft.azureactivedirectory": "identity", + "microsoft.managedservices": "identity", + "microsoft.authorization": "governance", + "microsoft.resources": "iac", + "microsoft.solutions": "iac", + "microsoft.blueprint": "governance", + "microsoft.management": "governance", + "microsoft.containerservice": "containers", + "microsoft.containerregistry": "containers", + "microsoft.app": "containers", + "microsoft.containerinstance": "containers", + "microsoft.kubernetes": "containers", + "microsoft.kubernetesconfiguration": "containers", + "microsoft.redhatopenshift": "containers", + "microsoft.web": "serverless", + "microsoft.apimanagement": "serverless", + "microsoft.servicebus": "integration", + "microsoft.eventhub": "integration", + "microsoft.eventgrid": "integration", + "microsoft.logic": "integration", + "microsoft.relay": "integration", + "microsoft.notificationhubs": "integration", + "microsoft.signalrservice": "integration", + "microsoft.devices": "integration", + "microsoft.insights": "logging", + "microsoft.operationalinsights": "logging", + "microsoft.alertsmanagement": "logging", + "microsoft.monitor": "logging", + "microsoft.dashboard": "logging", + "microsoft.automation": "operations", + "microsoft.maintenance": "operations", + "microsoft.appconfiguration": "operations", + "microsoft.guestconfiguration": "operations", + "microsoft.security": "security", + "microsoft.securityinsights": "security", + "microsoft.operationsmanagement": "security", + "microsoft.recoveryservices": "backup", + "microsoft.dataprotection": "backup", + "microsoft.devcenter": "cicd", + "microsoft.devtestlab": "cicd", +} + +_ALWAYS_KEPT = ( + "microsoft.resources/subscriptions", + "microsoft.resources/subscriptions/resourcegroups", + "microsoft.resources/resourcegroups", +) + + +def family_of_task(task: str) -> str: + return AZURE_TASK_FAMILIES.get(task, "other") + + +def family_of_type(arm_type: str | None) -> str: + """Service family of an ARM resource type (``other`` when unknown).""" + rtype = (arm_type or "").lower() + if rtype in _TYPE_FAMILIES: + return _TYPE_FAMILIES[rtype] + for full, fam in _TYPE_FAMILIES.items(): + if rtype.startswith(full + "/"): + return fam + return _NAMESPACE_FAMILIES.get(rtype.split("/", 1)[0], "other") + + +def _wanted(name: str, family: str, inc: set[str], exc: set[str]) -> bool: + if name in exc or family in exc: + return False + if "all" in inc or name in inc or family in inc: + return True + # in-cluster mapping rides on the containers family, like EKS + return name == "aks_kubernetes" and "containers" in inc + + +def select_azure_tasks( + names: Iterable[str] | None = None, + include: list[str] | None = None, + exclude: list[str] | None = None, +) -> list[str]: + """Filter Azure task names by task or family name. + + ``include`` defaults to everything (``["all"]``); ``exclude`` always + wins, except for :data:`CORE_TASKS`, which always run. "kubernetes" is + a pseudo-family that only toggles in-cluster AKS mapping. + """ + inc = {i.lower() for i in (include or ["all"])} + exc = {e.lower() for e in (exclude or [])} + out: list[str] = [] + for name in names if names is not None else AZURE_TASK_FAMILIES: + if name in CORE_TASKS or _wanted(name, family_of_task(name), inc, exc): + out.append(name) + return out + + +def row_filter(include: list[str] | None, exclude: list[str] | None): + """Predicate keeping the breadth rows of selected families, or None for all.""" + inc = {i.lower() for i in (include or ["all"])} + exc = {e.lower() for e in (exclude or [])} + if "all" in inc and not exc: + return None + + def keep(row: dict) -> bool: + rtype = str((row or {}).get("type") or "").lower() + if rtype in _ALWAYS_KEPT: + return True + family = family_of_type(rtype) + return _wanted(rtype, family, inc, exc) + + return keep + + +class TaskRecorder: + """Per-task coverage records (SUCCESS / FAILED / PARTIAL / SKIPPED). + + Args: + coverage: CollectionCoverage receiving the records (None: count only). + """ + + def __init__(self, coverage: Any = None) -> None: + self.coverage = coverage + self.records: dict[str, tuple[str, int, str | None]] = {} + + def record( + self, + task: str, + status: str, + count: int = 0, + error: str | None = None, + started: float | None = None, + ) -> None: + from cloudg.coverage import ServiceStatus + + self.records[task] = (status, count, error) + if self.coverage is not None: + self.coverage.record( + task, + ServiceStatus(status), + asset_count=count, + error=error, + duration_ms=int((time.time() - started) * 1000) if started else None, + ) + + def skip(self, task: str, reason: str) -> None: + self.record(task, "SKIPPED", error=reason) + + def outcome( + self, task: str, count: int, failures: list[str], calls: int, started: float | None + ) -> None: + """SUCCESS, PARTIAL (some calls failed) or FAILED (every call failed).""" + if not failures: + self.record(task, "SUCCESS", count, None, started) + return + status = "FAILED" if calls and len(failures) >= calls and not count else "PARTIAL" + shown = "; ".join(dict.fromkeys(failures))[:500] + self.record(task, status, count, shown, started) + + +__all__ = [ + "AZURE_TASK_FAMILIES", + "CORE_TASKS", + "TaskRecorder", + "family_of_task", + "family_of_type", + "row_filter", + "select_azure_tasks", +] diff --git a/cloudg/inventory/azure_graph/governance.py b/cloudg/inventory/azure_graph/governance.py new file mode 100644 index 0000000..03c4885 --- /dev/null +++ b/cloudg/inventory/azure_graph/governance.py @@ -0,0 +1,731 @@ +"""Extractors for governance: policy, RBAC role definitions, locks, IaC, delegation. + +- Policy assignments / exemptions (``policyresources``) become ``GUARDRAIL`` + assets that ``GOVERNS / COMPLIANCE_GOVERNS`` their scope, the same shape + the tenant hierarchy builds, so both copies dedupe. +- Policy definitions and initiatives become ``ORG_POLICY`` assets. A + definition whose effective effect is ``deny`` (or ``denyAction``) for an + enforced assignment ``GOVERNS / SCP_RESTRICTS`` the assignment scope, the + Azure analogue of an AWS service control policy. +- Non-compliant policy states: assignment ``GOVERNS / FINDING_AFFECTS`` the + resource. +- RBAC role definitions become ``IAM_POLICY`` assets with their actions; + each principal holding one gets ``IAM_POLICY_ATTACHMENT`` with + ``ROLE_HAS_POLICY`` (service principals, managed identities), + ``USER_HAS_POLICY`` or ``GROUP_HAS_POLICY``. +- Deny assignments ``GOVERNS / POLICY_DENIES_ACTION`` their scope and limit + their principals (``PERMISSION_BOUNDARY_LIMITS``). +- Resource locks ``GOVERNS / POLICY_DENIES_ACTION`` the locked scope. +- ARM deployments, deployment stacks and managed applications ``MANAGES / + OWNED_BY`` the resources they created; template specs are what they + deploy (``DEPENDS_ON``). +- Azure Lighthouse delegations: the managing tenant's principals + ``GRANTS_ACCESS / CROSS_ACCOUNT_TRUST`` the delegated scope. +- Galleries shared with other subscriptions or tenants, and the + community gallery flag (``CROSS_ACCOUNT_TRUST``). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + _GUID_TAIL_RE, + _get, + _list, + _lower, + _path, + _props, + _rid, + normalize_scope, + principal_ref, + subscription_ref, +) +from cloudg.inventory.azure_graph.registry import _Draft, extractor, relation_row +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +_DENY_EFFECTS = {"deny", "denyaction"} +# Principal type -> the ontology relation of its role-definition attachment +_ATTACHMENT_RELATIONSHIP = {"user": "USER_HAS_POLICY", "group": "GROUP_HAS_POLICY"} +_MAX_STATE_RELATIONS = 500 +_PRIVILEGED_ACTIONS = ( + "*", + "microsoft.authorization/*", + "microsoft.authorization/*/write", + "microsoft.authorization/roleassignments/write", + "microsoft.authorization/roledefinitions/write", +) + + +def extension_owner(extension_id: Any) -> str | None: + """``/providers///`` -> ```` (``/`` for root).""" + rid = str(extension_id or "") + cut = rid.lower().rfind("/providers/") + if cut < 0: + return None + return normalize_scope(rid[:cut]) if cut > 0 else "/" + + +# ── Azure Policy ── + + +def _effect_of(definition: _Draft | None, params: dict[str, Any]) -> str | None: + """Effective effect of a policy definition under an assignment's parameters.""" + if definition is None: + return None + effect = _path(definition.props, "policyRule", "then", "effect") + if not isinstance(effect, str): + return None + text = effect.strip() + if text.lower().startswith("[parameters("): + name = text.split("'")[1] if "'" in text else "" + value = _get(_get(params, name) or {}, "value") + if value is None: + value = _get( + _get(_get(definition.props, "parameters") or {}, name) or {}, "defaultValue" + ) + return str(value) if value is not None else None + return text + + +def _assignment_effects(b: "AzureAssetBuilder", d: _Draft) -> list[str]: + params = _get(d.props, "parameters") or {} + definition_id = _get(d.props, "policyDefinitionId") + definition = b.draft(definition_id) + if definition is None: + return [] + if definition.type == "microsoft.authorization/policysetdefinitions": + effects = [] + for member in _list(_get(definition.props, "policyDefinitions")): + effect = _effect_of(b.draft(_get(member, "policyDefinitionId")), params) + if effect: + effects.append(effect) + return effects + effect = _effect_of(definition, params) + return [effect] if effect else [] + + +def _policy_restrictions(b: "AzureAssetBuilder") -> None: + """Deny-effect definitions restrict the scopes of their enforced assignments.""" + for d in b.drafts_of("microsoft.authorization/policyassignments"): + effects = _assignment_effects(b, d) + d.md["effects"] = sorted({e for e in effects if e}) + denies = any(_lower(e) in _DENY_EFFECTS for e in effects) + d.md["denies"] = denies + scope = d.md.get("scope") + if not denies or not d.md.get("enforced", True) or not scope or scope == "/": + continue + definition = _get(d.props, "policyDefinitionId") + b.defer( + definition, + rel( + scope, + EdgeType.GOVERNS, + "SCP_RESTRICTS", + description=f"deny policy assigned by {d.name}", + assignment_id=d.id, + not_scopes=d.md.get("not_scopes"), + ), + ) + counts = b.state.get("policy_noncompliant") or {} + for key, n in counts.items(): + assignment = b.draft(key) + if assignment is not None: + assignment.md["non_compliant_resources"] = n + + +@extractor("microsoft.authorization/policyassignments") +def _x_policy_assignment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + scope = normalize_scope(_get(p, "scope")) or extension_owner(d.id) + definition = _get(p, "policyDefinitionId") + enforcement = _get(p, "enforcementMode") or "Default" + not_scopes = [normalize_scope(s) for s in _list(_get(p, "notScopes")) if s] + d.name = _get(p, "displayName") or d.name + d.md.update( + { + "guardrail_kind": "azure-policy-assignment", + "scope": scope, + "policy_definition_id": definition, + "initiative": "/policysetdefinitions/" in _lower(definition), + "enforcement_mode": enforcement, + "enforced": _lower(enforcement) != "donotenforce", + "not_scopes": not_scopes, + "description": _get(p, "description"), + } + ) + if scope and scope != "/": + d.add( + rel( + scope, + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description=f"policy {d.name}", + enforcement_mode=enforcement, + not_scopes=not_scopes, + ) + ) + d.add(rel(definition, EdgeType.REFERENCES, "DEPENDS_ON", description="policy definition")) + b.at_build(_policy_restrictions) + + +@extractor("microsoft.authorization/policyexemptions") +def _x_policy_exemption(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + scope = extension_owner(d.id) + assignment = _get(p, "policyAssignmentId") + category = _get(p, "exemptionCategory") + d.name = _get(p, "displayName") or d.name + d.md.update( + { + "guardrail_kind": "azure-policy-exemption", + "exemption": True, + "scope": scope, + "policy_assignment_id": assignment, + "exemption_category": category, + "expires_on": _get(p, "expiresOn"), + } + ) + if scope: + d.add( + rel( + scope, + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description=f"exemption {d.name}", + exemption=True, + category=category, + ) + ) + d.add( + rel( + assignment.lower() if isinstance(assignment, str) else None, + EdgeType.REFERENCES, + "DEPENDS_ON", + description="exempted assignment", + ) + ) + + +@extractor("microsoft.authorization/policydefinitions") +def _x_policy_definition(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.name = _get(p, "displayName") or d.name + effect = _path(p, "policyRule", "then", "effect") + d.md.update( + { + "policy_kind": "azure-policy-definition", + "policy_type": _get(p, "policyType"), + "mode": _get(p, "mode"), + "category": _path(p, "metadata", "category"), + "effect": effect, + "default_effect": _effect_of(d, {}), + "parameters": sorted((_get(p, "parameters") or {}).keys()), + } + ) + guid = _GUID_TAIL_RE.search(d.id) + if guid: + d.alias(f"azure-policy-definition:{guid.group(1).lower()}") + + +@extractor("microsoft.authorization/policysetdefinitions") +def _x_policy_set(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.name = _get(p, "displayName") or d.name + members = [_get(m, "policyDefinitionId") for m in _list(_get(p, "policyDefinitions"))] + d.md.update( + { + "policy_kind": "azure-policy-initiative", + "policy_type": _get(p, "policyType"), + "category": _path(p, "metadata", "category"), + "member_count": len(members), + } + ) + for member in members[:500]: + d.add(rel(member, EdgeType.REFERENCES, "DEPENDS_ON", description="initiative member")) + + +def add_policy_states(b: "AzureAssetBuilder", rows: list[dict[str, Any]]) -> int: + """Non-compliant policy states: the assignment GOVERNS / FINDING_AFFECTS the resource.""" + counts: dict[str, int] = b.state.setdefault("policy_noncompliant", {}) + linked: dict[str, int] = {} + for row in rows: + assignment = row.get("policyAssignmentId") or _get( + row.get("properties") or {}, "policyAssignmentId" + ) + resource = row.get("resourceId") or _get(row.get("properties") or {}, "resourceId") + if not assignment or not resource: + continue + key = str(assignment).lower() + counts[key] = counts.get(key, 0) + 1 + if linked.get(key, 0) >= _MAX_STATE_RELATIONS: + continue + linked[key] = linked.get(key, 0) + 1 + b.defer( + key, + rel( + resource, + EdgeType.GOVERNS, + "FINDING_AFFECTS", + description="non-compliant resource", + compliance_state=row.get("complianceState") or "NonCompliant", + policy_definition_id=row.get("policyDefinitionId"), + effect=row.get("policyDefinitionAction"), + ), + ) + b.at_build(_policy_restrictions) + return len(rows) + + +# ── RBAC role definitions ── + + +def _role_privileged(actions: list[str]) -> bool: + lowered = {a.lower() for a in actions} + return any(a in lowered for a in _PRIVILEGED_ACTIONS) + + +def _role_definition_asset(guid: str, row: dict[str, Any], subscription_id: str) -> CloudAsset: + props = row.get("properties") or {} + perms = [p for p in _list(_get(props, "permissions")) if isinstance(p, dict)] + actions = [a for p in perms for a in _list(_get(p, "actions")) if isinstance(a, str)] + data_actions = [a for p in perms for a in _list(_get(p, "dataActions")) if isinstance(a, str)] + custom = _lower(_get(props, "type")) == "customrole" + rid = str(row.get("id") or "") + arn = rid if custom else f"/providers/Microsoft.Authorization/roleDefinitions/{guid}" + name = _get(props, "roleName") or guid + return CloudAsset( + arn=arn, + name=name, + asset_type=AssetType.IAM_POLICY, + provider=CloudProvider.AZURE, + region="global", + account_id=subscription_id if custom else None, + metadata={ + "resource_type": "microsoft.authorization/roledefinitions", + "role_definition_id": guid, + "role_name": name, + "role_type": _get(props, "type"), + "custom": custom, + "description": _get(props, "description"), + "actions": actions[:200], + "not_actions": [a for p in perms for a in _list(_get(p, "notActions"))][:200], + "data_actions": data_actions[:200], + "not_data_actions": [a for p in perms for a in _list(_get(p, "notDataActions"))][:200], + "assignable_scopes": _list(_get(props, "assignableScopes"))[:50], + "privileged": _role_privileged(actions), + "aliases": list( + dict.fromkeys( + [ + f"azure-role:{guid}", + f"{subscription_ref(subscription_id)}/providers/" + f"Microsoft.Authorization/roleDefinitions/{guid}", + rid, + ] + ) + ), + }, + ) + + +def _custom_role_in_scope(row: dict[str, Any], subscription_id: str) -> bool: + props = row.get("properties") or {} + if _lower(_get(props, "type")) != "customrole": + return False + sub = subscription_ref(subscription_id).lower() + scopes = [str(s).lower() for s in _list(_get(props, "assignableScopes"))] + return any(s == sub or s.startswith(sub + "/") for s in scopes) + + +def role_definition_assets( + b: "AzureAssetBuilder", definitions: dict[str, dict[str, Any]], role_rows: list[dict] +) -> None: + """IAM_POLICY assets for assigned and in-scope custom roles, plus attachments.""" + used: dict[str, list[tuple[str, Any, str | None]]] = {} + for ra in role_rows: + props = ra.get("properties") or {} + pid = ra.get("principalId") or _get(props, "principalId") + rd = ra.get("roleDefinitionId") or _get(props, "roleDefinitionId") or "" + m = _GUID_TAIL_RE.search(str(rd)) + guid = str(ra.get("roleGuid") or (m.group(1) if m else "")).lower() + if pid and guid in definitions: + ptype = ra.get("principalType") or _get(props, "principalType") + scope = normalize_scope(ra.get("scope") or _get(props, "scope")) + used.setdefault(guid, []).append((pid, ptype, scope)) + attachments: list[tuple[str, Any, dict[str, Any] | None]] = [] + for guid, row in definitions.items(): + if guid not in used and not _custom_role_in_scope(row, b.subscription_id): + continue + asset = _role_definition_asset(guid, row, b.subscription_id) + b.add_extra(asset) + seen: set[str] = set() + for pid, ptype, scope in used.get(guid, []): + if pid.lower() in seen: + continue + seen.add(pid.lower()) + relationship = _ATTACHMENT_RELATIONSHIP.get(_lower(ptype), "ROLE_HAS_POLICY") + attachments.append( + ( + pid, + ptype, + rel( + f"azure-role:{guid}", + EdgeType.IAM_POLICY_ATTACHMENT, + relationship, + description=f"has role {asset.name}", + scope=scope, + ), + ) + ) + b.state["role_attachments"] = attachments + + +# ── PIM eligible role assignments ── + + +@relation_row("microsoft.authorization/roleeligibilityscheduleinstances") +def _eligible_assignment(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + """A PIM eligible assignment: the same grant as an active one, marked eligible.""" + p = row.get("properties") or {} + expanded = _get(p, "expandedProperties") or {} + b.add_role_assignments( + [ + { + "id": row.get("id"), + "roleDefinitionId": _get(p, "roleDefinitionId"), + "roleName": _path(expanded, "roleDefinition", "displayName"), + "principalId": _get(p, "principalId"), + "principalType": _get(p, "principalType"), + "scope": _get(p, "scope"), + "properties": {"condition": _get(p, "condition")}, + } + ], + eligible=True, + ) + + +# ── deny assignments, locks ── + + +@extractor("microsoft.authorization/denyassignments") +def _x_deny_assignment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + scope = normalize_scope(_get(p, "scope")) or extension_owner(d.id) + d.name = _get(p, "denyAssignmentName") or d.name + perms = [x for x in _list(_get(p, "permissions")) if isinstance(x, dict)] + principals = [x for x in _list(_get(p, "principals")) if isinstance(x, dict)] + excluded = [x for x in _list(_get(p, "excludePrincipals")) if isinstance(x, dict)] + everyone = any(str(_get(x, "id")) == "00000000-0000-0000-0000-000000000000" for x in principals) + d.md.update( + { + "guardrail_kind": "azure-deny-assignment", + "scope": scope, + "denied_actions": [a for x in perms for a in _list(_get(x, "actions"))][:200], + "not_actions": [a for x in perms for a in _list(_get(x, "notActions"))][:200], + "applies_to_everyone": everyone, + "excluded_principals": [_get(x, "id") for x in excluded], + "system_protected": _get(p, "isSystemProtected"), + "applies_to_children": _get(p, "doNotApplyToChildScopes") is not True, + } + ) + if scope and scope != "/": + d.add( + rel( + scope, + EdgeType.GOVERNS, + "POLICY_DENIES_ACTION", + description=f"deny assignment {d.name}", + ) + ) + for x in principals: + pid = _get(x, "id") + if not pid or everyone: + continue + b.note_principal(pid, _get(x, "type")) + d.add( + rel( + principal_ref(pid), + EdgeType.REFERENCES, + "PERMISSION_BOUNDARY_LIMITS", + reverse=True, + description="actions denied by deny assignment", + ) + ) + + +@extractor("microsoft.authorization/locks") +def _x_lock(d: _Draft, b: "AzureAssetBuilder") -> None: + level = _get(d.props, "level") + scope = extension_owner(d.id) + d.md.update( + { + "guardrail_kind": "azure-resource-lock", + "lock_level": level, + "scope": scope, + "notes": _get(d.props, "notes"), + "owners": [_get(o, "applicationId") for o in _list(_get(d.props, "owners"))], + } + ) + d.add( + rel( + scope, + EdgeType.GOVERNS, + "POLICY_DENIES_ACTION", + description="delete blocked" if _lower(level) == "cannotdelete" else "read-only lock", + lock_level=level, + ) + ) + + +# ── deployments, stacks, template specs, managed applications ── + + +def _managed_resources(d: _Draft, ids: list[Any], description: str) -> None: + for rid in ids[:500]: + d.add(rel(_rid(rid), EdgeType.MANAGES, "OWNED_BY", description=description)) + + +@extractor("microsoft.resources/deployments") +def _x_deployment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + link = _get(p, "templateLink") or {} + d.md.update( + { + "iac_kind": "arm-deployment", + "provisioning_state": _get(p, "provisioningState"), + "timestamp": _get(p, "timestamp"), + "mode": _get(p, "mode"), + "template_hash": _get(p, "templateHash"), + "template_uri": _get(link, "uri"), + "duration": _get(p, "duration"), + } + ) + _managed_resources(d, _list(_get(p, "outputResources")), "deployed resource") + d.add(rel(_get(link, "id"), EdgeType.REFERENCES, "DEPENDS_ON", description="template spec")) + + +@extractor("microsoft.resources/deploymentstacks") +def _x_deployment_stack(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + deny = _get(p, "denySettings") or {} + mode = _get(deny, "mode") + managed = [_get(r, "id") for r in _list(_get(p, "resources"))] + d.md.update( + { + "iac_kind": "deployment-stack", + "provisioning_state": _get(p, "provisioningState"), + "deny_mode": mode, + "action_on_unmanage": _get(p, "actionOnUnmanage"), + "managed_resource_count": len(managed), + "deployment_scope": _get(p, "deploymentScope"), + } + ) + _managed_resources(d, managed, "stack-managed resource") + if _lower(mode) not in ("", "none"): + for rid in managed[:500]: + if isinstance(rid, str): + d.add( + rel( + rid, + EdgeType.GOVERNS, + "POLICY_DENIES_ACTION", + description=f"deny settings {mode}", + ) + ) + d.add(rel(_get(p, "deploymentId"), EdgeType.REFERENCES, "DEPENDS_ON")) + d.add(rel(_path(p, "templateLink", "id"), EdgeType.REFERENCES, "DEPENDS_ON")) + + +@extractor("microsoft.resources/templatespecs", "microsoft.resources/templatespecs/versions") +def _x_template_spec(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["iac_kind"] = "template-spec" + d.md["description"] = _get(d.props, "description") + + +@extractor("microsoft.solutions/applications") +def _x_managed_application(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + publisher = _get(p, "publisherTenantId") + d.md.update( + { + "managed_resource_group": _get(p, "managedResourceGroupId"), + "publisher_tenant_id": publisher, + "application_definition_id": _get(p, "applicationDefinitionId"), + } + ) + d.add( + rel( + _get(p, "managedResourceGroupId"), + EdgeType.MANAGES, + "OWNED_BY", + description="managed resource group", + ) + ) + d.add(rel(_get(p, "applicationDefinitionId"), EdgeType.REFERENCES, "DEPENDS_ON")) + for auth in _list(_get(p, "authorizations")) + _list( + _path(p, "jitAccessPolicy", "jitApprovers") + ): + pid = _get(auth, "principalId") or _get(auth, "id") + if not pid: + continue + b.note_principal(pid, None) + d.add( + rel( + principal_ref(pid), + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="managed application publisher access", + role_definition_id=_get(auth, "roleDefinitionId"), + publisher_tenant_id=publisher, + ) + ) + + +@extractor("microsoft.solutions/applicationdefinitions") +def _x_managed_app_definition(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["lock_level"] = _get(p, "lockLevel") + for auth in _list(_get(p, "authorizations")): + pid = _get(auth, "principalId") + if pid: + b.note_principal(pid, None) + d.add( + rel( + principal_ref(pid), + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="publisher authorization on managed resource groups", + role_definition_id=_get(auth, "roleDefinitionId"), + ) + ) + + +# ── Azure Lighthouse ── + + +@extractor("microsoft.managedservices/registrationassignments") +def _x_lighthouse_assignment(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + definition = _get(p, "registrationDefinition") or {} + dp = _props(definition) + scope = extension_owner(d.id) + tenant = _get(dp, "managedByTenantId") + d.name = _get(dp, "registrationDefinitionName") or d.name + authorizations = [a for a in _list(_get(dp, "authorizations")) if isinstance(a, dict)] + eligible = [a for a in _list(_get(dp, "eligibleAuthorizations")) if isinstance(a, dict)] + d.md.update( + { + "delegation_kind": "azure-lighthouse", + "managed_by_tenant_id": tenant, + "managed_by_tenant_name": _get(dp, "managedByTenantName"), + "scope": scope, + "registration_definition_id": _get(p, "registrationDefinitionId"), + "authorization_count": len(authorizations), + "eligible_authorization_count": len(eligible), + } + ) + d.add( + rel( + scope, + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + description=f"delegated to tenant {tenant}", + managed_by_tenant_id=tenant, + ) + ) + for auth, is_eligible in [(a, False) for a in authorizations] + [(a, True) for a in eligible]: + pid = _get(auth, "principalId") + if not pid: + continue + b.note_principal(pid, None) + d.add( + rel( + principal_ref(pid), + EdgeType.ASSUMES_ROLE, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description=_get(auth, "principalIdDisplayName") or "Lighthouse principal", + role_definition_id=_get(auth, "roleDefinitionId"), + managed_by_tenant_id=tenant, + eligible=is_eligible or None, + ) + ) + + +# ── galleries ── + + +@extractor("microsoft.compute/galleries") +def _x_gallery(d: _Draft, b: "AzureAssetBuilder") -> None: + sharing = _get(d.props, "sharingProfile") or {} + permission = _get(sharing, "permissions") + d.md["sharing"] = permission + d.md["community"] = _get(sharing, "communityGalleryInfo") is not None + if _lower(permission) == "community": + d.exposed = True + for group in _list(_get(sharing, "groups")): + kind = _lower(_get(group, "type")) + for gid in _list(_get(group, "ids")): + target = subscription_ref(gid) if kind == "subscriptions" else f"entra:tenant/{gid}" + d.add( + rel( + target, + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description=f"gallery shared with {kind}", + ) + ) + + +@extractor("microsoft.compute/galleries/images/versions") +def _x_gallery_image_version(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + source = _path(p, "storageProfile", "source", "id") or _path( + p, "storageProfile", "osDiskImage", "source", "id" + ) + regions = [_get(r, "name") for r in _list(_path(p, "publishingProfile", "targetRegions"))] + d.md["target_regions"] = [r for r in regions if r] + d.add(rel(source, EdgeType.REFERENCES, "DEPENDS_ON", description="image source")) + + +@extractor("microsoft.compute/snapshots", "microsoft.compute/disks") +def _x_disk_sharing(d: _Draft, b: "AzureAssetBuilder") -> None: + """Snapshot source (the disk BACKUP_TO its snapshot) and disk / snapshot exposure.""" + p = d.props + policy = _get(p, "networkAccessPolicy") + d.md["network_access_policy"] = policy + d.md["public_network_access"] = _get(p, "publicNetworkAccess") + d.md["max_shares"] = _get(p, "maxShares") + d.add( + rel(_get(p, "diskAccessId"), EdgeType.REFERENCES, "DEPENDS_ON", description="disk access") + ) + # AllowAll: anyone holding a SAS export URL can download the data + d.md["export_open"] = ( + _lower(policy) == "allowall" + and _lower(_get(p, "publicNetworkAccess") or "enabled") != "disabled" + ) + if d.type == "microsoft.compute/snapshots": + d.md["incremental"] = _get(p, "incremental") + d.add( + rel( + _path(p, "creationData", "sourceResourceId"), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="snapshot of", + ) + ) + + +__all__ = [ + "add_policy_states", + "extension_owner", + "role_definition_assets", +] diff --git a/cloudg/inventory/azure_graph/helpers.py b/cloudg/inventory/azure_graph/helpers.py index 4fd2d9b..1c3ac6a 100644 --- a/cloudg/inventory/azure_graph/helpers.py +++ b/cloudg/inventory/azure_graph/helpers.py @@ -154,6 +154,22 @@ def host_of(url: Any) -> str | None: return host.lower().rstrip(".") if host else None +def issuer_kind(issuer: Any) -> str: + """What issues a federated credential's tokens: ``aks``, ``github`` or ``external``. + + Decided on the issuer URL's parsed hostname, never on a substring of the + URL, so ``https://evil.example/oidc.prod-aks.azure.com`` is external. AKS + issuers are ``https://.oic.prod-aks.azure.com///`` + (and the older ``https://oidc.prod-aks.azure.com//``). + """ + host = host_of(issuer) or "" + if host == "oidc.prod-aks.azure.com" or host.endswith(".oic.prod-aks.azure.com"): + return "aks" + if host == "token.actions.githubusercontent.com": + return "github" + return "external" + + def registry_host(image: Any) -> str | None: """Registry host of a container image reference (None for Docker Hub).""" if not isinstance(image, str) or "/" not in image: diff --git a/cloudg/inventory/azure_graph/hybrid.py b/cloudg/inventory/azure_graph/hybrid.py new file mode 100644 index 0000000..446a3bb --- /dev/null +++ b/cloudg/inventory/azure_graph/hybrid.py @@ -0,0 +1,866 @@ +"""Extractors for hybrid and edge networking, DNS, WAF and Virtual Network Manager. + +- Virtual WAN, virtual hubs, hub VNet connections, hub route tables and + routing intent (``TRANSIT_ROUTED`` to VNets, firewalls and NVAs), + ExpressRoute circuits / gateways / ports, vWAN VPN gateways and sites, + point-to-site gateways, local network gateways, Route Server BGP peers. +- IP groups and public IP prefixes. +- Public and private DNS record sets (``DNS_RESOLVED`` to alias targets, + IPs and CNAME hosts), DNS Private Resolver endpoints, forwarding + rulesets, rules and VNet links, Traffic Manager endpoints. +- WAF policies (mode, managed rule sets, the gateways / listeners / Front + Door endpoints they protect), Front Door (Standard / Premium) routes, + origin groups, rule sets and rules, secrets and custom domain TLS. +- Application Gateway listeners as ``LB_LISTENER`` assets serving traffic + to the backends their routing rules reach. +- Private Link service visibility and auto-approval (``GRANTS_ACCESS``). +- Virtual Network Manager: network groups and members, connectivity + configurations (hub-and-spoke / mesh ``TRANSIT_ROUTED``), security admin + rule collections (``PROTECTS / PROTECTED_BY_NACL`` the groups). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + _get, + _list, + _lower, + _path, + _props, + _rid, + _rids, + host_of, + owner_resource_id, + parent_resource_id, + subscription_ref, +) +from cloudg.inventory.azure_graph.registry import _Draft, extractor, relation_row +from cloudg.schema.models import EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + + +def _contained_by(d: _Draft, parent: Any, description: str | None = None) -> None: + d.add(rel(_rid(parent), EdgeType.CONTAINS, reverse=True, description=description)) + + +def _transit(d: _Draft, target: Any, description: str, **props: Any) -> None: + d.add(rel(target, EdgeType.ROUTE, "TRANSIT_ROUTED", description=description, **props)) + + +# ── Virtual WAN ── + + +@extractor("microsoft.network/virtualwans") +def _x_vwan(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "wan_type": _get(p, "type"), + "branch_to_branch": _get(p, "allowBranchToBranchTraffic"), + "hub_count": len(_list(_get(p, "virtualHubs"))), + } + ) + for hub in _rids(_get(p, "virtualHubs")): + d.add(rel(hub, EdgeType.CONTAINS, description="virtual hub")) + + +@extractor("microsoft.network/virtualhubs") +def _x_virtual_hub(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + wan = _rid(_get(p, "virtualWan")) + route_server = not wan and bool(_get(p, "virtualRouterIps")) + d.md.update( + { + "address_prefix": _get(p, "addressPrefix"), + "route_server": route_server, + "routing_state": _get(p, "routingState"), + "hub_routing_preference": _get(p, "hubRoutingPreference"), + "virtual_router_ips": _list(_get(p, "virtualRouterIps")), + "virtual_router_asn": _get(p, "virtualRouterAsn"), + } + ) + d.alias(*_list(_get(p, "virtualRouterIps"))) + _contained_by(d, wan, "virtual WAN") + for key in ("azureFirewall", "vpnGateway", "expressRouteGateway", "p2SVpnGateway"): + d.add(rel(_rid(_get(p, key)), EdgeType.CONTAINS, description=key)) + for conn in _list(_get(p, "virtualNetworkConnections")): # legacy embedded connections + _transit(d, _rid(_get(_props(conn), "remoteVirtualNetwork")), "hub VNet connection") + d.add(rel(_rid(_get(p, "routeTable")), EdgeType.ATTACHED_TO, description="hub route table")) + + +@extractor("microsoft.network/virtualhubs/hubvirtualnetworkconnections") +def _x_hub_connection(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + vnet = _rid(_get(p, "remoteVirtualNetwork")) + routing = _get(p, "routingConfiguration") or {} + d.md.update( + { + "remote_virtual_network": vnet, + "internet_security": _get(p, "enableInternetSecurity"), + "allow_hub_to_remote_vnet_transit": _get(p, "allowHubToRemoteVnetTransit"), + } + ) + _transit(d, vnet, "hub VNet connection") + d.add( + rel( + _rid(_get(routing, "associatedRouteTable")), + EdgeType.ATTACHED_TO, + description="associated route table", + ) + ) + for rt in _rids(_path(routing, "propagatedRouteTables", "ids")): + d.add(rel(rt, EdgeType.REFERENCES, "DEPENDS_ON", description="propagates to")) + hub = parent_resource_id(d.id) + if hub and vnet: + b.defer(hub, rel(vnet, EdgeType.ROUTE, "TRANSIT_ROUTED", description="hub VNet connection")) + + +def _route_targets(d: _Draft, routes: Any) -> list[dict[str, Any]]: + out = [] + for route in _list(routes): + rp = _props(route) + next_hop = _get(rp, "nextHop") + entry = { + "name": _get(route, "name"), + "destinations": _list(_get(rp, "destinations")), + "destination_type": _get(rp, "destinationType"), + "next_hop_type": _get(rp, "nextHopType"), + "next_hop": next_hop, + } + out.append(entry) + _transit( + d, + next_hop if isinstance(next_hop, str) else None, + f"route {','.join(map(str, entry['destinations']))} via next hop", + destinations=entry["destinations"], + ) + return out + + +@extractor("microsoft.network/virtualhubs/hubroutetables") +def _x_hub_route_table(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["routes"] = _route_targets(d, _get(p, "routes")) + d.md["labels"] = _list(_get(p, "labels")) + for conn in _list(_get(p, "associatedConnections")): + d.add(rel(conn, EdgeType.ATTACHED_TO, description="associated connection")) + + +@extractor("microsoft.network/virtualhubs/routingintent") +def _x_routing_intent(d: _Draft, b: "AzureAssetBuilder") -> None: + policies = [] + hub = parent_resource_id(d.id) + for policy in _list(_get(d.props, "routingPolicies")): + next_hop = _get(policy, "nextHop") + destinations = _list(_get(policy, "destinations")) + policies.append({"name": _get(policy, "name"), "destinations": destinations}) + _transit(d, next_hop, f"routing intent {destinations}", destinations=destinations) + if hub and isinstance(next_hop, str): + b.defer( + hub, + rel( + next_hop, + EdgeType.ROUTE, + "TRANSIT_ROUTED", + description="routing intent next hop", + destinations=destinations, + ), + ) + d.md["routing_policies"] = policies + d.md["inspects_internet"] = any("Internet" in p["destinations"] for p in policies) + d.md["inspects_private"] = any("PrivateTraffic" in p["destinations"] for p in policies) + + +@extractor("microsoft.network/virtualhubs/bgpconnections") +def _x_bgp_connection(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + peer = _get(p, "peerIp") + d.md.update( + {"peer_ip": peer, "peer_asn": _get(p, "peerAsn"), "state": _get(p, "connectionState")} + ) + _transit(d, peer, "BGP peer", peer_asn=_get(p, "peerAsn")) + d.add(rel(_rid(_get(p, "hubVirtualNetworkConnection")), EdgeType.REFERENCES, "DEPENDS_ON")) + + +@extractor("microsoft.network/virtualhubs/ipconfigurations") +def _x_hub_ip_configuration(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.add(rel(_rid(_get(p, "subnet")), EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + d.add(rel(_rid(_get(p, "publicIPAddress")), EdgeType.ATTACHED_TO, reverse=True)) + d.alias(_get(p, "privateIPAddress")) + + +# ── ExpressRoute, VPN ── + + +@extractor("microsoft.network/expressroutecircuits") +def _x_er_circuit(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + provider = _get(p, "serviceProviderProperties") or {} + peerings = [] + for peering in _list(_get(p, "peerings")): + pp = _props(peering) + peerings.append( + { + "name": _get(peering, "name"), + "type": _get(pp, "peeringType"), + "peer_asn": _get(pp, "peerASN"), + "vlan": _get(pp, "vlanId"), + "state": _get(pp, "state"), + } + ) + for conn in _list(_get(pp, "connections")): # Global Reach + other = _rid(_get(_props(conn), "peerExpressRouteCircuitPeering")) + _transit(d, owner_resource_id(other), "Global Reach") + d.md.update( + { + "provider": _get(provider, "serviceProviderName"), + "peering_location": _get(provider, "peeringLocation"), + "bandwidth_mbps": _get(provider, "bandwidthInMbps"), + "peerings": peerings, + "provider_state": _get(p, "serviceProviderProvisioningState"), + "global_reach": _get(p, "globalReachEnabled"), + "authorization_count": len(_list(_get(p, "authorizations"))), + } + ) + d.add(rel(_rid(_get(p, "expressRoutePort")), EdgeType.ATTACHED_TO, reverse=True)) + + +@extractor("microsoft.network/expressroutegateways") +def _x_er_gateway(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + _contained_by(d, _get(p, "virtualHub"), "virtual hub") + for conn in _list(_get(p, "expressRouteConnections")): + peering = _rid(_get(_props(conn), "expressRouteCircuitPeering")) + _transit(d, owner_resource_id(peering), "ExpressRoute connection") + + +@extractor("microsoft.network/vpngateways", "microsoft.network/p2svpngateways") +def _x_vwan_vpn_gateway(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + _contained_by(d, _get(p, "virtualHub"), "virtual hub") + for conn in _list(_get(p, "connections")): + _transit(d, _rid(_get(_props(conn), "remoteVpnSite")), "site-to-site connection") + for cfg in _list(_get(p, "ipConfigurations")): + d.alias(_get(cfg, "publicIpAddress"), _get(cfg, "privateIpAddress")) + d.add(rel(_rid(_get(p, "vpnServerConfiguration")), EdgeType.REFERENCES, "DEPENDS_ON")) + if _get(p, "p2SConnectionConfigurations"): + d.exposed = True + d.md["bgp_asn"] = _path(p, "bgpSettings", "asn") + + +@extractor("microsoft.network/vpnsites") +def _x_vpn_site(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.alias(_get(p, "ipAddress")) + for link in _list(_get(p, "vpnSiteLinks")): + lp = _props(link) + d.alias(_get(lp, "ipAddress"), host_of(_get(lp, "fqdn"))) + d.md["address_space"] = _list(_path(p, "addressSpace", "addressPrefixes")) + d.md["device_vendor"] = _path(p, "deviceProperties", "deviceVendor") + _contained_by(d, _get(p, "virtualWan"), "virtual WAN") + + +@extractor("microsoft.network/localnetworkgateways") +def _x_local_gateway(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.alias(_get(p, "gatewayIpAddress"), host_of(_get(p, "fqdn"))) + d.md.update( + { + "gateway_ip": _get(p, "gatewayIpAddress"), + "fqdn": _get(p, "fqdn"), + "address_space": _list(_path(p, "localNetworkAddressSpace", "addressPrefixes")), + "bgp_asn": _path(p, "bgpSettings", "asn"), + "bgp_peering_address": _path(p, "bgpSettings", "bgpPeeringAddress"), + } + ) + + +@extractor("microsoft.network/virtualnetworkgateways") +def _x_vnet_gateway_detail(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + client = _get(p, "vpnClientConfiguration") or {} + d.md.update( + { + "gateway_type": _get(p, "gatewayType"), + "vpn_type": _get(p, "vpnType"), + "active_active": _get(p, "activeActive"), + "bgp_enabled": _get(p, "enableBgp"), + "bgp_asn": _path(p, "bgpSettings", "asn"), + "point_to_site": bool(client), + "vpn_client_protocols": _list(_get(client, "vpnClientProtocols")), + "vpn_client_aad_tenant": _get(client, "aadTenant"), + } + ) + + +# ── IP groups, prefixes ── + + +@extractor("microsoft.network/ipgroups") +def _x_ip_group(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["ip_addresses"] = _list(_get(d.props, "ipAddresses"))[:500] + for user in _rids(_get(d.props, "firewalls")) + _rids(_get(d.props, "firewallPolicies")): + d.add(rel(user, EdgeType.REFERENCES, "DEPENDS_ON", reverse=True, description="IP group")) + + +@extractor("microsoft.network/publicipprefixes") +def _x_pip_prefix(d: _Draft, b: "AzureAssetBuilder") -> None: + d.exposed = True + d.md["ip_prefix"] = _get(d.props, "ipPrefix") + d.md["prefix_length"] = _get(d.props, "prefixLength") + for pip in _rids(_get(d.props, "publicIPAddresses")): + d.add(rel(pip, EdgeType.CONTAINS, description="address from prefix")) + d.add(rel(_rid(_get(d.props, "natGateway")), EdgeType.ATTACHED_TO)) + + +# ── DNS ── + +DNS_RECORD_TYPES = ("a", "aaaa", "cname", "mx", "ns", "txt", "srv", "caa", "ptr", "soa", "ds") +DNS_RECORD_ARM_TYPES = tuple( + f"microsoft.network/{zone}/{rec}" + for zone in ("dnszones", "privatednszones") + for rec in DNS_RECORD_TYPES +) + + +@extractor("microsoft.network/dnszones", "microsoft.network/privatednszones") +def _x_dns_zone(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + private = d.type.endswith("privatednszones") + d.md.update( + { + "zone_name": d.name, + "private": private, + "record_set_count": _get(p, "numberOfRecordSets"), + "name_servers": _list(_get(p, "nameServers")), + "vnet_link_count": _get(p, "numberOfVirtualNetworkLinks"), + } + ) + for vnet in _rids(_get(p, "registrationVirtualNetworks")) + _rids( + _get(p, "resolutionVirtualNetworks") + ): + d.add(rel(vnet, EdgeType.REFERENCES, "DNS_RESOLVED")) + + +@extractor(*DNS_RECORD_ARM_TYPES) +def _x_dns_record(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + rtype = d.type.rsplit("/", 1)[-1].upper() + fqdn = str(_get(p, "fqdn") or "").rstrip(".").lower() or None + values: list[str] = [] + for rec in _list(_get(p, "ARecords")) + _list(_get(p, "aRecords")): + values.append(_get(rec, "ipv4Address")) + for rec in _list(_get(p, "AAAARecords")) + _list(_get(p, "aaaaRecords")): + values.append(_get(rec, "ipv6Address")) + cname = _get(_get(p, "CNAMERecord") or _get(p, "cnameRecord") or {}, "cname") + if cname: + values.append(str(cname).rstrip(".").lower()) + for rec in _list(_get(p, "MXRecords")) + _list(_get(p, "mxRecords")): + values.append(str(_get(rec, "exchange") or "").rstrip(".").lower()) + for rec in _list(_get(p, "NSRecords")) + _list(_get(p, "nsRecords")): + values.append(str(_get(rec, "nsdname") or "").rstrip(".").lower()) + values = [v for v in dict.fromkeys(values) if v] + if fqdn: + d.name = fqdn + target = _rid(_get(p, "targetResource")) + d.md.update( + { + "record_type": rtype, + "fqdn": fqdn, + "ttl": _get(p, "TTL") or _get(p, "ttl"), + "values": values[:50], + "alias_target": target, + "private_zone": "/privatednszones/" in d.id.lower(), + } + ) + d.add(rel(target, EdgeType.ROUTE, "DNS_RESOLVED", description="alias record")) + if rtype in ("A", "AAAA", "CNAME"): + for value in values: + d.add(rel(value, EdgeType.ROUTE, "DNS_RESOLVED", description=f"{rtype} record")) + if rtype == "SOA" or (rtype == "NS" and str(d.row.get("name")) == "@"): + d.md["zone_apex"] = True + + +@extractor("microsoft.network/dnsresolvers") +def _x_dns_resolver(d: _Draft, b: "AzureAssetBuilder") -> None: + d.add(rel(_rid(_get(d.props, "virtualNetwork")), EdgeType.REFERENCES, "DNS_RESOLVED")) + + +@extractor( + "microsoft.network/dnsresolvers/inboundendpoints", + "microsoft.network/dnsresolvers/outboundendpoints", +) +def _x_dns_resolver_endpoint(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["direction"] = "inbound" if d.type.endswith("inboundendpoints") else "outbound" + subnets = [_rid(_get(c, "subnet")) for c in _list(_get(p, "ipConfigurations"))] + subnets.append(_rid(_get(p, "subnet"))) + for subnet in subnets: + d.add(rel(subnet, EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + for cfg in _list(_get(p, "ipConfigurations")): + d.alias(_get(cfg, "privateIpAddress")) + + +@extractor("microsoft.network/dnsforwardingrulesets") +def _x_forwarding_ruleset(d: _Draft, b: "AzureAssetBuilder") -> None: + for ep in _rids(_get(d.props, "dnsResolverOutboundEndpoints")): + d.add(rel(ep, EdgeType.REFERENCES, "DEPENDS_ON", description="outbound endpoint")) + + +@extractor("microsoft.network/dnsforwardingrulesets/forwardingrules") +def _x_forwarding_rule(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + targets = [_get(t, "ipAddress") for t in _list(_get(p, "targetDnsServers"))] + d.md.update( + { + "domain_name": _get(p, "domainName"), + "state": _get(p, "forwardingRuleState"), + "target_dns_servers": [t for t in targets if t], + } + ) + for ip in targets: + d.add( + rel(ip, EdgeType.ROUTE, "DNS_RESOLVED", description=f"forward {_get(p, 'domainName')}") + ) + + +@relation_row("microsoft.network/dnsforwardingrulesets/virtualnetworklinks") +def _ruleset_vnet_link(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + vnet = _rid(_get(row.get("properties") or {}, "virtualNetwork")) + b.defer( + parent_resource_id(row.get("id")), + rel(vnet, EdgeType.REFERENCES, "DNS_RESOLVED", description="ruleset VNet link"), + ) + + +@extractor("microsoft.network/trafficmanagerprofiles") +def _x_traffic_manager(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + fqdn = _path(p, "dnsConfig", "fqdn") + d.alias(fqdn.lower() if isinstance(fqdn, str) else None) + d.exposed = _lower(_get(p, "profileStatus") or "enabled") == "enabled" + endpoints = [] + for ep in _list(_get(p, "endpoints")): + ep_props = _props(ep) + target = _get(ep_props, "targetResourceId") or ( + str(_get(ep_props, "target") or "").lower() or None + ) + endpoints.append({"name": _get(ep, "name"), "target": target}) + d.add( + rel( + target, + EdgeType.LOAD_BALANCER_TARGET, + "SERVES_TRAFFIC_TO", + description=f"{_get(ep, 'type') or 'endpoint'}", + status=_get(ep_props, "endpointStatus"), + ) + ) + d.md.update( + { + "fqdn": fqdn, + "routing_method": _get(p, "trafficRoutingMethod"), + "endpoints": endpoints, + } + ) + + +# ── WAF policies ── + + +def _waf_settings(d: _Draft) -> None: + p = d.props + settings = _get(p, "policySettings") or {} + managed = _list(_path(p, "managedRules", "managedRuleSets")) + custom = _path(p, "customRules", "rules") or _get(p, "customRules") + mode = _get(settings, "mode") + d.md.update( + { + "waf_mode": mode, + "waf_enabled": _lower(_get(settings, "enabledState") or _get(settings, "state")) + != "disabled", + "detection_only": _lower(mode) == "detection", + "managed_rule_sets": [ + f"{_get(m, 'ruleSetType')} {_get(m, 'ruleSetVersion')}".strip() for m in managed + ], + "custom_rule_count": len(_list(custom)) if isinstance(custom, list) else None, + } + ) + + +@extractor( + "microsoft.network/frontdoorwebapplicationfirewallpolicies", + "microsoft.network/applicationgatewaywebapplicationfirewallpolicies", + "microsoft.cdn/cdnwebapplicationfirewallpolicies", +) +def _x_waf_policy(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + _waf_settings(d) + protected = _rids(_get(p, "applicationGateways")) + _rids(_get(p, "httpListeners")) + protected += [owner_resource_id(x) for x in _rids(_get(p, "pathBasedRules"))] + protected += [owner_resource_id(x) for x in _rids(_get(p, "frontendEndpointLinks"))] + protected += [owner_resource_id(x) for x in _rids(_get(p, "endpointLinks"))] + for target in dict.fromkeys(t for t in protected if t): + d.add(rel(target, EdgeType.PROTECTS, "PROTECTED_BY_WAF", description="WAF policy")) + for link in _rids(_get(p, "securityPolicyLinks")): + d.add(rel(link, EdgeType.REFERENCES, "DEPENDS_ON", reverse=True)) + + +# ── Front Door Standard / Premium children ── + + +@extractor("microsoft.cdn/profiles") +def _x_cdn_profile(d: _Draft, b: "AzureAssetBuilder") -> None: + sku = d.row.get("sku") + d.md["sku"] = _get(sku, "name") if isinstance(sku, dict) else sku + d.alias(_get(d.props, "frontDoorId")) + + +@extractor("microsoft.cdn/profiles/afdendpoints/routes") +def _x_afd_route(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + group = _rid(_get(p, "originGroup")) + d.md.update( + { + "patterns": _list(_get(p, "patternsToMatch")), + "forwarding_protocol": _get(p, "forwardingProtocol"), + "https_redirect": _get(p, "httpsRedirect"), + "supported_protocols": _list(_get(p, "supportedProtocols")), + "link_to_default_domain": _get(p, "linkToDefaultDomain"), + "caching": bool(_get(p, "cacheConfiguration")), + } + ) + d.add( + rel(group, EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO", description="origin group") + ) + for domain in _rids(_get(p, "customDomains")): + d.add(rel(domain, EdgeType.REFERENCES, "DEPENDS_ON", description="custom domain")) + for rs in _rids(_get(p, "ruleSets")): + d.add(rel(rs, EdgeType.REFERENCES, "DEPENDS_ON", description="rule set")) + endpoint = parent_resource_id(d.id) + if endpoint and group: + b.defer( + endpoint, + rel(group, EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO", description="route"), + ) + + +@extractor("microsoft.cdn/profiles/origingroups") +def _x_afd_origin_group(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["health_probe"] = _get(p, "healthProbeSettings") + d.md["session_affinity"] = _get(p, "sessionAffinityState") + + +@extractor("microsoft.cdn/profiles/rulesets/rules") +def _x_afd_rule(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + actions = [] + for action in _list(_get(p, "actions")): + name = _get(action, "name") + actions.append(name) + params = _get(action, "parameters") or {} + group = _rid(_path(params, "originGroupOverride", "originGroup")) + d.add(rel(group, EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO", description=name)) + d.md["actions"] = actions + d.md["conditions"] = [_get(c, "name") for c in _list(_get(p, "conditions"))] + d.md["order"] = _get(p, "order") + + +@extractor("microsoft.cdn/profiles/secrets") +def _x_afd_secret(d: _Draft, b: "AzureAssetBuilder") -> None: + source = _path(d.props, "parameters", "secretSource", "id") + d.md["secret_type"] = _path(d.props, "parameters", "type") + vault = None + if isinstance(source, str) and "/providers/microsoft.keyvault/vaults/" in source.lower(): + vault = owner_resource_id(source) + d.add( + rel( + vault, + EdgeType.REFERENCES, + "CERTIFICATE_SECURES", + description="certificate from Key Vault", + key_vault_secret=source, + ) + ) + + +@extractor("microsoft.cdn/profiles/customdomains") +def _x_afd_custom_domain_tls(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + tls = _get(p, "tlsSettings") or {} + d.md["certificate_type"] = _get(tls, "certificateType") + d.md["minimum_tls_version"] = _get(tls, "minimumTlsVersion") + d.md["validation_state"] = _get(p, "domainValidationState") + d.add(rel(_rid(_get(tls, "secret")), EdgeType.REFERENCES, "CERTIFICATE_SECURES", reverse=True)) + d.add(rel(_rid(_get(p, "azureDnsZone")), EdgeType.REFERENCES, "DNS_RESOLVED")) + + +# ── Application Gateway listeners ── + + +def _appgw_pool_targets(p: dict[str, Any]) -> dict[str, list[tuple[str, str]]]: + """Backend pool id (lowercase) -> [(target, relationship)].""" + pools: dict[str, list[tuple[str, str]]] = {} + for pool in _list(_get(p, "backendAddressPools")): + pid = _rid(pool) + if not pid: + continue + pp = _props(pool) + targets: list[tuple[str, str]] = [] + for ipc in _rids(_get(pp, "backendIPConfigurations")): + owner = owner_resource_id(ipc) + if owner: + targets.append((owner, "LB_TARGETS_INSTANCE")) + for addr in _list(_get(pp, "backendAddresses")): + target = _get(addr, "fqdn") or _get(addr, "ipAddress") + if target: + targets.append((str(target).lower(), "SERVES_TRAFFIC_TO")) + pools[pid.lower()] = targets + return pools + + +def _listener_pools(p: dict[str, Any]) -> dict[str, list[str]]: + """Listener id (lowercase) -> backend pool ids reached by its routing rules.""" + path_maps: dict[str, list[str]] = {} + for upm in _list(_get(p, "urlPathMaps")): + up = _props(upm) + pools = [_rid(_get(up, "defaultBackendAddressPool"))] + pools += [_rid(_get(_props(r), "backendAddressPool")) for r in _list(_get(up, "pathRules"))] + path_maps[str(_rid(upm)).lower()] = [x for x in pools if x] + out: dict[str, list[str]] = {} + for rule in _list(_get(p, "requestRoutingRules")): + rp = _props(rule) + listener = _rid(_get(rp, "httpListener")) + if not listener: + continue + pools = [_rid(_get(rp, "backendAddressPool"))] + pools += path_maps.get(str(_rid(_get(rp, "urlPathMap"))).lower(), []) + out.setdefault(listener.lower(), []).extend(x for x in pools if x) + return out + + +def _public_frontends(p: dict[str, Any]) -> set[str]: + return { + str(_rid(fe)).lower() + for fe in _list(_get(p, "frontendIPConfigurations")) + if _rid(_get(_props(fe), "publicIPAddress")) + } + + +@extractor("microsoft.network/applicationgateways") +def _x_appgw_listeners(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + pools = _appgw_pool_targets(p) + reached = _listener_pools(p) + public = _public_frontends(p) + ports = { + str(_rid(fp)).lower(): _get(_props(fp), "port") for fp in _list(_get(p, "frontendPorts")) + } + certs = { + str(_rid(c)).lower(): _get(_props(c), "keyVaultSecretId") + for c in _list(_get(p, "sslCertificates")) + } + listeners = [] + for listener in _list(_get(p, "httpListeners")): + lid = _rid(listener) + if not lid: + continue + lp = _props(listener) + listeners.append(_get(listener, "name")) + targets: list[tuple[str, str]] = [] + for pool in reached.get(lid.lower(), []): + targets.extend(pools.get(pool.lower(), [])) + b.add_row( + { + "id": lid, + "name": _get(listener, "name") or lid.rsplit("/", 1)[-1], + "type": "microsoft.network/applicationgateways/httplisteners", + "location": d.region, + "subscriptionId": d.account_id, + "properties": { + **lp, + "_frontendPort": ports.get(str(_rid(_get(lp, "frontendPort"))).lower()), + "_public": str(_rid(_get(lp, "frontendIPConfiguration"))).lower() in public, + "_targets": [list(t) for t in dict.fromkeys(targets)], + "_certificate": certs.get(str(_rid(_get(lp, "sslCertificate"))).lower()), + }, + }, + synthetic=True, + ) + d.md["listeners"] = listeners + + +@extractor("microsoft.network/applicationgateways/httplisteners") +def _x_appgw_listener(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + hosts = [h for h in [_get(p, "hostName")] + _list(_get(p, "hostNames")) if h] + d.md.update( + { + "protocol": _get(p, "protocol"), + "port": _get(p, "_frontendPort"), + "host_names": hosts, + "public": bool(_get(p, "_public")), + "require_sni": _get(p, "requireServerNameIndication"), + } + ) + d.exposed = bool(_get(p, "_public")) + for target, relationship in _get(p, "_targets") or []: + d.add(rel(target, EdgeType.LOAD_BALANCER_TARGET, relationship, description="routing rule")) + d.add( + rel( + _rid(_get(p, "firewallPolicy")), + EdgeType.PROTECTS, + "PROTECTED_BY_WAF", + reverse=True, + description="listener WAF policy", + ) + ) + cert = _get(p, "_certificate") + d.add(rel(host_of(cert), EdgeType.REFERENCES, "CERTIFICATE_SECURES", key_vault_secret=cert)) + d.props = {k: v for k, v in p.items() if not k.startswith("_")} + + +# ── private endpoint DNS ── + + +@extractor("microsoft.network/privateendpoints") +def _x_private_endpoint_ips(d: _Draft, b: "AzureAssetBuilder") -> None: + """Private IPs of an endpoint, so private DNS A records resolve to it.""" + for cfg in _list(_get(d.props, "customDnsConfigs")): + d.alias(*[str(ip) for ip in _list(_get(cfg, "ipAddresses"))]) + for cfg in _list(_get(d.props, "ipConfigurations")): + d.alias(_get(_props(cfg), "privateIPAddress")) + + +# ── Private Link service visibility ── + + +@extractor("microsoft.network/privatelinkservices") +def _x_pls_visibility(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + visible = [str(s) for s in _list(_path(p, "visibility", "subscriptions"))] + auto = {str(s).lower() for s in _list(_path(p, "autoApproval", "subscriptions"))} + d.md["visibility"] = visible + d.md["auto_approval"] = sorted(auto) + d.md["fqdns"] = _list(_get(p, "fqdns")) + d.alias(*[str(f).lower() for f in _list(_get(p, "fqdns"))]) + if "*" in visible: + d.md["public_visibility"] = True + for sub in visible: + if sub == "*": + continue + d.add( + rel( + subscription_ref(sub), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="may connect private endpoints", + auto_approved=sub.lower() in auto or None, + cross_account=sub.lower() != d.account_id.lower() or None, + ) + ) + + +# ── Virtual Network Manager ── + + +@extractor("microsoft.network/networkmanagers") +def _x_network_manager(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + scopes = _get(p, "networkManagerScopes") or {} + d.md["scope_accesses"] = _list(_get(p, "networkManagerScopeAccesses")) + for sub in _list(_get(scopes, "subscriptions")): + d.add(rel(sub, EdgeType.GOVERNS, "COMPLIANCE_GOVERNS", description="network manager scope")) + for mg in _list(_get(scopes, "managementGroups")): + d.add(rel(mg, EdgeType.GOVERNS, "COMPLIANCE_GOVERNS", description="network manager scope")) + + +@extractor("microsoft.network/networkmanagers/networkgroups") +def _x_network_group(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["member_type"] = _get(d.props, "memberType") + + +@relation_row("microsoft.network/networkmanagers/networkgroups/staticmembers") +def _static_member(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + member = _get(row.get("properties") or {}, "resourceId") + b.defer( + parent_resource_id(row.get("id")), + rel(member, EdgeType.CONTAINS, description="static member"), + ) + + +@extractor("microsoft.network/networkmanagers/connectivityconfigurations") +def _x_connectivity_config(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + topology = _get(p, "connectivityTopology") + d.md.update( + { + "topology": topology, + "global_mesh": _get(p, "isGlobal"), + "delete_existing_peering": _get(p, "deleteExistingPeering"), + } + ) + for hub in _list(_get(p, "hubs")): + _transit(d, _get(hub, "resourceId"), f"{topology} hub") + for group in _list(_get(p, "appliesToGroups")): + d.add( + rel( + _get(group, "networkGroupId"), + EdgeType.ROUTE, + "TRANSIT_ROUTED", + description=f"{topology} member group", + use_hub_gateway=_get(group, "useHubGateway"), + direct_connectivity=_get(group, "groupConnectivity"), + ) + ) + + +@extractor("microsoft.network/networkmanagers/securityadminconfigurations/rulecollections") +def _x_admin_rule_collection(d: _Draft, b: "AzureAssetBuilder") -> None: + for group in _list(_get(d.props, "appliesToGroups")): + d.add( + rel( + _get(group, "networkGroupId"), + EdgeType.PROTECTS, + "PROTECTED_BY_NACL", + description="security admin rules", + ) + ) + + +@relation_row("microsoft.network/networkmanagers/securityadminconfigurations/rulecollections/rules") +def _admin_rule(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + collection = parent_resource_id(row.get("id")) + if not collection: + return + b.state.setdefault("admin_rules", {}).setdefault(collection.lower(), []).append( + { + "name": row.get("name"), + "access": _get(p, "access"), + "direction": _get(p, "direction"), + "priority": _get(p, "priority"), + "protocol": _get(p, "protocol"), + "sources": [_get(s, "addressPrefix") for s in _list(_get(p, "sources"))], + "destination_ports": _list(_get(p, "destinationPortRanges")), + } + ) + b.at_build(_attach_admin_rules) + + +def _attach_admin_rules(b: "AzureAssetBuilder") -> None: + for collection, rules in (b.state.get("admin_rules") or {}).items(): + d = b.draft(collection) + if d is not None: + d.md["rules"] = rules[:200] + d.md["denies"] = any(_lower(r.get("access")) == "deny" for r in rules) + + +__all__ = ["DNS_RECORD_ARM_TYPES"] diff --git a/cloudg/inventory/azure_graph/integration.py b/cloudg/inventory/azure_graph/integration.py new file mode 100644 index 0000000..2e203b4 --- /dev/null +++ b/cloudg/inventory/azure_graph/integration.py @@ -0,0 +1,928 @@ +"""Extractors for API Management, messaging, functions, workflows and web apps. + +- API Management APIs, backends (``SERVES_TRAFFIC_TO`` the Functions, App + Service, Logic Apps or hosts behind them), products, named values + (``READS_FROM`` Key Vault), self-hosted gateways, OAuth / OpenID + servers, and API / service policies: ``validate-jwt`` and + ``validate-azure-ad-token`` become ``AUTHORIZER`` assets, and + ``set-backend-service`` routes are followed. +- Service Bus queues, topics and subscriptions (``forwardTo`` and + dead-letter forwarding ``WRITES_TO``), Event Hubs (Capture + ``STREAMS_TO`` storage; Kafka endpoint flag). +- Function bindings: triggers ``INVOKES / TRIGGERED_BY`` the function, + output bindings ``WRITES_TO``, input bindings ``READS_FROM``, timer + triggers as ``SCHEDULE`` assets (``MANAGES / SCHEDULED_BY``). +- Logic Apps actions and triggers (``INVOKES`` functions, workflows, HTTP + hosts and API connections; recurrence ``SCHEDULED_BY``), API connections + and the hosts they reach. +- Stream Analytics inputs (``READS_FROM``) and outputs (``WRITES_TO``). +- Event Grid subscriptions that react to Key Vault near-expiry events: + the vault ``INVOKES / ROTATES_SECRET`` the handler. +- Web app site config (access restrictions, images, FTP / basic auth), + source control (``READS_FROM`` the repository), Static Web Apps and their + linked backends, Azure Monitor private link scopes. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, Iterable + +from cloudg.inventory._util import walk_strings +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + _docker_image, + _get, + _list, + _lower, + _path, + _rid, + host_of, + parent_resource_id, + registry_host, +) +from cloudg.inventory.azure_graph.registry import ( + _Draft, + _network_rule_grant, + extractor, + relation_row, +) +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +_ARM_URL_PREFIX = "https://management.azure.com" +_MAX_POLICY_CHARS = 200_000 + + +def arm_id_of(value: Any) -> str | None: + """ARM resource ID from an ID or a ``https://management.azure.com/...`` URL.""" + if not isinstance(value, str) or not value: + return None + text = value.strip() + if text.lower().startswith(_ARM_URL_PREFIX): + text = text[len(_ARM_URL_PREFIX) :] + text = text.split("?", 1)[0] + return text if text.lower().startswith("/subscriptions/") else None + + +def repository_ref(b: "AzureAssetBuilder", url: Any) -> str | None: + """Identifier of an external source repository, adding its asset once.""" + if not isinstance(url, str) or "://" not in url and not url.startswith("git@"): + return None + norm = url.strip().rstrip("/").removesuffix(".git").lower() + if not norm: + return None + norm = re.sub(r"^https?://[^@/]+@", "https://", norm) # never keep URL credentials + arn = f"repo:{norm}" + seen: set[str] = b.state.setdefault("repositories", set()) + if arn not in seen: + seen.add(arn) + b.add_extra( + CloudAsset( + arn=arn, + name=norm.split("://", 1)[-1], + asset_type=AssetType.SOURCE_CONNECTION, + provider=CloudProvider.AZURE, + region="global", + account_id=None, + metadata={ + "repository_url": norm, + "external": True, + "host": host_of(norm), + "discovered_via": "repository-reference", + }, + ) + ) + return arn + + +def _set_md_later(b: "AzureAssetBuilder", owner: str | None, values: dict[str, Any]) -> None: + """Merge metadata into another draft at build time (it may not exist yet).""" + if not owner: + return + b.state.setdefault("late_md", {}).setdefault(owner.lower(), {}).update( + {k: v for k, v in values.items() if v not in (None, "", [], {})} + ) + b.at_build(_apply_late_md) + + +def _apply_late_md(b: "AzureAssetBuilder") -> None: + for owner, values in (b.state.get("late_md") or {}).items(): + d = b.draft(owner) + if d is not None: + d.md.update(values) + + +# ── API Management ── + +_JWT_RE = re.compile(r"|)", re.S | re.I) +_ATTR_RE = re.compile(r'([a-zA-Z-]+)\s*=\s*"([^"]*)"') +_BACKEND_RE = re.compile(r"]*)>", re.I) +_SEND_RE = re.compile(r"\s*([^<\s]+)\s*", re.S | re.I) +_URL_RE = re.compile(r'url="([^"]+)"', re.I) + + +def _attrs(text: str) -> dict[str, str]: + return {k.lower(): v for k, v in _ATTR_RE.findall(text)} + + +def _api_hosts(d: _Draft, b: "AzureAssetBuilder", url: Any) -> None: + host = host_of(url) + d.add( + rel(host, EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO", description="API backend URL") + ) + + +@extractor("microsoft.apimanagement/service/apis") +def _x_apim_api(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + service = parent_resource_id(d.id) + d.name = _get(p, "displayName") or d.name + d.md.update( + { + "api_type": _get(p, "type") or _get(p, "apiType"), + "path": _get(p, "path"), + "protocols": _list(_get(p, "protocols")), + "service_url": _get(p, "serviceUrl"), + "subscription_required": _get(p, "subscriptionRequired"), + "revision": _get(p, "apiRevision"), + "is_current": _get(p, "isCurrent"), + "service_id": service, + } + ) + _api_hosts(d, b, _get(p, "serviceUrl")) + parent = b.draft(service) + if parent is not None and parent.exposed: + d.exposed = True + + +def _policy_relations( + b: "AzureAssetBuilder", owner: str, service: str | None, xml: str +) -> dict[str, Any]: + """Relations / metadata of one APIM policy document (never stored itself).""" + xml = xml[:_MAX_POLICY_CHARS] + md: dict[str, Any] = {} + jwt = [] + for kind, body in _JWT_RE.findall(xml): + attrs = _attrs(body.split(">", 1)[0]) + idps = [host_of(u) for u in _URL_RE.findall(body)] + jwt.append( + { + "kind": f"validate-{kind.lower()}", + "header": attrs.get("header-name"), + "tenant_id": attrs.get("tenant-id"), + "openid_hosts": [h for h in dict.fromkeys(idps) if h], + } + ) + if jwt: + md["jwt_validation"] = jwt + b.add_row( + { + "id": f"{owner}/authorizers/jwt", + "name": "JWT validation", + "type": "microsoft.apimanagement/service/apis/authorizers", + "subscriptionId": b.subscription_id, + "properties": {"validations": jwt}, + }, + synthetic=True, + ) + b.defer( + owner, + rel( + f"{owner}/authorizers/jwt", + EdgeType.REFERENCES, + "DEPENDS_ON", + description="authorizer", + ), + ) + for match in _BACKEND_RE.findall(xml): + attrs = _attrs(match) + backend = attrs.get("backend-id") + if backend and service and "@" not in backend and "{" not in backend: + b.defer( + owner, + rel( + f"{service}/backends/{backend}", + EdgeType.LOAD_BALANCER_TARGET, + "SERVES_TRAFFIC_TO", + description="set-backend-service", + ), + ) + base = attrs.get("base-url") + if base and "{" not in base and "@" not in base: + b.defer(owner, rel(host_of(base), EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO")) + for url in _SEND_RE.findall(xml): + if "{" not in url and "@" not in url: + b.defer( + owner, rel(host_of(url), EdgeType.INVOKES, "INVOKES", description="send-request") + ) + lowered = xml.lower() + md["ip_filter"] = " None: + xml = _get(row.get("properties") or {}, "value") + owner = parent_resource_id(row.get("id")) + if not isinstance(xml, str) or not owner: + return + # service-level policy: the owner is the service; API policy: the API's parent + service_level = str(row.get("type", "")).lower().endswith("/service/policies") + service = owner if service_level else parent_resource_id(owner) + _set_md_later(b, owner, _policy_relations(b, owner, service, xml)) + + +@extractor("microsoft.apimanagement/service/apis/authorizers") +def _x_apim_authorizer(d: _Draft, b: "AzureAssetBuilder") -> None: + validations = _get(d.props, "validations") or [] + d.md["validations"] = validations + for v in validations: + for host in v.get("openid_hosts") or []: + d.add(rel(host, EdgeType.REFERENCES, "DEPENDS_ON", description="OpenID configuration")) + + +@extractor("microsoft.apimanagement/service/backends") +def _x_apim_backend(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + target = arm_id_of(_get(p, "resourceId")) + d.md.update( + { + "url": _get(p, "url"), + "protocol": _get(p, "protocol"), + "backend_type": _get(p, "type"), + "resource_id": target, + "validate_certificate_chain": _path(p, "tls", "validateCertificateChain"), + "circuit_breaker": bool(_get(p, "circuitBreaker")), + } + ) + d.add( + rel( + target, + EdgeType.LOAD_BALANCER_TARGET, + "SERVES_TRAFFIC_TO", + description="backend resource", + ) + ) + if not target: + _api_hosts(d, b, _get(p, "url")) + for member in _list(_path(p, "pool", "services")): + d.add( + rel( + _rid(member), + EdgeType.LOAD_BALANCER_TARGET, + "SERVES_TRAFFIC_TO", + description="pool member", + ) + ) + # credentials are header / query names only; never keep their values + d.props = {k: v for k, v in p.items() if _lower(k) != "credentials"} + + +@extractor("microsoft.apimanagement/service/products") +def _x_apim_product(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.name = _get(p, "displayName") or d.name + d.md.update( + { + "state": _get(p, "state"), + "subscription_required": _get(p, "subscriptionRequired"), + "approval_required": _get(p, "approvalRequired"), + } + ) + + +@relation_row("microsoft.apimanagement/service/products/apis") +def _apim_product_api(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + product = parent_resource_id(row.get("id")) + service = parent_resource_id(product) + name = row.get("name") or str(row.get("id", "")).rsplit("/", 1)[-1] + if product and service: + b.defer(product, rel(f"{service}/apis/{name}", EdgeType.REFERENCES, "DEPENDS_ON")) + + +@extractor("microsoft.apimanagement/service/namedvalues") +def _x_apim_named_value(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + secret_id = _path(p, "keyVault", "secretIdentifier") + d.md.update( + { + "secret": _get(p, "secret"), + "key_vault_secret": secret_id, + "display_name": _get(p, "displayName"), + } + ) + d.add( + rel(host_of(secret_id), EdgeType.REFERENCES, "READS_FROM", description="Key Vault secret") + ) + d.props = {k: v for k, v in p.items() if _lower(k) != "value"} + + +@extractor("microsoft.apimanagement/service/gateways") +def _x_apim_gateway(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["self_hosted"] = True + d.md["location"] = _path(d.props, "locationData", "name") + + +@extractor( + "microsoft.apimanagement/service/authorizationservers", + "microsoft.apimanagement/service/openidconnectproviders", +) +def _x_apim_idp(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + for key in ("authorizationEndpoint", "tokenEndpoint", "metadataEndpoint"): + d.add(rel(host_of(_get(p, key)), EdgeType.REFERENCES, "DEPENDS_ON", description=key)) + d.props = {k: v for k, v in p.items() if "secret" not in k.lower()} + + +# ── Service Bus, Event Hubs ── + + +def _namespace_host(entity_id: str) -> tuple[str | None, str | None]: + """(namespace name, namespace host) of a Service Bus / Event Hubs entity.""" + ns = parent_resource_id(entity_id) + if ns and "/topics/" in ns.lower(): + ns = parent_resource_id(ns) + name = ns.rsplit("/", 1)[-1].lower() if ns else None + return name, f"{name}.servicebus.windows.net" if name else None + + +def _entity_aliases(d: _Draft, kind: str) -> None: + ns, host = _namespace_host(d.id) + if host: + d.alias(f"{host}/{d.name.lower()}", f"servicebus:{ns}/{d.name.lower()}") + d.alias(f"{kind}:{d.name.lower()}") + + +def _forward(d: _Draft, key: str, description: str) -> None: + target = _get(d.props, key) + if not isinstance(target, str) or not target: + return + ns, host = _namespace_host(d.id) + ref = target.lower() if "://" not in target else host_of(target) + if ns and "/" not in str(ref) and "." not in str(ref): + ref = f"servicebus:{ns}/{ref}" + d.add(rel(ref, EdgeType.REFERENCES, "WRITES_TO", description=description)) + + +@extractor("microsoft.servicebus/namespaces/queues", "microsoft.servicebus/namespaces/topics") +def _x_servicebus_entity(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + queue = d.type.endswith("/queues") + _entity_aliases(d, "servicebus-queue" if queue else "servicebus-topic") + d.md.update( + { + "status": _get(p, "status"), + "requires_session": _get(p, "requiresSession"), + "max_delivery_count": _get(p, "maxDeliveryCount"), + "dead_lettering_on_expiration": _get(p, "deadLetteringOnMessageExpiration"), + "partitioned": _get(p, "enablePartitioning"), + "message_count": _get(p, "messageCount"), + "subscription_count": _get(p, "subscriptionCount"), + } + ) + _forward(d, "forwardTo", "auto-forward") + _forward(d, "forwardDeadLetteredMessagesTo", "dead-letter forward") + + +@extractor("microsoft.servicebus/namespaces/topics/subscriptions") +def _x_servicebus_subscription(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "status": _get(p, "status"), + "max_delivery_count": _get(p, "maxDeliveryCount"), + "requires_session": _get(p, "requiresSession"), + } + ) + _forward(d, "forwardTo", "auto-forward") + _forward(d, "forwardDeadLetteredMessagesTo", "dead-letter forward") + + +@extractor("microsoft.eventhub/namespaces/eventhubs") +def _x_event_hub(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + _entity_aliases(d, "eventhub") + capture = _get(p, "captureDescription") or {} + dest = _path(capture, "destination", "properties") or {} + d.md.update( + { + "partition_count": _get(p, "partitionCount"), + "retention_days": _get(p, "messageRetentionInDays") + or _path(p, "retentionDescription", "retentionTimeInHours"), + "status": _get(p, "status"), + "capture_enabled": _get(capture, "enabled"), + } + ) + if _get(capture, "enabled"): + storage = _get(dest, "storageAccountResourceId") + lake = _get(dest, "dataLakeAccountName") + d.add( + rel( + storage or (f"{str(lake).lower()}.dfs.core.windows.net" if lake else None), + EdgeType.LOGS_TO, + "STREAMS_TO", + description="Event Hubs Capture", + container=_get(dest, "blobContainer"), + ) + ) + + +@extractor("microsoft.eventhub/namespaces") +def _x_event_hub_namespace_kafka(d: _Draft, b: "AzureAssetBuilder") -> None: + kafka = _get(d.props, "kafkaEnabled") + d.md["kafka_enabled"] = kafka + if kafka: + d.md["message_broker_protocol"] = "kafka" + d.alias(f"{d.name.lower()}.servicebus.windows.net:9093") + + +# ── Functions ── + +_TRIGGER_TARGETS = { + "servicebustrigger": ("servicebus", "queueName", "topicName"), + "servicebus": ("servicebus", "queueName", "topicName"), + "eventhubtrigger": ("eventhub", "eventHubName", None), + "eventhub": ("eventhub", "eventHubName", None), +} + + +def _binding_target(binding: dict[str, Any]) -> str | None: + kind = _lower(_get(binding, "type")) + spec = _TRIGGER_TARGETS.get(kind) + if spec is None: + return None + family, first, second = spec + if family == "servicebus": + queue = _get(binding, first) + if queue: + return f"servicebus-queue:{str(queue).lower()}" + topic = _get(binding, second) if second else None + return f"servicebus-topic:{str(topic).lower()}" if topic else None + hub = _get(binding, first) + return f"eventhub:{str(hub).lower()}" if hub and "%" not in str(hub) else None + + +@extractor("microsoft.web/sites/functions") +def _x_function(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + bindings = [x for x in _list(_path(p, "config", "bindings")) if isinstance(x, dict)] + triggers = [x for x in bindings if _lower(_get(x, "type")).endswith("trigger")] + d.md.update( + { + "function_name": d.name.rsplit("/", 1)[-1], + "language": _get(p, "language"), + "disabled": _get(p, "isDisabled"), + "invoke_url": _get(p, "invoke_url_template"), + "triggers": [_get(t, "type") for t in triggers], + "bindings": [ + { + "type": _get(x, "type"), + "direction": _get(x, "direction"), + "name": _get(x, "name"), + } + for x in bindings + ], + } + ) + d.alias(host_of(_get(p, "invoke_url_template"))) + for binding in bindings: + kind = _lower(_get(binding, "type")) + direction = _lower(_get(binding, "direction")) + target = _binding_target(binding) + if kind == "httptrigger": + d.md["http_auth_level"] = _get(binding, "authLevel") + if _lower(_get(binding, "authLevel")) == "anonymous": + d.md["anonymous_http"] = True + elif kind == "timertrigger": + _timer(d, b, binding) + elif kind.endswith("trigger"): + d.add( + rel(target, EdgeType.INVOKES, "TRIGGERED_BY", reverse=True, description=f"{kind}") + ) + elif direction == "out": + d.add(rel(target, EdgeType.REFERENCES, "WRITES_TO", description=f"{kind} output")) + elif direction == "in": + d.add(rel(target, EdgeType.REFERENCES, "READS_FROM", description=f"{kind} input")) + + +def _timer(d: _Draft, b: "AzureAssetBuilder", binding: dict[str, Any]) -> None: + schedule = _get(binding, "schedule") + b.add_row( + { + "id": f"{d.id}/timers/{_get(binding, 'name') or 'timer'}", + "name": f"timer {schedule}", + "type": "microsoft.web/sites/functions/timers", + "location": d.region, + "subscriptionId": d.account_id, + "properties": {"schedule": schedule, "runOnStartup": _get(binding, "runOnStartup")}, + }, + synthetic=True, + ) + + +@extractor("microsoft.web/sites/functions/timers", "microsoft.logic/workflows/triggers") +def _x_schedule(d: _Draft, b: "AzureAssetBuilder") -> None: + target = parent_resource_id(d.id) + d.md["schedule"] = _get(d.props, "schedule") or _get(d.props, "recurrence") + d.add(rel(target, EdgeType.MANAGES, "SCHEDULED_BY", description="schedule")) + + +# ── Logic Apps, API connections ── + +_CONN_KEY_RE = re.compile(r"\$connections'[\])]*\['([^']+)'\]") + + +def _walk_actions(actions: Any) -> Iterable[tuple[str, dict[str, Any]]]: + stack = [actions] + while stack: + current = stack.pop() + if not isinstance(current, dict): + continue + for name, action in current.items(): + if not isinstance(action, dict): + continue + yield name, action + stack.append(_get(action, "actions")) + stack.append(_path(action, "else", "actions")) + stack.append(_path(action, "default", "actions")) + for case in (_get(action, "cases") or {}).values(): + stack.append(_get(case, "actions")) + + +def _connection_id(item: dict[str, Any], connections: dict[str, Any]) -> str | None: + ref = _path(item, "inputs", "host", "connection", "name") + m = _CONN_KEY_RE.search(str(ref or "")) + if not m: + return None + return _get(_get(connections, m.group(1)) or {}, "connectionId") + + +def _logic_action_target(item: dict[str, Any], connections: dict[str, Any]) -> str | None: + kind = _lower(_get(item, "type")) + inputs = _get(item, "inputs") or {} + if kind in ("apiconnection", "apiconnectionwebhook"): + return _connection_id(item, connections) + if kind == "http": + uri = _get(inputs, "uri") + return host_of(uri) if isinstance(uri, str) and "@" not in uri else None + if kind == "function": + return _path(inputs, "function", "id") + if kind == "workflow": + return _path(inputs, "host", "workflow", "id") + return None + + +@extractor("microsoft.logic/workflows") +def _x_logic_app(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + definition = _get(p, "definition") or {} + connections = _path(p, "parameters", "$connections", "value") or {} + triggers = _get(definition, "triggers") or {} + action_count = 0 + for _name, action in _walk_actions(_get(definition, "actions")): + action_count += 1 + d.add( + rel( + _logic_action_target(action, connections), + EdgeType.INVOKES, + "INVOKES", + description=f"{_get(action, 'type')} action", + ) + ) + trigger_kinds = [] + for name, trigger in triggers.items() if isinstance(triggers, dict) else []: + kind = _lower(_get(trigger, "type")) + trigger_kinds.append(kind) + if kind == "recurrence": + b.add_row( + { + "id": f"{d.id}/triggers/{name}", + "name": f"{name} ({_path(trigger, 'recurrence', 'frequency')})", + "type": "microsoft.logic/workflows/triggers", + "location": d.region, + "subscriptionId": d.account_id, + "properties": {"recurrence": _get(trigger, "recurrence")}, + }, + synthetic=True, + ) + elif kind == "request": + d.md["http_trigger"] = True + else: + source = _connection_id(trigger, connections) + d.add( + rel( + source, + EdgeType.INVOKES, + "TRIGGERED_BY", + reverse=True, + description=f"{kind} trigger", + ) + ) + for conn in connections.values() if isinstance(connections, dict) else []: + d.add( + rel( + _get(conn, "connectionId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="API connection", + ) + ) + d.md.update( + { + "state": _get(p, "state"), + "trigger_types": trigger_kinds, + "action_count": action_count, + "allowed_caller_ips": [ + _get(r, "addressRange") + for r in _list(_path(p, "accessControl", "triggers", "allowedCallerIpAddresses")) + ], + } + ) + d.add(rel(_rid(_get(p, "integrationAccount")), EdgeType.REFERENCES, "DEPENDS_ON")) + + +@extractor("microsoft.web/connections") +def _x_api_connection(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + api = _path(p, "api", "id") or "" + d.md.update( + { + "api": str(api).rsplit("/", 1)[-1] or _path(p, "api", "name"), + "status": [_get(s, "status") for s in _list(_get(p, "statuses"))], + "display_name": _get(p, "displayName"), + } + ) + hosts = [] + for text in walk_strings(_get(p, "parameterValues") or {}): + host = host_of(text) if "://" in text else None + if host and host not in hosts: + hosts.append(host) + for host in hosts[:10]: + d.add(rel(host, EdgeType.REFERENCES, "DEPENDS_ON", description="connection endpoint")) + # parameter values and the signed-in user are not inventory + d.props = { + k: v + for k, v in p.items() + if _lower(k) not in ("parametervalues", "parametervalueset", "authenticateduser") + } + + +# ── Stream Analytics ── + + +def _stream_target(props: dict[str, Any]) -> str | None: + ds = _get(props, "datasource") or {} + kind = _lower(_get(ds, "type")) + dp = _get(ds, "properties") or {} + ns = _lower(_get(dp, "serviceBusNamespace")) + if kind == "microsoft.servicebus/eventhub" or kind == "microsoft.eventhub/eventhub": + return f"{ns}.servicebus.windows.net/{_lower(_get(dp, 'eventHubName'))}" if ns else None + if kind == "microsoft.servicebus/queue": + return f"{ns}.servicebus.windows.net/{_lower(_get(dp, 'queueName'))}" if ns else None + if kind == "microsoft.servicebus/topic": + return f"{ns}.servicebus.windows.net/{_lower(_get(dp, 'topicName'))}" if ns else None + if kind in ("microsoft.storage/blob", "microsoft.storage/table", "microsoft.storage/datalake"): + accounts = _list(_get(dp, "storageAccounts")) or [{"accountName": _get(dp, "accountName")}] + name = _lower(_get(accounts[0], "accountName")) if accounts else "" + suffix = {"microsoft.storage/table": "table", "microsoft.storage/datalake": "dfs"}.get( + kind, "blob" + ) + return f"{name}.{suffix}.core.windows.net" if name else None + if kind == "microsoft.devices/iothubs": + name = _lower(_get(dp, "iotHubNamespace")) + return f"{name}.azure-devices.net" if name else None + if kind.startswith("microsoft.sql/server"): + server = _lower(_get(dp, "server")) + return server if "." in server else (f"{server}.database.windows.net" if server else None) + if kind == "microsoft.storage/documentdb": + acct = _lower(_get(dp, "accountId")) + return f"{acct}.documents.azure.com" if acct else None + if kind == "microsoft.azurefunction": + app = _lower(_get(dp, "functionAppName")) + return f"{app}.azurewebsites.net" if app else None + return host_of(_get(dp, "url") or _get(dp, "endpoint")) + + +@relation_row( + "microsoft.streamanalytics/streamingjobs/inputs", + "microsoft.streamanalytics/streamingjobs/outputs", +) +def _stream_io(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + job = parent_resource_id(row.get("id")) + props = row.get("properties") or {} + target = _stream_target(props) + output = str(row.get("type", "")).lower().endswith("/outputs") + b.defer( + job, + rel( + target, + EdgeType.REFERENCES, + "WRITES_TO" if output else "READS_FROM", + description=f"{'output' if output else 'input'} {row.get('name')}", + datasource=_path(props, "datasource", "type"), + ), + ) + + +@extractor("microsoft.streamanalytics/streamingjobs") +def _x_stream_job(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["job_type"] = _get(p, "jobType") + d.md["state"] = _get(p, "jobState") + d.add(rel(_path(p, "cluster", "id"), EdgeType.CONTAINS, reverse=True, description="cluster")) + for io in _list(_get(p, "inputs")) + _list(_get(p, "outputs")): + if isinstance(io, dict) and io.get("id"): + _stream_io(io, b) + + +# ── Event Grid: Key Vault rotation handlers ── + +_ROTATION_EVENTS = ( + "microsoft.keyvault.secretnearexpiry", + "microsoft.keyvault.secretexpired", + "microsoft.keyvault.certificatenearexpiry", + "microsoft.keyvault.keynearexpiry", +) + + +def _subscription_target(p: dict[str, Any]) -> str | None: + dest = _get(p, "destination") or {} + return _path(dest, "properties", "resourceId") or host_of( + _path(dest, "properties", "endpointUrl") + ) + + +def _rotation_handlers(b: "AzureAssetBuilder") -> None: + for sub_id, target in b.state.get("rotation_subscriptions", []): + topic = parent_resource_id(sub_id) + topic_draft = b.draft(topic) + source = _get(topic_draft.props, "source") if topic_draft is not None else topic + if isinstance(source, str) and "/microsoft.keyvault/vaults/" in source.lower(): + b.defer( + source, + rel( + target, + EdgeType.INVOKES, + "ROTATES_SECRET", + description="near-expiry event handler", + event_subscription=sub_id, + ), + ) + + +@extractor( + "microsoft.eventgrid/systemtopics/eventsubscriptions", + "microsoft.eventgrid/eventsubscriptions", +) +def _x_rotation_subscription(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + events = [str(e).lower() for e in _list(_path(p, "filter", "includedEventTypes"))] + target = _subscription_target(p) + if d.type == "microsoft.eventgrid/eventsubscriptions": # extension subscription on a resource + d.md["included_event_types"] = events + d.add(rel(target, EdgeType.INVOKES, "INVOKES", description="event delivery")) + source = parent_resource_id(d.id) + if source and target: + b.defer( + source, rel(target, EdgeType.INVOKES, "INVOKES", description=f"events {d.name}") + ) + if target and any(e in _ROTATION_EVENTS for e in events): + b.state.setdefault("rotation_subscriptions", []).append((d.id, target)) + b.at_build(_rotation_handlers) + + +# ── Web apps, Static Web Apps, Monitor private link scopes ── + + +@relation_row("microsoft.web/sites/config", "microsoft.web/sites/slots/config") +def _site_config(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + site = parent_resource_id(row.get("id")) + p = row.get("properties") or {} + if not site or not str(row.get("name", "")).lower().endswith("web"): + return + images = [ + i for i in (_docker_image(_get(p, k)) for k in ("linuxFxVersion", "windowsFxVersion")) if i + ] + for image in images: + b.defer(site, rel(registry_host(image), EdgeType.USES_IMAGE, "RUNS_ON", image=image)) + rules = [] + for key in ("ipSecurityRestrictions", "scmIpSecurityRestrictions"): + for r in _list(_get(p, key)): + subnet = _get(r, "vnetSubnetResourceId") + rules.append( + { + "scm": key.startswith("scm"), + "action": _get(r, "action"), + "ip": _get(r, "ipAddress"), + "subnet": subnet, + "service_tag": _get(r, "tag"), + } + ) + if subnet and _lower(_get(r, "action")) == "allow": + b.defer(site, _network_rule_grant(subnet, "access restriction")) + _set_md_later( + b, + site, + { + "access_restrictions": rules, + "ftps_state": _get(p, "ftpsState"), + "min_tls_version": _get(p, "minTlsVersion"), + "remote_debugging": _get(p, "remoteDebuggingEnabled"), + "vnet_route_all": _get(p, "vnetRouteAllEnabled"), + "acr_managed_identity": _get(p, "acrUseManagedIdentityCreds"), + "http20": _get(p, "http20Enabled"), + }, + ) + + +@relation_row("microsoft.web/sites/sourcecontrols", "microsoft.web/sites/slots/sourcecontrols") +def _site_source_control(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + site = parent_resource_id(row.get("id")) + p = row.get("properties") or {} + repo = repository_ref(b, _get(p, "repoUrl")) + b.defer( + site, + rel( + repo, + EdgeType.REFERENCES, + "READS_FROM", + description="deployment source", + branch=_get(p, "branch"), + github_action=_get(p, "isGitHubAction"), + ), + ) + + +@relation_row("microsoft.web/sites/basicpublishingcredentialspolicies") +def _site_basic_auth(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + site = parent_resource_id(row.get("id")) + kind = str(row.get("name") or "").lower() + _set_md_later(b, site, {f"basic_auth_{kind}": _get(row.get("properties") or {}, "allow")}) + + +@extractor("microsoft.web/staticsites") +def _x_static_site(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.exposed = True + host = _get(p, "defaultHostname") + d.alias(host.lower() if isinstance(host, str) else None) + for domain in _list(_get(p, "customDomains")): + d.alias(str(domain).lower()) + d.md.update( + { + "default_host_name": host, + "repository_url": _get(p, "repositoryUrl"), + "branch": _get(p, "branch"), + "staging_policy": _get(p, "stagingEnvironmentPolicy"), + } + ) + d.add(rel(repository_ref(b, _get(p, "repositoryUrl")), EdgeType.REFERENCES, "READS_FROM")) + for backend in _list(_get(p, "linkedBackends")): + d.add( + rel( + _get(backend, "backendResourceId"), + EdgeType.LOAD_BALANCER_TARGET, + "SERVES_TRAFFIC_TO", + ) + ) + + +@relation_row( + "microsoft.web/staticsites/linkedbackends", + "microsoft.web/staticsites/userprovidedfunctionapps", +) +def _static_site_backend(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + target = _get(p, "backendResourceId") or _get(p, "functionAppResourceId") + b.defer( + parent_resource_id(row.get("id")), + rel( + target, EdgeType.LOAD_BALANCER_TARGET, "SERVES_TRAFFIC_TO", description="linked backend" + ), + ) + + +@extractor("microsoft.insights/privatelinkscopes") +def _x_monitor_pls(d: _Draft, b: "AzureAssetBuilder") -> None: + settings = _get(d.props, "accessModeSettings") or {} + d.md["ingestion_access_mode"] = _get(settings, "ingestionAccessMode") + d.md["query_access_mode"] = _get(settings, "queryAccessMode") + + +@relation_row("microsoft.insights/privatelinkscopes/scopedresources") +def _monitor_scoped_resource(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + linked = _get(row.get("properties") or {}, "linkedResourceId") + b.defer( + parent_resource_id(row.get("id")), + rel(linked, EdgeType.REFERENCES, "DEPENDS_ON", description="scoped resource"), + ) + + +__all__ = ["arm_id_of", "repository_ref"] diff --git a/cloudg/inventory/azure_graph/ops.py b/cloudg/inventory/azure_graph/ops.py new file mode 100644 index 0000000..a0204ff --- /dev/null +++ b/cloudg/inventory/azure_graph/ops.py @@ -0,0 +1,1007 @@ +"""Extractors for operations, monitoring, backup, containers and identity depth. + +- Metric, log-search, activity-log and smart-detector alerts + (``MONITORS / MONITORED_BY`` their scopes, ``INVOKES`` action groups); + action groups (``INVOKES`` webhooks, functions, Logic Apps, runbooks, + Event Hubs; e-mail addresses and phone numbers are dropped). +- Diagnostic settings (the resource and the setting ``LOGS_TO`` the + workspace / storage, ``STREAMS_TO`` Event Hubs), data collection rules + and associations, maintenance configurations and their assignments + (``SCHEDULED_BY``), Automation runbooks, schedules, job schedules, + hybrid worker groups and update deployments, Arc machines and + extensions, App Configuration replicas. +- Backup: Recovery Services protected items and policies, Azure Backup + instances and policies (``BACKUP_TO``), Site Recovery replicated items + (``REPLICATES_TO``), vault immutability, soft delete and CMK. +- Containers: ACR replications (``REPLICATES_TO``), webhooks (``INVOKES``), + tasks (``READS_FROM`` source repos, ``WRITES_TO`` the registry, timer + ``SCHEDULED_BY``), scope maps and tokens (``GRANTS_ACCESS``); AKS Entra + admin groups, OIDC issuer, trusted access, cluster extensions and Flux + configurations (``READS_FROM`` Git). +- Identity: user-assigned identity federated credentials (``IAM_TRUST`` + from the AKS issuer or an external issuer), Key Vault keys and secrets + (metadata only: no values) with rotation policies. +- Compute: Batch pools, Spring Apps, availability sets, proximity + placement and host groups; Azure Deployment Environments (dev centers, + projects, catalogs, environment types). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + LOG_ANALYTICS_PREFIX, + _get, + _list, + _lower, + _path, + _rid, + _rids, + host_of, + issuer_kind, + parent_resource_id, + principal_ref, + subscription_ref, +) +from cloudg.inventory.azure_graph.registry import _Draft, _keyvault_key_ref, extractor, relation_row +from cloudg.schema.models import EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +_MODEL_LOG_CATEGORIES = {"requestresponse", "audit", "trace", "allmetrics"} + + +def _owner_of(extension_id: Any) -> str | None: + rid = str(extension_id or "") + cut = rid.lower().rfind("/providers/") + return rid[:cut] if cut > 0 else None + + +# ── alerts and action groups ── + + +def _alert_common(d: _Draft, scopes: list[Any], groups: list[Any], description: str) -> None: + for scope in scopes: + d.add(rel(scope, EdgeType.MONITORS, "MONITORED_BY", description=description)) + for group in groups: + d.add(rel(group, EdgeType.INVOKES, "INVOKES", description="action group")) + d.md["enabled"] = _get(d.props, "enabled") + d.md["severity"] = _get(d.props, "severity") + d.md["scopes"] = scopes[:50] + + +@extractor("microsoft.insights/metricalerts") +def _x_metric_alert(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + groups = [_get(a, "actionGroupId") for a in _list(_get(p, "actions"))] + d.md["criteria"] = [_get(c, "metricName") for c in _list(_path(p, "criteria", "allOf"))] + _alert_common(d, _list(_get(p, "scopes")), groups, "metric alert") + + +@extractor("microsoft.insights/scheduledqueryrules") +def _x_log_alert(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + actions = _get(p, "actions") or {} + groups = _list(_get(actions, "actionGroups")) if isinstance(actions, dict) else [] + groups += [_get(a, "actionGroupId") for a in _list(actions) if isinstance(a, dict)] + _alert_common(d, _list(_get(p, "scopes")), groups, "log search alert") + + +@extractor("microsoft.insights/activitylogalerts") +def _x_activity_alert(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + groups = [_get(a, "actionGroupId") for a in _list(_path(p, "actions", "actionGroups"))] + d.md["conditions"] = [ + f"{_get(c, 'field')}={_get(c, 'equals')}" for c in _list(_path(p, "condition", "allOf")) + ][:20] + _alert_common(d, _list(_get(p, "scopes")), groups, "activity log alert") + + +@extractor("microsoft.alertsmanagement/smartdetectoralertrules") +def _x_smart_alert(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + groups = _list(_path(p, "actionGroups", "groupIds")) + _alert_common(d, _list(_get(p, "scope")), groups, "smart detector") + + +_RECEIVERS = ( + ("webhookReceivers", "serviceUri", True), + ("azureFunctionReceivers", "functionAppResourceId", False), + ("logicAppReceivers", "resourceId", False), + ("automationRunbookReceivers", "automationAccountId", False), + ("armRoleReceivers", None, False), +) + + +@extractor("microsoft.insights/actiongroups") +def _x_action_group(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + counts: dict[str, int] = {} + for key, field, is_url in _RECEIVERS: + receivers = _list(_get(p, key)) + counts[key] = len(receivers) + if not field: + continue + for r in receivers: + target = _get(r, field) + d.add( + rel( + host_of(target) if is_url else target, + EdgeType.INVOKES, + "INVOKES", + description=key.removesuffix("Receivers"), + ) + ) + for r in _list(_get(p, "eventHubReceivers")): + ns, hub = _lower(_get(r, "eventHubNameSpace")), _lower(_get(r, "eventHubName")) + d.add( + rel( + f"{ns}.servicebus.windows.net/{hub}" if ns and hub else None, + EdgeType.INVOKES, + "STREAMS_TO", + description="Event Hubs receiver", + ) + ) + for key in ("emailReceivers", "smsReceivers", "voiceReceivers", "azureAppPushReceivers"): + counts[key] = len(_list(_get(p, key))) + d.md["receiver_counts"] = {k: v for k, v in counts.items() if v} + d.md["enabled"] = _get(p, "enabled") + # contact details are personal data, not inventory + d.props = { + k: v + for k, v in p.items() + if _lower(k) + not in ("emailreceivers", "smsreceivers", "voicereceivers", "azureapppushreceivers") + } + + +# ── diagnostic settings, data collection ── + + +@extractor("microsoft.insights/diagnosticsettings") +def _x_diagnostic_setting(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import _set_md_later + + p = d.props + source = _owner_of(d.id) + categories = [ + str(_get(x, "category") or _get(x, "categoryGroup") or "") + for x in _list(_get(p, "logs")) + if _get(x, "enabled") + ] + d.md.update( + { + "source_resource": source, + "log_categories": categories, + "metrics": any(_get(m, "enabled") for m in _list(_get(p, "metrics"))), + } + ) + targets = [ + (_get(p, "workspaceId"), "LOGS_TO", EdgeType.LOGS_TO), + (_get(p, "storageAccountId"), "LOGS_TO", EdgeType.LOGS_TO), + (_get(p, "marketplacePartnerId"), "LOGS_TO", EdgeType.LOGS_TO), + ] + rule = _get(p, "eventHubAuthorizationRuleId") + hub_ns = None + if isinstance(rule, str) and rule: + cut = rule.lower().find("/authorizationrules/") + hub_ns = rule[:cut] if cut > 0 else rule + hub = _get(p, "eventHubName") + hub_target = f"{hub_ns}/eventhubs/{hub}" if hub_ns and hub else hub_ns + targets.append((hub_target, "STREAMS_TO", EdgeType.LOGS_TO)) + for target, relationship, edge in targets: + if not target: + continue + d.add(rel(target, edge, relationship, description="diagnostic setting")) + if source: + b.defer( + source, rel(target, edge, relationship, description=f"diagnostic setting {d.name}") + ) + if source and "/microsoft.cognitiveservices/accounts/" in source.lower(): + logged = {c.lower() for c in categories} & _MODEL_LOG_CATEGORIES + _set_md_later(b, source, {"model_request_logging": bool(logged) or None}) + if source: + _set_md_later(b, source, {"diagnostic_settings": True}) + + +@extractor("microsoft.insights/datacollectionrules") +def _x_dcr(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + dest = _get(p, "destinations") or {} + for ws in _list(_get(dest, "logAnalytics")): + d.add(rel(_get(ws, "workspaceResourceId"), EdgeType.LOGS_TO, "LOGS_TO")) + for acct in _list(_get(dest, "storageBlobsDirect")) + _list(_get(dest, "storageAccounts")): + d.add(rel(_get(acct, "storageAccountResourceId"), EdgeType.LOGS_TO, "LOGS_TO")) + for hub in _list(_get(dest, "eventHubsDirect")) + _list(_get(dest, "eventHubs")): + d.add(rel(_get(hub, "eventHubResourceId"), EdgeType.LOGS_TO, "STREAMS_TO")) + for mon in _list(_get(dest, "monitoringAccounts")): + d.add(rel(_get(mon, "accountResourceId"), EdgeType.LOGS_TO, "LOGS_TO")) + d.add(rel(_get(p, "dataCollectionEndpointId"), EdgeType.REFERENCES, "DEPENDS_ON")) + d.md["streams"] = sorted( + {s for f in _list(_get(p, "dataFlows")) for s in _list(_get(f, "streams"))} + ) + + +@relation_row("microsoft.insights/datacollectionruleassociations") +def _dcr_association(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + resource = _owner_of(row.get("id")) + rule = _get(p, "dataCollectionRuleId") + if not resource: + return + b.defer(resource, rel(rule, EdgeType.LOGS_TO, "LOGS_TO", description="data collection rule")) + b.defer(resource, rel(_get(p, "dataCollectionEndpointId"), EdgeType.REFERENCES, "DEPENDS_ON")) + b.defer(rule, rel(resource, EdgeType.MONITORS, "MONITORED_BY", description="collects from")) + + +@extractor("microsoft.maintenance/maintenanceconfigurations") +def _x_maintenance_config(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + window = _get(p, "maintenanceWindow") or {} + d.md.update( + { + "maintenance_scope": _get(p, "maintenanceScope"), + "recur_every": _get(window, "recurEvery"), + "start": _get(window, "startDateTime"), + "duration": _get(window, "duration"), + } + ) + + +@relation_row("microsoft.maintenance/configurationassignments") +def _maintenance_assignment(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + config = _get(p, "maintenanceConfigurationId") + target = _get(p, "resourceId") or _owner_of(row.get("id")) + b.defer( + config, rel(target, EdgeType.MANAGES, "SCHEDULED_BY", description="maintenance assignment") + ) + + +# ── Automation ── + + +def _account_of(child_id: str | None) -> str | None: + return parent_resource_id(child_id) + + +@extractor("microsoft.automation/automationaccounts/runbooks") +def _x_runbook(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "runbook_type": _get(p, "runbookType"), + "state": _get(p, "state"), + "log_verbose": _get(p, "logVerbose"), + } + ) + + +@extractor("microsoft.automation/automationaccounts/schedules") +def _x_automation_schedule(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "frequency": _get(p, "frequency"), + "interval": _get(p, "interval"), + "enabled": _get(p, "isEnabled"), + "next_run": _get(p, "nextRun"), + } + ) + + +@relation_row("microsoft.automation/automationaccounts/jobschedules") +def _job_schedule(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + account = _account_of(row.get("id")) + runbook = _get(_get(p, "runbook") or {}, "name") + schedule = _get(_get(p, "schedule") or {}, "name") + if not account or not runbook: + return + runbook_id = f"{account}/runbooks/{runbook}" + if schedule: + b.defer( + f"{account}/schedules/{schedule}", + rel(runbook_id, EdgeType.MANAGES, "SCHEDULED_BY", description="job schedule"), + ) + worker = _get(p, "runOn") + if worker: + b.defer( + runbook_id, + rel( + f"{account}/hybridRunbookWorkerGroups/{worker}", + EdgeType.REFERENCES, + "RUNS_ON", + description="hybrid worker group", + ), + ) + + +@extractor("microsoft.automation/automationaccounts/hybridrunbookworkergroups") +def _x_worker_group(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["group_type"] = _get(d.props, "groupType") + d.md["credential"] = _path(d.props, "credential", "name") + + +@relation_row( + "microsoft.automation/automationaccounts/hybridrunbookworkergroups/hybridrunbookworkers" +) +def _hybrid_worker(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + b.defer( + parent_resource_id(row.get("id")), + rel( + _get(p, "vmResourceId"), + EdgeType.CONTAINS, + "CLUSTER_CONTAINS_SERVICE", + description="hybrid worker", + ), + ) + + +@extractor("microsoft.automation/automationaccounts/softwareupdateconfigurations") +def _x_update_configuration(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + cfg = _get(p, "updateConfiguration") or {} + d.md["schedule"] = _path(p, "scheduleInfo", "frequency") + for vm in _list(_get(cfg, "azureVirtualMachines")) + _list(_get(cfg, "nonAzureComputerNames")): + d.add(rel(vm, EdgeType.MANAGES, "SCHEDULED_BY", description="update deployment")) + for query in _list(_path(cfg, "targets", "azureQueries")): + for scope in _list(_get(query, "scope")): + d.add( + rel(scope, EdgeType.MANAGES, "SCHEDULED_BY", description="update deployment scope") + ) + + +@relation_row("microsoft.automation/automationaccounts/webhooks") +def _automation_webhook(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + account = _account_of(row.get("id")) + runbook = _path(p, "runbook", "name") + if account and runbook: + from cloudg.inventory.azure_graph.integration import _set_md_later + + _set_md_later( + b, + f"{account}/runbooks/{runbook}", + {"webhook_trigger": True, "webhook_expiry": _get(p, "expiryTime")}, + ) + + +@extractor("microsoft.automation/automationaccounts") +def _x_automation_account(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md["local_auth_disabled"] = _get(p, "disableLocalAuth") + _keyvault_key_ref(d, _path(p, "encryption", "keyVaultProperties", "keyvaultUri")) + + +# ── Arc, App Configuration ── + + +@extractor("microsoft.hybridcompute/machines") +def _x_arc_machine(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "arc": True, + "os_name": _get(p, "osName"), + "agent_version": _get(p, "agentVersion"), + "status": _get(p, "status"), + "machine_fqdn": _get(p, "machineFqdn"), + } + ) + d.add( + rel( + _get(p, "privateLinkScopeResourceId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="Arc private link scope", + ) + ) + + +@extractor( + "microsoft.hybridcompute/machines/extensions", "microsoft.compute/virtualmachines/extensions" +) +def _x_machine_extension(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "publisher": _get(p, "publisher"), + "extension_type": _get(p, "type"), + "version": _get(p, "typeHandlerVersion"), + } + ) + ws = _path(p, "settings", "workspaceId") + if ws: + d.add(rel(f"{LOG_ANALYTICS_PREFIX}{str(ws).lower()}", EdgeType.LOGS_TO, "LOGS_TO")) + + +@extractor("microsoft.appconfiguration/configurationstores/replicas") +def _x_appconfig_replica(d: _Draft, b: "AzureAssetBuilder") -> None: + d.alias(host_of(_get(d.props, "endpoint"))) + store = parent_resource_id(d.id) + if store: + b.defer( + store, + rel( + d.id, EdgeType.REFERENCES, "REPLICATES_TO", description="App Configuration replica" + ), + ) + + +# ── backup and disaster recovery ── + + +@extractor("microsoft.recoveryservices/vaults", "microsoft.dataprotection/backupvaults") +def _x_vault_protection(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + security = _get(p, "securitySettings") or {} + redundancy = _get(p, "redundancySettings") or {} + d.md.update( + { + "immutability": _path(security, "immutabilitySettings", "state"), + "soft_delete": _path(security, "softDeleteSettings", "softDeleteState") + or _path(security, "softDeleteSettings", "state"), + "cross_region_restore": _get(redundancy, "crossRegionRestore"), + "storage_redundancy": _get(redundancy, "standardTierStorageRedundancy"), + "multi_user_authorization": _get(security, "multiUserAuthorization"), + } + ) + _keyvault_key_ref(d, _path(p, "encryption", "keyVaultProperties", "keyUri"), "vault CMK") + + +def _vault_of(child_id: str | None) -> str | None: + rid = str(child_id or "") + for marker in ( + "/backupfabrics/", + "/replicationfabrics/", + "/backuppolicies/", + "/backupinstances/", + "/replicationpolicies/", + ): + cut = rid.lower().find(marker) + if cut > 0: + return rid[:cut] + return parent_resource_id(rid) + + +@relation_row( + "microsoft.recoveryservices/vaults/backupfabrics/protectioncontainers/protecteditems", + "microsoft.dataprotection/backupvaults/backupinstances", +) +def _protected_item(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + source = ( + _get(p, "sourceResourceId") + or _path(p, "dataSourceInfo", "resourceID") + or _path(p, "dataSourceInfo", "resourceId") + ) + policy = _get(p, "policyId") or _path(p, "policyInfo", "policyId") + vault = _vault_of(row.get("id")) + state = ( + _get(p, "protectionState") + or _path(p, "currentProtectionState") + or _get(p, "protectionStatus") + ) + if not source: + return + details = { + "workload": _get(p, "workloadType") or _path(p, "dataSourceInfo", "datasourceType"), + "state": state, + "last_backup": _get(p, "lastBackupTime"), + } + b.defer( + source, + rel( + vault, + EdgeType.REFERENCES, + "BACKUP_TO", + description="protected by Azure Backup", + **details, + ), + ) + b.defer( + policy or vault, + rel(source, EdgeType.MANAGES, "BACKUP_TO", description="backup protection", **details), + ) + b.state.setdefault("backup_items", {}).setdefault(str(vault).lower(), 0) + b.state["backup_items"][str(vault).lower()] += 1 + + +@extractor( + "microsoft.recoveryservices/vaults/backuppolicies", + "microsoft.dataprotection/backupvaults/backuppolicies", +) +def _x_backup_policy(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + vault = _vault_of(d.id) + retention = _get(p, "retentionPolicy") or {} + d.md.update( + { + "management_type": _get(p, "backupManagementType") or _get(p, "objectType"), + "schedule": _path(p, "schedulePolicy", "scheduleRunFrequency"), + "daily_retention_days": _path(retention, "dailySchedule", "retentionDuration", "count"), + "protected_items": _get(p, "protectedItemsCount"), + "datasource_types": _list(_get(p, "datasourceTypes")), + } + ) + d.add(rel(vault, EdgeType.REFERENCES, "BACKUP_TO", description="policy target vault")) + + +@relation_row( + "microsoft.recoveryservices/vaults/replicationfabrics/replicationprotectioncontainers/replicationprotecteditems" +) +def _asr_item(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + details = _get(p, "providerSpecificDetails") or {} + source = _get(details, "fabricObjectId") or _get(details, "vmId") or _get(p, "fabricObjectId") + target = ( + _get(details, "recoveryAzureVMId") + or _get(details, "selectedRecoveryAzureNetworkId") + or _get(details, "recoveryAzureResourceGroupId") + ) + vault = _vault_of(row.get("id")) + if not isinstance(source, str) or not source.startswith("/"): + return + b.defer( + source, + rel( + target or vault, + EdgeType.REFERENCES, + "REPLICATES_TO", + description="Site Recovery replication", + health=_get(p, "replicationHealth"), + recovery_location=_get(details, "recoveryAzureLocation") or _get(p, "recoveryFabricId"), + ), + ) + b.defer( + vault, + rel(source, EdgeType.MANAGES, "REPLICATES_TO", description="Site Recovery protected item"), + ) + + +@extractor("microsoft.recoveryservices/vaults/replicationpolicies") +def _x_asr_policy(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["recovery_point_history_minutes"] = _path( + d.props, "providerSpecificDetails", "recoveryPointHistory" + ) + + +# ── containers ── + + +@extractor("microsoft.containerregistry/registries/replications") +def _x_acr_replication(d: _Draft, b: "AzureAssetBuilder") -> None: + registry = parent_resource_id(d.id) + d.md["zone_redundancy"] = _get(d.props, "zoneRedundancy") + d.md["region_endpoint_enabled"] = _get(d.props, "regionEndpointEnabled") + if registry: + b.defer( + registry, + rel( + d.id, + EdgeType.REFERENCES, + "REPLICATES_TO", + description="geo-replication", + location=d.region, + ), + ) + + +@extractor("microsoft.containerregistry/registries/webhooks") +def _x_acr_webhook(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "actions": _list(_get(p, "actions")), + "scope": _get(p, "scope"), + "status": _get(p, "status"), + } + ) + uri = _get(p, "serviceUri") + d.add(rel(host_of(uri), EdgeType.INVOKES, "INVOKES", description="registry webhook")) + d.props = {k: v for k, v in p.items() if _lower(k) not in ("serviceuri", "customheaders")} + + +@extractor("microsoft.containerregistry/registries/tasks") +def _x_acr_task(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import repository_ref + + p = d.props + registry = parent_resource_id(d.id) + step = _get(p, "step") or {} + trigger = _get(p, "trigger") or {} + repos = [_get(step, "contextPath")] + [ + _path(t, "sourceRepository", "repositoryUrl") + for t in _list(_get(trigger, "sourceTriggers")) + ] + for url in dict.fromkeys(r for r in repos if isinstance(r, str) and "://" in r): + d.add( + rel( + repository_ref(b, url), EdgeType.REFERENCES, "READS_FROM", description="task source" + ) + ) + d.add(rel(registry, EdgeType.REFERENCES, "WRITES_TO", description="pushes images")) + timers = [_get(t, "schedule") for t in _list(_get(trigger, "timerTriggers"))] + d.md.update( + { + "status": _get(p, "status"), + "timer_schedules": timers, + "base_image_trigger": bool(_get(trigger, "baseImageTrigger")), + "agent_pool": _get(p, "agentPoolName"), + } + ) + if timers and registry: + d.md["scheduled"] = True + + +@extractor("microsoft.containerregistry/registries/scopemaps") +def _x_acr_scope_map(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["actions"] = _list(_get(d.props, "actions"))[:200] + + +@extractor("microsoft.containerregistry/registries/tokens") +def _x_acr_token(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + registry = parent_resource_id(d.id) + d.md["status"] = _get(p, "status") + d.add(rel(_get(p, "scopeMapId"), EdgeType.REFERENCES, "DEPENDS_ON", description="scope map")) + d.add( + rel( + registry, + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + description="repository-scoped token", + ) + ) + d.props = {k: v for k, v in p.items() if _lower(k) != "credentials"} + + +@extractor("microsoft.containerservice/managedclusters") +def _x_aks_access(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + issuer = _path(p, "oidcIssuerProfile", "issuerURL") + if isinstance(issuer, str) and issuer: + d.alias(issuer.rstrip("/"), issuer.rstrip("/") + "/") + d.md.update( + { + "oidc_issuer": issuer, + "workload_identity": _path(p, "securityProfile", "workloadIdentity", "enabled"), + "entra_integration": bool(_get(p, "aadProfile")), + "image_cleaner": _path(p, "securityProfile", "imageCleaner", "enabled"), + "defender_enabled": _path( + p, "securityProfile", "defender", "securityMonitoring", "enabled" + ), + "sku_tier": _get(d.row.get("sku") or {}, "tier"), + } + ) + for group in _list(_path(p, "aadProfile", "adminGroupObjectIDs")): + b.note_principal(group, "Group") + d.add( + rel( + principal_ref(group), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="AKS cluster admin group", + role="cluster-admin", + privileged=True, + ) + ) + + +@relation_row("microsoft.containerservice/managedclusters/trustedaccessrolebindings") +def _aks_trusted_access(row: dict[str, Any], b: "AzureAssetBuilder") -> None: + p = row.get("properties") or {} + b.defer( + parent_resource_id(row.get("id")), + rel( + _get(p, "sourceResourceId"), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="trusted access", + roles=_list(_get(p, "roles")), + ), + ) + + +@extractor("microsoft.kubernetesconfiguration/extensions") +def _x_cluster_extension(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "extension_type": _get(p, "extensionType"), + "version": _get(p, "version") or _get(p, "currentVersion"), + "release_train": _get(p, "releaseTrain"), + "state": _get(p, "provisioningState"), + "cluster_id": _owner_of(d.id), + } + ) + + +@extractor( + "microsoft.kubernetesconfiguration/fluxconfigurations", + "microsoft.kubernetesconfiguration/sourcecontrolconfigurations", +) +def _x_flux(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import repository_ref + + p = d.props + cluster = _owner_of(d.id) + git = _get(p, "gitRepository") or {} + url = ( + _get(git, "url") + or _get(p, "repositoryUrl") + or _path(p, "bucket", "url") + or _path(p, "ociRepository", "url") + ) + d.md.update( + { + "source_kind": _get(p, "sourceKind"), + "repository_url": url, + "branch": _path(git, "repositoryRef", "branch"), + "kustomizations": sorted((_get(p, "kustomizations") or {}).keys()), + "compliance_state": _get(p, "complianceState"), + } + ) + d.add( + rel(repository_ref(b, url), EdgeType.REFERENCES, "READS_FROM", description="GitOps source") + ) + d.add(rel(cluster, EdgeType.MANAGES, "OWNED_BY", description="GitOps deploys to")) + d.props = {k: v for k, v in p.items() if _lower(k) not in ("configurationprotectedsettings",)} + + +# ── identity depth ── + + +@extractor("microsoft.managedidentity/userassignedidentities") +def _x_uami_client(d: _Draft, b: "AzureAssetBuilder") -> None: + client = _get(d.props, "clientId") + if isinstance(client, str) and client: + d.alias(f"azure-client-id:{client.lower()}") + + +@extractor("microsoft.managedidentity/userassignedidentities/federatedidentitycredentials") +def _x_federated_credential(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + identity = parent_resource_id(d.id) + issuer = str(_get(p, "issuer") or "").rstrip("/") + subject = _get(p, "subject") + kind = issuer_kind(issuer) + aks = kind == "aks" + d.md.update( + { + "issuer": issuer, + "subject": subject, + "audiences": _list(_get(p, "audiences")), + "issuer_kind": kind, + } + ) + d.add( + rel( + identity, + EdgeType.IAM_TRUST, + "ROLE_ASSUMES_ROLE" if aks else "CROSS_ACCOUNT_TRUST", + description=f"tokens from {issuer} for {subject}", + subject=subject, + ) + ) + if identity and issuer: + b.defer( + identity, + rel( + issuer, + EdgeType.IAM_TRUST, + "ROLE_ASSUMES_ROLE", + reverse=True, + description="workload identity federation", + subject=subject, + ), + ) + + +@extractor("microsoft.keyvault/vaults/keys") +def _x_kv_key(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + policy = _get(p, "rotationPolicy") or {} + actions = [_lower(_path(a, "action", "type")) for a in _list(_get(policy, "lifetimeActions"))] + d.md.update( + { + "key_type": _get(p, "kty"), + "key_size": _get(p, "keySize"), + "enabled": _path(p, "attributes", "enabled"), + "expires": _path(p, "attributes", "exp"), + "auto_rotation": "rotate" in actions, + "rotation_expiry": _path(policy, "attributes", "expiryTime"), + "exportable": _path(p, "attributes", "exportable"), + } + ) + uri = _get(p, "keyUri") + d.alias(uri, _get(p, "keyUriWithVersion")) + + +@extractor("microsoft.keyvault/vaults/secrets") +def _x_kv_secret(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.md.update( + { + "content_type": _get(p, "contentType"), + "enabled": _path(p, "attributes", "enabled"), + "expires": _path(p, "attributes", "exp"), + "has_expiry": _path(p, "attributes", "exp") is not None, + } + ) + d.alias(_get(p, "secretUri")) + # values are never inventory, even if a response carried one + d.props = {k: v for k, v in p.items() if _lower(k) != "value"} + + +# ── compute ── + + +@extractor("microsoft.batch/batchaccounts") +def _x_batch_account(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.add( + rel( + _path(p, "autoStorage", "storageAccountId"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="auto storage", + ) + ) + d.add( + rel( + _path(p, "keyVaultReference", "id"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="Key Vault", + ) + ) + d.alias(host_of(_get(p, "accountEndpoint"))) + d.md["pool_allocation_mode"] = _get(p, "poolAllocationMode") + + +@extractor("microsoft.batch/batchaccounts/pools") +def _x_batch_pool(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + net = _get(p, "networkConfiguration") or {} + d.add(rel(_get(net, "subnetId"), EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + public = _path(net, "publicIPAddressConfiguration", "provision") + d.md.update( + { + "vm_size": _get(p, "vmSize"), + "public_ip": public, + "current_nodes": _get(p, "currentDedicatedNodes"), + } + ) + d.exposed = _lower(public) not in ("", "nopublicipaddresses") + + +@extractor("microsoft.appplatform/spring") +def _x_spring(d: _Draft, b: "AzureAssetBuilder") -> None: + net = _get(d.props, "networkProfile") or {} + for key in ("serviceRuntimeSubnetId", "appSubnetId"): + d.add(rel(_get(net, key), EdgeType.CONTAINS, "SUBNET_CONTAINS_INSTANCE", reverse=True)) + + +@extractor("microsoft.appplatform/spring/apps") +def _x_spring_app(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.alias(host_of(_get(p, "url")), str(_get(p, "fqdn") or "").lower() or None) + d.md["public"] = _get(p, "public") + d.exposed = bool(_get(p, "public")) + + +@extractor( + "microsoft.compute/availabilitysets", + "microsoft.compute/proximityplacementgroups", + "microsoft.compute/hostgroups", +) +def _x_placement_group(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + members = _rids(_get(p, "virtualMachines")) + _rids(_get(p, "virtualMachineScaleSets")) + d.md.update( + { + "member_count": len(members), + "fault_domains": _get(p, "platformFaultDomainCount"), + "update_domains": _get(p, "platformUpdateDomainCount"), + } + ) + for member in members: + d.add(rel(member, EdgeType.CONTAINS, description="placement member")) + d.add(rel(_rid(_get(p, "proximityPlacementGroup")), EdgeType.CONTAINS, reverse=True)) + for host in _rids(_get(p, "hosts")): + d.add(rel(host, EdgeType.CONTAINS, description="dedicated host")) + + +# ── Azure Deployment Environments ── + + +@extractor("microsoft.devcenter/projects") +def _x_devcenter_project(d: _Draft, b: "AzureAssetBuilder") -> None: + d.add( + rel(_get(d.props, "devCenterId"), EdgeType.CONTAINS, reverse=True, description="dev center") + ) + d.md["max_dev_boxes_per_user"] = _get(d.props, "maxDevBoxesPerUser") + + +@extractor("microsoft.devcenter/devcenters/catalogs", "microsoft.devcenter/projects/catalogs") +def _x_devcenter_catalog(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.integration import repository_ref + + p = d.props + source = _get(p, "gitHub") or _get(p, "adoGit") or {} + url = _get(source, "uri") + d.md.update( + {"repository_url": url, "branch": _get(source, "branch"), "path": _get(source, "path")} + ) + d.add( + rel( + repository_ref(b, url), + EdgeType.REFERENCES, + "READS_FROM", + description="environment catalog", + ) + ) + d.add( + rel( + host_of(_get(source, "secretIdentifier")), + EdgeType.REFERENCES, + "READS_FROM", + description="Git token", + ) + ) + + +@extractor("microsoft.devcenter/projects/environmenttypes") +def _x_environment_type(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + target = _get(p, "deploymentTargetId") + d.md["status"] = _get(p, "status") + d.add(rel(target, EdgeType.MANAGES, "OWNED_BY", description="deployment target")) + role_map = _path(p, "creatorRoleAssignment", "roles") or {} + d.md["creator_roles"] = sorted(role_map.keys()) if isinstance(role_map, dict) else [] + for oid in (_get(p, "userRoleAssignments") or {}).keys(): + b.note_principal(oid, None) + d.add( + rel( + principal_ref(oid), + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + reverse=True, + description="environment type user role", + ) + ) + + +@extractor("microsoft.devcenter/devcenters") +def _x_devcenter(d: _Draft, b: "AzureAssetBuilder") -> None: + d.md["dev_center_uri"] = _get(d.props, "devCenterUri") + d.alias(host_of(_get(d.props, "devCenterUri"))) + + +# ── subscription-level activity log export ── + + +@extractor("microsoft.insights/diagnosticsettings") +def _x_activity_log(d: _Draft, b: "AzureAssetBuilder") -> None: + owner = _owner_of(d.id) + if owner and owner.rstrip("/").count("/") == 2 and owner.lower().startswith("/subscriptions/"): + d.md["activity_log_export"] = True + d.add( + rel( + subscription_ref(owner.rsplit("/", 1)[-1]), + EdgeType.MONITORS, + "MONITORED_BY", + description="activity log export", + ) + ) + + +__all__: list[str] = [] diff --git a/cloudg/inventory/azure_graph/query.py b/cloudg/inventory/azure_graph/query.py index 7741937..d65a8da 100644 --- a/cloudg/inventory/azure_graph/query.py +++ b/cloudg/inventory/azure_graph/query.py @@ -11,6 +11,7 @@ from __future__ import annotations +import re import time from dataclasses import dataclass from types import SimpleNamespace @@ -177,11 +178,24 @@ def containers_query(subscription_id: str) -> str: def role_assignments_query(subscription_id: str) -> str: + return _role_assignments_query(f"| where subscriptionId =~ '{_kql(subscription_id)}' ") + + +def management_group_role_assignments_query() -> str: + """Role assignments made at management group or tenant root scope (run tenant-wide).""" + return _role_assignments_query( + "| where isempty(subscriptionId) " + "| where tostring(properties.scope) startswith '/providers/Microsoft.Management/' " + "or tostring(properties.scope) == '/' " + ) + + +def _role_assignments_query(where: str) -> str: guid = r"@'([0-9a-fA-F-]{36})$'" return ( "authorizationresources " "| where type =~ 'microsoft.authorization/roleassignments' " - f"| where subscriptionId =~ '{_kql(subscription_id)}' " + f"{where}" "| extend roleDefinitionId = tostring(properties.roleDefinitionId), " "principalId = tostring(properties.principalId), " "principalType = tostring(properties.principalType), scope = tostring(properties.scope) " @@ -201,3 +215,138 @@ def defender_query(subscription_id: str) -> str: f"| where subscriptionId =~ '{_kql(subscription_id)}' " "| project id, name, type, subscriptionId, properties | order by id asc" ) + + +# ── Azure parity: extra Resource Graph tables ── + +_ROW_COLUMNS = ( + "id, name, type, kind, location, resourceGroup, subscriptionId, tags, identity, properties" +) + + +def _kql_type(value: str) -> str: + """An ARM type literal for KQL: letters, digits, ``.``, ``_``, ``-`` and ``/`` only.""" + return re.sub(r"[^A-Za-z0-9._/\-]", "", value or "") + + +def _types_clause(types: tuple[str, ...]) -> str: + if not types: + return "" + quoted = ", ".join(f"'{_kql_type(t)}'" for t in types) + return f"| where type in~ ({quoted}) " + + +def table_query(table: str, subscription_id: str, types: tuple[str, ...] = ()) -> str: + """ARM-shaped rows of one Resource Graph table for one subscription.""" + return ( + f"{_kql(table)} | where subscriptionId =~ '{_kql(subscription_id)}' " + f"{_types_clause(types)}| project {_ROW_COLUMNS} | order by id asc" + ) + + +def role_definitions_query() -> str: + """Every role definition visible to the caller (built-in and custom).""" + guid = r"@'([0-9a-fA-F-]{36})$'" + return ( + "authorizationresources | where type =~ 'microsoft.authorization/roledefinitions' " + f"| extend roleGuid = tolower(extract({guid}, 1, id)) " + "| project id, name, type, roleGuid, properties | order by id asc" + ) + + +POLICY_TYPES = ( + "microsoft.authorization/policyassignments", + "microsoft.authorization/policyexemptions", +) +POLICY_DEFINITION_TYPES = ( + "microsoft.authorization/policydefinitions", + "microsoft.authorization/policysetdefinitions", +) + + +def policy_definitions_query(names: list[str]) -> str: + """Custom policy definitions / initiatives, plus the built-in ones named in ``names``.""" + named = ", ".join(f"'{_kql(n)}'" for n in names[:800] if _kql(n)) + builtin = f" or name in~ ({named})" if named else "" + return ( + f"policyresources {_types_clause(POLICY_DEFINITION_TYPES)}" + f"| where tostring(properties.policyType) !~ 'BuiltIn'{builtin} " + "| project id, name, type, subscriptionId, properties | order by id asc" + ) + + +def policy_states_query(subscription_id: str) -> str: + """Non-compliant policy states of one subscription (latest evaluation).""" + return ( + "policyresources | where type =~ 'microsoft.policyinsights/policystates' " + f"| where subscriptionId =~ '{_kql(subscription_id)}' " + "| where tostring(properties.complianceState) =~ 'NonCompliant' " + "| project id, type, subscriptionId, resourceId = tostring(properties.resourceId), " + "policyAssignmentId = tostring(properties.policyAssignmentId), " + "policyDefinitionId = tostring(properties.policyDefinitionId), " + "policyDefinitionAction = tostring(properties.policyDefinitionAction), " + "complianceState = tostring(properties.complianceState) | order by id asc" + ) + + +def security_assessments_query(subscription_id: str) -> str: + """Defender for Cloud assessments and sub-assessments, one lean row each.""" + return ( + "securityresources " + "| where type in~ ('microsoft.security/assessments', " + "'microsoft.security/assessments/subassessments') " + f"| where subscriptionId =~ '{_kql(subscription_id)}' " + "| extend resourceId = tostring(coalesce(properties.resourceDetails.Id, " + "properties.resourceDetails.id, properties.resourceDetails.ResourceId)), " + "code = tostring(properties.status.code), " + "severity = tostring(coalesce(properties.metadata.severity, properties.status.severity)), " + "title = tostring(properties.displayName) " + "| project id, type, subscriptionId, resourceId, code, severity, title, " + "additionalData = properties.additionalData | order by id asc" + ) + + +def secure_score_query(subscription_id: str) -> str: + return ( + "securityresources " + "| where type in~ ('microsoft.security/securescores', " + "'microsoft.security/securescores/securescorecontrols') " + f"| where subscriptionId =~ '{_kql(subscription_id)}' " + "| project id, name, type, subscriptionId, properties | order by id asc" + ) + + +def security_alerts_query(subscription_id: str) -> str: + """Active (not dismissed or resolved) Defender for Cloud alerts.""" + return ( + "securityresources | where type =~ 'microsoft.security/locations/alerts' " + f"| where subscriptionId =~ '{_kql(subscription_id)}' " + "| where tostring(properties.Status) !in~ ('Dismissed', 'Resolved') " + "| project id, name, type, subscriptionId, properties | order by id asc" + ) + + +BACKUP_TYPES = ( + "microsoft.recoveryservices/vaults/backupfabrics/protectioncontainers/protecteditems", + "microsoft.recoveryservices/vaults/backuppolicies", + "microsoft.recoveryservices/vaults/replicationfabrics/replicationprotectioncontainers/" + "replicationprotecteditems", + "microsoft.recoveryservices/vaults/replicationpolicies", + "microsoft.dataprotection/backupvaults/backupinstances", + "microsoft.dataprotection/backupvaults/backuppolicies", +) + +# (task, table, types) of the best-effort ARM-shaped tables +ARG_TABLE_TASKS: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ("policy", "policyresources", POLICY_TYPES), + ("backup_items", "recoveryservicesresources", BACKUP_TYPES), + ("kubernetes_configuration", "kubernetesconfigurationresources", ()), + ("network_resources", "networkresources", ()), + ( + "guest_configuration", + "guestconfigurationresources", + ("microsoft.guestconfiguration/guestconfigurationassignments",), + ), + ("maintenance", "maintenanceresources", ("microsoft.maintenance/configurationassignments",)), + ("insights", "insightsresources", ("microsoft.insights/datacollectionruleassociations",)), +) diff --git a/cloudg/inventory/azure_graph/registry.py b/cloudg/inventory/azure_graph/registry.py index 5d671bb..ea026d4 100644 --- a/cloudg/inventory/azure_graph/registry.py +++ b/cloudg/inventory/azure_graph/registry.py @@ -63,6 +63,16 @@ def alias(self, *values: Any) -> None: "microsoft.containerservice/managedclusters/agentpools": "CLUSTER_CONTAINS_SERVICE", "microsoft.web/sites/slots": "CLUSTER_CONTAINS_SERVICE", "microsoft.web/sites/functions": "CLUSTER_CONTAINS_SERVICE", + # Azure parity + "microsoft.kubernetesconfiguration/extensions": "CLUSTER_CONTAINS_SERVICE", + "microsoft.kubernetesconfiguration/fluxconfigurations": "CLUSTER_CONTAINS_SERVICE", + "microsoft.kubernetesconfiguration/sourcecontrolconfigurations": "CLUSTER_CONTAINS_SERVICE", + "microsoft.app/containerapps/revisions": "CLUSTER_CONTAINS_SERVICE", + "microsoft.appplatform/spring/apps": "CLUSTER_CONTAINS_SERVICE", + "microsoft.machinelearningservices/workspaces/onlineendpoints/deployments": ( + "CLUSTER_CONTAINS_SERVICE" + ), + # end Azure parity } @@ -77,6 +87,23 @@ def deco(fn: Extractor) -> Extractor: return deco +# Rows of these types never become assets: their handler only adds relations +# or metadata to other assets (links, memberships, assignments). +RowHandler = Callable[[dict[str, Any], "AzureAssetBuilder"], None] +_ROW_HANDLERS: dict[str, RowHandler] = {} + + +def relation_row(*types: str) -> Callable[[RowHandler], RowHandler]: + """Register a handler for rows that only carry relations (no asset).""" + + def deco(fn: RowHandler) -> RowHandler: + for t in types: + _ROW_HANDLERS[t.lower()] = fn + return fn + + return deco + + def _set_exposure(d: _Draft, default_public: bool = False) -> None: """PaaS exposure: public network access on and no default-deny ACL.""" pna = _get(d.props, "publicNetworkAccess") diff --git a/cloudg/inventory/azure_graph/rest.py b/cloudg/inventory/azure_graph/rest.py new file mode 100644 index 0000000..e54ff62 --- /dev/null +++ b/cloudg/inventory/azure_graph/rest.py @@ -0,0 +1,254 @@ +"""ARM, Microsoft Graph and data-plane REST reads for what Resource Graph misses. + +Resource Graph returns tracked resources, but not most proxy (child) +resources (APIM APIs, Service Bus queues, DNS record sets, function +bindings, diagnostic settings, ...) and nothing from Entra ID. Those are +read with plain HTTPS GETs authenticated by the azure-identity credential +cloudg already holds, so no extra SDK is needed: + +- :class:`AzureRestClient` pages through ``nextLink`` / ``@odata.nextLink``, + only follows links on its own host, retries throttled (429 / 503) + responses after ``Retry-After``, and paces every call through the shared + rate limiter (``azure//*/`` buckets). +- :func:`default_arm_client_factory`, :func:`default_msgraph_client_factory` + and :func:`default_dataplane_client_factory` build clients for + ``management.azure.com``, ``graph.microsoft.com`` and per-service data + planes. Collectors take them as injectable ``credential -> client`` + factories, so tests pass fakes and never touch the network. + +Clients only issue ``GET`` (and ``POST`` for read-only ``list*`` actions +such as ``listClusterUserCredential``). Response bodies are returned as +dicts; secret values are never requested. +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Callable + +from cloudg.exceptions import CredentialError +from cloudg.resilience.governor import get_governor +from cloudg.resilience.limiter import Scope + +logger = logging.getLogger("cloudg.inventory.azure_graph") + +ARM_ENDPOINT = "https://management.azure.com" +GRAPH_ENDPOINT = "https://graph.microsoft.com" +_ARM_SCOPE = f"{ARM_ENDPOINT}/.default" +_GRAPH_SCOPE = f"{GRAPH_ENDPOINT}/.default" +_RETRY_STATUSES = (429, 503) + + +class AzureRestError(Exception): + """A REST call answered with an HTTP error status.""" + + def __init__(self, status_code: int, url: str, message: str = "") -> None: + self.status_code = status_code + self.url = url + super().__init__(f"HTTP {status_code} for {url.split('?', 1)[0]}: {message[:300]}") + + @property + def forbidden(self) -> bool: + return self.status_code in (401, 403) + + @property + def not_applicable(self) -> bool: + """404 / 400 / 405 / 409: the child kind does not exist for this parent or SKU.""" + return self.status_code in (400, 404, 405, 409) + + +def _retry_after(headers: Any) -> float | None: + if headers is None: + return None + for key in ("Retry-After", "retry-after", "x-ms-retry-after-ms"): + value = headers.get(key) if hasattr(headers, "get") else None + if value is None: + continue + try: + seconds = float(value) + except (TypeError, ValueError): + continue + return seconds / 1000.0 if key.endswith("-ms") else seconds + return None + + +class AzureRestClient: + """Minimal paginated JSON REST client authenticated by a TokenCredential. + + Args: + credential: azure-identity TokenCredential. + base_url: ``https://host`` every relative path is joined to. + scope: OAuth scope requested for the bearer token. + service: Rate-limit bucket name (``arm``, ``msgraph``, ``dataplane``). + timeout: Socket timeout of one call, in seconds. + max_retries: Retries of throttled calls. + sleep: Injectable sleep (tests). + """ + + def __init__( + self, + credential: Any, + *, + base_url: str = ARM_ENDPOINT, + scope: str = _ARM_SCOPE, + service: str = "arm", + timeout: float = 30.0, + max_retries: int = 4, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + if not base_url.startswith("https://"): + raise ValueError("Azure REST base URL must be https") + self._credential = credential + self.base_url = base_url.rstrip("/") + self._scope = scope + self._timeout = timeout + self._max_retries = max_retries + self._sleep = sleep + self._limit_scope = Scope("azure", None, None, service) + self._token: str | None = None + self._expires = 0.0 + self._lock = threading.Lock() + + # ── auth ── + + def _bearer(self) -> str: + with self._lock: + if self._token and time.time() < self._expires - 120: + return self._token + get_token = getattr(self._credential, "get_token", None) + if not callable(get_token): + raise CredentialError("azure", "the credential cannot issue access tokens") + try: + token = get_token(self._scope) + except Exception as exc: + raise CredentialError("azure", f"token for {self._scope} refused: {exc}") from exc + self._token = str(getattr(token, "token", "") or "") + self._expires = float(getattr(token, "expires_on", 0) or time.time() + 600) + return self._token + + # ── transport ── + + def url(self, path: str, params: dict[str, Any] | None = None) -> str: + url = path if path.startswith("https://") else self.base_url + "/" + path.lstrip("/") + if params: + sep = "&" if "?" in url else "?" + url += sep + urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) + return url + + def _send(self, url: str, method: str, body: Any) -> dict[str, Any]: + if not url.startswith(self.base_url + "/"): + raise ValueError(f"refusing to call {url.split('?', 1)[0]}: not on {self.base_url}") + data = None if body is None else json.dumps(body).encode("utf-8") + headers = {"Authorization": f"Bearer {self._bearer()}", "Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) # nosec B310 + with urllib.request.urlopen(req, timeout=self._timeout) as resp: # nosec B310 - https only + raw = resp.read() + if not raw: + return {} + out = json.loads(raw.decode("utf-8")) + return out if isinstance(out, dict) else {"value": out} + + def request(self, path: str, *, method: str = "GET", params=None, body=None) -> dict[str, Any]: + """One call, paced and retried when throttled. + + Raises: + AzureRestError: The service answered with an error status. + CredentialError: No token could be obtained. + """ + url = self.url(path, params) + gov = get_governor() + attempt = 0 + while True: + if gov.provider_enabled("azure"): + gov.check(self._limit_scope) + gov.limiter.acquire_sync(self._limit_scope, sleep=self._sleep) + try: + result = self._send(url, method, body) + except urllib.error.HTTPError as exc: + text = "" + try: + text = exc.read().decode("utf-8", "replace") + except Exception: # pragma: no cover - defensive + pass + err = AzureRestError(exc.code, url, text) + if exc.code in _RETRY_STATUSES and attempt < self._max_retries: + hint = _retry_after(exc.headers) + gov.on_throttle(self._limit_scope, hint, str(err)[:200]) + attempt += 1 + gov.record_retry(self._limit_scope) + self._sleep(min(max(hint or 0.0, 2.0**attempt), 60.0)) + continue + gov.on_final_error(self._limit_scope, err, "Azure REST call") + raise err from None + gov.on_success(self._limit_scope) + return result + + def get(self, path: str, api_version: str | None = None, **params: Any) -> dict[str, Any]: + if api_version: + params["api-version"] = api_version + return self.request(path, params=params or None) + + def post(self, path: str, api_version: str | None = None, body: Any = None) -> dict[str, Any]: + params = {"api-version": api_version} if api_version else None + return self.request(path, method="POST", params=params, body=body if body else {}) + + def list( + self, + path: str, + api_version: str | None = None, + *, + max_items: int = 5000, + **params: Any, + ) -> list[dict[str, Any]]: + """Every item of a list endpoint (``value``), following next links on this host.""" + if api_version: + params["api-version"] = api_version + items: list[dict[str, Any]] = [] + url: str | None = self.url(path, params or None) + pages = 0 + while url and pages < 500 and len(items) < max_items: + pages += 1 + data = self.request(url) + for item in data.get("value", []) or []: + if isinstance(item, dict): + items.append(item) + url = data.get("nextLink") or data.get("@odata.nextLink") + if url and not str(url).startswith(self.base_url + "/"): + break + return items[:max_items] + + +def default_arm_client_factory(credential: Any) -> AzureRestClient: + """REST client for Azure Resource Manager (``management.azure.com``).""" + return AzureRestClient(credential, base_url=ARM_ENDPOINT, scope=_ARM_SCOPE, service="arm") + + +def default_msgraph_client_factory(credential: Any) -> AzureRestClient: + """REST client for Microsoft Graph v1.0 (``graph.microsoft.com``).""" + return AzureRestClient( + credential, base_url=f"{GRAPH_ENDPOINT}", scope=_GRAPH_SCOPE, service="msgraph" + ) + + +def default_dataplane_client_factory(credential: Any, base_url: str, scope: str) -> AzureRestClient: + """REST client for a service data plane (AI Search, Synapse dev endpoint, ...).""" + return AzureRestClient(credential, base_url=base_url, scope=scope, service="dataplane") + + +__all__ = [ + "ARM_ENDPOINT", + "GRAPH_ENDPOINT", + "AzureRestClient", + "AzureRestError", + "default_arm_client_factory", + "default_dataplane_client_factory", + "default_msgraph_client_factory", +] diff --git a/cloudg/inventory/azure_graph/security.py b/cloudg/inventory/azure_graph/security.py new file mode 100644 index 0000000..d5eed28 --- /dev/null +++ b/cloudg/inventory/azure_graph/security.py @@ -0,0 +1,534 @@ +"""Defender for Cloud and Microsoft Sentinel: coverage, findings and placeholders. + +- Defender plans (``microsoft.security/pricings``): an enabled plan + ``MONITORS / MONITORED_BY`` the subscription (as before) and every + resource of the types it covers, unless a resource-level pricing turns it + off. Plans the subscription does not list become "not enabled" + placeholders, like the AWS ``_disabled_asset`` ones. +- Assessments and sub-assessments (``securityresources``): one + ``SECURITY_HUB`` asset per subscription ("Defender for Cloud + recommendations", with the secure score) ``MONITORS`` each assessed + resource, ``FINDING_AFFECTS`` when it has unhealthy recommendations or + vulnerabilities (container image findings included). +- Alerts: one ``THREAT_DETECTOR`` asset ``MONITORS / FINDING_AFFECTS`` each + resource an active alert names. +- Sentinel: onboarded workspaces (``microsoft.securityinsights``), their + data connectors (``LOGS_TO`` the workspace, ``MONITORS`` their source, + AWS connectors ``ASSUMES_ROLE / CROSS_ACCOUNT_TRUST`` the AWS role), + analytics rules and automation rules (``INVOKES`` playbooks); a "not + enabled" placeholder when no workspace of the subscription has Sentinel. +- Guest configuration assignments ``GOVERNS`` their machine. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.azure_graph.helpers import ( + _get, + _list, + _lower, + _path, + host_of, + subscription_ref, +) +from cloudg.inventory.azure_graph.registry import _Draft, extractor +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +# Defender plan (lowercase pricing name) -> resource types it protects +PLAN_RESOURCE_TYPES: dict[str, tuple[str, ...]] = { + "virtualmachines": ( + "microsoft.compute/virtualmachines", + "microsoft.compute/virtualmachinescalesets", + "microsoft.hybridcompute/machines", + ), + "sqlservers": ("microsoft.sql/servers", "microsoft.sql/managedinstances"), + "sqlservervirtualmachines": ("microsoft.sqlvirtualmachine/sqlvirtualmachines",), + "appservices": ("microsoft.web/sites",), + "storageaccounts": ("microsoft.storage/storageaccounts",), + "keyvaults": ("microsoft.keyvault/vaults", "microsoft.keyvault/managedhsms"), + "containers": ( + "microsoft.containerservice/managedclusters", + "microsoft.containerregistry/registries", + "microsoft.kubernetes/connectedclusters", + ), + "kubernetesservice": ("microsoft.containerservice/managedclusters",), + "containerregistry": ("microsoft.containerregistry/registries",), + "opensourcerelationaldatabases": ( + "microsoft.dbforpostgresql/flexibleservers", + "microsoft.dbforpostgresql/servers", + "microsoft.dbformysql/flexibleservers", + "microsoft.dbformysql/servers", + "microsoft.dbformariadb/servers", + ), + "cosmosdbs": ("microsoft.documentdb/databaseaccounts",), + "api": ("microsoft.apimanagement/service",), + "ai": ("microsoft.cognitiveservices/accounts",), +} + +# Plans every subscription can turn on; missing ones get a placeholder +STANDARD_PLANS: tuple[str, ...] = ( + "VirtualMachines", + "SqlServers", + "SqlServerVirtualMachines", + "AppServices", + "StorageAccounts", + "KeyVaults", + "Arm", + "OpenSourceRelationalDatabases", + "Containers", + "CosmosDbs", + "CloudPosture", + "Api", + "AI", +) +_MAX_COVERED = 5000 +_MAX_TITLES = 20 + + +def _security_alias(service: str, subscription_id: str) -> str: + return f"azure-security:{service}:{subscription_id}" + + +def _disabled_asset( + subscription_id: str, + service: str, + asset_type: AssetType, + label: str, + reason: str = "not enabled", +) -> CloudAsset: + """Placeholder for a security service that is not on in this subscription.""" + return CloudAsset( + arn=f"cloudg:azure:{service}:{subscription_id}:not-enabled", + name=f"{label} ({reason})", + asset_type=asset_type, + provider=CloudProvider.AZURE, + region="global", + account_id=subscription_id, + metadata={ + "security_service": service, + "enabled": False, + "status": reason, + "aliases": [_security_alias(service, subscription_id)], + }, + ) + + +def defender_coverage( + b: "AzureAssetBuilder", overrides: dict[str, bool], pricings_seen: bool +) -> None: + """Per-resource MONITORS of enabled plans; placeholders for plans not listed.""" + present: set[str] = set() + for asset in b.extras: + md = asset.metadata + if md.get("resource_type") != "microsoft.security/pricings": + continue + plan = str(md.get("security_service", "")).removeprefix("defender-") + present.add(plan) + if not md.get("enabled"): + continue + relations = md.setdefault("relations", []) + covered = 0 + for d in b.drafts_of(*PLAN_RESOURCE_TYPES.get(plan, ())): + if overrides.get(f"{d.id.lower()}|{plan}") is False or covered >= _MAX_COVERED: + continue + covered += 1 + relations.append( + rel(d.id, EdgeType.MONITORS, "MONITORED_BY", description=f"Defender plan {plan}") + ) + md["covered_resources"] = covered + if not pricings_seen: + return + for plan in STANDARD_PLANS: + if plan.lower() not in present: + b.add_extra( + _disabled_asset( + b.subscription_id, + f"defender-{plan.lower()}", + AssetType.THREAT_DETECTOR, + f"Defender for Cloud: {plan}", + ) + ) + + +# ── assessments, secure score, alerts ── + + +def _cspm_asset(b: "AzureAssetBuilder") -> CloudAsset: + asset = b.state.get("defender_cspm_asset") + if asset is None: + sub = b.subscription_id + asset = CloudAsset( + arn=f"cloudg:azure:defender-cspm:{sub}", + name="Defender for Cloud recommendations", + asset_type=AssetType.SECURITY_HUB, + provider=CloudProvider.AZURE, + region="global", + account_id=sub, + metadata={ + "security_service": "defender-cspm", + "enabled": True, + "resource_type": "microsoft.security/assessments", + "aliases": [_security_alias("defender-cspm", sub)], + "relations": [ + rel(subscription_ref(sub), EdgeType.MONITORS, "MONITORED_BY"), + ], + }, + ) + b.state["defender_cspm_asset"] = asset + b.add_extra(asset) + return asset + + +def _bucket(agg: dict[str, dict[str, Any]], rid: str) -> dict[str, Any]: + return agg.setdefault( + rid, + { + "healthy": 0, + "unhealthy": 0, + "not_applicable": 0, + "high": 0, + "vulnerabilities": 0, + "high_vulnerabilities": 0, + "titles": [], + "images": [], + }, + ) + + +def _count_assessment(entry: dict[str, Any], row: dict[str, Any]) -> None: + code = _lower(row.get("code")) + severity = _lower(row.get("severity")) + title = row.get("title") + if _lower(row.get("type")).endswith("/subassessments"): + if code == "unhealthy": + entry["vulnerabilities"] += 1 + entry["high_vulnerabilities"] += severity in ("high", "critical") + extra = row.get("additionalData") or {} + image = _get(extra, "repositoryName") or _path( + extra, "artifactDetails", "repositoryName" + ) + if image and image not in entry["images"] and len(entry["images"]) < _MAX_TITLES: + entry["images"].append(image) + return + if code == "healthy": + entry["healthy"] += 1 + elif code == "unhealthy": + entry["unhealthy"] += 1 + entry["high"] += severity == "high" + if title and title not in entry["titles"] and len(entry["titles"]) < _MAX_TITLES: + entry["titles"].append(title) + else: + entry["not_applicable"] += 1 + + +def add_assessments(b: "AzureAssetBuilder", rows: list[dict[str, Any]]) -> int: + """Aggregate assessments / sub-assessments into per-resource relations.""" + agg: dict[str, dict[str, Any]] = {} + for row in rows: + if not isinstance(row, dict): + continue + if _lower(row.get("type")) not in ( + "microsoft.security/assessments", + "microsoft.security/assessments/subassessments", + ): + continue + rid = row.get("resourceId") or _path(row.get("properties") or {}, "resourceDetails", "Id") + if not isinstance(rid, str) or not rid.startswith("/"): + continue + _count_assessment(_bucket(agg, rid.rstrip("/")), row) + if not agg: + return 0 + asset = _cspm_asset(b) + relations = asset.metadata.setdefault("relations", []) + totals = {"healthy": 0, "unhealthy": 0, "vulnerabilities": 0} + for rid, entry in list(agg.items())[:_MAX_COVERED]: + for key in totals: + totals[key] += entry[key] + finding = entry["unhealthy"] or entry["vulnerabilities"] + relations.append( + rel( + rid, + EdgeType.MONITORS, + "FINDING_AFFECTS" if finding else "MONITORED_BY", + description="Defender for Cloud assessment", + **{k: v for k, v in entry.items() if v}, + ) + ) + asset.metadata.update( + { + "assessed_resources": len(agg), + "healthy_assessments": totals["healthy"], + "unhealthy_assessments": totals["unhealthy"], + "vulnerability_findings": totals["vulnerabilities"], + } + ) + return len(rows) + + +def add_secure_score(b: "AzureAssetBuilder", rows: list[dict[str, Any]]) -> int: + score: dict[str, Any] | None = None + controls: list[dict[str, Any]] = [] + for row in rows: + rtype = _lower(row.get("type")) + props = row.get("properties") or {} + sc = _get(props, "score") or {} + if rtype == "microsoft.security/securescores": + score = { + "current": _get(sc, "current"), + "max": _get(sc, "max"), + "percentage": _get(sc, "percentage"), + } + elif rtype == "microsoft.security/securescores/securescorecontrols": + controls.append( + { + "control": _get(props, "displayName"), + "current": _get(sc, "current"), + "max": _get(sc, "max"), + "unhealthy_resources": _get(props, "unhealthyResourceCount"), + } + ) + if score is None and not controls: + return 0 + md = _cspm_asset(b).metadata + if score is not None: + md["secure_score"] = score + if controls: + controls.sort(key=lambda c: -(c.get("unhealthy_resources") or 0)) + md["secure_score_controls"] = controls[:30] + return len(rows) + + +def _alert_resources(props: dict[str, Any]) -> list[str]: + out: list[str] = [] + for ident in _list(_get(props, "resourceIdentifiers")): + rid = _get(ident, "azureResourceId") or _get(ident, "AzureResourceId") + if isinstance(rid, str) and rid.startswith("/") and rid not in out: + out.append(rid.rstrip("/")) + entity = _get(props, "compromisedEntity") + if not out and isinstance(entity, str) and entity.startswith("/subscriptions/"): + out.append(entity) + return out + + +def add_security_alerts(b: "AzureAssetBuilder", rows: list[dict[str, Any]]) -> int: + """Active alerts: one THREAT_DETECTOR asset FINDING_AFFECTS each named resource.""" + agg: dict[str, dict[str, Any]] = {} + severities: dict[str, int] = {} + rows = [r for r in rows if _lower(r.get("type")) == "microsoft.security/locations/alerts"] + for row in rows: + props = row.get("properties") or {} + sev = str(_get(props, "severity") or _get(props, "Severity") or "Unknown") + severities[sev] = severities.get(sev, 0) + 1 + name = _get(props, "alertDisplayName") or _get(props, "AlertDisplayName") or row.get("name") + for rid in _alert_resources(props): + entry = agg.setdefault(rid, {"alerts": 0, "high": 0, "names": []}) + entry["alerts"] += 1 + entry["high"] += _lower(sev) == "high" + if name and name not in entry["names"] and len(entry["names"]) < _MAX_TITLES: + entry["names"].append(name) + if not rows: + return 0 + sub = b.subscription_id + relations = [rel(subscription_ref(sub), EdgeType.MONITORS, "MONITORED_BY")] + relations += [ + rel(rid, EdgeType.MONITORS, "FINDING_AFFECTS", description="Defender alert", **entry) + for rid, entry in list(agg.items())[:_MAX_COVERED] + ] + b.add_extra( + CloudAsset( + arn=f"cloudg:azure:defender-alerts:{sub}", + name="Defender for Cloud alerts", + asset_type=AssetType.THREAT_DETECTOR, + provider=CloudProvider.AZURE, + region="global", + account_id=sub, + metadata={ + "security_service": "defender-alerts", + "enabled": True, + "active_alerts": sum(severities.values()), + "alerts_by_severity": severities, + "affected_resources": len(agg), + "aliases": [_security_alias("defender-alerts", sub)], + "relations": [r for r in relations if r], + }, + ) + ) + return len(rows) + + +# ── Microsoft Sentinel ── + + +def _sentinel_row(workspace: str, subscription_id: str, region: str) -> dict[str, Any]: + return { + "id": f"{workspace}/providers/Microsoft.SecurityInsights/onboardingStates/default", + "name": "default", + "type": "microsoft.securityinsights/onboardingstates", + "location": region, + "subscriptionId": subscription_id, + "properties": {}, + } + + +@extractor("microsoft.operationsmanagement/solutions") +def _x_solution(d: _Draft, b: "AzureAssetBuilder") -> None: + ws = _get(d.props, "workspaceResourceId") + product = _lower(_get(d.row.get("plan") or {}, "product")) + d.md["workspace_id"] = ws + d.add(rel(ws, EdgeType.REFERENCES, "DEPENDS_ON", description="solution workspace")) + if isinstance(ws, str) and ( + d.name.lower().startswith("securityinsights(") or product.endswith("/securityinsights") + ): + b.add_row(_sentinel_row(ws, d.account_id, d.region), synthetic=True) + + +@extractor("microsoft.securityinsights/onboardingstates") +def _x_sentinel(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.governance import extension_owner + + ws = extension_owner(d.id) + d.name = f"Microsoft Sentinel ({(ws or '').rsplit('/', 1)[-1]})" + d.md.update({"security_service": "sentinel", "enabled": True, "workspace_id": ws}) + d.alias(_security_alias("sentinel", d.account_id)) + d.add(rel(ws, EdgeType.REFERENCES, "DEPENDS_ON", description="Sentinel workspace")) + d.add(rel(subscription_ref(d.account_id), EdgeType.MONITORS, "MONITORED_BY")) + + +def sentinel_placeholder(b: "AzureAssetBuilder") -> None: + """ "Microsoft Sentinel (not enabled)" when no workspace has Sentinel.""" + if not b.drafts_of("microsoft.securityinsights/onboardingstates"): + b.add_extra( + _disabled_asset( + b.subscription_id, "sentinel", AssetType.THREAT_DETECTOR, "Microsoft Sentinel" + ) + ) + + +def _connector_relations(d: _Draft, ws: str | None) -> None: + p = d.props + kind = _lower(d.row.get("kind")) + d.md["connector_kind"] = d.row.get("kind") + d.add(rel(ws, EdgeType.LOGS_TO, "LOGS_TO", description="Sentinel data connector")) + sub = _get(p, "subscriptionId") + if sub: + d.add(rel(subscription_ref(sub), EdgeType.MONITORS, "MONITORED_BY")) + tenant = _get(p, "tenantId") + if tenant and kind in ("azureactivedirectory", "office365", "microsoftthreatprotection"): + d.add(rel(f"entra:tenant/{tenant}", EdgeType.MONITORS, "MONITORED_BY")) + role = _get(p, "awsRoleArn") or _get(p, "roleArn") + if isinstance(role, str) and role.startswith("arn:aws"): + d.add( + rel( + role, + EdgeType.ASSUMES_ROLE, + "CROSS_ACCOUNT_TRUST", + description="Sentinel AWS connector role", + ) + ) + for url in _list(_get(p, "sqsUrls")): + d.add(rel(url, EdgeType.REFERENCES, "READS_FROM", description="S3 notification queue")) + + +@extractor("microsoft.securityinsights/dataconnectors") +def _x_sentinel_connector(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.governance import extension_owner + + _connector_relations(d, extension_owner(d.id)) + + +@extractor("microsoft.securityinsights/alertrules") +def _x_sentinel_rule(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.governance import extension_owner + + p = d.props + d.name = _get(p, "displayName") or d.name + d.md.update( + { + "rule_kind": d.row.get("kind"), + "enabled": _get(p, "enabled"), + "severity": _get(p, "severity"), + "tactics": _list(_get(p, "tactics")), + "query_frequency": _get(p, "queryFrequency"), + } + ) + d.add( + rel(extension_owner(d.id), EdgeType.MONITORS, "MONITORED_BY", description="analytics rule") + ) + + +@extractor("microsoft.securityinsights/automationrules") +def _x_sentinel_automation(d: _Draft, b: "AzureAssetBuilder") -> None: + p = d.props + d.name = _get(p, "displayName") or d.name + d.md["order"] = _get(p, "order") + for action in _list(_get(p, "actions")): + app = _path(action, "actionConfiguration", "logicAppResourceId") + d.add(rel(app, EdgeType.INVOKES, "INVOKES", description="playbook")) + + +# ── guest configuration ── + + +@extractor("microsoft.guestconfiguration/guestconfigurationassignments") +def _x_guest_configuration(d: _Draft, b: "AzureAssetBuilder") -> None: + from cloudg.inventory.azure_graph.governance import extension_owner + + p = d.props + status = _get(p, "complianceStatus") + machine = _get(p, "targetResourceId") or extension_owner(d.id) + d.md.update( + { + "guardrail_kind": "guest-configuration", + "compliance_status": status, + "configuration": _path(p, "guestConfiguration", "name"), + "assignment_type": _path(p, "guestConfiguration", "assignmentType"), + "last_checked": _get(p, "lastComplianceStatusChecked"), + } + ) + d.add( + rel( + machine, + EdgeType.GOVERNS, + "FINDING_AFFECTS" if _lower(status) == "noncompliant" else "COMPLIANCE_GOVERNS", + description="guest configuration", + compliance_status=status, + ) + ) + + +# ── Defender for Cloud automations (export to Event Hubs / Log Analytics / Logic Apps) ── + + +@extractor("microsoft.security/automations") +def _x_security_automation(d: _Draft, b: "AzureAssetBuilder") -> None: + for action in _list(_get(d.props, "actions")): + kind = _lower(_get(action, "actionType")) + target = ( + _get(action, "logicAppResourceId") + or _get(action, "eventHubResourceId") + or _get(action, "workspaceResourceId") + ) + if kind == "logicapp": + d.add(rel(target, EdgeType.INVOKES, "INVOKES", description="workflow automation")) + elif kind == "eventhub": + d.add(rel(target, EdgeType.LOGS_TO, "STREAMS_TO", description="continuous export")) + elif kind == "workspace": + d.add(rel(target, EdgeType.LOGS_TO, "LOGS_TO", description="continuous export")) + else: + d.add(rel(host_of(_get(action, "uri")), EdgeType.INVOKES, "INVOKES")) + + +__all__ = [ + "PLAN_RESOURCE_TYPES", + "STANDARD_PLANS", + "add_assessments", + "add_secure_score", + "add_security_alerts", + "defender_coverage", + "sentinel_placeholder", +] diff --git a/cloudg/inventory/azure_graph/tables.py b/cloudg/inventory/azure_graph/tables.py new file mode 100644 index 0000000..404e2f4 --- /dev/null +++ b/cloudg/inventory/azure_graph/tables.py @@ -0,0 +1,196 @@ +"""Best-effort Resource Graph tables beyond ``resources``. + +Each table is one task (:mod:`.families`); a failing query is recorded as a +FAILED coverage record and never stops the subscription. ARM-shaped rows +go through the normal builder pipeline, so through the extractors (and +relation-only row handlers, such as maintenance and data collection rule +associations) registered for their types. The aggregate tables (policy +states, Defender assessments, alerts, secure score) have handlers that +attach relations to existing assets instead of creating one asset per row. +""" + +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING, Any, Callable + +from cloudg.inventory.azure_graph.helpers import _get, _lower, management_group_ref, normalize_scope +from cloudg.inventory.azure_graph.query import ( + ARG_TABLE_TASKS, + management_group_role_assignments_query, + policy_definitions_query, + policy_states_query, + role_definitions_query, + run_query, + secure_score_query, + security_alerts_query, + security_assessments_query, + table_query, +) + +if TYPE_CHECKING: + from cloudg.inventory.azure_graph.builder import AzureAssetBuilder + +logger = logging.getLogger("cloudg.inventory.azure_graph") + +# Aggregate tables are capped: they can hold one row per CVE per image +_AGGREGATE_PAGES = 20 + + +def _ancestors(container_rows: list[dict[str, Any]], subscription_id: str) -> list[str]: + """Management group names above the subscription, nearest first.""" + for row in container_rows: + if _lower(row.get("type")) != "microsoft.resources/subscriptions": + continue + chain = _get(row.get("properties") or {}, "managementGroupAncestorsChain") or [] + return [str(_get(a, "name")) for a in chain if isinstance(a, dict) and _get(a, "name")] + return [] + + +def _inherited_rows(rows: list[dict[str, Any]], ancestors: list[str]) -> list[dict[str, Any]]: + """Management-group / root role assignments that apply to this subscription.""" + wanted = {management_group_ref(a).lower() for a in ancestors} + out = [] + for row in rows: + scope = normalize_scope(row.get("scope") or _get(row.get("properties") or {}, "scope")) + if not scope: + continue + above = scope == "/" or scope.lower().startswith( + "/providers/microsoft.management/managementgroups/" + ) + # without the ancestor chain only root assignments are known to apply + if above and (scope == "/" or scope.lower() in wanted): + out.append(row) + return out + + +def _referenced_definition_names(builder: "AzureAssetBuilder") -> list[str]: + names: list[str] = [] + for d in builder.drafts_of("microsoft.authorization/policyassignments"): + ref = _get(d.props, "policyDefinitionId") + if isinstance(ref, str) and ref: + names.append(ref.rstrip("/").rsplit("/", 1)[-1]) + return list(dict.fromkeys(names)) + + +class _TableRun: + """Runs one subscription's best-effort tables and records each as a task.""" + + def __init__( + self, + client: Any, + builder: "AzureAssetBuilder", + subscription_id: str, + on: Callable[[str], bool], + recorder: Any, + ) -> None: + self.client = client + self.b = builder + self.sub = subscription_id + self.on = on + self.rec = recorder + + def query(self, query: str, *, tenant: bool = False, max_pages: int | None = None) -> list: + kwargs: dict[str, Any] = {} if tenant else {"subscriptions": [self.sub]} + if max_pages: + kwargs["max_pages"] = max_pages + return run_query(self.client, query, **kwargs) + + def task(self, name: str, fn: Callable[[], int]) -> None: + if not self.on(name): + return + started = time.time() + try: + count = fn() + except Exception as exc: + logger.warning("Azure Resource Graph task %s failed for %s: %s", name, self.sub, exc) + self.rec.record(name, "FAILED", error=str(exc)[:500], started=started) + return + self.rec.record(name, "SUCCESS", count, started=started) + + # ── tasks ── + + def arm_table(self, table: str, types: tuple[str, ...]) -> int: + rows = self.query(table_query(table, self.sub, types)) + if types: + wanted = set(types) + rows = [r for r in rows if _lower(r.get("type")) in wanted] + self.b.add_rows(rows, discovered_via="resource-graph") + return len(rows) + + def policy_definitions(self) -> int: + rows = self.query( + policy_definitions_query(_referenced_definition_names(self.b)), tenant=True + ) + before = len(self.b.drafts) + self.b.add_rows(rows, discovered_via="resource-graph") + return len(self.b.drafts) - before + + def policy_states(self) -> int: + from cloudg.inventory.azure_graph.governance import add_policy_states + + rows = self.query(policy_states_query(self.sub), max_pages=_AGGREGATE_PAGES) + return add_policy_states(self.b, rows) + + def assessments(self) -> int: + from cloudg.inventory.azure_graph.security import add_assessments + + rows = self.query(security_assessments_query(self.sub), max_pages=_AGGREGATE_PAGES) + return add_assessments(self.b, rows) + + def secure_score(self) -> int: + from cloudg.inventory.azure_graph.security import add_secure_score + + return add_secure_score(self.b, self.query(secure_score_query(self.sub))) + + def alerts(self) -> int: + from cloudg.inventory.azure_graph.security import add_security_alerts + + rows = self.query(security_alerts_query(self.sub), max_pages=_AGGREGATE_PAGES) + return add_security_alerts(self.b, rows) + + def inherited_roles(self, container_rows: list[dict[str, Any]]) -> int: + rows = self.query(management_group_role_assignments_query(), tenant=True) + rows = _inherited_rows( + [r for r in rows if _get(r, "principalId")], _ancestors(container_rows, self.sub) + ) + self.b.add_role_assignments(rows, inherited_from="management group") + return len(rows) + + def role_definitions(self) -> int: + rows = self.query(role_definitions_query(), tenant=True) + self.b.add_role_definitions(rows) + return len(rows) + + +def collect_tables( + client: Any, + builder: "AzureAssetBuilder", + subscription_id: str, + *, + on: Callable[[str], bool], + recorder: Any, + container_rows: list[dict[str, Any]], + role_assignments: bool = True, +) -> None: + """Run every best-effort Resource Graph task selected by ``on``.""" + run = _TableRun(client, builder, subscription_id, on, recorder) + if role_assignments: + run.task("management_group_role_assignments", lambda: run.inherited_roles(container_rows)) + run.task("role_definitions", run.role_definitions) + for task, table, types in ARG_TABLE_TASKS: + if task == "policy": + run.task( + "policy", + lambda t=table, ty=types: run.arm_table(t, ty) + run.policy_definitions(), + ) + run.task("policy_states", run.policy_states) + continue + run.task(task, lambda t=table, ty=types: run.arm_table(t, ty)) + run.task("security_assessments", run.assessments) + run.task("secure_score", run.secure_score) + run.task("security_alerts", run.alerts) + + +__all__ = ["collect_tables"] diff --git a/cloudg/inventory/azure_hierarchy.py b/cloudg/inventory/azure_hierarchy.py index f763062..eabc322 100644 --- a/cloudg/inventory/azure_hierarchy.py +++ b/cloudg/inventory/azure_hierarchy.py @@ -352,12 +352,78 @@ def _policy_assets(rows: list[dict[str, Any]]) -> list[CloudAsset]: return assets +# Azure parity: subscription vending +def _subscription_alias_assets( + credential: Any, arm_client_factory: Callable[[Any], Any] | None, errors: list[str] | None +) -> list[CloudAsset]: + """Subscription aliases (vending): ``PROVISIONED_PRODUCT`` that ``MANAGES`` its subscription. + + Best-effort. A credential that cannot list aliases (403, or no token) + is logged, not reported: most callers have no vending rights at all. + """ + from cloudg.exceptions import CredentialError + from cloudg.inventory.azure_graph.rest import AzureRestError, default_arm_client_factory + + try: + client = (arm_client_factory or default_arm_client_factory)(credential) + rows = client.list("/providers/Microsoft.Subscription/aliases", "2021-10-01") + except (CredentialError, AzureRestError) as exc: + if isinstance(exc, AzureRestError) and not (exc.forbidden or exc.not_applicable): + if errors is not None: + errors.append(f"subscription aliases: {exc}") + logger.info("Azure subscription aliases not mapped: %s", exc) + return [] + except Exception as exc: + if errors is not None: + errors.append(f"subscription aliases: {exc}") + return [] + assets = [] + for row in rows: + rid = row.get("id") + props = row.get("properties") or {} + sub = _get(props, "subscriptionId") + if not rid: + continue + assets.append( + _asset( + rid, + row.get("name") or rid.rsplit("/", 1)[-1], + AssetType.PROVISIONED_PRODUCT, + { + "resource_type": "microsoft.subscription/aliases", + "subscription_id": sub, + "provisioning_state": _get(props, "provisioningState"), + "workload": _get(props, "workload"), + "billing_scope": _get(props, "billingScope"), + }, + [ + rel( + subscription_ref(sub) if sub else None, + EdgeType.MANAGES, + "OWNED_BY", + description="subscription vending alias", + ), + _contained_by_group( + str(_get(props, "managementGroupId") or "").rsplit("/", 1)[-1] or None + ), + ], + [rid.lower()], + ) + ) + return assets + + +# end Azure parity + + def discover_azure_hierarchy( credential: Any, graph_client_factory: Callable[[Any], Any] | None = None, *, include_policies: bool = True, errors: list[str] | None = None, + subscription_aliases: bool = True, + arm_client_factory: Callable[[Any], Any] | None = None, ) -> list[CloudAsset]: """Discover the tenant's management groups, subscriptions and policy. @@ -369,6 +435,9 @@ def discover_azure_hierarchy( defaults to ``azure.mgmt.resourcegraph.ResourceGraphClient``. include_policies: Also map policy assignments and exemptions. errors: Optional list receiving non-fatal failure messages. + subscription_aliases: Also map subscription vending aliases + (``Microsoft.Subscription/aliases``) through ARM REST. + arm_client_factory: ``credential -> ARM REST client`` for the aliases. Returns: ORG_UNIT, CLOUD_ACCOUNT and GUARDRAIL assets with declared relations. @@ -401,6 +470,8 @@ def discover_azure_hierarchy( if errors is not None: errors.append(f"policyresources (subscriptions): {exc2}") assets.extend(_policy_assets(policy_rows)) + if subscription_aliases: + assets.extend(_subscription_alias_assets(credential, arm_client_factory, errors)) logger.info( "Azure hierarchy: %d management groups, %d subscriptions, %d policy objects", len(groups), diff --git a/cloudg/inventory/azure_kubernetes.py b/cloudg/inventory/azure_kubernetes.py new file mode 100644 index 0000000..71b2862 --- /dev/null +++ b/cloudg/inventory/azure_kubernetes.py @@ -0,0 +1,102 @@ +"""Read-only in-cluster mapping for AKS, the same way EKS clusters are mapped. + +For each AKS cluster with Microsoft Entra integration, cloudg: + +1. reads the cluster API server URL and CA certificate from the user + kubeconfig ARM returns for ``listClusterUserCredential`` (a read-only + list action; for Entra-integrated clusters the kubeconfig holds no + secret, and only the ``clusters`` section is ever parsed), +2. requests an Entra token for the AKS server application + (``6dae42f8-4368-4678-94ff-3960e28e3630/.default``) from the same + azure-identity credential, exactly what ``kubelogin`` does, and +3. reads namespaces, workloads, services, ingresses and service accounts + with :func:`cloudg.inventory.kubernetes.map_cluster_objects`, issuing + only ``GET`` requests with TLS verified against the cluster CA. + +The identity needs Azure Kubernetes Service RBAC Reader (Azure RBAC +clusters) or a Kubernetes ``view`` binding. Clusters with local accounts +only (no Entra integration) are skipped: their credentials are secrets. +Private clusters are tried and reported FAILED when the API server is not +reachable from where cloudg runs. ``az aks command invoke`` is not used: +it starts a pod in the cluster, which is not read-only. +""" + +from __future__ import annotations + +import base64 +import logging +from typing import Any, Callable + +import yaml + +from cloudg.inventory.azure_graph.helpers import _get, _path +from cloudg.inventory.kubernetes import KubernetesReader, map_cluster_objects +from cloudg.schema.models import CloudAsset, CloudProvider + +logger = logging.getLogger(__name__) + +AKS_SERVER_APP_ID = "6dae42f8-4368-4678-94ff-3960e28e3630" +AKS_SCOPE = f"{AKS_SERVER_APP_ID}/.default" +_AKS_API = "2024-05-01" + + +def cluster_endpoint(arm: Any, cluster_id: str) -> tuple[str, str | None]: + """(API server URL, base64 CA bundle) from the cluster's user kubeconfig.""" + data = arm.post(f"{cluster_id}/listClusterUserCredential", _AKS_API) + for entry in data.get("kubeconfigs") or []: + raw = entry.get("value") + if not raw: + continue + config = yaml.safe_load(base64.b64decode(raw).decode("utf-8")) or {} + for cluster in config.get("clusters") or []: + spec = cluster.get("cluster") or {} + server = spec.get("server") + if server: + return str(server), spec.get("certificate-authority-data") + raise ValueError("no API server in the cluster kubeconfig") + + +def aks_mappable(cluster: dict[str, Any]) -> str | None: + """Why a cluster cannot be read in-cluster (None when it can).""" + props = cluster.get("properties") or {} + if not _get(props, "aadProfile"): + return "no Microsoft Entra integration (local accounts only)" + if _lower_state(_path(props, "powerState", "code")) == "stopped": + return "cluster is stopped" + return None + + +def _lower_state(value: Any) -> str: + return str(value or "").lower() + + +def collect_aks_workloads( + credential: Any, + arm: Any, + cluster: dict[str, Any], + *, + timeout: int = 10, + reader_factory: Callable[[str, str | None, str, int], Any] | None = None, +) -> list[CloudAsset]: + """Map the Kubernetes objects of one AKS cluster (blocking; run in a thread). + + Args: + credential: azure-identity TokenCredential. + arm: ARM REST client (``post(path, api_version)``). + cluster: The cluster row (``id``, ``location``, ``subscriptionId``, ``properties``). + reader_factory: ``(server, ca_b64, token, timeout) -> reader`` (tests). + """ + server, ca = cluster_endpoint(arm, cluster["id"]) + token = credential.get_token(AKS_SCOPE).token + factory = reader_factory or (lambda s, c, t, to: KubernetesReader(s, c, t, timeout=to)) + reader = factory(server, ca, token, timeout) + return map_cluster_objects( + reader, + cluster["id"], + cluster.get("location") or "global", + cluster.get("subscriptionId"), + provider=CloudProvider.AZURE, + ) + + +__all__ = ["AKS_SCOPE", "aks_mappable", "cluster_endpoint", "collect_aks_workloads"] diff --git a/cloudg/inventory/catalogs/azure_arm_types.yaml b/cloudg/inventory/catalogs/azure_arm_types.yaml index 4856a1b..b9395d8 100644 --- a/cloudg/inventory/catalogs/azure_arm_types.yaml +++ b/cloudg/inventory/catalogs/azure_arm_types.yaml @@ -131,3 +131,173 @@ arm_types: "microsoft.management/managementgroups": ORG_UNIT "microsoft.authorization/policyassignments": GUARDRAIL "microsoft.authorization/policyexemptions": GUARDRAIL + + # Azure parity: child resources, extra Resource Graph tables, REST listings + # governance and identity + "microsoft.authorization/policydefinitions": ORG_POLICY + "microsoft.authorization/policysetdefinitions": ORG_POLICY + "microsoft.authorization/locks": GUARDRAIL + "microsoft.authorization/denyassignments": GUARDRAIL + "microsoft.authorization/roledefinitions": IAM_POLICY + "microsoft.guestconfiguration/guestconfigurationassignments": GUARDRAIL + "microsoft.resources/deployments": IAC_STACK + "microsoft.resources/deploymentstacks": IAC_STACK + "microsoft.resources/templatespecs": PRODUCT_PORTFOLIO + "microsoft.resources/templatespecs/versions": PRODUCT_PORTFOLIO + "microsoft.solutions/applications": PROVISIONED_PRODUCT + "microsoft.solutions/applicationdefinitions": PRODUCT_PORTFOLIO + "microsoft.subscription/aliases": PROVISIONED_PRODUCT + "microsoft.managedservices/registrationassignments": RESOURCE_SHARE + "microsoft.managedservices/registrationdefinitions": RESOURCE_SHARE + "microsoft.azureactivedirectory/b2cdirectories": USER_POOL + "microsoft.azureactivedirectory/ciamdirectories": USER_POOL + "microsoft.managedidentity/userassignedidentities/federatedidentitycredentials": IDENTITY_PROVIDER + "microsoft.keyvault/vaults/keys": KMS_KEY + "microsoft.keyvault/vaults/secrets": SECRET + # compute + "microsoft.compute/availabilitysets": PLACEMENT_GROUP + "microsoft.compute/proximityplacementgroups": PLACEMENT_GROUP + "microsoft.compute/hostgroups": PLACEMENT_GROUP + "microsoft.compute/hostgroups/hosts": PLACEMENT_GROUP + "microsoft.compute/galleries": MACHINE_IMAGE + "microsoft.batch/batchaccounts/pools": BATCH_ENVIRONMENT + "microsoft.appplatform/spring": ECS_CLUSTER + "microsoft.appplatform/spring/apps": CONTAINER_SERVICE + # hybrid and transit networking + "microsoft.network/virtualhubs/hubvirtualnetworkconnections": PEERING_CONNECTION + "microsoft.network/virtualhubs/hubroutetables": ROUTE_TABLE + "microsoft.network/virtualhubs/routingintent": ROUTE_TABLE + "microsoft.network/virtualhubs/bgpconnections": PEERING_CONNECTION + "microsoft.network/virtualhubs/ipconfigurations": NETWORK_INTERFACE + "microsoft.network/vpnsites": CUSTOMER_GATEWAY + "microsoft.network/p2svpngateways": VPN_GATEWAY + "microsoft.network/expressrouteports": DIRECT_CONNECT + "microsoft.network/networkmanagers/networkgroups": SERVICE_NETWORK + "microsoft.network/networkmanagers/connectivityconfigurations": SERVICE_NETWORK + "microsoft.network/networkmanagers/securityadminconfigurations": NACL + "microsoft.network/networkmanagers/securityadminconfigurations/rulecollections": NACL + "microsoft.network/networkmanagers/routingconfigurations": ROUTE_TABLE + "microsoft.network/applicationgateways/httplisteners": LB_LISTENER + "microsoft.insights/privatelinkscopes": ENDPOINT_SERVICE + # DNS + "microsoft.network/dnszones/a": DNS_RECORD + "microsoft.network/dnszones/aaaa": DNS_RECORD + "microsoft.network/dnszones/cname": DNS_RECORD + "microsoft.network/dnszones/mx": DNS_RECORD + "microsoft.network/dnszones/ns": DNS_RECORD + "microsoft.network/dnszones/txt": DNS_RECORD + "microsoft.network/dnszones/srv": DNS_RECORD + "microsoft.network/dnszones/caa": DNS_RECORD + "microsoft.network/dnszones/ptr": DNS_RECORD + "microsoft.network/dnszones/soa": DNS_RECORD + "microsoft.network/dnszones/ds": DNS_RECORD + "microsoft.network/privatednszones/a": DNS_RECORD + "microsoft.network/privatednszones/aaaa": DNS_RECORD + "microsoft.network/privatednszones/cname": DNS_RECORD + "microsoft.network/privatednszones/mx": DNS_RECORD + "microsoft.network/privatednszones/ptr": DNS_RECORD + "microsoft.network/privatednszones/soa": DNS_RECORD + "microsoft.network/privatednszones/srv": DNS_RECORD + "microsoft.network/privatednszones/txt": DNS_RECORD + "microsoft.network/dnsresolvers/inboundendpoints": DNS_RESOLVER + "microsoft.network/dnsresolvers/outboundendpoints": DNS_RESOLVER + "microsoft.network/dnsforwardingrulesets": DNS_RESOLVER + "microsoft.network/dnsforwardingrulesets/forwardingrules": DNS_RESOLVER + # Front Door Standard / Premium + "microsoft.cdn/profiles/afdendpoints/routes": LB_LISTENER + "microsoft.cdn/profiles/origingroups": TARGET_GROUP + "microsoft.cdn/profiles/origingroups/origins": TARGET_GROUP + "microsoft.cdn/profiles/rulesets": EDGE_FUNCTION + "microsoft.cdn/profiles/rulesets/rules": EDGE_FUNCTION + "microsoft.cdn/profiles/secrets": CERTIFICATE + "microsoft.cdn/profiles/securitypolicies": WAF_WEB_ACL + # API Management + "microsoft.apimanagement/service/apis": API_GATEWAY + "microsoft.apimanagement/service/apis/authorizers": AUTHORIZER + "microsoft.apimanagement/service/backends": TARGET_GROUP + "microsoft.apimanagement/service/products": PRODUCT_PORTFOLIO + "microsoft.apimanagement/service/namedvalues": PARAMETER + "microsoft.apimanagement/service/gateways": API_GATEWAY + "microsoft.apimanagement/service/authorizationservers": AUTHORIZER + "microsoft.apimanagement/service/openidconnectproviders": IDENTITY_PROVIDER + # serverless and integration + "microsoft.web/sites/functions/timers": SCHEDULE + "microsoft.web/connections": SOURCE_CONNECTION + "microsoft.logic/workflows/triggers": SCHEDULE + "microsoft.servicebus/namespaces/queues": MESSAGE_QUEUE + "microsoft.servicebus/namespaces/topics": NOTIFICATION_TOPIC + "microsoft.servicebus/namespaces/topics/subscriptions": MESSAGE_QUEUE + "microsoft.eventhub/namespaces/eventhubs": DATA_STREAM + "microsoft.eventgrid/eventsubscriptions": EVENT_RULE + # analytics and data + "microsoft.datafactory/factories/pipelines": ETL_JOB + "microsoft.datafactory/factories/linkedservices": SOURCE_CONNECTION + "microsoft.datafactory/factories/datasets": DATA_CATALOG + "microsoft.datafactory/factories/integrationruntimes": BATCH_ENVIRONMENT + "microsoft.datafactory/factories/triggers": SCHEDULE + "microsoft.synapse/workspaces/pipelines": ETL_JOB + "microsoft.synapse/workspaces/linkedservices": SOURCE_CONNECTION + "microsoft.synapse/workspaces/datasets": DATA_CATALOG + "microsoft.synapse/workspaces/integrationruntimes": BATCH_ENVIRONMENT + "microsoft.synapse/workspaces/triggers": SCHEDULE + "microsoft.synapse/workspaces/sqlpools": DATA_WAREHOUSE + "microsoft.synapse/workspaces/bigdatapools": BIG_DATA_CLUSTER + "microsoft.kusto/clusters/databases": DATA_WAREHOUSE + "microsoft.sql/managedinstances/databases": AZURE_SQL + "microsoft.documentdb/databaseaccounts/regions": DYNAMODB_TABLE + "microsoft.storage/storageaccounts/localusers": IDENTITY_USER + "microsoft.netapp/netappaccounts/capacitypools": FILE_SYSTEM + "microsoft.netapp/netappaccounts/capacitypools/volumes": FILE_SYSTEM + "microsoft.netapp/netappaccounts/snapshotpolicies": BACKUP_PLAN + "microsoft.netapp/netappaccounts/backuppolicies": BACKUP_PLAN + "microsoft.netapp/netappaccounts/backupvaults": BACKUP_VAULT + # AI and ML + "microsoft.cognitiveservices/accounts/deployments": ML_MODEL + "microsoft.cognitiveservices/accounts/raipolicies": AI_GUARDRAIL + "microsoft.cognitiveservices/accounts/projects": ML_WORKSPACE + "microsoft.cognitiveservices/accounts/connections": SOURCE_CONNECTION + "microsoft.cognitiveservices/accounts/projects/connections": SOURCE_CONNECTION + "microsoft.machinelearningservices/workspaces/onlineendpoints/deployments": ML_MODEL + "microsoft.machinelearningservices/workspaces/serverlessendpoints": ML_ENDPOINT + "microsoft.machinelearningservices/workspaces/computes": BATCH_ENVIRONMENT + "microsoft.machinelearningservices/workspaces/datastores": SOURCE_CONNECTION + "microsoft.machinelearningservices/workspaces/connections": SOURCE_CONNECTION + "microsoft.search/searchservices/datasources": SOURCE_CONNECTION + "microsoft.search/searchservices/indexers": ETL_JOB + "microsoft.search/searchservices/indexes": KNOWLEDGE_BASE + "microsoft.search/searchservices/skillsets": ETL_JOB + # operations and monitoring + "microsoft.alertsmanagement/smartdetectoralertrules": ALARM + "microsoft.insights/datacollectionendpoints": LOG_SINK + "microsoft.automation/automationaccounts/runbooks": RUNBOOK + "microsoft.automation/automationaccounts/schedules": SCHEDULE + "microsoft.automation/automationaccounts/hybridrunbookworkergroups": NODE_GROUP + "microsoft.automation/automationaccounts/softwareupdateconfigurations": SCHEDULE + "microsoft.maintenance/maintenanceconfigurations": SCHEDULE + "microsoft.appconfiguration/configurationstores/replicas": PARAMETER + # containers + "microsoft.kubernetes/connectedclusters": AKS_CLUSTER + "microsoft.kubernetesconfiguration/extensions": CLUSTER_ADDON + "microsoft.kubernetesconfiguration/fluxconfigurations": DEPLOYMENT_GROUP + "microsoft.kubernetesconfiguration/sourcecontrolconfigurations": DEPLOYMENT_GROUP + "microsoft.containerregistry/registries/replications": CONTAINER_REGISTRY + "microsoft.containerregistry/registries/webhooks": EVENT_RULE + "microsoft.containerregistry/registries/tasks": BUILD_PROJECT + "microsoft.containerregistry/registries/scopemaps": IAM_POLICY + "microsoft.containerregistry/registries/tokens": ACCESS_KEY + # security + "microsoft.securityinsights/onboardingstates": THREAT_DETECTOR + "microsoft.securityinsights/dataconnectors": LOG_SINK + "microsoft.securityinsights/alertrules": ALARM + "microsoft.securityinsights/automationrules": EVENT_RULE + # backup and recovery + "microsoft.recoveryservices/vaults/backuppolicies": BACKUP_PLAN + "microsoft.recoveryservices/vaults/replicationpolicies": BACKUP_PLAN + "microsoft.dataprotection/backupvaults/backuppolicies": BACKUP_PLAN + # CI/CD + "microsoft.devcenter/devcenters": PRODUCT_PORTFOLIO + "microsoft.devcenter/projects": DEPLOYMENT_GROUP + "microsoft.devcenter/devcenters/catalogs": SOURCE_CONNECTION + "microsoft.devcenter/projects/catalogs": SOURCE_CONNECTION + "microsoft.devcenter/projects/environmenttypes": DEPLOYMENT_GROUP + # end Azure parity diff --git a/cloudg/inventory/catalogs/gcp_asset_types.yaml b/cloudg/inventory/catalogs/gcp_asset_types.yaml index 959f98c..51c1c6b 100644 --- a/cloudg/inventory/catalogs/gcp_asset_types.yaml +++ b/cloudg/inventory/catalogs/gcp_asset_types.yaml @@ -122,7 +122,7 @@ deep: "file.googleapis.com/Instance": FILE_SYSTEM "file.googleapis.com/Backup": SNAPSHOT "bigquery.googleapis.com/Dataset": DATA_WAREHOUSE - "bigquery.googleapis.com/Table": OTHER + "bigquery.googleapis.com/Table": DATA_TABLE # GCP parity (was OTHER) "bigtableadmin.googleapis.com/Instance": DYNAMODB_TABLE "firestore.googleapis.com/Database": DYNAMODB_TABLE "spanner.googleapis.com/Instance": CLOUD_SQL @@ -155,3 +155,372 @@ deep: "logging.googleapis.com/LogSink": LOG_SINK "logging.googleapis.com/LogBucket": LOG_GROUP "monitoring.googleapis.com/AlertPolicy": ALARM + + # GCP parity: types mapped for depth parity with AWS. Types marked + # (API) are not in Cloud Asset Inventory; the deep collector reads them + # from the service API and gives them this asset type name. + "compute.googleapis.com/PacketMirroring": FLOW_LOG + "compute.googleapis.com/NodeGroup": AUTOSCALING_GROUP + "compute.googleapis.com/NodeTemplate": LAUNCH_TEMPLATE + "networkconnectivity.googleapis.com/RouteTable": ROUTE_TABLE + "networkconnectivity.googleapis.com/Group": SERVICE_NETWORK + "networkconnectivity.googleapis.com/InternalRange": PREFIX_LIST + "networkconnectivity.googleapis.com/PolicyBasedRoute": ROUTE_TABLE + "networkconnectivity.googleapis.com/RegionalEndpoint": VPC_ENDPOINT + "networksecurity.googleapis.com/FirewallEndpoint": NETWORK_FIREWALL + "networksecurity.googleapis.com/FirewallEndpointAssociation": NETWORK_FIREWALL + "networksecurity.googleapis.com/SecurityProfile": NETWORK_FIREWALL + "networksecurity.googleapis.com/SecurityProfileGroup": NETWORK_FIREWALL + "networksecurity.googleapis.com/TlsInspectionPolicy": NETWORK_FIREWALL + "networksecurity.googleapis.com/ServerTlsPolicy": CERTIFICATE + "networksecurity.googleapis.com/ClientTlsPolicy": CERTIFICATE + "networksecurity.googleapis.com/AuthorizationPolicy": WAF_WEB_ACL + "networksecurity.googleapis.com/AuthzPolicy": WAF_WEB_ACL + "networksecurity.googleapis.com/AddressGroup": PREFIX_LIST + "ids.googleapis.com/Endpoint": THREAT_DETECTOR + "networkservices.googleapis.com/Gateway": LOAD_BALANCER + "networkservices.googleapis.com/Mesh": SERVICE_NETWORK + "networkservices.googleapis.com/HttpRoute": LB_LISTENER + "networkservices.googleapis.com/GrpcRoute": LB_LISTENER + "networkservices.googleapis.com/TcpRoute": LB_LISTENER + "networkservices.googleapis.com/TlsRoute": LB_LISTENER + "networkservices.googleapis.com/ServiceBinding": SERVICE_REGISTRY + "networkservices.googleapis.com/EndpointPolicy": GUARDRAIL + "networkservices.googleapis.com/ServiceLbPolicy": LOAD_BALANCER + "networkservices.googleapis.com/EdgeCacheService": CDN + "networkservices.googleapis.com/EdgeCacheOrigin": CDN + "networkservices.googleapis.com/EdgeCacheKeyset": ACCESS_KEY + "certificatemanager.googleapis.com/CertificateMap": CERTIFICATE + "certificatemanager.googleapis.com/CertificateMapEntry": CERTIFICATE + "certificatemanager.googleapis.com/DnsAuthorization": DNS_RECORD + "certificatemanager.googleapis.com/CertificateIssuanceConfig": CERTIFICATE + "certificatemanager.googleapis.com/TrustConfig": CERTIFICATE + "privateca.googleapis.com/CaPool": CERTIFICATE + "dns.googleapis.com/ResourceRecordSet": DNS_RECORD # (API) + "identitytoolkit.googleapis.com/Config": USER_POOL + "identitytoolkit.googleapis.com/Tenant": USER_POOL + "identitytoolkit.googleapis.com/InboundSamlConfig": IDENTITY_PROVIDER + "identitytoolkit.googleapis.com/OauthIdpConfig": IDENTITY_PROVIDER + "identitytoolkit.googleapis.com/DefaultSupportedIdpConfig": IDENTITY_PROVIDER + "apigee.googleapis.com/Organization": API_GATEWAY + "apigee.googleapis.com/Instance": API_GATEWAY + "apigee.googleapis.com/Environment": API_GATEWAY + "apigee.googleapis.com/ApiProxy": API_GATEWAY + "apigee.googleapis.com/ApiProxyRevision": API_GATEWAY + "apigee.googleapis.com/TargetServer": API_DESTINATION # (API) + "apigateway.googleapis.com/Authorizer": AUTHORIZER # derived from ApiConfig documents + "artifactregistry.googleapis.com/DockerImage": CONTAINER_IMAGE + "binaryauthorization.googleapis.com/Policy": GUARDRAIL + "binaryauthorization.googleapis.com/Attestor": GUARDRAIL + "binaryauthorization.googleapis.com/PlatformPolicy": GUARDRAIL + "gkehub.googleapis.com/Fleet": FLEET + "gkehub.googleapis.com/Membership": FLEET + "gkehub.googleapis.com/Scope": FLEET + "gkehub.googleapis.com/MembershipBinding": FLEET + "gkehub.googleapis.com/Feature": CLUSTER_ADDON + "gkehub.googleapis.com/Namespace": K8S_NAMESPACE + "gkehub.googleapis.com/RBACRoleBinding": IAM_POLICY + "gkebackup.googleapis.com/BackupPlan": BACKUP_PLAN + "gkebackup.googleapis.com/Backup": SNAPSHOT + "gkebackup.googleapis.com/RestorePlan": BACKUP_PLAN + "containeranalysis.googleapis.com/VulnerabilitySummary": VULNERABILITY_SCANNER # (API) + "cloudscheduler.googleapis.com/Job": SCHEDULE # (API when not in the listing) + "eventarc.googleapis.com/ChannelConnection": EVENT_BUS + "eventarc.googleapis.com/MessageBus": EVENT_BUS + "eventarc.googleapis.com/GoogleApiSource": EVENT_BUS + "eventarc.googleapis.com/Enrollment": EVENT_RULE + "eventarc.googleapis.com/Pipeline": EVENT_PIPE + "managedkafka.googleapis.com/ConnectCluster": MESSAGE_BROKER + "managedkafka.googleapis.com/Connector": EVENT_PIPE + "managedkafka.googleapis.com/ConsumerGroup": MESSAGE_QUEUE + "pubsub.googleapis.com/Schema": DATA_CATALOG + "batch.googleapis.com/Job": JOB_DEFINITION + "spanner.googleapis.com/Database": CLOUD_SQL + "spanner.googleapis.com/Backup": SNAPSHOT + "spanner.googleapis.com/InstancePartition": CLOUD_SQL + "bigtableadmin.googleapis.com/Cluster": DYNAMODB_TABLE + "bigtableadmin.googleapis.com/Table": DATA_TABLE + "bigtableadmin.googleapis.com/AuthorizedView": DATA_TABLE + "bigtableadmin.googleapis.com/LogicalView": DATA_TABLE + "bigtableadmin.googleapis.com/MaterializedView": DATA_TABLE + "bigtableadmin.googleapis.com/Backup": SNAPSHOT + "bigtableadmin.googleapis.com/AppProfile": ROUTE_TABLE + "firestore.googleapis.com/Backup": SNAPSHOT + "firestore.googleapis.com/BackupSchedule": BACKUP_PLAN # (API) + "bigquery.googleapis.com/Routine": JOB_DEFINITION + "bigquery.googleapis.com/Model": ML_MODEL + "bigquery.googleapis.com/RowAccessPolicy": GUARDRAIL + "bigqueryconnection.googleapis.com/Connection": SOURCE_CONNECTION + "bigquerydatatransfer.googleapis.com/TransferConfig": DATA_TRANSFER + "dataplex.googleapis.com/Zone": DATA_CATALOG + "dataplex.googleapis.com/Asset": DATA_CATALOG + "dataplex.googleapis.com/Task": ETL_JOB + "dataplex.googleapis.com/DataScan": ETL_JOB + "dataplex.googleapis.com/EntryGroup": DATA_CATALOG + "datacatalog.googleapis.com/EntryGroup": DATA_CATALOG + "datacatalog.googleapis.com/TagTemplate": DATA_CATALOG + "datacatalog.googleapis.com/Taxonomy": DATA_CATALOG + "metastore.googleapis.com/Service": DATA_CATALOG + "datafusion.googleapis.com/DnsPeering": DNS_RESOLVER + "datastream.googleapis.com/Stream": DATA_STREAM + "datastream.googleapis.com/ConnectionProfile": SOURCE_CONNECTION + "datastream.googleapis.com/PrivateConnection": PEERING_CONNECTION + "storagetransfer.googleapis.com/TransferJob": DATA_TRANSFER + "netapp.googleapis.com/StoragePool": FILE_SYSTEM + "netapp.googleapis.com/Volume": FILE_SYSTEM + "netapp.googleapis.com/Replication": FILE_SYSTEM + "netapp.googleapis.com/BackupVault": BACKUP_VAULT + "netapp.googleapis.com/BackupPolicy": BACKUP_PLAN + "netapp.googleapis.com/Backup": SNAPSHOT + "netapp.googleapis.com/Snapshot": SNAPSHOT + "netapp.googleapis.com/ActiveDirectory": IDENTITY_PROVIDER + "netapp.googleapis.com/KmsConfig": KMS_KEY + "parametermanager.googleapis.com/Parameter": PARAMETER + "parametermanager.googleapis.com/ParameterVersion": PARAMETER + "aiplatform.googleapis.com/ReasoningEngine": AI_AGENT + "aiplatform.googleapis.com/Index": KNOWLEDGE_BASE + "aiplatform.googleapis.com/IndexEndpoint": ML_ENDPOINT + "aiplatform.googleapis.com/CustomJob": JOB_DEFINITION + "aiplatform.googleapis.com/HyperparameterTuningJob": JOB_DEFINITION + "aiplatform.googleapis.com/NasJob": JOB_DEFINITION + "aiplatform.googleapis.com/BatchPredictionJob": JOB_DEFINITION + "aiplatform.googleapis.com/TuningJob": JOB_DEFINITION + "aiplatform.googleapis.com/TrainingPipeline": ETL_JOB + "aiplatform.googleapis.com/PipelineJob": ETL_JOB + "aiplatform.googleapis.com/FeatureGroup": DATA_CATALOG + "aiplatform.googleapis.com/Featurestore": DATA_CATALOG + "aiplatform.googleapis.com/FeatureOnlineStore": ML_ENDPOINT + "aiplatform.googleapis.com/Dataset": DATA_CATALOG + "aiplatform.googleapis.com/NotebookRuntime": ML_WORKSPACE + "aiplatform.googleapis.com/NotebookRuntimeTemplate": LAUNCH_TEMPLATE + "discoveryengine.googleapis.com/DataStore": KNOWLEDGE_BASE + "discoveryengine.googleapis.com/Collection": KNOWLEDGE_BASE + "discoveryengine.googleapis.com/Engine": AI_AGENT + "discoveryengine.googleapis.com/Assistant": AI_AGENT + "dialogflow.googleapis.com/Agent": AI_AGENT + "dialogflow.googleapis.com/KnowledgeBase": KNOWLEDGE_BASE + "modelarmor.googleapis.com/Template": AI_GUARDRAIL + "cloudresourcemanager.googleapis.com/TagKey": TAG + "cloudresourcemanager.googleapis.com/TagValue": TAG + "cloudresourcemanager.googleapis.com/TagBinding": TAG + "cloudresourcemanager.googleapis.com/Lien": GUARDRAIL + "orgpolicy.googleapis.com/CustomConstraint": ORG_POLICY + "assuredworkloads.googleapis.com/Workload": LANDING_ZONE + "accesscontextmanager.googleapis.com/AuthorizedOrgsDesc": GUARDRAIL + "accesscontextmanager.googleapis.com/AccessLevel": GUARDRAIL + "config.googleapis.com/Deployment": IAC_STACK + "config.googleapis.com/Preview": IAC_STACK + "deploymentmanager.googleapis.com/Deployment": IAC_STACK # (API) + "monitoring.googleapis.com/NotificationChannel": NOTIFICATION_TOPIC + "monitoring.googleapis.com/UptimeCheckConfig": ALARM + "osconfig.googleapis.com/PatchDeployment": SCHEDULE + "osconfig.googleapis.com/OSPolicyAssignment": RUNBOOK + "osconfig.googleapis.com/VulnerabilityReport": VULNERABILITY_SCANNER + "backupdr.googleapis.com/BackupVault": BACKUP_VAULT + "backupdr.googleapis.com/BackupPlan": BACKUP_PLAN + "backupdr.googleapis.com/BackupPlanAssociation": BACKUP_PLAN + "backupdr.googleapis.com/DataSource": BACKUP_VAULT + "backupdr.googleapis.com/Backup": SNAPSHOT + "backupdr.googleapis.com/ManagementServer": BACKUP_VAULT + "securitycenter.googleapis.com/SecurityCenter": SECURITY_HUB # (API) + "securitycenter.googleapis.com/Source": SECURITY_HUB # (API) + "securitycentermanagement.googleapis.com/SecurityCenterService": SECURITY_HUB # (API) + "securitycenter.googleapis.com/NotificationConfig": EVENT_RULE + "securitycenter.googleapis.com/BigQueryExport": LOG_SINK + "securitycenter.googleapis.com/MuteConfig": SECURITY_HUB + "securitycenter.googleapis.com/ResourceValueConfig": SECURITY_HUB + "securitycenter.googleapis.com/EventThreatDetectionSettings": THREAT_DETECTOR + "securitycenter.googleapis.com/ContainerThreatDetectionSettings": THREAT_DETECTOR + "securitycenter.googleapis.com/VirtualMachineThreatDetectionSettings": THREAT_DETECTOR + "securitycenter.googleapis.com/SecurityHealthAnalyticsSettings": CONFIG_RECORDER + "securitycenter.googleapis.com/WebSecurityScannerSettings": VULNERABILITY_SCANNER + "dlp.googleapis.com/DiscoveryConfig": DATA_SECURITY_SCANNER + "dlp.googleapis.com/JobTrigger": DATA_SECURITY_SCANNER + "dlp.googleapis.com/InspectTemplate": DATA_SECURITY_SCANNER + "dlp.googleapis.com/DeidentifyTemplate": DATA_SECURITY_SCANNER + "dlp.googleapis.com/StoredInfoType": DATA_SECURITY_SCANNER + "logging.googleapis.com/AuditConfig": CLOUDTRAIL # (API: IAM policy auditConfigs) + "recommender.googleapis.com/IamPolicyRecommender": ACCESS_ANALYZER # (API) + "iam.googleapis.com/PolicyV2": IAM_POLICY + "iam.googleapis.com/DenyPolicy": IAM_POLICY # (API) + "iam.googleapis.com/PrincipalAccessBoundaryPolicy": IAM_POLICY + "iam.googleapis.com/PolicyBinding": IAM_POLICY + "iam.googleapis.com/OauthClient": IDENTITY_PROVIDER + "iam.googleapis.com/OauthClientCredential": ACCESS_KEY + "iam.googleapis.com/WorkloadIdentityPoolNamespace": IDENTITY_PROVIDER + "iam.googleapis.com/WorkloadIdentityPoolManagedIdentity": SERVICE_PRINCIPAL + "iam.googleapis.com/WorkforcePoolProviderKey": ACCESS_KEY + "iam.googleapis.com/WorkloadIdentityPoolProviderKey": ACCESS_KEY + "privilegedaccessmanager.googleapis.com/Entitlement": PERMISSION_SET + "cloudidentity.googleapis.com/Group": IDENTITY_GROUP # (API) + "cloudbuild.googleapis.com/WorkerPool": CAPACITY_PROVIDER + "cloudbuild.googleapis.com/Connection": SOURCE_CONNECTION + "cloudbuild.googleapis.com/Repository": SOURCE_CONNECTION + "cloudbuild.googleapis.com/GithubEnterpriseConfig": SOURCE_CONNECTION + "cloudbuild.googleapis.com/BitbucketServerConfig": SOURCE_CONNECTION + "developerconnect.googleapis.com/Connection": SOURCE_CONNECTION + "developerconnect.googleapis.com/GitRepositoryLink": SOURCE_CONNECTION + # end GCP parity + +# GCP parity: service families for inventory.services / --services. Keys +# are the API service of an asset type (compute.googleapis.com/... -> +# compute); type_families overrides single types. Anything else is "other". +families: + compute: compute + batch: compute + appengine: serverless + container: containers + k8s.io: containers + apps.k8s.io: containers + batch.k8s.io: containers + networking.k8s.io: containers + extensions.k8s.io: containers + rbac.authorization.k8s.io: containers + artifactregistry: containers + binaryauthorization: containers + gkehub: containers + gkebackup: containers + containeranalysis: containers + vpcaccess: network + networkconnectivity: network + networkservices: network + dns: dns + storage: storage + file: storage + netapp: storage + sqladmin: data + spanner: data + bigtableadmin: data + firestore: data + alloydb: data + redis: data + memcache: data + bigquery: data + bigqueryconnection: data + bigquerydatatransfer: data + dataplex: data + datacatalog: data + metastore: data + datafusion: data + dataproc: data + dataflow: data + composer: data + datastream: data + storagetransfer: data + iam: identity + cloudidentity: identity + privilegedaccessmanager: identity + apikeys: identity + identitytoolkit: identity + cloudkms: identity + secretmanager: identity + parametermanager: identity + privateca: identity + certificatemanager: identity + run: serverless + cloudfunctions: serverless + workflows: serverless + apigateway: serverless + apigee: serverless + pubsub: integration + cloudtasks: integration + cloudscheduler: integration + eventarc: integration + managedkafka: integration + securitycenter: security + securitycentermanagement: security + dlp: security + ids: security + networksecurity: security + recommender: security + accesscontextmanager: security + logging: logging + monitoring: logging + config: iac + deploymentmanager: iac + aiplatform: ml + notebooks: ml + discoveryengine: ml + dialogflow: ml + modelarmor: ml + backupdr: backup + osconfig: ops + cloudbuild: cicd + clouddeploy: cicd + developerconnect: cicd + cloudresourcemanager: governance + orgpolicy: governance + assuredworkloads: governance + +type_families: + "cloudresourcemanager.googleapis.com/Organization": hierarchy + "cloudresourcemanager.googleapis.com/Folder": hierarchy + "cloudresourcemanager.googleapis.com/Project": hierarchy + "compute.googleapis.com/Network": network + "compute.googleapis.com/Subnetwork": network + "compute.googleapis.com/Firewall": network + "compute.googleapis.com/FirewallPolicy": network + "compute.googleapis.com/NetworkFirewallPolicy": network + "compute.googleapis.com/RegionNetworkFirewallPolicy": network + "compute.googleapis.com/Route": network + "compute.googleapis.com/Router": network + "compute.googleapis.com/VpnGateway": network + "compute.googleapis.com/TargetVpnGateway": network + "compute.googleapis.com/VpnTunnel": network + "compute.googleapis.com/ExternalVpnGateway": network + "compute.googleapis.com/Interconnect": network + "compute.googleapis.com/InterconnectAttachment": network + "compute.googleapis.com/ServiceAttachment": network + "compute.googleapis.com/Address": network + "compute.googleapis.com/GlobalAddress": network + "compute.googleapis.com/ForwardingRule": network + "compute.googleapis.com/GlobalForwardingRule": network + "compute.googleapis.com/BackendService": network + "compute.googleapis.com/RegionBackendService": network + "compute.googleapis.com/BackendBucket": network + "compute.googleapis.com/TargetHttpProxy": network + "compute.googleapis.com/TargetHttpsProxy": network + "compute.googleapis.com/RegionTargetHttpProxy": network + "compute.googleapis.com/RegionTargetHttpsProxy": network + "compute.googleapis.com/TargetGrpcProxy": network + "compute.googleapis.com/TargetSslProxy": network + "compute.googleapis.com/TargetTcpProxy": network + "compute.googleapis.com/RegionTargetTcpProxy": network + "compute.googleapis.com/UrlMap": network + "compute.googleapis.com/RegionUrlMap": network + "compute.googleapis.com/TargetPool": network + "compute.googleapis.com/TargetInstance": network + "compute.googleapis.com/NetworkEndpointGroup": network + "compute.googleapis.com/GlobalNetworkEndpointGroup": network + "compute.googleapis.com/RegionNetworkEndpointGroup": network + "compute.googleapis.com/PacketMirroring": network + "compute.googleapis.com/SslCertificate": identity + "compute.googleapis.com/RegionSslCertificate": identity + "compute.googleapis.com/SecurityPolicy": security + "compute.googleapis.com/RegionSecurityPolicy": security + "compute.googleapis.com/NetworkEdgeSecurityService": security + "compute.googleapis.com/Disk": storage + "compute.googleapis.com/RegionDisk": storage + "compute.googleapis.com/Snapshot": storage + "compute.googleapis.com/ResourcePolicy": backup + "file.googleapis.com/Backup": backup + "netapp.googleapis.com/BackupVault": backup + "netapp.googleapis.com/BackupPolicy": backup + "netapp.googleapis.com/Backup": backup + "firestore.googleapis.com/Backup": backup + "firestore.googleapis.com/BackupSchedule": backup + "gkebackup.googleapis.com/BackupPlan": backup + "gkebackup.googleapis.com/Backup": backup + "gkebackup.googleapis.com/RestorePlan": backup + "spanner.googleapis.com/Backup": backup + "bigtableadmin.googleapis.com/Backup": backup + "binaryauthorization.googleapis.com/Policy": security + "binaryauthorization.googleapis.com/Attestor": security + "binaryauthorization.googleapis.com/PlatformPolicy": security + "containeranalysis.googleapis.com/VulnerabilitySummary": security + "osconfig.googleapis.com/VulnerabilityReport": security + "logging.googleapis.com/AuditConfig": security + "accesscontextmanager.googleapis.com/ServicePerimeter": governance + "accesscontextmanager.googleapis.com/AccessLevel": governance + "accesscontextmanager.googleapis.com/AuthorizedOrgsDesc": governance +# end GCP parity diff --git a/cloudg/inventory/gcp_deep.py b/cloudg/inventory/gcp_deep.py index f24dc74..1317236 100644 --- a/cloudg/inventory/gcp_deep.py +++ b/cloudg/inventory/gcp_deep.py @@ -2,15 +2,28 @@ The base :class:`GCPCollector` enumerates every resource with Cloud Asset Inventory ``ListAssets`` (full resource JSON) and declares typed relations -through :mod:`cloudg.inventory.gcp_relations`. This subclass deepens the map: +through :mod:`cloudg.inventory.gcp_relations`. This subclass deepens the map +to the level of the AWS deep collector: - A much richer GCP-type -> normalised AssetType mapping, so resources classify as real asset types instead of OTHER. - IAM policy bindings via ``SearchAllIamPolicies``: every member becomes a principal asset (the collected service account, or a non-external ``gcp-principal:`` placeholder) carrying GRANTS_ACCESS / - ASSUMES_ROLE relations to the bound resource. ``allUsers`` and + ASSUMES_ROLE relations to the bound resource and USER / GROUP / + ROLE_HAS_POLICY relations to the bound role. ``allUsers`` and ``allAuthenticatedUsers`` mark the resource internet-exposed. +- API-backed sources (:mod:`cloudg.inventory.gcp_sources`) for what Cloud + Asset Inventory does not return: Cloud DNS records, Security Command + Center, Cloud Identity groups, deny and Principal Access Boundary + policies, PAM, audit-log configs, Cloud Monitoring, OS Config, Artifact + Analysis, Backup and DR and more. Each one is best-effort with its own + coverage record. +- Cross-asset passes (:mod:`cloudg.inventory.gcp_sources.derived`): tag + bindings, secret rotation handlers, resource-policy schedules, labels of + Deployment Manager / Config Connector resources, API Gateway authorizers. +- Per-family coverage (``gcp_assets_``) and ``inventory.services`` + / ``inventory.exclude_services`` filtering (:mod:`cloudg.inventory.gcp_tasks`). Edges are resolved by the inventory mapper's linker, so this collector does not link locally. @@ -21,18 +34,28 @@ import asyncio import logging import time +from dataclasses import dataclass, field, fields from typing import Any -from cloudg.inventory.catalogs import asset_type_map, load_catalog from cloudg.collectors.gcp import _GCP_ASSET_TYPE_MAP, GCPCollector from cloudg.coverage import ServiceStatus -from cloudg.resilience.errors import describe_error +from cloudg.inventory.catalogs import asset_type_map, load_catalog from cloudg.inventory.gcp_relations import apply_iam_policies, merge_gcp_principals +from cloudg.inventory.gcp_sources import GCPRestClient, GCPSourceOptions, SourceContext, run_sources +from cloudg.inventory.gcp_sources import derived +from cloudg.inventory.gcp_tasks import gcp_family, is_selected, type_selected +from cloudg.resilience.errors import describe_error from cloudg.schema.models import AssetType, CloudAsset logger = logging.getLogger(__name__) -__all__ = ["GCPDeepInventoryCollector", "_DEEP_GCP_TYPE_MAP", "merge_gcp_principals"] +__all__ = [ + "GCPDeepInventoryCollector", + "GCPDeepOptions", + "_DEEP_GCP_TYPE_MAP", + "deep_options_from_config", + "merge_gcp_principals", +] _C = "compute.googleapis.com/" @@ -43,14 +66,129 @@ } +@dataclass +class GCPDeepOptions: + """Keyword options the deep collector adds to :class:`GCPCollectorOptions`. + + Attributes: + services: Families / services / sources to map (default: all). + exclude_services: Families / services / sources to skip (wins over ``services``). + rest_client: Injected REST client for the API-backed sources (tests); + otherwise one is built from ``credentials``. + deep_sources: Run the API-backed sources at all. + rest_timeout: Per-request timeout of the REST client, in seconds. + max_images_per_repository: Newest Docker images kept per repository. + source_options: :class:`GCPSourceOptions` (or a dict of its fields). + """ + + services: list[str] | None = None + exclude_services: list[str] | None = None + rest_client: Any = None + deep_sources: bool = True + rest_timeout: float = 60.0 + max_images_per_repository: int | None = 100 + source_options: Any = field(default=None) + + +def deep_options_from_config(inventory: Any = None, gcp: Any = None) -> dict[str, Any]: + """Deep collector keyword options from ``inventory.*`` and ``gcp.*`` config.""" + out: dict[str, Any] = {} + if inventory is not None: + out.update( + services=list(getattr(inventory, "services", None) or ["all"]), + exclude_services=list(getattr(inventory, "exclude_services", None) or []), + max_images_per_repository=getattr(inventory, "max_images_per_repository", 100), + ) + if gcp is not None: + out.update( + deep_sources=getattr(gcp, "deep_sources", True), + rest_timeout=getattr(gcp, "rest_timeout_seconds", 60.0), + source_options={ + "customer_id": getattr(gcp, "cloud_identity_customer_id", None), + "max_dns_records_per_zone": getattr(gcp, "max_dns_records_per_zone", 5000), + "max_vulnerability_occurrences": getattr( + gcp, "max_vulnerability_occurrences", 5000 + ), + "max_groups": getattr(gcp, "max_groups", 2000), + "max_group_members": getattr(gcp, "max_group_members", 5000), + "expand_predefined_roles": getattr(gcp, "expand_predefined_roles", True), + "max_role_expansions": getattr(gcp, "max_role_expansions", 500), + "max_scheduler_locations": getattr(gcp, "max_scheduler_locations", 25), + "concurrency": getattr(gcp, "source_concurrency", 4), + }, + ) + return out + + class GCPDeepInventoryCollector(GCPCollector): - """GCP collector with a deeper type taxonomy and IAM-binding relations.""" + """GCP collector with a deeper type taxonomy, IAM bindings and API-backed sources.""" _link_locally_default = False + def __init__(self, project_id: str | None, credentials: Any = None, **options: Any) -> None: + names = {f.name for f in fields(GCPDeepOptions)} + deep = GCPDeepOptions(**{k: options.pop(k) for k in list(options) if k in names}) + super().__init__(project_id, credentials, **options) + self._services = list(deep.services or ["all"]) + self._exclude_services = list(deep.exclude_services or []) + self._deep_sources = deep.deep_sources + self._max_images = deep.max_images_per_repository + opts = deep.source_options + if isinstance(opts, dict): + opts = GCPSourceOptions(**opts) + self._source_options = opts or GCPSourceOptions() + rest = deep.rest_client + if rest is None and credentials is not None and deep.deep_sources: + rest = GCPRestClient(credentials, timeout=deep.rest_timeout) + self._rest = rest + def _resolve_asset_type(self, gcp_type: str) -> AssetType: return _DEEP_GCP_TYPE_MAP.get(gcp_type, AssetType.OTHER) + # ------------------------------------------------------------------ + # Service filtering and per-family coverage + # ------------------------------------------------------------------ + + def _selected(self, name: str, family: str) -> bool: + return is_selected(name, family, self._services, self._exclude_services) + + def _keep(self, rec: dict[str, Any], pid: str | None, number: str | None) -> bool: + if not type_selected(rec["asset_type"], self._services, self._exclude_services): + return False + return super()._keep(rec, pid, number) + + def _listing_status(self) -> tuple[ServiceStatus, str | None]: + """Status of this run's asset listing, for the per-family records.""" + by_service = {s.service: s for s in self.coverage.services} + if "gcp_search_resources" in by_service: + return ServiceStatus.PARTIAL, "summary data only (ListAssets unavailable)" + listing = by_service.get("gcp_list_assets") + if listing is not None and listing.status != ServiceStatus.SUCCESS: + return ServiceStatus.PARTIAL, listing.error + return ServiceStatus.SUCCESS, None + + def _record_families(self, assets: list[CloudAsset]) -> None: + counts: dict[str, int] = {} + for a in assets: + fam = gcp_family(str(a.metadata.get("gcp_asset_type") or "")) + counts[fam] = counts.get(fam, 0) + 1 + known = {gcp_family(t) for t in _DEEP_GCP_TYPE_MAP} | set(counts) + status, error = self._listing_status() + for fam in sorted(known): + if not self._selected(fam, fam): + continue + self._record(f"gcp_assets_{fam}", status, counts.get(fam, 0), error) + + def _build_assets(self, records: list[dict[str, Any]]) -> list[CloudAsset]: + records = derived.cap_images(records, self._max_images) + assets = super()._build_assets(records) + self._record_families(assets) + return assets + + # ------------------------------------------------------------------ + # IAM policies (SearchAllIamPolicies) + # ------------------------------------------------------------------ + def _search_iam(self) -> tuple[list[dict[str, Any]], Exception | None]: client = self._get_client() request = {"scope": self._scope, "page_size": 500} @@ -65,9 +203,7 @@ def _keep_policy(self, pol: dict[str, Any]) -> bool: number = project.removeprefix("projects/") return bool({number, self._number_to_id.get(number)} & self._project_filter) - async def _enrich(self, assets: list[CloudAsset]) -> list[CloudAsset]: - if not self._include_iam: - return [] + async def _enrich_iam(self, assets: list[CloudAsset], ctx: SourceContext) -> list[CloudAsset]: start = time.time() policies, error = await asyncio.to_thread(self._search_iam) policies = [p for p in policies if isinstance(p, dict) and self._keep_policy(p)] @@ -80,5 +216,51 @@ async def _enrich(self, assets: list[CloudAsset]) -> list[CloudAsset]: else: self._record("gcp_iam_policies", ServiceStatus.SUCCESS, len(policies), start=start) created = apply_iam_policies(assets, policies) + ctx.state["iam_roles"] = sorted( + { + b.get("role") + for p in policies + for b in (p.get("policy") or {}).get("bindings") or [] + if isinstance(b, dict) and isinstance(b.get("role"), str) + } + ) logger.info("GCP IAM: %d policies, %d new principal assets", len(policies), len(created)) return created + + # ------------------------------------------------------------------ + # Enrichment and cross-asset passes + # ------------------------------------------------------------------ + + async def _enrich(self, assets: list[CloudAsset]) -> list[CloudAsset]: + ctx = SourceContext(self, assets, self._rest, self._source_options) + inc, exc = self._services, self._exclude_services + created: list[CloudAsset] = [] + if self._deep_sources: + created += await run_sources(ctx, inc, exc, phases=[0]) + if self._include_iam and self._selected("iam_policies", "identity"): + found = await self._enrich_iam(assets + created, ctx) + ctx.add(found) + created += found + if self._deep_sources: + created += await run_sources(ctx, inc, exc, phases=[1, 2, 3]) + return created + + def _finalize(self, assets: list[CloudAsset]) -> list[CloudAsset]: + assets = self._derived("tag_bindings", derived.fold_tag_bindings, assets) or assets + for name, fn in derived.PASSES: + self._derived(name, fn, assets) + assets = assets + (self._derived("authorizers", derived.authorizer_assets, assets) or []) + assets = assets + ( + self._derived("principals", derived.ensure_principal_placeholders, assets) or [] + ) + return super()._finalize(assets) + + def _derived(self, name: str, fn: Any, assets: list[CloudAsset]) -> Any: + """Run one cross-asset pass; a failure is recorded, not raised.""" + try: + return fn(assets) + except Exception as exc: + logger.warning("GCP pass %s failed: %s", name, exc) + logger.debug("GCP pass %s failed", name, exc_info=True) + self._record(f"gcp_derived_{name}", ServiceStatus.FAILED, error=describe_error(exc)) + return None diff --git a/cloudg/inventory/gcp_hierarchy.py b/cloudg/inventory/gcp_hierarchy.py index 150241e..78b8623 100644 --- a/cloudg/inventory/gcp_hierarchy.py +++ b/cloudg/inventory/gcp_hierarchy.py @@ -129,6 +129,15 @@ def perimeter_assets(results: list[dict[str, Any]]) -> list[CloudAsset]: ) for ref, dry in governed ] + relations += [ + rel( + full_name(level, "accesscontextmanager"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="access level the perimeter admits", + ) + for level in md.get("access_levels") or [] + ] md.update( gcp_asset_type="accesscontextmanager.googleapis.com/ServicePerimeter", access_policy=relative_name(name).split("/servicePerimeters/", 1)[0], @@ -148,6 +157,46 @@ def perimeter_assets(results: list[dict[str, Any]]) -> list[CloudAsset]: return assets +def access_level_assets(results: list[dict[str, Any]]) -> list[CloudAsset]: + """GUARDRAIL assets for Access Context Manager access levels (``ContentType.ACCESS_POLICY``). + + Perimeters that require a level depend on it (see :func:`perimeter_assets`). + """ + assets: list[CloudAsset] = [] + for res in results: + level = res.get("access_level") or res.get("accessLevel") + if not isinstance(level, dict) or not level: + continue + name = res.get("name") or full_name(level.get("name"), "accesscontextmanager") or "" + basic = level.get("basic") or {} + conditions = [c for c in basic.get("conditions") or [] if isinstance(c, dict)] + + def values(*keys: str) -> list[str]: + return sorted({v for c in conditions for k in keys for v in c.get(k) or []}) + + assets.append( + CloudAsset( + arn=name, + name=level.get("title") or name.rsplit("/", 1)[-1], + asset_type=AssetType.GUARDRAIL, + provider=CloudProvider.GCP, + region="global", + metadata={ + "gcp_asset_type": "accesscontextmanager.googleapis.com/AccessLevel", + "title": level.get("title"), + "access_policy": relative_name(name).split("/accessLevels/", 1)[0], + "combining_function": basic.get("combining_function") + or basic.get("combiningFunction"), + "ip_subnetworks": values("ip_subnetworks", "ipSubnetworks"), + "regions": values("regions"), + "members": values("members"), + "custom": bool(level.get("custom")), + }, + ) + ) + return assets + + def _org_placeholder(org: str) -> CloudAsset: """The configured organization, when the listing did not return it.""" return CloudAsset( @@ -256,6 +305,7 @@ def discover_gcp_hierarchy( results, error = collector._list_assets("ACCESS_POLICY") perimeters = perimeter_assets(results) assets.extend(perimeters) + assets.extend(access_level_assets(results)) _record_part( collector.coverage, "gcp_vpc_service_controls", (results, error), len(perimeters), start ) diff --git a/cloudg/inventory/gcp_relations/__init__.py b/cloudg/inventory/gcp_relations/__init__.py index e8204ea..d84c3e2 100644 --- a/cloudg/inventory/gcp_relations/__init__.py +++ b/cloudg/inventory/gcp_relations/__init__.py @@ -41,6 +41,24 @@ resource_manager, serverless, ) + +# Added for AWS depth parity; several register second extractors for types +# the modules above already handle (the registry runs both). +from cloudg.inventory.gcp_relations import ( # noqa: F401, E402 + apis, + cicd, + containers_ext, + data_ext, + edge, + governance, + hybrid, + identity_ext, + integration, + ml, + netsec, + ops, + security_ext, +) from cloudg.inventory.gcp_relations.context import ( EXTRACTORS, Extracted, diff --git a/cloudg/inventory/gcp_relations/_refs.py b/cloudg/inventory/gcp_relations/_refs.py new file mode 100644 index 0000000..c9e1c77 --- /dev/null +++ b/cloudg/inventory/gcp_relations/_refs.py @@ -0,0 +1,156 @@ +"""Reference helpers shared by the extractors added for AWS depth parity. + +Each helper turns one GCP reference shape into the Cloud Asset Inventory +full resource name the linker indexes, or returns None. +""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.gcp_relations.names import full_name, url_alias + +_CRM = "//cloudresourcemanager.googleapis.com/" +_BQ_DOTTED_RE = re.compile(r"^(?:bq://)?([^.:/\s]+)[.:]([^.\s/]+)(?:\.([^.\s/]+))?$") +_BQ_PATH_RE = re.compile(r"projects/([^/]+)/datasets/([^/]+)(?:/tables/([^/]+))?") +_WORKFLOW_EXEC_RE = re.compile( + r"^https://workflowexecutions\.googleapis\.com/v\d[^/]*/" + r"(projects/[^/]+/locations/[^/]+/workflows/[^/:]+)" +) +_RUN_JOB_V2_RE = re.compile( + r"^https://run\.googleapis\.com/v2/(projects/[^/]+/locations/[^/]+/jobs/[^/:]+)" +) +_RUN_JOB_V1_RE = re.compile( + r"^https://([a-z0-9-]+)-run\.googleapis\.com/apis/run\.googleapis\.com/v1/" + r"namespaces/([^/]+)/jobs/([^/:]+)" +) +_LB_SERVICE_RE = re.compile(r"/locations/([^/]+)/backendServices/") + + +def crm_ref(value: Any) -> str | None: + """Project / folder / organization (``projects/x``, ``folders/1``, bare id) as a full name.""" + if not isinstance(value, str) or not value: + return None + s = value.strip() + if s.startswith("//"): + return s + if re.fullmatch(r"(projects|folders|organizations)/[^/]+", s): + return f"{_CRM}{s}" + if re.fullmatch(r"[a-z][a-z0-9-]{4,28}[a-z0-9]|\d+", s): + return f"{_CRM}projects/{s}" + return None + + +def parent_of(name: str, segment: str) -> str | None: + """``name`` cut before ``//`` (the parent resource), or None.""" + marker = f"/{segment}/" + if not isinstance(name, str) or marker not in name: + return None + return name.split(marker, 1)[0] + + +def gcs_ref(value: Any) -> str | None: + """Cloud Storage bucket of a ``gs://`` URI, bucket name or storage URL.""" + if not isinstance(value, str) or not value: + return None + s = value.strip() + if s.startswith("gs://"): + return full_name(s) + if "://" in s or s.startswith("//"): + return full_name(s) + if re.fullmatch(r"[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]", s): + return f"//storage.googleapis.com/{s}" + return None + + +def bq_ref(value: Any, default_project: str | None = None) -> str | None: + """BigQuery dataset or table from ``p.d.t``, ``p:d.t``, ``bq://p.d.t`` or a path.""" + if not isinstance(value, str) or not value: + return None + s = value.strip().strip("`") + m = _BQ_PATH_RE.search(s) + if m: + project, dataset, table = m.groups() + else: + m = _BQ_DOTTED_RE.match(s) + if not m: + return None + project, dataset, table = m.groups() + if not project: + project = default_project + if not project or not dataset: + return None + out = f"//bigquery.googleapis.com/projects/{project}/datasets/{dataset}" + return f"{out}/tables/{table}" if table else out + + +def bq_dataset_ref(project: Any, dataset: Any) -> str | None: + if isinstance(project, str) and project and isinstance(dataset, str) and dataset: + return f"//bigquery.googleapis.com/projects/{project}/datasets/{dataset}" + return None + + +def url_target(uri: Any) -> str | None: + """What an HTTP target URL calls. + + Workflow execution and Cloud Run job ``:run`` URLs map to the workflow or + job; anything else is reduced to ``https://host`` (the alias Cloud Run, + Cloud Functions and API Gateway assets carry). + """ + if not isinstance(uri, str) or not uri: + return None + m = _WORKFLOW_EXEC_RE.match(uri) + if m: + return f"//workflows.googleapis.com/{m.group(1)}" + m = _RUN_JOB_V2_RE.match(uri) + if m: + return f"//run.googleapis.com/{m.group(1)}" + m = _RUN_JOB_V1_RE.match(uri) + if m: + region, project, job = m.groups() + return f"//run.googleapis.com/projects/{project}/locations/{region}/jobs/{job}" + return url_alias(uri) + + +def backend_service_ref(value: Any) -> str | None: + """Backend service named the Network Services way (``.../locations/global/...``).""" + ref = full_name(value, "compute") + if not ref: + return None + m = _LB_SERVICE_RE.search(ref) + if not m: + return ref + loc = m.group(1) + repl = "/global/backendServices/" if loc == "global" else f"/regions/{loc}/backendServices/" + return ref[: m.start()] + repl + ref[m.end() :] + + +def host_target(host: Any) -> str | None: + """``https://host`` for a bare host name (ports and paths dropped).""" + if not isinstance(host, str) or not host: + return None + if "://" in host: + return url_alias(host) + h = host.strip().split("/", 1)[0].split(":", 1)[0].lower() + return f"https://{h}" if h and "." in h else None + + +def secret_version_ref(value: Any) -> str | None: + """Secret Manager secret of a secret (version) name.""" + if not isinstance(value, str) or "/secrets/" not in value: + return None + return full_name(value.split("/versions/", 1)[0], "secretmanager") + + +__all__ = [ + "backend_service_ref", + "bq_dataset_ref", + "bq_ref", + "crm_ref", + "gcs_ref", + "host_target", + "parent_of", + "secret_version_ref", + "url_target", +] diff --git a/cloudg/inventory/gcp_relations/apis.py b/cloudg/inventory/gcp_relations/apis.py new file mode 100644 index 0000000..0c2c43b --- /dev/null +++ b/cloudg/inventory/gcp_relations/apis.py @@ -0,0 +1,247 @@ +"""APIs: Apigee organizations, instances, environments and proxies, and the +JWT providers and backends inside API Gateway configs.""" + +from __future__ import annotations + +import base64 +import binascii +import json +from typing import Any + +import yaml + +from cloudg.inventory.gcp_relations._refs import crm_ref, host_target, parent_of, url_target +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, dig, full_name, network_ref, sa_email +from cloudg.schema.models import EdgeType + +_MAX_DOC_BYTES = 512 * 1024 + + +# --------------------------------------------------------------------------- +# Apigee +# --------------------------------------------------------------------------- + + +@_extractor("apigee.googleapis.com/Organization") +def _apigee_org(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + project = d.get("projectId") or ctx.project_id + out.add( + network_ref(d.get("authorizedNetwork"), project), + EdgeType.ATTACHED_TO, + description="Apigee runtime peered network", + ) + out.add(crm_ref(project), EdgeType.REFERENCES, "DEPENDS_ON", description="Apigee host project") + out.metadata.update( + runtime_type=d.get("runtimeType"), + billing_type=d.get("billingType"), + disable_vpc_peering=d.get("disableVpcPeering"), + analytics_region=d.get("analyticsRegion"), + apigee_org=ctx.name.rsplit("/", 1)[-1], + ) + + +@_extractor("apigee.googleapis.com/Instance") +def _apigee_instance(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "instances"), EdgeType.CONTAINS, reverse=True) + out.add( + full_name(d.get("serviceAttachment"), "compute"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="runtime PSC service attachment", + ) + for project in _list(d.get("consumerAcceptList")): + out.add( + crm_ref(project), + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="PSC consumer accept list", + ) + out.alias(d.get("host")) + out.metadata.update(host=d.get("host"), peering_cidr_range=d.get("peeringCidrRange")) + + +@_extractor("apigee.googleapis.com/Environment") +def _apigee_env(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "environments"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + deployment_type=d.get("deploymentType"), + api_proxy_type=d.get("apiProxyType"), + environment_type=d.get("type"), + ) + + +@_extractor("apigee.googleapis.com/ApiProxy") +def _apigee_proxy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "apis"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + latest_revision=d.get("latestRevisionId"), revisions=list(d.get("revision") or []) + ) + + +@_extractor("apigee.googleapis.com/ApiProxyRevision") +def _apigee_proxy_revision(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "revisions"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + target_servers=list(d.get("targetServers") or []), + target_endpoints=list(d.get("targetEndpoints") or []), + proxy_endpoints=list(d.get("proxyEndpoints") or []), + policies=list(d.get("policies") or [])[:100], + ) + + +@_extractor("apigee.googleapis.com/TargetServer") +def _apigee_target_server(ctx: GCPContext, out: Extracted) -> None: + """Target servers come from the Apigee API (not Cloud Asset Inventory).""" + d = ctx.data + out.add(parent_of(ctx.name, "targetservers"), EdgeType.CONTAINS, reverse=True) + out.add( + host_target(d.get("host")), + EdgeType.ROUTE, + "SERVES_TRAFFIC_TO", + description="Apigee target server", + port=d.get("port"), + ) + out.metadata.update( + host=d.get("host"), + port=d.get("port"), + protocol=d.get("protocol"), + tls_enabled=bool(dig(d, "sSLInfo", "enabled")), + enabled=d.get("isEnabled"), + ) + + +# --------------------------------------------------------------------------- +# API Gateway configs: JWT providers (authorizers) and backends +# --------------------------------------------------------------------------- + + +def _decode_document(doc: Any) -> dict[str, Any] | None: + """An OpenAPI / service config file's ``contents`` (base64 YAML or JSON) as a dict.""" + if not isinstance(doc, dict): + return None + contents = doc.get("contents") + if not isinstance(contents, str) or not contents or len(contents) > _MAX_DOC_BYTES: + return None + try: + text = base64.b64decode(contents, validate=False).decode("utf-8", "replace") + except (binascii.Error, ValueError): + text = contents + for loader in (json.loads, yaml.safe_load): + try: + parsed = loader(text) + except Exception: # nosec B112 -- try the next parser; untrusted, partial docs are normal + continue + if isinstance(parsed, dict): + return parsed + return None + + +def _openapi_providers(spec: dict[str, Any]) -> list[dict[str, Any]]: + """``securityDefinitions`` (OpenAPI 2) / ``components.securitySchemes`` with x-google-issuer.""" + schemes = dict(spec.get("securityDefinitions") or {}) + schemes.update(dig(spec, "components", "securitySchemes", default={}) or {}) + out = [] + for name, scheme in schemes.items(): + if not isinstance(scheme, dict) or not scheme.get("x-google-issuer"): + continue + out.append( + { + "name": str(name), + "issuer": scheme.get("x-google-issuer"), + "jwks_uri": scheme.get("x-google-jwks_uri"), + "audiences": [ + a.strip() for a in str(scheme.get("x-google-audiences") or "").split(",") if a + ], + } + ) + return out + + +def _service_config_providers(cfg: dict[str, Any]) -> list[dict[str, Any]]: + """``authentication.providers`` of a gRPC service config.""" + out = [] + for p in _list(dig(cfg, "authentication", "providers")): + if isinstance(p, dict) and p.get("issuer"): + out.append( + { + "name": str(p.get("id") or p["issuer"]), + "issuer": p.get("issuer"), + "jwks_uri": p.get("jwksUri") or p.get("jwks_uri"), + "audiences": [a.strip() for a in str(p.get("audiences") or "").split(",") if a], + } + ) + return out + + +def _openapi_backends(spec: dict[str, Any]) -> list[str]: + backends = [] + top = dig(spec, "x-google-backend", "address") + if top: + backends.append(top) + for ops in (spec.get("paths") or {}).values(): + if not isinstance(ops, dict): + continue + for op in ops.values(): + addr = dig(op, "x-google-backend", "address") if isinstance(op, dict) else None + if addr and addr not in backends: + backends.append(addr) + return backends + + +def _service_config_backends(cfg: dict[str, Any]) -> list[str]: + return [ + r["address"] + for r in _list(dig(cfg, "backend", "rules")) + if isinstance(r, dict) and r.get("address") + ] + + +def jwt_issuer_target(issuer: Any) -> str | None: + """What a JWT issuer names: a service account, Identity Platform project or host.""" + if not isinstance(issuer, str) or not issuer: + return None + email = sa_email(issuer) + if email: + return f"serviceAccount:{email}" + if issuer.startswith("https://securetoken.google.com/"): + return issuer.rstrip("/") + return host_target(issuer) + + +@_extractor("apigateway.googleapis.com/ApiConfig") +def _api_config_documents(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + providers: list[dict[str, Any]] = [] + backends: list[str] = [] + for item in _list(d.get("openapiDocuments")): + spec = _decode_document(dig(item, "document") if isinstance(item, dict) else None) + if spec: + providers += _openapi_providers(spec) + backends += _openapi_backends(spec) + for item in _list(d.get("managedServiceConfigs")): + cfg = _decode_document(item) + if cfg: + providers += _service_config_providers(cfg) + backends += _service_config_backends(cfg) + for addr in dict.fromkeys(backends): + out.add(url_target(addr), EdgeType.ROUTE, "SERVES_TRAFFIC_TO", description="API backend") + if providers: + out.metadata["jwt_providers"] = providers + if backends: + out.metadata["backends"] = [url_target(b) for b in dict.fromkeys(backends)] + + +@_extractor("apigateway.googleapis.com/Authorizer") +def _api_authorizer(ctx: GCPContext, out: Extracted) -> None: + """JWT authorizer of an API config (built from the config's documents).""" + d = ctx.data + out.add(parent_of(ctx.name, "authorizers"), EdgeType.CONTAINS, reverse=True) + out.add(jwt_issuer_target(d.get("issuer")), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata.update(issuer=d.get("issuer"), audiences=list(d.get("audiences") or [])) diff --git a/cloudg/inventory/gcp_relations/cicd.py b/cloudg/inventory/gcp_relations/cicd.py new file mode 100644 index 0000000..284df76 --- /dev/null +++ b/cloudg/inventory/gcp_relations/cicd.py @@ -0,0 +1,137 @@ +"""CI/CD: Cloud Build private worker pools, repository connections (Cloud +Build 2nd gen and Developer Connect) and the source / pool a build trigger +uses.""" + +from __future__ import annotations + +from typing import Any, Iterator + +from cloudg.inventory.gcp_relations._refs import parent_of, secret_version_ref +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, dig, full_name, url_alias +from cloudg.schema.models import EdgeType + + +def _secret_versions(value: Any, depth: int = 0) -> Iterator[str]: + """Every ``...SecretVersion`` value in a connection config.""" + if depth > 5: + return + if isinstance(value, dict): + for k, v in value.items(): + if isinstance(v, str) and (k.endswith("SecretVersion") or k.endswith("secretVersion")): + yield v + else: + yield from _secret_versions(v, depth + 1) + elif isinstance(value, list): + for item in value: + yield from _secret_versions(item, depth + 1) + + +def _connection_secrets(out: Extracted, d: dict[str, Any]) -> None: + for version in dict.fromkeys(_secret_versions(d)): + out.add( + secret_version_ref(version), + EdgeType.REFERENCES, + "READS_FROM", + description="repository credentials", + ) + + +def _provider(d: dict[str, Any]) -> str | None: + return next( + (k.removesuffix("Config") for k in d if k.endswith("Config") and isinstance(d[k], dict)), + None, + ) + + +@_extractor("cloudbuild.googleapis.com/WorkerPool") +def _worker_pool(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + cfg = d.get("privatePoolV1Config") or {} + net = cfg.get("networkConfig") or {} + out.add(full_name(net.get("peeredNetwork"), "compute"), EdgeType.PEERING, "VPC_PEERED") + out.add( + full_name(dig(cfg, "privateServiceConnect", "networkAttachment"), "compute"), + EdgeType.ATTACHED_TO, + ) + out.metadata.update( + egress=net.get("egressOption"), + machine_type=dig(cfg, "workerConfig", "machineType"), + public_egress=net.get("egressOption") != "NO_PUBLIC_EGRESS", + ) + + +@_extractor( + "cloudbuild.googleapis.com/Connection", + "developerconnect.googleapis.com/Connection", + "cloudbuild.googleapis.com/GithubEnterpriseConfig", + "cloudbuild.googleapis.com/BitbucketServerConfig", +) +def _repo_connection(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + _connection_secrets(out, d) + out.add(full_name(d.get("peeredNetwork"), "compute"), EdgeType.PEERING, "VPC_PEERED") + host = ( + d.get("hostUri") + or dig(d, "githubEnterpriseConfig", "hostUri") + or dig(d, "gitlabConfig", "hostUri") + or dig(d, "gitlabEnterpriseConfig", "hostUri") + or dig(d, "bitbucketDataCenterConfig", "hostUri") + ) + out.metadata.update( + provider=_provider(d), + host=url_alias(host), + installation_state=dig(d, "installationState", "stage"), + disabled=d.get("disabled"), + ) + + +@_extractor( + "cloudbuild.googleapis.com/Repository", "developerconnect.googleapis.com/GitRepositoryLink" +) +def _repo_link(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = parent_of(ctx.name, "repositories") or parent_of(ctx.name, "gitRepositoryLinks") + out.add(parent, EdgeType.CONTAINS, reverse=True) + uri = d.get("remoteUri") or d.get("cloneUri") + out.metadata["remote_uri"] = uri + + +@_extractor("developerconnect.googleapis.com/InsightsConfig") +def _insights_config(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("appHubApplication"), "apphub"), EdgeType.MONITORS, "MONITORED_BY") + for ref in _list(d.get("artifactConfigs")): + uri = ref.get("uri") if isinstance(ref, dict) else None + out.add( + full_name(uri, "artifactregistry") if uri else None, EdgeType.MONITORS, "MONITORED_BY" + ) + + +@_extractor("cloudbuild.googleapis.com/BuildTrigger") +def _build_trigger_sources(ctx: GCPContext, out: Extracted) -> None: + """2nd gen repository, Developer Connect link and private pool of a trigger.""" + d = ctx.data + out.add( + full_name(dig(d, "repositoryEventConfig", "repository"), "cloudbuild"), + EdgeType.REFERENCES, + "READS_FROM", + description="source repository", + ) + out.add( + full_name(dig(d, "developerConnectEventConfig", "gitRepositoryLink"), "developerconnect"), + EdgeType.REFERENCES, + "READS_FROM", + description="source repository link", + ) + out.add( + full_name(dig(d, "sourceToBuild", "repository"), "cloudbuild"), + EdgeType.REFERENCES, + "READS_FROM", + ) + out.add( + full_name(dig(d, "build", "options", "pool", "name"), "cloudbuild"), + EdgeType.REFERENCES, + "RUNS_ON", + description="private worker pool", + ) diff --git a/cloudg/inventory/gcp_relations/containers_ext.py b/cloudg/inventory/gcp_relations/containers_ext.py new file mode 100644 index 0000000..e3f4949 --- /dev/null +++ b/cloudg/inventory/gcp_relations/containers_ext.py @@ -0,0 +1,291 @@ +"""Containers beyond GKE: Artifact Registry images and upstreams, Binary +Authorization, GKE fleets (Hub) and Backup for GKE.""" + +from __future__ import annotations + +import re + +from cloudg.inventory.gcp_relations._refs import crm_ref, parent_of, secret_version_ref +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, _num, dig, full_name +from cloudg.schema.models import EdgeType + +_CLUSTER_KEY_RE = re.compile(r"^([a-z0-9-]+)\.(.+)$") + + +# --------------------------------------------------------------------------- +# Artifact Registry +# --------------------------------------------------------------------------- + + +@_extractor("artifactregistry.googleapis.com/DockerImage") +def _docker_image(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "dockerImages"), EdgeType.CONTAINS, reverse=True) + uri = d.get("uri") + if isinstance(uri, str) and uri: + out.alias(uri, f"https://{uri}") + out.metadata.update( + uri=uri, + tags=list(d.get("tags") or []), + image_size_bytes=_num(d.get("imageSizeBytes")), + upload_time=d.get("uploadTime"), + build_time=d.get("buildTime"), + media_type=d.get("mediaType"), + ) + + +@_extractor("artifactregistry.googleapis.com/Repository") +def _ar_repo_upstreams(ctx: GCPContext, out: Extracted) -> None: + """Virtual repository upstreams, remote upstream credentials and scanning.""" + d = ctx.data + virtual = d.get("virtualRepositoryConfig") or {} + for policy in _list(virtual.get("upstreamPolicies")): + if isinstance(policy, dict): + out.add( + full_name(policy.get("repository"), "artifactregistry"), + EdgeType.REFERENCES, + "READS_FROM", + description="virtual repository upstream", + priority=policy.get("priority"), + ) + remote = d.get("remoteRepositoryConfig") or {} + out.add( + secret_version_ref( + dig( + remote, + "upstreamCredentials", + "usernamePasswordCredentials", + "passwordSecretVersion", + ) + ), + EdgeType.REFERENCES, + "READS_FROM", + description="remote upstream credentials", + ) + upstream = None + for key, cfg in remote.items(): + if key.endswith("Repository") and isinstance(cfg, dict): + upstream = ( + cfg.get("publicRepository") + or dig(cfg, "customRepository", "uri") + or dig(cfg, "commonRepository", "uri") + ) + scan = d.get("vulnerabilityScanningConfig") or {} + out.metadata.update( + upstream=upstream, + remote_description=remote.get("description"), + vulnerability_scanning=scan.get("enablementState") or scan.get("enablementConfig"), + cleanup_policy_count=len(d.get("cleanupPolicies") or {}), + ) + + +# --------------------------------------------------------------------------- +# Binary Authorization +# --------------------------------------------------------------------------- + + +def _admission_rule(rule: object) -> dict: + if not isinstance(rule, dict): + return {} + return { + "evaluation_mode": rule.get("evaluationMode"), + "enforcement_mode": rule.get("enforcementMode"), + "attestors": list(rule.get("requireAttestationsBy") or []), + } + + +@_extractor("binaryauthorization.googleapis.com/Policy") +def _binauthz_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + pid = ctx.project_id + attestors: list[str] = [] + for key, rule in (d.get("clusterAdmissionRules") or {}).items(): + m = _CLUSTER_KEY_RE.match(str(key)) + if m and pid: + loc, cluster = m.groups() + out.add( + f"//container.googleapis.com/projects/{pid}/locations/{loc}/clusters/{cluster}", + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description="cluster admission rule", + **_admission_rule(rule), + ) + attestors += _admission_rule(rule).get("attestors", []) + default = _admission_rule(d.get("defaultAdmissionRule")) + attestors += default.get("attestors", []) + out.add( + crm_ref(f"projects/{pid}") if pid else None, + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description="default admission rule for every cluster in the project", + **default, + ) + for att in dict.fromkeys(attestors): + out.add(full_name(att, "binaryauthorization"), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata.update( + default_rule=default, + global_policy_evaluation_mode=d.get("globalPolicyEvaluationMode"), + allowlist_pattern_count=len(_list(d.get("admissionWhitelistPatterns"))), + ) + + +@_extractor("binaryauthorization.googleapis.com/Attestor") +def _binauthz_attestor(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + note = d.get("userOwnedGrafeasNote") or {} + out.sa(note.get("delegationServiceAccountEmail"), ctx, "attestor verification identity") + out.metadata.update( + note=note.get("noteReference"), public_key_count=len(_list(note.get("publicKeys"))) + ) + + +@_extractor("binaryauthorization.googleapis.com/PlatformPolicy") +def _binauthz_platform_policy(ctx: GCPContext, out: Extracted) -> None: + gke = ctx.data.get("gkePolicy") or {} + out.metadata["check_set_count"] = len(_list(gke.get("checkSets"))) + + +# --------------------------------------------------------------------------- +# GKE fleets (gkehub) +# --------------------------------------------------------------------------- + + +@_extractor("gkehub.googleapis.com/Membership") +def _fleet_membership(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + endpoint = d.get("endpoint") or {} + for key in ( + "gkeCluster", + "onPremCluster", + "multiCloudCluster", + "edgeCluster", + "applianceCluster", + ): + out.add( + full_name(dig(endpoint, key, "resourceLink")), + EdgeType.MANAGES, + "OWNED_BY", + description="fleet member cluster", + ) + authority = d.get("authority") or {} + out.metadata.update( + workload_identity_pool=authority.get("workloadIdentityPool"), + issuer=authority.get("issuer"), + unique_id=d.get("uniqueId"), + ) + + +@_extractor("gkehub.googleapis.com/Feature") +def _fleet_feature(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for membership in (d.get("membershipSpecs") or {}).keys(): + out.add( + full_name(membership, "gkehub"), + EdgeType.MANAGES, + "OWNED_BY", + description=f"fleet feature {ctx.name.rsplit('/', 1)[-1]}", + ) + out.metadata["feature_state"] = dig(d, "resourceState", "state") + + +@_extractor("gkehub.googleapis.com/MembershipBinding") +def _fleet_binding(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "bindings"), EdgeType.CONTAINS, reverse=True) + out.add(full_name(ctx.data.get("scope"), "gkehub"), EdgeType.REFERENCES, "DEPENDS_ON") + + +@_extractor("gkehub.googleapis.com/Namespace", "gkehub.googleapis.com/RBACRoleBinding") +def _fleet_scoped(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + scope = parent_of(ctx.name, "namespaces") or parent_of(ctx.name, "rbacrolebindings") + if scope and "/namespaces/" in scope: + scope = parent_of(scope, "namespaces") + out.add(scope, EdgeType.CONTAINS, reverse=True) + out.metadata.update( + user=d.get("user"), + group=d.get("group"), + role=dig(d, "role", "predefinedRole") or dig(d, "role", "customRole"), + ) + + +# --------------------------------------------------------------------------- +# Backup for GKE +# --------------------------------------------------------------------------- + + +@_extractor("gkebackup.googleapis.com/BackupPlan") +def _gke_backup_plan(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("cluster"), "container"), + EdgeType.MANAGES, + "BACKUP_TO", + description="Backup for GKE plan", + ) + cfg = d.get("backupConfig") or {} + out.metadata.update( + cron_schedule=dig(d, "backupSchedule", "cronSchedule"), + retention_days=_num(dig(d, "retentionPolicy", "backupRetainDays")), + all_namespaces=cfg.get("allNamespaces"), + include_secrets=cfg.get("includeSecrets"), + include_volume_data=cfg.get("includeVolumeData"), + deactivated=d.get("deactivated"), + ) + + +@_extractor("gkebackup.googleapis.com/Backup") +def _gke_backup(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "backups"), EdgeType.CONTAINS, reverse=True) + out.metadata["backup_state"] = ctx.data.get("state") + + +@_extractor("gkebackup.googleapis.com/RestorePlan") +def _gke_restore_plan(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("cluster"), "container"), EdgeType.REFERENCES, "DEPENDS_ON") + out.add(full_name(d.get("backupPlan"), "gkebackup"), EdgeType.REFERENCES, "DEPENDS_ON") + + +# --------------------------------------------------------------------------- +# Kubernetes namespaces and RBAC roles (Cloud Asset Inventory GKE objects) +# --------------------------------------------------------------------------- + + +@_extractor("k8s.io/Namespace") +def _k8s_namespace(ctx: GCPContext, out: Extracted) -> None: + out.add( + parent_of(ctx.name, "k8s"), + EdgeType.CONTAINS, + "CLUSTER_CONTAINS_SERVICE", + reverse=True, + description="cluster namespace", + ) + out.metadata["phase"] = dig(ctx.data, "status", "phase") + + +@_extractor("rbac.authorization.k8s.io/ClusterRole", "rbac.authorization.k8s.io/Role") +def _k8s_role(ctx: GCPContext, out: Extracted) -> None: + rules = [r for r in _list(ctx.data.get("rules")) if isinstance(r, dict)] + out.add(parent_of(ctx.name, "k8s"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + rule_count=len(rules), + wildcard=any( + "*" in (r.get("verbs") or []) or "*" in (r.get("resources") or []) for r in rules + ), + reads_secrets=any( + "secrets" in (r.get("resources") or []) + and bool(set(r.get("verbs") or []) & {"get", "list", "watch", "*"}) + for r in rules + ), + ) + + +@_extractor("gkehub.googleapis.com/Fleet", "gkehub.googleapis.com/Scope") +def _fleet(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + display_name=d.get("displayName"), + default_cluster_config=sorted((d.get("defaultClusterConfig") or {}).keys()) or None, + namespace_labels=d.get("namespaceLabels"), + ) diff --git a/cloudg/inventory/gcp_relations/context.py b/cloudg/inventory/gcp_relations/context.py index e1b9bf9..c99cd46 100644 --- a/cloudg/inventory/gcp_relations/context.py +++ b/cloudg/inventory/gcp_relations/context.py @@ -102,28 +102,38 @@ def expose( Extractor = Callable[[GCPContext, Extracted], None] -EXTRACTORS: dict[str, Extractor] = {} +# Asset type -> extractors, run in registration order. Several modules can +# register for the same type; each one adds to the same Extracted. +EXTRACTORS: dict[str, list[Extractor]] = {} def _extractor(*asset_types: str) -> Callable[[Extractor], Extractor]: def register(fn: Extractor) -> Extractor: for t in asset_types: - EXTRACTORS[t] = fn + fns = EXTRACTORS.setdefault(t, []) + if fn not in fns: + fns.append(fn) return fn return register def extract(ctx: GCPContext) -> Extracted: - """Run the extractor for ``ctx.asset_type`` plus the generic CMEK scan.""" + """Run every extractor registered for ``ctx.asset_type``, then the generic CMEK scan. + + A failing extractor is recorded in ``extraction_error`` and does not stop + the others registered for the same type. + """ out = Extracted() - fn = EXTRACTORS.get(ctx.asset_type) - if fn is not None: + errors: list[str] = [] + for fn in EXTRACTORS.get(ctx.asset_type, ()): try: fn(ctx, out) except Exception as exc: # one malformed resource must not stop the sweep logger.debug("GCP extractor failed for %s: %s", ctx.name, exc, exc_info=True) - out.metadata["extraction_error"] = f"{type(exc).__name__}: {exc}" + errors.append(f"{type(exc).__name__}: {exc}") + if errors: + out.metadata["extraction_error"] = "; ".join(errors) if asset_type_service(ctx.asset_type) == "cloudkms": return out keys = kms_refs(ctx.data) diff --git a/cloudg/inventory/gcp_relations/data_ext.py b/cloudg/inventory/gcp_relations/data_ext.py new file mode 100644 index 0000000..7a1ce08 --- /dev/null +++ b/cloudg/inventory/gcp_relations/data_ext.py @@ -0,0 +1,779 @@ +"""Data services: Spanner, Bigtable, Firestore, BigQuery tables, connections +and transfers, Dataplex and Data Catalog, Data Fusion, Datastream, Storage +Transfer, NetApp Volumes, Filestore backups, Dataproc Metastore, Secret +Manager rotation and Parameter Manager.""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.gcp_relations._refs import ( + bq_dataset_ref, + gcs_ref, + host_target, + parent_of, + secret_version_ref, +) +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import ( + _list, + _num, + dig, + full_name, + network_ref, +) +from cloudg.schema.models import EdgeType + +# --------------------------------------------------------------------------- +# Spanner +# --------------------------------------------------------------------------- + + +@_extractor("spanner.googleapis.com/Instance") +def _spanner_instance(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + instance_config=str(d.get("config") or "").rsplit("/", 1)[-1] or None, + node_count=_num(d.get("nodeCount")), + processing_units=_num(d.get("processingUnits")), + autoscaling=bool(d.get("autoscalingConfig")), + edition=d.get("edition"), + ) + + +@_extractor("spanner.googleapis.com/Database") +def _spanner_database(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + parent_of(ctx.name, "databases"), + EdgeType.CONTAINS, + "CLUSTER_CONTAINS_SERVICE", + reverse=True, + ) + out.add( + full_name(dig(d, "restoreInfo", "backupInfo", "backup"), "spanner"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="restored from backup", + ) + out.metadata.update( + dialect=d.get("databaseDialect"), + drop_protection=bool(d.get("enableDropProtection")), + version_retention_period=d.get("versionRetentionPeriod"), + ) + + +@_extractor("spanner.googleapis.com/Backup") +def _spanner_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("database"), "spanner"), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="database backup", + ) + out.metadata.update(expire_time=d.get("expireTime"), size_bytes=_num(d.get("sizeBytes"))) + + +@_extractor("spanner.googleapis.com/InstancePartition") +def _spanner_partition(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "instancePartitions"), EdgeType.CONTAINS, reverse=True) + + +# --------------------------------------------------------------------------- +# Bigtable +# --------------------------------------------------------------------------- + + +@_extractor("bigtableadmin.googleapis.com/Instance") +def _bigtable_instance(ctx: GCPContext, out: Extracted) -> None: + out.metadata.update(instance_type=ctx.data.get("type"), edition=ctx.data.get("edition")) + + +@_extractor("bigtableadmin.googleapis.com/Cluster") +def _bigtable_cluster(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + parent_of(ctx.name, "clusters"), EdgeType.CONTAINS, "CLUSTER_CONTAINS_SERVICE", reverse=True + ) + out.metadata.update( + serve_nodes=_num(d.get("serveNodes")), + storage_type=d.get("defaultStorageType"), + autoscaling=bool(dig(d, "clusterConfig", "clusterAutoscalingConfig")), + ) + + +@_extractor( + "bigtableadmin.googleapis.com/Table", + "bigtableadmin.googleapis.com/LogicalView", + "bigtableadmin.googleapis.com/MaterializedView", +) +def _bigtable_table(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = ( + parent_of(ctx.name, "tables") + or parent_of(ctx.name, "logicalViews") + or parent_of(ctx.name, "materializedViews") + ) + out.add(parent, EdgeType.CONTAINS, reverse=True) + out.metadata.update( + deletion_protection=d.get("deletionProtection"), + change_stream=bool(d.get("changeStreamConfig")), + ) + + +@_extractor("bigtableadmin.googleapis.com/AuthorizedView") +def _bigtable_authorized_view(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "authorizedViews"), EdgeType.CONTAINS, reverse=True) + + +@_extractor("bigtableadmin.googleapis.com/Backup") +def _bigtable_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("sourceTable"), "bigtableadmin"), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="table backup", + ) + out.metadata.update(expire_time=d.get("expireTime"), backup_type=d.get("backupType")) + + +@_extractor("bigtableadmin.googleapis.com/AppProfile") +def _bigtable_app_profile(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + instance = parent_of(ctx.name, "appProfiles") + out.add(instance, EdgeType.CONTAINS, reverse=True) + clusters = [] + single = dig(d, "singleClusterRouting", "clusterId") + if single: + clusters.append(single) + clusters += list(dig(d, "multiClusterRoutingUseAny", "clusterIds", default=[]) or []) + for cluster in clusters: + if instance: + out.add(f"{instance}/clusters/{cluster}", EdgeType.ROUTE, "SERVES_TRAFFIC_TO") + out.metadata["routing"] = "single" if single else "multi" + + +# --------------------------------------------------------------------------- +# Firestore +# --------------------------------------------------------------------------- + + +@_extractor("firestore.googleapis.com/Database") +def _firestore_database(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + database_type=d.get("type"), + concurrency_mode=d.get("concurrencyMode"), + point_in_time_recovery=d.get("pointInTimeRecoveryEnablement"), + delete_protection=d.get("deleteProtectionState"), + edition=d.get("databaseEdition"), + ) + + +@_extractor("firestore.googleapis.com/Backup") +def _firestore_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("database"), "firestore"), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="database backup", + ) + out.metadata.update(expire_time=d.get("expireTime"), backup_state=d.get("state")) + + +@_extractor("firestore.googleapis.com/BackupSchedule") +def _firestore_backup_schedule(ctx: GCPContext, out: Extracted) -> None: + """Backup schedules come from the Firestore API (not Cloud Asset Inventory).""" + d = ctx.data + out.add( + parent_of(ctx.name, "backupSchedules"), + EdgeType.MANAGES, + "BACKUP_TO", + description="scheduled database backups", + ) + out.metadata.update( + retention=d.get("retention"), + recurrence="weekly" if d.get("weeklyRecurrence") else "daily", + ) + + +# --------------------------------------------------------------------------- +# BigQuery tables, connections and transfers +# --------------------------------------------------------------------------- + +_SQL_REF_RE = re.compile(r"`([A-Za-z0-9_-]+(?:[.:][A-Za-z0-9_$-]+){1,2})`") +_SQL_FROM_RE = re.compile( + r"\b(?:FROM|JOIN)\s+([A-Za-z0-9_-]+\.[A-Za-z0-9_]+(?:\.[A-Za-z0-9_$]+)?)\b", re.IGNORECASE +) + + +def sql_table_refs(query: Any, default_project: str | None) -> list[str]: + """BigQuery tables a SQL query reads (backticked or after FROM / JOIN).""" + if not isinstance(query, str) or not query: + return [] + refs: list[str] = [] + for raw in _SQL_REF_RE.findall(query) + _SQL_FROM_RE.findall(query): + parts = re.split(r"[.:]", raw) + if len(parts) == 2: + parts = [default_project or "", *parts] + if len(parts) != 3 or not parts[0]: + continue + ref = f"//bigquery.googleapis.com/projects/{parts[0]}/datasets/{parts[1]}/tables/{parts[2]}" + if ref not in refs: + refs.append(ref) + return refs + + +def _bq_connection_ref(value: Any) -> str | None: + """``projects/p/locations/l/connections/c`` or ``p.l.c`` as a full name.""" + if not isinstance(value, str) or not value: + return None + if "/connections/" in value: + return full_name(value, "bigqueryconnection") + parts = value.split(".") + if len(parts) == 3: + return ( + f"//bigqueryconnection.googleapis.com/projects/{parts[0]}" + f"/locations/{parts[1]}/connections/{parts[2]}" + ) + return None + + +def _table_reference(ref: Any) -> str | None: + if not isinstance(ref, dict): + return None + pid, ds, tbl = ref.get("projectId"), ref.get("datasetId"), ref.get("tableId") + if pid and ds and tbl: + return f"//bigquery.googleapis.com/projects/{pid}/datasets/{ds}/tables/{tbl}" + return None + + +@_extractor("bigquery.googleapis.com/Table") +def _bq_table(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "tables"), EdgeType.CONTAINS, reverse=True) + project = dig(d, "tableReference", "projectId") or ctx.project_id + query = dig(d, "view", "query") or dig(d, "materializedView", "query") + for ref in sql_table_refs(query, project): + if ref != ctx.name: + out.add(ref, EdgeType.REFERENCES, "READS_FROM", description="view query source") + ext = d.get("externalDataConfiguration") or {} + for uri in _list(ext.get("sourceUris")): + out.add(gcs_ref(str(uri).split("*", 1)[0]), EdgeType.REFERENCES, "READS_FROM") + biglake = d.get("biglakeConfiguration") or {} + out.add(gcs_ref(biglake.get("storageUri")), EdgeType.REFERENCES, "READS_FROM") + for conn in (ext.get("connectionId"), biglake.get("connectionId")): + out.add(_bq_connection_ref(conn), EdgeType.REFERENCES, "DEPENDS_ON") + for key in ("snapshotDefinition", "cloneDefinition"): + out.add( + _table_reference(dig(d, key, "baseTableReference")), + EdgeType.REFERENCES, + "BACKUP_TO" if key == "snapshotDefinition" else "REPLICATES_TO", + reverse=True, + ) + out.metadata.update( + table_type=d.get("type"), + num_rows=_num(d.get("numRows")), + num_bytes=_num(d.get("numBytes")), + partitioned=bool(d.get("timePartitioning") or d.get("rangePartitioning")), + require_partition_filter=d.get("requirePartitionFilter"), + expiration_time=_num(d.get("expirationTime")), + ) + + +@_extractor("bigquery.googleapis.com/Routine") +def _bq_routine(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "routines"), EdgeType.CONTAINS, reverse=True) + project = dig(d, "routineReference", "projectId") or ctx.project_id + for ref in sql_table_refs(d.get("definitionBody"), project): + out.add(ref, EdgeType.REFERENCES, "READS_FROM") + out.add( + _bq_connection_ref(dig(d, "remoteFunctionOptions", "connection")), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + out.metadata.update(routine_type=d.get("routineType"), language=d.get("language")) + + +@_extractor("bigquery.googleapis.com/Model") +def _bq_model(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "models"), EdgeType.CONTAINS, reverse=True) + out.metadata["model_type"] = ctx.data.get("modelType") + + +@_extractor("bigquery.googleapis.com/RowAccessPolicy") +def _bq_row_access(ctx: GCPContext, out: Extracted) -> None: + out.add( + parent_of(ctx.name, "rowAccessPolicies"), + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description="row-level security", + ) + out.metadata["filter_predicate"] = ctx.data.get("filterPredicate") + + +@_extractor("bigqueryconnection.googleapis.com/Connection") +def _bq_connection(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + sql = d.get("cloudSql") or {} + parts = str(sql.get("instanceId") or "").split(":") + if len(parts) == 3: + out.add( + f"//cloudsql.googleapis.com/projects/{parts[0]}/instances/{parts[2]}", + EdgeType.REFERENCES, + "READS_FROM", + database=sql.get("database"), + ) + out.add( + full_name(dig(d, "cloudSpanner", "database"), "spanner"), + EdgeType.REFERENCES, + "READS_FROM", + ) + role = dig(d, "aws", "accessRole", "iamRoleId") + out.add( + role, + EdgeType.ASSUMES_ROLE, + "CROSS_ACCOUNT_TRUST", + description="BigQuery Omni assumes the AWS role", + ) + for key in ("cloudSql", "cloudResource", "spark"): + out.sa(dig(d, key, "serviceAccountId"), ctx, "connection identity") + out.add( + full_name(dig(d, "spark", "metastoreService"), "metastore"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + kind = next( + ( + k + for k in ("cloudSql", "aws", "azure", "cloudSpanner", "cloudResource", "spark") + if d.get(k) + ), + None, + ) + out.metadata.update( + connection_type=kind, + azure_tenant=dig(d, "azure", "customerTenantId"), + aws_role=role, + ) + + +@_extractor("bigquerydatatransfer.googleapis.com/TransferConfig") +def _bq_transfer(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + bq_dataset_ref(ctx.project_id, d.get("destinationDatasetId")), + EdgeType.REFERENCES, + "WRITES_TO", + description="transfer destination dataset", + ) + params = d.get("params") or {} + for key in ("data_path_template", "data_path"): + value = params.get(key) + if isinstance(value, str) and value.startswith("gs://"): + out.add(gcs_ref(value.split("*", 1)[0]), EdgeType.REFERENCES, "READS_FROM") + elif isinstance(value, str) and value.startswith("s3://"): + out.add(f"arn:aws:s3:::{value[5:].split('/', 1)[0]}", EdgeType.REFERENCES, "READS_FROM") + for ref in sql_table_refs(params.get("query"), ctx.project_id): + out.add(ref, EdgeType.REFERENCES, "READS_FROM", description="scheduled query source") + out.add( + full_name(d.get("notificationPubsubTopic"), "pubsub"), + EdgeType.INVOKES, + "INVOKES", + description="run notifications", + ) + out.metadata.update( + data_source=d.get("dataSourceId"), + schedule=d.get("schedule"), + disabled=d.get("disabled"), + ) + + +# --------------------------------------------------------------------------- +# Dataplex, Data Catalog, Dataproc Metastore +# --------------------------------------------------------------------------- + + +def _dataplex_resource(spec: Any) -> str | None: + """A Dataplex asset's bucket or dataset.""" + if not isinstance(spec, dict): + return None + name = spec.get("name") + if not isinstance(name, str): + return None + if "/buckets/" in name: + return f"//storage.googleapis.com/{name.rsplit('/buckets/', 1)[1]}" + if "/datasets/" in name: + return full_name(name, "bigquery") + return full_name(name) + + +@_extractor("dataplex.googleapis.com/Lake") +def _dataplex_lake(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(dig(d, "metastore", "service"), "metastore"), EdgeType.REFERENCES, "DEPENDS_ON" + ) + out.sa(d.get("serviceAccount"), ctx, "lake service agent") + + +@_extractor("dataplex.googleapis.com/Zone") +def _dataplex_zone(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "zones"), EdgeType.CONTAINS, reverse=True) + out.metadata["zone_type"] = ctx.data.get("type") + + +@_extractor("dataplex.googleapis.com/Asset") +def _dataplex_asset(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "assets"), EdgeType.CONTAINS, reverse=True) + out.add( + _dataplex_resource(d.get("resourceSpec")), + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description="Dataplex governs the resource", + ) + out.metadata["resource_type"] = dig(d, "resourceSpec", "type") + + +@_extractor("dataplex.googleapis.com/Task") +def _dataplex_task(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "tasks"), EdgeType.CONTAINS, reverse=True) + out.sa(dig(d, "executionSpec", "serviceAccount"), ctx, "task identity") + spark = d.get("spark") or {} + for key in ("mainJarFileUri", "pythonScriptFile", "sqlScriptFile"): + uri = spark.get(key) + if isinstance(uri, str) and uri.startswith("gs://"): + out.add(gcs_ref(uri), EdgeType.REFERENCES, "READS_FROM") + out.metadata["trigger"] = dig(d, "triggerSpec", "type") + + +@_extractor("dataplex.googleapis.com/DataScan") +def _dataplex_scan(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + target = full_name(dig(d, "data", "resource")) or full_name( + dig(d, "data", "entity"), "dataplex" + ) + out.add( + target, + EdgeType.MONITORS, + "MONITORED_BY", + description=f"{d.get('type') or 'data'} scan", + ) + out.metadata.update( + scan_type=d.get("type"), trigger=list((dig(d, "executionSpec", "trigger") or {}).keys()) + ) + + +@_extractor( + "dataplex.googleapis.com/EntryGroup", + "datacatalog.googleapis.com/EntryGroup", + "datacatalog.googleapis.com/TagTemplate", + "datacatalog.googleapis.com/Taxonomy", +) +def _catalog_container(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + activated_policy_types=list(d.get("activatedPolicyTypes") or []) or None, + field_count=len(d.get("fields") or {}) or None, + publicly_readable=d.get("isPubliclyReadable"), + ) + + +@_extractor("metastore.googleapis.com/Service") +def _metastore(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("network"), "compute"), EdgeType.ATTACHED_TO) + for consumer in _list(dig(d, "networkConfig", "consumers")): + if isinstance(consumer, dict): + out.in_subnet(full_name(consumer.get("subnetwork"), "compute")) + out.add(gcs_ref(d.get("artifactGcsUri")), EdgeType.REFERENCES, "WRITES_TO") + out.metadata.update( + tier=d.get("tier"), database_type=d.get("databaseType"), endpoint=d.get("endpointUri") + ) + + +# --------------------------------------------------------------------------- +# Data Fusion, Datastream, Storage Transfer +# --------------------------------------------------------------------------- + + +@_extractor("datafusion.googleapis.com/Instance") +def _datafusion(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + nc = d.get("networkConfig") or {} + out.add(network_ref(nc.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + for key in ("dataprocServiceAccount", "p4ServiceAccount", "serviceAccount"): + out.sa(d.get(key), ctx, f"Data Fusion {key}") + for url in (d.get("apiEndpoint"), d.get("serviceEndpoint")): + out.alias(host_target(url)) + private = bool(d.get("privateInstance")) + if not private: + out.expose("Data Fusion public instance endpoint", protocol="tcp", ports=["443"]) + out.metadata.update( + private_instance=private, + edition=d.get("type"), + version=d.get("version"), + enable_rbac=d.get("enableRbac"), + ) + + +@_extractor("datafusion.googleapis.com/DnsPeering") +def _datafusion_dns_peering(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "dnsPeerings"), EdgeType.CONTAINS, reverse=True) + out.add( + network_ref(d.get("targetNetwork"), d.get("targetProject")), EdgeType.ROUTE, "DNS_RESOLVED" + ) + + +@_extractor("datastream.googleapis.com/Stream") +def _datastream_stream(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(dig(d, "sourceConfig", "sourceConnectionProfile"), "datastream"), + EdgeType.REFERENCES, + "READS_FROM", + description="change data capture source", + ) + out.add( + full_name(dig(d, "destinationConfig", "destinationConnectionProfile"), "datastream"), + EdgeType.INVOKES, + "STREAMS_TO", + description="stream destination", + ) + out.metadata.update(stream_state=d.get("state"), backfill_all=bool(d.get("backfillAll"))) + + +@_extractor("datastream.googleapis.com/ConnectionProfile") +def _datastream_profile(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(gcs_ref(dig(d, "gcsProfile", "bucket")), EdgeType.REFERENCES, "DEPENDS_ON") + kind = None + for key in ("mysqlProfile", "postgresqlProfile", "oracleProfile", "sqlServerProfile"): + prof = d.get(key) + if isinstance(prof, dict): + kind = key.removesuffix("Profile") + out.add(prof.get("hostname"), EdgeType.REFERENCES, "DEPENDS_ON", port=prof.get("port")) + out.add( + full_name(dig(d, "privateConnectivity", "privateConnection"), "datastream"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + out.metadata.update( + profile_type=kind + or next( + (k.removesuffix("Profile") for k in ("gcsProfile", "bigqueryProfile") if k in d), None + ), + connectivity=next( + ( + k + for k in ( + "privateConnectivity", + "staticServiceIpConnectivity", + "forwardSshConnectivity", + ) + if k in d + ), + None, + ), + ) + + +@_extractor("datastream.googleapis.com/PrivateConnection") +def _datastream_private_connection(ctx: GCPContext, out: Extracted) -> None: + cfg = ctx.data.get("vpcPeeringConfig") or {} + out.add(full_name(cfg.get("vpc"), "compute"), EdgeType.PEERING, "VPC_PEERED") + out.metadata["subnet"] = cfg.get("subnet") + + +@_extractor("storagetransfer.googleapis.com/TransferJob") +def _transfer_job(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + spec = d.get("transferSpec") or d.get("replicationSpec") or {} + out.add(gcs_ref(dig(spec, "gcsDataSource", "bucketName")), EdgeType.REFERENCES, "READS_FROM") + out.add(gcs_ref(dig(spec, "gcsDataSink", "bucketName")), EdgeType.REFERENCES, "WRITES_TO") + s3 = spec.get("awsS3DataSource") or {} + if s3.get("bucketName"): + out.add(f"arn:aws:s3:::{s3['bucketName']}", EdgeType.REFERENCES, "READS_FROM") + out.add( + s3.get("roleArn"), + EdgeType.ASSUMES_ROLE, + "CROSS_ACCOUNT_TRUST", + description="Storage Transfer assumes the AWS role", + ) + azure = spec.get("azureBlobStorageDataSource") or {} + if azure.get("storageAccount"): + out.add( + host_target(f"{azure['storageAccount']}.blob.core.windows.net"), + EdgeType.REFERENCES, + "READS_FROM", + container=azure.get("container"), + ) + for src in (s3, azure): + out.add(secret_version_ref(src.get("credentialsSecret")), EdgeType.REFERENCES, "READS_FROM") + out.add( + full_name(dig(d, "notificationConfig", "pubsubTopic"), "pubsub"), + EdgeType.INVOKES, + "INVOKES", + ) + stream = dig(d, "eventStream", "name") + out.add(full_name(stream, "pubsub") or stream, EdgeType.REFERENCES, "READS_FROM") + out.metadata.update( + transfer_status=d.get("status"), + source_type=next((k for k in spec if k.endswith("DataSource")), None), + schedule_start=dig(d, "schedule", "scheduleStartDate"), + repeat_interval=dig(d, "schedule", "repeatInterval"), + ) + + +# --------------------------------------------------------------------------- +# NetApp Volumes, Filestore backups +# --------------------------------------------------------------------------- + + +@_extractor("netapp.googleapis.com/StoragePool") +def _netapp_pool(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + out.add(full_name(d.get("activeDirectory"), "netapp"), EdgeType.REFERENCES, "DEPENDS_ON") + out.add(full_name(d.get("kmsConfig"), "netapp"), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata.update( + service_level=d.get("serviceLevel"), capacity_gib=_num(d.get("capacityGib")) + ) + + +@_extractor("netapp.googleapis.com/Volume") +def _netapp_volume(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("storagePool"), "netapp"), EdgeType.CONTAINS, reverse=True) + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + backup = d.get("backupConfig") or {} + out.add(full_name(backup.get("backupVault"), "netapp"), EdgeType.REFERENCES, "BACKUP_TO") + for policy in _list(backup.get("backupPolicies")): + out.add(full_name(policy, "netapp"), EdgeType.REFERENCES, "DEPENDS_ON") + clients = sorted( + { + c.strip() + for r in _list(dig(d, "exportPolicy", "rules")) + if isinstance(r, dict) + for c in str(r.get("allowedClients") or "").split(",") + if c.strip() + } + ) + out.metadata.update( + protocols=list(d.get("protocols") or []), + capacity_gib=_num(d.get("capacityGib")), + allowed_clients=clients, + snapshot_policy=bool(dig(d, "snapshotPolicy", "enabled")), + ) + + +@_extractor("netapp.googleapis.com/Replication") +def _netapp_replication(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "replications"), EdgeType.CONTAINS, reverse=True) + out.add( + full_name(d.get("destinationVolume"), "netapp"), + EdgeType.REFERENCES, + "REPLICATES_TO", + description="volume replication", + ) + out.metadata.update(replication_schedule=d.get("replicationSchedule"), role=d.get("role")) + + +@_extractor("netapp.googleapis.com/Backup", "netapp.googleapis.com/Snapshot") +def _netapp_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = parent_of(ctx.name, "backups") or parent_of(ctx.name, "snapshots") + source = full_name(d.get("sourceVolume"), "netapp") + if "/snapshots/" in ctx.name: + source = source or parent + else: + out.add(parent, EdgeType.CONTAINS, reverse=True) + out.add(source, EdgeType.REFERENCES, "BACKUP_TO", reverse=True, description="volume backup") + + +@_extractor( + "netapp.googleapis.com/BackupVault", + "netapp.googleapis.com/BackupPolicy", + "netapp.googleapis.com/ActiveDirectory", + "netapp.googleapis.com/KmsConfig", +) +def _netapp_misc(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + domain=d.get("domain"), + enabled=d.get("enabled"), + daily_backup_limit=_num(d.get("dailyBackupLimit")), + vault_type=d.get("backupVaultType"), + ) + + +@_extractor("file.googleapis.com/Backup") +def _filestore_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("sourceInstance"), "file"), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="file share backup", + ) + out.metadata.update( + source_file_share=d.get("sourceFileShare"), storage_bytes=_num(d.get("storageBytes")) + ) + + +# --------------------------------------------------------------------------- +# Secret Manager rotation, Parameter Manager +# --------------------------------------------------------------------------- + + +@_extractor("secretmanager.googleapis.com/Secret") +def _secret_rotation(ctx: GCPContext, out: Extracted) -> None: + """Rotation schedule and notification topics (the rotator subscribes to them).""" + d = ctx.data + rotation = d.get("rotation") or {} + topics = [ + full_name(t.get("name"), "pubsub") for t in _list(d.get("topics")) if isinstance(t, dict) + ] + out.metadata.update( + rotation_period=rotation.get("rotationPeriod"), + next_rotation_time=rotation.get("nextRotationTime"), + rotation_topics=[t for t in topics if t] if rotation else [], + expire_time=d.get("expireTime"), + version_destroy_ttl=d.get("versionDestroyTtl"), + ) + + +@_extractor("parametermanager.googleapis.com/Parameter") +def _parameter(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + format=d.get("format"), + principal=dig(d, "policyMember", "iamPolicyUidPrincipal"), + ) + + +@_extractor("parametermanager.googleapis.com/ParameterVersion") +def _parameter_version(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "versions"), EdgeType.CONTAINS, reverse=True) + out.metadata["disabled"] = ctx.data.get("disabled") + + +__all__ = ["sql_table_refs"] + + +@_extractor("cloudkms.googleapis.com/KeyRing") +def _key_ring(ctx: GCPContext, out: Extracted) -> None: + out.metadata["key_ring"] = ctx.name.rsplit("/", 1)[-1] diff --git a/cloudg/inventory/gcp_relations/edge.py b/cloudg/inventory/gcp_relations/edge.py new file mode 100644 index 0000000..42f1fe0 --- /dev/null +++ b/cloudg/inventory/gcp_relations/edge.py @@ -0,0 +1,292 @@ +"""Edge: Cloud DNS record sets, Certificate Manager, Certificate Authority +Service, Cloud Run domain mappings and Identity Platform (the issuer of +Firebase / Identity Platform ID tokens).""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.gcp_relations._refs import host_target, parent_of, url_target +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, _num, dig, full_name, relative_name +from cloudg.schema.models import EdgeType + +_CM = "certificatemanager" +_MAX_RRDATAS = 50 + + +# --------------------------------------------------------------------------- +# Cloud DNS record sets (read from the Cloud DNS API) +# --------------------------------------------------------------------------- + + +def _routing_rrdatas(policy: Any) -> list[str]: + """Answers of a routing policy (weighted, geo, primary / backup, health-checked ILBs).""" + if not isinstance(policy, dict): + return [] + out: list[str] = [] + + def targets(block: Any) -> None: + if not isinstance(block, dict): + return + out.extend(str(r) for r in _list(block.get("rrdatas"))) + hc = block.get("healthCheckedTargets") or {} + for lb in _list(hc.get("internalLoadBalancers")): + if isinstance(lb, dict) and lb.get("ipAddress"): + out.append(str(lb["ipAddress"])) + out.extend(str(e) for e in _list(hc.get("externalEndpoints"))) + + for key in ("wrr", "geo"): + for item in _list(dig(policy, key, "items")): + targets(item) + pb = policy.get("primaryBackup") or {} + targets(pb.get("primaryTargets")) + for item in _list(dig(pb, "backupGeoTargets", "items")): + targets(item) + return out + + +@_extractor("dns.googleapis.com/ResourceRecordSet") +def _dns_record(ctx: GCPContext, out: Extracted) -> None: + """A record set: the zone contains it and it resolves to IPs / hosts (DNS_RESOLVED).""" + d = ctx.data + out.add(d.get("managedZone"), EdgeType.CONTAINS, reverse=True, description="zone record") + rtype = str(d.get("type") or "").upper() + answers = [str(r) for r in _list(d.get("rrdatas"))] + _routing_rrdatas(d.get("routingPolicy")) + for answer in dict.fromkeys(answers[:_MAX_RRDATAS]): + if rtype in ("A", "AAAA"): + target = answer + elif rtype in ("CNAME", "ALIAS"): + target = host_target(answer.rstrip(".")) + else: + continue + out.add(target, EdgeType.ROUTE, "DNS_RESOLVED", record_type=rtype) + out.metadata.update( + fqdn=str(d.get("name") or "").rstrip(".") or None, + record_type=rtype, + ttl=_num(d.get("ttl")), + rrdatas=answers[:_MAX_RRDATAS], + routing_policy=sorted((d.get("routingPolicy") or {}).keys()) or None, + ) + + +# --------------------------------------------------------------------------- +# Certificate Manager +# --------------------------------------------------------------------------- + + +@_extractor("certificatemanager.googleapis.com/Certificate") +def _cm_certificate(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + managed = d.get("managed") or {} + for auth in _list(managed.get("dnsAuthorizations")): + out.add( + full_name(auth, _CM), EdgeType.REFERENCES, "DEPENDS_ON", description="DNS authorization" + ) + out.add( + full_name(managed.get("issuanceConfig"), _CM), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="issuance config", + ) + out.metadata.update( + domains=list(managed.get("domains") or d.get("sanDnsnames") or []), + managed=bool(managed), + managed_state=managed.get("state"), + scope=d.get("scope"), + expire_time=d.get("expireTime"), + ) + + +@_extractor("certificatemanager.googleapis.com/CertificateMap") +def _cm_map(ctx: GCPContext, out: Extracted) -> None: + for target in _list(ctx.data.get("gclbTargets")): + if not isinstance(target, dict): + continue + for key in ("targetHttpsProxy", "targetSslProxy"): + out.add( + full_name(target.get(key), "compute"), + EdgeType.REFERENCES, + "CERTIFICATE_SECURES", + reverse=True, + description="load balancer uses the certificate map", + ) + + +@_extractor("certificatemanager.googleapis.com/CertificateMapEntry") +def _cm_map_entry(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "certificateMapEntries"), EdgeType.CONTAINS, reverse=True) + for cert in _list(d.get("certificates")): + out.add(full_name(cert, _CM), EdgeType.REFERENCES, "CERTIFICATE_SECURES") + out.metadata.update(hostname=d.get("hostname"), matcher=d.get("matcher")) + + +@_extractor("certificatemanager.googleapis.com/DnsAuthorization") +def _cm_dns_auth(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + domain=d.get("domain"), + dns_record=dig(d, "dnsResourceRecord", "name"), + auth_type=d.get("type"), + ) + + +@_extractor("certificatemanager.googleapis.com/CertificateIssuanceConfig") +def _cm_issuance(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name( + dig(d, "certificateAuthorityConfig", "certificateAuthorityServiceConfig", "caPool"), + "privateca", + ), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="issuing CA pool", + ) + out.metadata.update(lifetime=d.get("lifetime"), key_algorithm=d.get("keyAlgorithm")) + + +@_extractor("certificatemanager.googleapis.com/TrustConfig") +def _cm_trust_config(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + trust_store_count=len(_list(d.get("trustStores"))), + allowlisted_certificate_count=len(_list(d.get("allowlistedCertificates"))), + ) + + +# --------------------------------------------------------------------------- +# Certificate Authority Service +# --------------------------------------------------------------------------- + + +@_extractor("privateca.googleapis.com/CertificateAuthority") +def _ca(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "certificateAuthorities"), EdgeType.CONTAINS, reverse=True) + out.add( + full_name(dig(d, "subordinateConfig", "certificateAuthority"), "privateca"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="issuing CA", + ) + bucket = d.get("gcsBucket") + if bucket: + out.add( + f"//storage.googleapis.com/{bucket}", + EdgeType.REFERENCES, + "WRITES_TO", + description="CA certificate and CRL publishing", + ) + out.metadata.update( + ca_type=d.get("type"), tier=d.get("tier"), key_algorithm=dig(d, "keySpec", "algorithm") + ) + + +@_extractor("privateca.googleapis.com/CaPool") +def _ca_pool(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + tier=d.get("tier"), + publish_ca_cert=dig(d, "publishingOptions", "publishCaCert"), + publish_crl=dig(d, "publishingOptions", "publishCrl"), + ) + + +# --------------------------------------------------------------------------- +# Cloud Run domain mappings +# --------------------------------------------------------------------------- + + +@_extractor("run.googleapis.com/DomainMapping") +def _run_domain_mapping(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + meta = d.get("metadata") or {} + domain = meta.get("name") or ctx.name.rsplit("/", 1)[-1] + route = dig(d, "spec", "routeName") + pid = ctx.project_id or meta.get("namespace") + if route and pid and ctx.location: + out.add( + f"//run.googleapis.com/projects/{pid}/locations/{ctx.location}/services/{route}", + EdgeType.ROUTE, + "SERVES_TRAFFIC_TO", + description=f"{domain} serves the service", + ) + out.alias(host_target(domain)) + records = [ + {"name": r.get("name"), "type": r.get("type"), "rrdata": r.get("rrdata")} + for r in _list(dig(d, "status", "resourceRecords")) + if isinstance(r, dict) + ] + out.metadata.update( + domain=domain, + route=route, + certificate_mode=dig(d, "spec", "certificateMode"), + resource_records=records, + ) + + +# --------------------------------------------------------------------------- +# Identity Platform +# --------------------------------------------------------------------------- + + +@_extractor("identitytoolkit.googleapis.com/Config") +def _idp_config(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parts = relative_name(ctx.name).split("/") + pid = ctx.project_id or (parts[1] if len(parts) > 1 and parts[0] == "projects" else None) + if pid: + # The issuer of the ID tokens this project's users carry + out.alias(f"https://securetoken.google.com/{pid}", f"securetoken.google.com/{pid}") + triggers = dig(d, "blockingFunctions", "triggers", default={}) or {} + for event, trig in triggers.items(): + if isinstance(trig, dict): + out.add( + url_target(trig.get("functionUri")), + EdgeType.INVOKES, + "INVOKES", + description=f"blocking function {event}", + ) + sign_in = d.get("signIn") or {} + out.metadata.update( + email_sign_in=bool(dig(sign_in, "email", "enabled")), + phone_sign_in=bool(dig(sign_in, "phoneNumber", "enabled")), + anonymous_sign_in=bool(dig(sign_in, "anonymous", "enabled")), + mfa_state=dig(d, "mfa", "state"), + authorized_domains=list(d.get("authorizedDomains") or []), + multi_tenant=bool(dig(d, "multiTenant", "allowTenants")), + ) + + +@_extractor("identitytoolkit.googleapis.com/Tenant") +def _idp_tenant(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + project = parent_of(ctx.name, "tenants") + if project: + out.add(f"{project}/config", EdgeType.CONTAINS, reverse=True, description="tenant") + out.metadata.update( + mfa_state=dig(d, "mfaConfig", "state"), + allow_password_signup=d.get("allowPasswordSignup"), + enable_anonymous_user=d.get("enableAnonymousUser"), + ) + + +@_extractor( + "identitytoolkit.googleapis.com/InboundSamlConfig", + "identitytoolkit.googleapis.com/OauthIdpConfig", + "identitytoolkit.googleapis.com/DefaultSupportedIdpConfig", +) +def _idp_provider(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + owner = None + for seg in ("inboundSamlConfigs", "oauthIdpConfigs", "defaultSupportedIdpConfigs"): + owner = owner or parent_of(ctx.name, seg) + if owner and "/tenants/" not in owner: + owner = f"{owner}/config" + out.add(owner, EdgeType.CONTAINS, reverse=True, description="sign-in provider") + out.metadata.update( + enabled=d.get("enabled"), + issuer=d.get("issuer") or dig(d, "idpConfig", "idpEntityId"), + ) diff --git a/cloudg/inventory/gcp_relations/governance.py b/cloudg/inventory/gcp_relations/governance.py new file mode 100644 index 0000000..a5c78df --- /dev/null +++ b/cloudg/inventory/gcp_relations/governance.py @@ -0,0 +1,246 @@ +"""Governance: Resource Manager tags and liens, custom organization policy +constraints, Assured Workloads, authorized organizations, Infrastructure +Manager and Deployment Manager deployments.""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.gcp_relations._refs import crm_ref, gcs_ref +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, dig, full_name, relative_name +from cloudg.schema.models import EdgeType + +_CRM = "//cloudresourcemanager.googleapis.com/" + + +# --------------------------------------------------------------------------- +# Tags and liens +# --------------------------------------------------------------------------- + + +@_extractor("cloudresourcemanager.googleapis.com/TagKey") +def _tag_key(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(crm_ref(d.get("parent")), EdgeType.CONTAINS, reverse=True) + out.alias(relative_name(ctx.name), d.get("namespacedName")) + out.add( + full_name(dig(d, "purposeData", "network"), "compute"), + EdgeType.ATTACHED_TO, + description="firewall tag network", + ) + out.metadata.update( + short_name=d.get("shortName"), + namespaced_name=d.get("namespacedName"), + purpose=d.get("purpose"), + ) + + +@_extractor("cloudresourcemanager.googleapis.com/TagValue") +def _tag_value(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = d.get("parent") + out.add( + f"{_CRM}{parent}" if isinstance(parent, str) and parent.startswith("tagKeys/") else None, + EdgeType.CONTAINS, + reverse=True, + ) + out.alias(relative_name(ctx.name), d.get("namespacedName")) + out.metadata.update(short_name=d.get("shortName"), namespaced_name=d.get("namespacedName")) + + +def tag_binding_parts(d: dict[str, Any]) -> tuple[str | None, str | None]: + """(bound resource, tag value) full names of a tag binding.""" + value = d.get("tagValue") + value_ref = ( + f"{_CRM}{value}" if isinstance(value, str) and value.startswith("tagValues/") else None + ) + parent = d.get("parent") + return (full_name(parent) if isinstance(parent, str) else None), value_ref + + +@_extractor("cloudresourcemanager.googleapis.com/TagBinding") +def _tag_binding(ctx: GCPContext, out: Extracted) -> None: + """Folded into a TAGGED_WITH relation on the bound resource by the deep collector.""" + d = ctx.data + resource, value = tag_binding_parts(d) + out.metadata.update( + bound_resource=resource, + tag_value=value, + tag_value_namespaced_name=d.get("tagValueNamespacedName"), + ) + + +@_extractor("cloudresourcemanager.googleapis.com/Lien") +def _lien(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + crm_ref(d.get("parent")), + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description="lien blocks deletion", + restrictions=list(d.get("restrictions") or []), + ) + out.metadata.update( + restrictions=list(d.get("restrictions") or []), + reason=d.get("reason"), + origin=d.get("origin"), + ) + + +# --------------------------------------------------------------------------- +# Organization policy constraints, Assured Workloads, authorized orgs +# --------------------------------------------------------------------------- + + +@_extractor("orgpolicy.googleapis.com/CustomConstraint") +def _custom_constraint(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.alias(relative_name(ctx.name)) + out.metadata.update( + resource_types=list(d.get("resourceTypes") or []), + method_types=list(d.get("methodTypes") or []), + condition=d.get("condition"), + action_type=d.get("actionType"), + ) + + +@_extractor("orgpolicy.googleapis.com/Policy") +def _org_policy_custom(ctx: GCPContext, out: Extracted) -> None: + """A policy enforcing a custom constraint depends on its definition.""" + short = relative_name(ctx.name).rsplit("/policies/", 1)[-1] + if not short.startswith("custom."): + return + org = next((a for a in ctx.ancestors if str(a).startswith("organizations/")), None) + if org: + out.add( + f"//orgpolicy.googleapis.com/{org}/customConstraints/{short}", + EdgeType.REFERENCES, + "DEPENDS_ON", + description="custom constraint definition", + ) + + +@_extractor("assuredworkloads.googleapis.com/Workload") +def _assured_workload(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for res in _list(d.get("resources")): + if not isinstance(res, dict) or not res.get("resourceId"): + continue + kind = "folders" if "FOLDER" in str(res.get("resourceType")) else "projects" + out.add( + f"{_CRM}{kind}/{res['resourceId']}", + EdgeType.GOVERNS, + "COMPLIANCE_GOVERNS", + description=f"{d.get('complianceRegime')} workload", + resource_type=res.get("resourceType"), + ) + out.metadata.update( + compliance_regime=d.get("complianceRegime"), + partner=d.get("partner"), + violation_notifications=d.get("violationNotificationsEnabled"), + acknowledged_violations=dig(d, "complianceStatus", "acknowledgedViolationCount"), + active_violations=dig(d, "complianceStatus", "activeViolationCount"), + ) + + +@_extractor("accesscontextmanager.googleapis.com/AuthorizedOrgsDesc") +def _authorized_orgs(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for org in _list(d.get("orgs")): + out.add( + crm_ref(org), + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + description=f"{d.get('authorizationType')} {d.get('assetType')}", + ) + out.metadata.update( + authorization_type=d.get("authorizationType"), + authorization_direction=d.get("authorizationDirection"), + authorized_asset_type=d.get("assetType"), + ) + + +@_extractor("accesscontextmanager.googleapis.com/ServicePerimeter") +def _perimeter_access_levels(ctx: GCPContext, out: Extracted) -> None: + status = ctx.data.get("status") or {} + for level in _list(status.get("accessLevels")): + out.add(full_name(level, "accesscontextmanager"), EdgeType.REFERENCES, "DEPENDS_ON") + + +@_extractor("accesscontextmanager.googleapis.com/AccessLevel") +def _access_level(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + conditions = _list(dig(d, "basic", "conditions")) + out.metadata.update( + title=d.get("title"), + combining_function=dig(d, "basic", "combiningFunction"), + ip_subnetworks=sorted( + {s for c in conditions if isinstance(c, dict) for s in c.get("ipSubnetworks") or []} + ), + regions=sorted( + {r for c in conditions if isinstance(c, dict) for r in c.get("regions") or []} + ), + custom=bool(d.get("custom")), + ) + + +# --------------------------------------------------------------------------- +# Infrastructure Manager and Deployment Manager (MANAGES) +# --------------------------------------------------------------------------- + + +@_extractor("config.googleapis.com/Deployment") +def _infra_manager_deployment(ctx: GCPContext, out: Extracted) -> None: + """Managed resources are listed by the deep collector (Infrastructure Manager API).""" + d = ctx.data + out.sa(d.get("serviceAccount"), ctx, "Terraform runner identity") + blueprint = d.get("terraformBlueprint") or {} + out.add( + gcs_ref(blueprint.get("gcsSource")), + EdgeType.REFERENCES, + "READS_FROM", + description="blueprint", + ) + out.add( + gcs_ref(d.get("artifactsGcsBucket")), + EdgeType.REFERENCES, + "WRITES_TO", + description="artifacts", + ) + out.add(full_name(d.get("workerPool"), "cloudbuild"), EdgeType.REFERENCES, "RUNS_ON") + for res in _list(d.get("managedResources")): + out.add( + full_name(res), EdgeType.MANAGES, "OWNED_BY", description="Terraform-managed resource" + ) + out.metadata.update( + latest_revision=d.get("latestRevision"), + git_repo=dig(blueprint, "gitSource", "repo"), + deployment_state=d.get("state"), + tf_version=d.get("tfVersion"), + managed_resource_count=len(_list(d.get("managedResources"))) or None, + ) + + +@_extractor("config.googleapis.com/Preview") +def _infra_manager_preview(ctx: GCPContext, out: Extracted) -> None: + out.add(full_name(ctx.data.get("deployment"), "config"), EdgeType.REFERENCES, "DEPENDS_ON") + + +@_extractor("deploymentmanager.googleapis.com/Deployment") +def _deployment_manager(ctx: GCPContext, out: Extracted) -> None: + """Deployment Manager deployment and the resources it created (Deployment Manager API).""" + d = ctx.data + for res in _list(d.get("resources")): + if isinstance(res, dict): + out.add( + full_name(res.get("url")), + EdgeType.MANAGES, + "OWNED_BY", + description=f"Deployment Manager {res.get('type')}", + ) + out.metadata.update( + manifest=str(d.get("manifest") or "").rsplit("/", 1)[-1] or None, + operation_status=dig(d, "operation", "status"), + resource_count=len(_list(d.get("resources"))), + ) diff --git a/cloudg/inventory/gcp_relations/hybrid.py b/cloudg/inventory/gcp_relations/hybrid.py new file mode 100644 index 0000000..bd1f747 --- /dev/null +++ b/cloudg/inventory/gcp_relations/hybrid.py @@ -0,0 +1,333 @@ +"""Hybrid connectivity and compute extras: NCC, Interconnect, external VPN +gateways, PSC accept lists, packet mirroring, sole-tenant nodes and +resource policies (snapshot and instance schedules).""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.gcp_relations._refs import crm_ref, parent_of +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, _num, _short, dig, full_name, network_ref +from cloudg.schema.models import EdgeType + +_C = "//compute.googleapis.com/" +NODE_GROUP_AFFINITY_KEY = "compute.googleapis.com/node-group-name" + + +# --------------------------------------------------------------------------- +# Network Connectivity Center +# --------------------------------------------------------------------------- + + +@_extractor("networkconnectivity.googleapis.com/Hub") +def _ncc_hub(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for vpc in _list(d.get("routingVpcs")): + if isinstance(vpc, dict): + out.add( + full_name(vpc.get("uri"), "compute"), + EdgeType.ROUTE, + "TRANSIT_ROUTED", + description="hub routing VPC", + ) + counts = { + str(c.get("spokeType")): _num(c.get("count")) + for c in _list(dig(d, "spokeSummary", "spokeTypeCounts")) + if isinstance(c, dict) + } + out.metadata.update( + policy_mode=d.get("policyMode"), + preset_topology=d.get("presetTopology"), + export_psc=d.get("exportPsc"), + spoke_type_counts=counts or None, + ) + + +@_extractor( + "networkconnectivity.googleapis.com/RouteTable", "networkconnectivity.googleapis.com/Group" +) +def _ncc_hub_child(ctx: GCPContext, out: Extracted) -> None: + hub = parent_of(ctx.name, "routeTables") or parent_of(ctx.name, "groups") + out.add(hub, EdgeType.CONTAINS, reverse=True, description="hub contains") + out.metadata["auto_accept_projects"] = list( + dig(ctx.data, "autoAccept", "autoAcceptProjects", default=[]) or [] + ) + + +@_extractor("networkconnectivity.googleapis.com/InternalRange") +def _internal_range(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + out.metadata.update( + ip_cidr_range=d.get("ipCidrRange"), usage=d.get("usage"), peering=d.get("peering") + ) + + +@_extractor("networkconnectivity.googleapis.com/PolicyBasedRoute") +def _policy_route(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + out.add(d.get("nextHopIlbIp"), EdgeType.ROUTE, "TRANSIT_ROUTED", description="next hop ILB") + out.metadata.update( + next_hop_ilb_ip=d.get("nextHopIlbIp"), + next_hop_other_routes=d.get("nextHopOtherRoutes"), + priority=_num(d.get("priority")), + filter=d.get("filter"), + ) + + +@_extractor("networkconnectivity.googleapis.com/RegionalEndpoint") +def _regional_endpoint(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + out.in_subnet(full_name(d.get("subnetwork"), "compute")) + out.alias(d.get("address")) + out.metadata.update( + target_google_api=d.get("targetGoogleApi"), + access_type=d.get("accessType"), + address=d.get("address"), + ) + + +# --------------------------------------------------------------------------- +# Interconnect, external VPN gateways, VPN peers +# --------------------------------------------------------------------------- + + +@_extractor("compute.googleapis.com/Interconnect") +def _interconnect(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + attachments = _list(d.get("interconnectAttachments")) + for att in attachments: + out.add(full_name(att, "compute"), EdgeType.CONTAINS, description="VLAN attachment") + out.metadata.update( + interconnect_type=d.get("interconnectType"), + link_type=d.get("linkType"), + location=_short(d.get("location")), + provisioned_link_count=_num(d.get("provisionedLinkCount")), + operational_status=d.get("operationalStatus"), + macsec_enabled=bool(d.get("macsecEnabled")), + admin_enabled=d.get("adminEnabled"), + attachment_count=len(attachments), + ) + + +@_extractor("compute.googleapis.com/ExternalVpnGateway") +def _external_vpn_gateway(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + ips = [ + i.get("ipAddress") or i.get("ipv6Address") + for i in _list(d.get("interfaces")) + if isinstance(i, dict) + ] + ips = [ip for ip in ips if ip] + out.alias(*ips) + out.metadata.update(redundancy_type=d.get("redundancyType"), peer_ips=ips) + + +@_extractor("compute.googleapis.com/VpnTunnel") +def _vpn_tunnel_peer(ctx: GCPContext, out: Extracted) -> None: + """Classic VPN tunnels name their peer only by IP: link the external gateway with that IP.""" + d = ctx.data + if d.get("peerExternalGateway") or d.get("peerGcpGateway"): + return + out.add(d.get("peerIp"), EdgeType.ROUTE, "TRANSIT_ROUTED", description="VPN peer address") + + +# --------------------------------------------------------------------------- +# Private Service Connect accept lists +# --------------------------------------------------------------------------- + + +@_extractor("compute.googleapis.com/ServiceAttachment") +def _psc_accept_lists(ctx: GCPContext, out: Extracted) -> None: + """Consumer projects / networks on the accept list may connect (GRANTS_ACCESS).""" + d = ctx.data + for entry in _list(d.get("consumerAcceptLists")): + if not isinstance(entry, dict): + continue + consumer = crm_ref(entry.get("projectIdOrNum")) or full_name( + entry.get("networkUrl"), "compute" + ) + out.add( + consumer, + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="PSC consumer accept list", + connection_limit=_num(entry.get("connectionLimit")), + ) + for ep in _list(d.get("connectedEndpoints")): + if isinstance(ep, dict): + out.add( + full_name(ep.get("endpoint"), "compute"), + EdgeType.ROUTE, + "SERVES_TRAFFIC_TO", + reverse=True, + description="connected PSC endpoint", + status=ep.get("status"), + ) + out.metadata["consumer_reject_lists"] = list(d.get("consumerRejectLists") or []) + + +# --------------------------------------------------------------------------- +# Packet mirroring +# --------------------------------------------------------------------------- + + +@_extractor("compute.googleapis.com/PacketMirroring") +def _packet_mirroring(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(dig(d, "network", "url"), "compute"), EdgeType.ATTACHED_TO) + out.add( + full_name(dig(d, "collectorIlb", "url"), "compute"), + EdgeType.INVOKES, + "STREAMS_TO", + description="mirrored traffic collector", + ) + mirrored = d.get("mirroredResources") or {} + for key in ("subnetworks", "instances"): + for item in _list(mirrored.get(key)): + if isinstance(item, dict): + out.add( + full_name(item.get("url"), "compute"), + EdgeType.MONITORS, + "MONITORED_BY", + description="traffic mirrored", + ) + out.metadata.update( + enable=d.get("enable"), + mirrored_tags=list(mirrored.get("tags") or []), + filter=d.get("filter"), + ) + + +# --------------------------------------------------------------------------- +# Sole-tenant nodes +# --------------------------------------------------------------------------- + + +@_extractor("compute.googleapis.com/NodeTemplate") +def _node_template(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + node_type=d.get("nodeType"), + server_binding=dig(d, "serverBinding", "type"), + cpu_overcommit=d.get("cpuOvercommitType"), + ) + + +@_extractor("compute.googleapis.com/NodeGroup") +def _node_group(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("nodeTemplate"), "compute"), EdgeType.REFERENCES, "DEPENDS_ON") + share = d.get("shareSettings") or {} + for project in (share.get("projectMap") or {}).keys(): + out.add( + crm_ref(project), + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="node group shared with project", + ) + out.metadata.update( + size=_num(d.get("size")), + autoscaling_mode=dig(d, "autoscalingPolicy", "mode"), + maintenance_policy=d.get("maintenancePolicy"), + share_type=share.get("shareType"), + ) + + +@_extractor("compute.googleapis.com/Instance") +def _instance_extras(ctx: GCPContext, out: Extracted) -> None: + """Numeric-id aliases, sole-tenant placement and attached resource policies.""" + d = ctx.data + zone = ctx.location + vm_id = _num(d.get("id")) + if vm_id and zone: + for project in {ctx.project_id, ctx.project_number} - {None}: + out.alias(f"{_C}projects/{project}/zones/{zone}/instances/{vm_id}") + if vm_id and ctx.project_id: + out.alias(f"gcp-instance-id:{ctx.project_id}/{vm_id}") + for aff in _list(dig(d, "scheduling", "nodeAffinities")): + if not isinstance(aff, dict) or aff.get("key") != NODE_GROUP_AFFINITY_KEY: + continue + if aff.get("operator") not in (None, "IN"): + continue + for group in _list(aff.get("values")): + if ctx.project_id and zone and isinstance(group, str): + out.add( + f"{_C}projects/{ctx.project_id}/zones/{zone}/nodeGroups/{group}", + EdgeType.REFERENCES, + "RUNS_ON", + description="sole-tenant node group", + ) + policies = [p for p in (full_name(r, "compute") for r in _list(d.get("resourcePolicies"))) if p] + if policies: + out.metadata["resource_policies"] = policies + if vm_id: + out.metadata["instance_id"] = vm_id + + +# --------------------------------------------------------------------------- +# Resource policies: snapshot schedules, instance schedules, placement +# --------------------------------------------------------------------------- + + +def resource_policy_kind(d: dict[str, Any]) -> str: + for key, kind in ( + ("snapshotSchedulePolicy", "snapshot_schedule"), + ("instanceSchedulePolicy", "instance_schedule"), + ("groupPlacementPolicy", "group_placement"), + ("diskConsistencyGroupPolicy", "disk_consistency_group"), + ("workloadPolicy", "workload"), + ): + if d.get(key) is not None: + return kind + return "other" + + +def _schedule_summary(schedule: dict[str, Any]) -> str | None: + for key in ("hourlySchedule", "dailySchedule", "weeklySchedule"): + s = schedule.get(key) + if isinstance(s, dict): + return key.removesuffix("Schedule") + return None + + +@_extractor("compute.googleapis.com/ResourcePolicy") +def _resource_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + kind = resource_policy_kind(d) + md: dict[str, Any] = {"policy_kind": kind} + snap = d.get("snapshotSchedulePolicy") or {} + if snap: + md.update( + schedule=_schedule_summary(snap.get("schedule") or {}), + retention_days=_num(dig(snap, "retentionPolicy", "maxRetentionDays")), + on_source_disk_delete=dig(snap, "retentionPolicy", "onSourceDiskDelete"), + storage_locations=list(dig(snap, "snapshotProperties", "storageLocations", default=[])), + guest_flush=dig(snap, "snapshotProperties", "guestFlush"), + ) + sched = d.get("instanceSchedulePolicy") or {} + if sched: + md.update( + vm_start=dig(sched, "vmStartSchedule", "schedule"), + vm_stop=dig(sched, "vmStopSchedule", "schedule"), + time_zone=sched.get("timeZone"), + ) + out.metadata.update(md) + + +@_extractor("compute.googleapis.com/Disk", "compute.googleapis.com/RegionDisk") +def _disk_policies(ctx: GCPContext, out: Extracted) -> None: + """A disk only takes snapshot schedule policies: the policy backs it up.""" + for ref in _list(ctx.data.get("resourcePolicies")): + out.add( + full_name(ref, "compute"), + EdgeType.MANAGES, + "BACKUP_TO", + reverse=True, + description="snapshot schedule", + ) diff --git a/cloudg/inventory/gcp_relations/iam.py b/cloudg/inventory/gcp_relations/iam.py index fa51673..58c9a90 100644 --- a/cloudg/inventory/gcp_relations/iam.py +++ b/cloudg/inventory/gcp_relations/iam.py @@ -217,6 +217,33 @@ def principal_asset(member: str, discovered_via: str = "iam_policy") -> CloudAss ) +_HAS_POLICY = { + "user": "USER_HAS_POLICY", + "group": "GROUP_HAS_POLICY", + "domain": "GROUP_HAS_POLICY", +} + + +def has_policy_relationship(principal_type: Any) -> str: + """USER_HAS_POLICY / GROUP_HAS_POLICY / ROLE_HAS_POLICY for a principal type. + + Service accounts, federated principals and Kubernetes service accounts + are workload identities, the GCP counterpart of an AWS role. + """ + return _HAS_POLICY.get(str(principal_type or ""), "ROLE_HAS_POLICY") + + +def role_ref(role: Any) -> str | None: + """A role name (``roles/x``, ``projects/p/roles/x``) as its IAM full name.""" + if not isinstance(role, str) or not role: + return None + if role.startswith("//"): + return role + if role.startswith(("roles/", "projects/", "organizations/")) and "roles/" in role: + return f"//iam.googleapis.com/{role}" + return None + + def _index(assets: Iterable[CloudAsset]) -> dict[str, CloudAsset]: idx: dict[str, CloudAsset] = {} for a in assets: @@ -349,6 +376,21 @@ def emit(self) -> None: condition=", ".join(info["conditions"]) or None, ), ) + relationship = has_policy_relationship(principal.metadata.get("principal_type")) + for role in info["roles"]: + _append_relation( + principal, + rel( + role_ref(role), + EdgeType.IAM_POLICY_ATTACHMENT, + relationship, + description=f"bound to {role}", + ), + ) + + def roles(self) -> list[str]: + """Every role bound by the policies seen so far.""" + return list(dict.fromkeys(r for info in self.grants.values() for r in info["roles"])) def apply_iam_policies( diff --git a/cloudg/inventory/gcp_relations/identity_ext.py b/cloudg/inventory/gcp_relations/identity_ext.py new file mode 100644 index 0000000..fc991db --- /dev/null +++ b/cloudg/inventory/gcp_relations/identity_ext.py @@ -0,0 +1,359 @@ +"""Identity beyond allow bindings: identity pools, IAM deny policies, +Principal Access Boundary policies and bindings, Privileged Access Manager +entitlements, Cloud Identity groups and role permission sets. + +Principals named here are referenced by their +:func:`~cloudg.inventory.gcp_relations.iam.principal_identifier`; the deep +collector creates placeholder assets for the ones that were not collected. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import unquote + +from cloudg.inventory.gcp_relations._refs import crm_ref, parent_of +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.iam import ( + has_policy_relationship, + principal_identifier, + role_ref, +) +from cloudg.inventory.gcp_relations.names import ( + PRINCIPAL_PREFIX, + PUBLIC_MEMBERS, + _list, + dig, + full_name, + relative_name, + sa_ref, +) +from cloudg.schema.models import EdgeType + +_MAX_PERMISSIONS = 200 + +_V2_PREFIXES = ( + ("principal://goog/subject/", "user:"), + ("principalSet://goog/group/", "group:"), + ("principal://iam.googleapis.com/projects/-/serviceAccounts/", "serviceAccount:"), + ( + "principalSet://goog/cloudIdentityCustomerId/", + "principalSet://goog/cloudIdentityCustomerId/", + ), +) + + +def v1_member(principal: Any) -> str | None: + """IAM v2 principal identifier (deny policies) as a v1 member string. + + ``principalSet://goog/public:all`` becomes ``allUsers``; workforce and + workload identity principals are kept as they are. + """ + if not isinstance(principal, str) or not principal: + return None + if principal == "principalSet://goog/public:all": + return "allUsers" + for prefix, member in _V2_PREFIXES: + if principal.startswith(prefix): + rest = principal[len(prefix) :] + return member + rest if member.endswith(":") else principal + return principal + + +def principal_target(member: Any) -> str | None: + """Relation target for an IAM member (None for public members).""" + if not isinstance(member, str) or not member or member in PUBLIC_MEMBERS: + return None + if member.startswith("serviceAccount:") and "[" not in member: + return sa_ref(member.split(":", 1)[1]) + return principal_identifier(member) + + +# --------------------------------------------------------------------------- +# Identity pools and OAuth clients +# --------------------------------------------------------------------------- + + +@_extractor("iam.googleapis.com/WorkforcePool") +def _workforce_pool(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(crm_ref(d.get("parent")), EdgeType.CONTAINS, reverse=True) + out.alias(relative_name(ctx.name)) + out.metadata.update( + disabled=bool(d.get("disabled")), + session_duration=d.get("sessionDuration"), + access_restricted=bool(d.get("accessRestrictions")), + pool_kind="workforce", + ) + + +@_extractor("iam.googleapis.com/WorkloadIdentityPool") +def _workload_pool(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.alias(relative_name(ctx.name)) + out.metadata.update(disabled=bool(d.get("disabled")), mode=d.get("mode"), pool_kind="workload") + + +@_extractor( + "iam.googleapis.com/WorkloadIdentityPoolNamespace", + "iam.googleapis.com/WorkloadIdentityPoolManagedIdentity", + "iam.googleapis.com/WorkforcePoolProviderKey", + "iam.googleapis.com/WorkloadIdentityPoolProviderKey", + "iam.googleapis.com/OauthClientCredential", +) +def _pool_child(ctx: GCPContext, out: Extracted) -> None: + for seg in ("managedIdentities", "namespaces", "keys", "credentials"): + parent = parent_of(ctx.name, seg) + if parent: + out.add(parent, EdgeType.CONTAINS, reverse=True) + return + + +@_extractor("iam.googleapis.com/OauthClient") +def _oauth_client(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + client_type=d.get("clientType"), + allowed_grant_types=list(d.get("allowedGrantTypes") or []), + allowed_scopes=list(d.get("allowedScopes") or []), + redirect_uri_count=len(_list(d.get("allowedRedirectUris"))), + disabled=bool(d.get("disabled")), + ) + + +@_extractor("iam.googleapis.com/Role") +def _role_permissions(ctx: GCPContext, out: Extracted) -> None: + """Permission list of a custom or (Roles API) predefined role.""" + perms = [p for p in _list(ctx.data.get("includedPermissions")) if isinstance(p, str)] + out.alias(relative_name(ctx.name)) + if perms: + out.metadata["included_permissions"] = perms[:_MAX_PERMISSIONS] + out.metadata["permissions_truncated"] = len(perms) > _MAX_PERMISSIONS or None + out.metadata["predefined"] = relative_name(ctx.name).startswith("roles/") + + +# --------------------------------------------------------------------------- +# IAM deny policies (iam.googleapis.com/PolicyV2) +# --------------------------------------------------------------------------- + + +def deny_attachment_point(name: str) -> str | None: + """``policies//denypolicies/x`` -> attachment full name.""" + rel_name = relative_name(name) + if not rel_name.startswith("policies/"): + return None + encoded = rel_name.split("/", 2)[1] + point = unquote(encoded) + return f"//{point}" if point and not point.startswith("//") else point or None + + +@_extractor("iam.googleapis.com/PolicyV2", "iam.googleapis.com/DenyPolicy") +def _deny_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + if d.get("kind") not in (None, "DenyPolicy", "denypolicies"): + return + attachment = deny_attachment_point(ctx.name) + rules: list[dict[str, Any]] = [] + for rule in _list(d.get("rules")): + deny = rule.get("denyRule") if isinstance(rule, dict) else None + if not isinstance(deny, dict): + continue + principals = [v1_member(p) for p in _list(deny.get("deniedPrincipals"))] + permissions = list(deny.get("deniedPermissions") or []) + rules.append( + { + "denied_principals": principals, + "exception_principals": [ + v1_member(p) for p in _list(deny.get("exceptionPrincipals")) + ], + "denied_permissions": permissions, + "exception_permissions": list(deny.get("exceptionPermissions") or []), + "condition": dig(deny, "denialCondition", "expression"), + } + ) + for member in principals: + out.add( + principal_target(member), + EdgeType.GOVERNS, + "POLICY_DENIES_ACTION", + description="IAM deny rule names this principal", + denied_permissions=permissions[:50], + attachment_point=attachment, + ) + all_perms = sorted({p for r in rules for p in r["denied_permissions"]}) + out.add( + attachment, + EdgeType.GOVERNS, + "POLICY_DENIES_ACTION", + description="IAM deny policy attached here", + denied_permissions=all_perms[:50], + ) + out.metadata.update( + policy_kind="deny", + attachment_point=attachment, + rules=rules, + denies_public=any("allUsers" in r["denied_principals"] for r in rules) or None, + ) + + +# --------------------------------------------------------------------------- +# Principal Access Boundary +# --------------------------------------------------------------------------- + + +@_extractor("iam.googleapis.com/PrincipalAccessBoundaryPolicy") +def _pab_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + details = d.get("details") or {} + resources: list[str] = [] + for rule in _list(details.get("rules")): + if isinstance(rule, dict) and rule.get("effect") in (None, "ALLOW"): + resources += [r for r in _list(rule.get("resources")) if isinstance(r, str)] + for res in dict.fromkeys(resources): + out.add( + full_name(res), + EdgeType.GOVERNS, + "PERMISSION_BOUNDARY_LIMITS", + description="bound principals can only reach resources under this one", + ) + out.metadata.update( + policy_kind="principal_access_boundary", + allowed_resources=list(dict.fromkeys(resources)), + enforcement_version=details.get("enforcementVersion"), + ) + + +def binding_policy_ref(d: dict[str, Any]) -> str | None: + policy = d.get("policy") + if not isinstance(policy, str) or not policy: + return None + return policy if policy.startswith("//") else f"//iam.googleapis.com/{policy}" + + +@_extractor("iam.googleapis.com/PolicyBinding") +def _policy_binding(ctx: GCPContext, out: Extracted) -> None: + """Binds a policy (Principal Access Boundary) to a principal set. + + The edge from the principal set to the policy is added on the policy + asset by the deep collector, since a binding cannot declare it. + """ + d = ctx.data + target = dig(d, "target", "principalSet") or dig(d, "target", "resource") + out.add(binding_policy_ref(d), EdgeType.REFERENCES, "DEPENDS_ON", description="bound policy") + out.add(full_name(target), EdgeType.GOVERNS, "PERMISSION_BOUNDARY_LIMITS") + out.metadata.update( + policy_kind=d.get("policyKind"), + policy=binding_policy_ref(d), + target_principal_set=full_name(target), + condition=dig(d, "condition", "expression"), + ) + + +# --------------------------------------------------------------------------- +# Privileged Access Manager +# --------------------------------------------------------------------------- + + +@_extractor("privilegedaccessmanager.googleapis.com/Entitlement") +def _pam_entitlement(ctx: GCPContext, out: Extracted) -> None: + """Eligible principals may elevate to the entitlement's roles on its resource.""" + d = ctx.data + access = dig(d, "privilegedAccess", "gcpIamAccess", default={}) or {} + roles = [b.get("role") for b in _list(access.get("roleBindings")) if isinstance(b, dict)] + roles = [r for r in roles if r] + resource = full_name(access.get("resource")) + out.add( + resource, + EdgeType.GRANTS_ACCESS, + "POLICY_ALLOWS_ACTION", + description="roles granted while an entitlement grant is active", + roles=roles, + role=roles[0] if roles else None, + just_in_time=True, + ) + for role in roles: + out.add(role_ref(role), EdgeType.IAM_POLICY_ATTACHMENT, "ROLE_HAS_POLICY") + eligible = [ + p + for e in _list(d.get("eligibleUsers")) + if isinstance(e, dict) + for p in _list(e.get("principals")) + ] + for member in eligible: + out.add( + principal_target(member), + EdgeType.ASSUMES_ROLE, + "ROLE_ASSUMES_ROLE", + reverse=True, + description="eligible to request this entitlement", + ) + approvers = [ + p + for step in _list(dig(d, "approvalWorkflow", "manualApprovals", "steps")) + if isinstance(step, dict) + for a in _list(step.get("approvers")) + if isinstance(a, dict) + for p in _list(a.get("principals")) + ] + for member in approvers: + out.add(principal_target(member), EdgeType.REFERENCES, "DEPENDS_ON", description="approver") + out.metadata.update( + roles=roles, + resource=resource, + resource_type=access.get("resourceType"), + eligible_principals=eligible, + approvers=approvers, + approval_required=bool(approvers), + max_request_duration=d.get("maxRequestDuration"), + entitlement_state=d.get("state"), + ) + + +# --------------------------------------------------------------------------- +# Cloud Identity groups (from the Cloud Identity API) +# --------------------------------------------------------------------------- + + +@_extractor("cloudidentity.googleapis.com/Group") +def _cloud_identity_group(ctx: GCPContext, out: Extracted) -> None: + """A Google group, its members (CONTAINS) and the IAM member string aliases.""" + d = ctx.data + email = str(dig(d, "groupKey", "id") or "").lower() + if email: + out.alias(email, f"group:{email}", f"{PRINCIPAL_PREFIX}group:{email}") + members: list[dict[str, Any]] = [] + for m in _list(d.get("memberships")): + if not isinstance(m, dict): + continue + key = str(dig(m, "preferredMemberKey", "id") or "").lower() + mtype = str(m.get("type") or "").upper() + prefix = {"GROUP": "group", "SERVICE_ACCOUNT": "serviceAccount"}.get(mtype, "user") + if not key: + continue + roles = [r.get("name") for r in _list(m.get("roles")) if isinstance(r, dict)] + members.append({"member": f"{prefix}:{key}", "roles": roles}) + out.add( + principal_target(f"{prefix}:{key}"), + EdgeType.CONTAINS, + description=f"member of {email or ctx.name}", + roles=roles, + ) + labels = d.get("labels") or {} + out.metadata.update( + email=email or None, + principal_type="group", + member_count=len(members), + members=members[:500], + security_group="cloudidentity.googleapis.com/groups.security" in labels, + dynamic=bool(d.get("dynamicGroupMetadata")), + ) + + +__all__ = [ + "binding_policy_ref", + "deny_attachment_point", + "has_policy_relationship", + "principal_target", + "role_ref", + "v1_member", +] diff --git a/cloudg/inventory/gcp_relations/integration.py b/cloudg/inventory/gcp_relations/integration.py new file mode 100644 index 0000000..a7fc7b7 --- /dev/null +++ b/cloudg/inventory/gcp_relations/integration.py @@ -0,0 +1,368 @@ +"""Integration: Cloud Scheduler, Eventarc channels and Eventarc Advanced, +Managed Kafka topics and Connect, Pub/Sub snapshots and schemas, the calls +inside Workflows definitions, and Batch jobs.""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.gcp_relations._refs import ( + bq_ref, + gcs_ref, + parent_of, + url_target, +) +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import ( + _list, + _num, + dig, + full_name, + image_repository, +) +from cloudg.schema.models import EdgeType + +# --------------------------------------------------------------------------- +# Cloud Scheduler +# --------------------------------------------------------------------------- + + +@_extractor("cloudscheduler.googleapis.com/Job") +def _scheduler_job(ctx: GCPContext, out: Extracted) -> None: + """A scheduler job runs its target on a schedule (INVOKES / SCHEDULED_BY).""" + d = ctx.data + http = d.get("httpTarget") or {} + out.add( + url_target(http.get("uri")), + EdgeType.INVOKES, + "SCHEDULED_BY", + description="scheduled HTTP target", + http_method=http.get("httpMethod"), + ) + out.sa( + dig(http, "oidcToken", "serviceAccountEmail") + or dig(http, "oauthToken", "serviceAccountEmail"), + ctx, + "scheduler job identity", + ) + out.add( + full_name(dig(d, "pubsubTarget", "topicName"), "pubsub"), + EdgeType.INVOKES, + "SCHEDULED_BY", + description="scheduled Pub/Sub message", + ) + ae = d.get("appEngineHttpTarget") or {} + service = dig(ae, "appEngineRouting", "service") or ("default" if ae else None) + if service and ctx.project_id: + out.add( + f"//appengine.googleapis.com/apps/{ctx.project_id}/services/{service}", + EdgeType.INVOKES, + "SCHEDULED_BY", + description="scheduled App Engine request", + ) + out.metadata.update( + schedule=d.get("schedule"), + time_zone=d.get("timeZone"), + target_type="http" if http else "pubsub" if d.get("pubsubTarget") else "appengine", + target=url_target(http.get("uri")), + attempt_deadline=d.get("attemptDeadline"), + retry_count=_num(dig(d, "retryConfig", "retryCount")), + ) + + +# --------------------------------------------------------------------------- +# Eventarc +# --------------------------------------------------------------------------- + + +@_extractor("eventarc.googleapis.com/Channel") +def _eventarc_channel(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("pubsubTopic"), "pubsub"), + EdgeType.INVOKES, + "STREAMS_TO", + description="channel transport topic", + ) + out.metadata.update(provider=d.get("provider"), channel_state=d.get("state")) + + +@_extractor("eventarc.googleapis.com/ChannelConnection") +def _eventarc_channel_connection(ctx: GCPContext, out: Extracted) -> None: + out.add(full_name(ctx.data.get("channel"), "eventarc"), EdgeType.REFERENCES, "DEPENDS_ON") + + +@_extractor("eventarc.googleapis.com/Trigger") +def _eventarc_trigger_channel(ctx: GCPContext, out: Extracted) -> None: + out.add( + full_name(ctx.data.get("channel"), "eventarc"), + EdgeType.INVOKES, + "TRIGGERED_BY", + reverse=True, + description="third-party channel events", + ) + + +@_extractor("eventarc.googleapis.com/GoogleApiSource") +def _eventarc_api_source(ctx: GCPContext, out: Extracted) -> None: + out.add( + full_name(ctx.data.get("destination"), "eventarc"), + EdgeType.INVOKES, + "STREAMS_TO", + description="Google API events to the message bus", + ) + + +@_extractor("eventarc.googleapis.com/Enrollment") +def _eventarc_enrollment(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("messageBus"), "eventarc"), + EdgeType.INVOKES, + "TRIGGERED_BY", + reverse=True, + description="message bus delivers matching events", + ) + out.add(full_name(d.get("destination"), "eventarc"), EdgeType.INVOKES, "INVOKES") + out.metadata["cel_match"] = d.get("celMatch") + + +@_extractor("eventarc.googleapis.com/Pipeline") +def _eventarc_pipeline(ctx: GCPContext, out: Extracted) -> None: + for dest in _list(ctx.data.get("destinations")): + if not isinstance(dest, dict): + continue + out.add(full_name(dest.get("workflow"), "workflows"), EdgeType.INVOKES, "INVOKES") + out.add(full_name(dest.get("messageBus"), "eventarc"), EdgeType.INVOKES, "STREAMS_TO") + out.add(full_name(dest.get("topic"), "pubsub"), EdgeType.INVOKES, "STREAMS_TO") + out.add(url_target(dig(dest, "httpEndpoint", "uri")), EdgeType.INVOKES, "INVOKES") + auth = dest.get("authenticationConfig") or {} + out.sa( + dig(auth, "googleOidc", "serviceAccount") or dig(auth, "oauthToken", "serviceAccount"), + ctx, + "pipeline delivery identity", + ) + + +@_extractor("eventarc.googleapis.com/MessageBus") +def _eventarc_bus(ctx: GCPContext, out: Extracted) -> None: + out.metadata["log_severity"] = dig(ctx.data, "loggingConfig", "logSeverity") + + +# --------------------------------------------------------------------------- +# Managed Service for Apache Kafka +# --------------------------------------------------------------------------- + + +@_extractor("managedkafka.googleapis.com/Topic") +def _kafka_topic(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "topics"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + partition_count=_num(d.get("partitionCount")), + replication_factor=_num(d.get("replicationFactor")), + ) + + +@_extractor("managedkafka.googleapis.com/ConsumerGroup") +def _kafka_consumer_group(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "consumerGroups"), EdgeType.CONTAINS, reverse=True) + cluster = parent_of(ctx.name, "consumerGroups") + for topic in (ctx.data.get("topics") or {}).keys(): + ref = full_name(topic, "managedkafka") if "/" in str(topic) else None + if not ref and cluster: + ref = f"{cluster}/topics/{topic}" + out.add(ref, EdgeType.REFERENCES, "READS_FROM", description="consumer group offsets") + + +@_extractor("managedkafka.googleapis.com/ConnectCluster") +def _kafka_connect_cluster(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("kafkaCluster"), "managedkafka"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="Kafka cluster the connectors use", + ) + for nc in _list(dig(d, "gcpConfig", "accessConfig", "networkConfigs")): + if isinstance(nc, dict): + out.in_subnet(full_name(nc.get("primarySubnet"), "compute")) + + +def _connector_targets(cfg: dict[str, Any], project: str | None) -> list[tuple[str | None, str]]: + """(target, relation) pairs a Kafka Connect connector's config names.""" + cls = str(cfg.get("connector.class") or "") + sink = "Sink" in cls + verb = "STREAMS_TO" if sink else "READS_FROM" + targets: list[tuple[str | None, str]] = [] + if cfg.get("cps.topic"): + targets.append( + ( + f"//pubsub.googleapis.com/projects/{cfg.get('cps.project') or project}/topics/{cfg['cps.topic']}", + verb, + ) + ) + if cfg.get("gcs.bucket.name"): + targets.append((gcs_ref(cfg["gcs.bucket.name"]), verb)) + if cfg.get("defaultDataset"): + targets.append((bq_ref(f"{cfg.get('project') or project}.{cfg['defaultDataset']}"), verb)) + return targets + + +@_extractor("managedkafka.googleapis.com/Connector") +def _kafka_connector(ctx: GCPContext, out: Extracted) -> None: + cfg = ctx.data.get("configs") or {} + out.add(parent_of(ctx.name, "connectors"), EdgeType.CONTAINS, reverse=True) + for target, verb in _connector_targets(cfg, ctx.project_id): + edge = EdgeType.INVOKES if verb == "STREAMS_TO" else EdgeType.REFERENCES + out.add(target, edge, verb, description="Kafka Connect connector") + out.metadata["connector_class"] = cfg.get("connector.class") + + +# --------------------------------------------------------------------------- +# Pub/Sub snapshots and schemas +# --------------------------------------------------------------------------- + + +@_extractor("pubsub.googleapis.com/Snapshot") +def _pubsub_snapshot(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("topic"), "pubsub"), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata["expire_time"] = d.get("expireTime") + + +@_extractor("pubsub.googleapis.com/Schema") +def _pubsub_schema(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update(schema_type=d.get("type"), revision_id=d.get("revisionId")) + + +# --------------------------------------------------------------------------- +# Workflows: what a definition calls +# --------------------------------------------------------------------------- + +_CALL_RE = re.compile(r"^\s*(?:-\s*)?call:\s*([A-Za-z0-9_.]+)") +_LIT = r"[A-Za-z0-9_.-]+" +_RES_RE = re.compile( + rf"(projects/{_LIT}/(?:locations/{_LIT}/)?" + r"(topics|subscriptions|secrets|workflows|functions|services|jobs|datasets|queues)" + rf"/{_LIT})" +) +_URL_RE = re.compile(r"https://[A-Za-z0-9.-]+(?:/[^\s\"'$}]*)?") +_BUCKET_RE = re.compile(rf"^\s*bucket:\s*[\"']?({_LIT})[\"']?\s*$") +_WRITE_VERBS = ("publish", "insert", "create", "write", "patch", "update", "put", "upload", "add") +_SERVICES = { + "topics": "pubsub", + "subscriptions": "pubsub", + "secrets": "secretmanager", + "workflows": "workflows", + "functions": "cloudfunctions", + "services": "run", + "jobs": "run", + "datasets": "bigquery", + "queues": "cloudtasks", +} +_COMPUTE = {"workflows", "functions", "services", "jobs"} +_MAX_SOURCE = 256 * 1024 + + +def _call_verb(call: str | None) -> str: + method = (call or "").rsplit(".", 1)[-1].lower() + return "WRITES_TO" if any(v in method for v in _WRITE_VERBS) else "READS_FROM" + + +def workflow_calls(source: str) -> list[tuple[str, EdgeType, str]]: + """(target, edge, relation) for each literal resource / URL a definition calls.""" + found: list[tuple[str, EdgeType, str]] = [] + call: str | None = None + for line in source[:_MAX_SOURCE].splitlines(): + m = _CALL_RE.match(line) + if m: + call = m.group(1) + for ref, collection in _RES_RE.findall(line): + target = full_name(ref, _SERVICES[collection]) + if collection in _COMPUTE: + found.append((target, EdgeType.INVOKES, "INVOKES")) + elif collection == "secrets": + found.append( + ( + full_name(ref.split("/versions/")[0], "secretmanager"), + EdgeType.REFERENCES, + "READS_FROM", + ) + ) + else: + found.append((target, EdgeType.REFERENCES, _call_verb(call))) + if call and call.startswith("http"): + for url in _URL_RE.findall(line): + found.append((url_target(url), EdgeType.INVOKES, "INVOKES")) + m = _BUCKET_RE.match(line) + if m and call and ".storage." in f".{call}.": + found.append((gcs_ref(m.group(1)), EdgeType.REFERENCES, _call_verb(call))) + return [(t, e, r) for t, e, r in dict.fromkeys(found) if t] + + +@_extractor("workflows.googleapis.com/Workflow") +def _workflow_definition(ctx: GCPContext, out: Extracted) -> None: + source = ctx.data.get("sourceContents") + if not isinstance(source, str) or not source: + return + connectors = sorted( + { + m.group(1).rsplit(".", 1)[0] + for m in map(_CALL_RE.match, source.splitlines()) + if m and m.group(1).startswith("googleapis.") + } + ) + for target, edge, relation in workflow_calls(source): + out.add(target, edge, relation, description="called from the workflow definition") + out.metadata["connectors"] = connectors + + +# --------------------------------------------------------------------------- +# Batch +# --------------------------------------------------------------------------- + + +@_extractor("batch.googleapis.com/Job") +def _batch_job(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + alloc = d.get("allocationPolicy") or {} + out.sa(dig(alloc, "serviceAccount", "email") or "default", ctx, "Batch VM identity") + no_external = True + for ni in _list(dig(alloc, "network", "networkInterfaces")): + if isinstance(ni, dict): + out.in_subnet(full_name(ni.get("subnetwork") or ni.get("network"), "compute")) + no_external = no_external and bool(ni.get("noExternalIpAddress")) + for inst in _list(alloc.get("instances")): + if isinstance(inst, dict): + out.add( + full_name(inst.get("instanceTemplate"), "compute"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + images = [] + for group in _list(d.get("taskGroups")): + spec = (group.get("taskSpec") or {}) if isinstance(group, dict) else {} + for runnable in _list(spec.get("runnables")): + image = dig(runnable, "container", "imageUri") if isinstance(runnable, dict) else None + if image: + images.append(image) + out.add(image_repository(image), EdgeType.USES_IMAGE, "RUNS_ON") + for vol in _list(spec.get("volumes")): + path = dig(vol, "gcs", "remotePath") if isinstance(vol, dict) else None + if path: + bucket = str(path).removeprefix("gs://").split("/", 1)[0] + out.add( + gcs_ref(bucket), EdgeType.REFERENCES, "READS_FROM", description="mounted bucket" + ) + for note in _list(d.get("notifications")): + if isinstance(note, dict): + out.add(full_name(note.get("pubsubTopic"), "pubsub"), EdgeType.INVOKES, "INVOKES") + out.metadata.update( + images=images, + job_state=dig(d, "status", "state"), + logs_destination=dig(d, "logsPolicy", "destination"), + no_external_ip=no_external if alloc.get("network") else None, + ) diff --git a/cloudg/inventory/gcp_relations/ml.py b/cloudg/inventory/gcp_relations/ml.py new file mode 100644 index 0000000..348bb88 --- /dev/null +++ b/cloudg/inventory/gcp_relations/ml.py @@ -0,0 +1,363 @@ +"""ML and generative AI: Vertex AI Agent Engine (reasoning engines), Vector +Search, Agent Builder (Discovery Engine, Dialogflow CX), Model Armor, +pipelines and training jobs, Feature Store and Colab Enterprise runtimes.""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.gcp_relations._refs import bq_ref, gcs_ref, parent_of +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, dig, full_name, image_repository +from cloudg.schema.models import EdgeType + +_AI = "aiplatform" + + +def _data_ref(uri: Any) -> str | None: + """A ``gs://`` or ``bq://`` data location.""" + if not isinstance(uri, str) or not uri: + return None + if uri.startswith("bq://"): + return bq_ref(uri) + if uri.startswith("gs://"): + return gcs_ref(uri.split("*", 1)[0]) + return None + + +# --------------------------------------------------------------------------- +# Agents and retrieval +# --------------------------------------------------------------------------- + + +@_extractor("aiplatform.googleapis.com/ReasoningEngine") +def _reasoning_engine(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + spec = d.get("spec") or {} + pkg = spec.get("packageSpec") or {} + for key in ("pickleObjectGcsUri", "dependencyFilesGcsUri", "requirementsGcsUri"): + out.add(_data_ref(pkg.get(key)), EdgeType.REFERENCES, "READS_FROM", description=key) + deploy = spec.get("deploymentSpec") or {} + for env in _list(deploy.get("secretEnv")): + secret = dig(env, "secretRef", "secret") if isinstance(env, dict) else None + if secret and ctx.project_id and "/" not in secret: + secret = f"projects/{ctx.project_id}/secrets/{secret}" + out.add(full_name(secret, "secretmanager"), EdgeType.REFERENCES, "READS_FROM") + out.add( + full_name(dig(deploy, "pscInterfaceConfig", "networkAttachment"), "compute"), + EdgeType.ATTACHED_TO, + description="PSC interface network attachment", + ) + out.sa(spec.get("serviceAccount") or d.get("serviceAccount"), ctx, "agent identity") + out.metadata.update( + agent_framework=spec.get("agentFramework"), + python_version=pkg.get("pythonVersion"), + class_methods=len(_list(spec.get("classMethods"))), + ) + + +@_extractor("aiplatform.googleapis.com/Index") +def _vector_index(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + _data_ref(dig(d, "metadata", "contentsDeltaUri")), + EdgeType.REFERENCES, + "READS_FROM", + description="index contents", + ) + out.metadata.update( + update_method=d.get("indexUpdateMethod"), + dimensions=dig(d, "metadata", "config", "dimensions"), + deployed_count=len(_list(d.get("deployedIndexes"))), + ) + + +@_extractor("aiplatform.googleapis.com/IndexEndpoint") +def _vector_index_endpoint(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for dep in _list(d.get("deployedIndexes")): + if isinstance(dep, dict): + out.add( + full_name(dep.get("index"), _AI), + EdgeType.ROUTE, + "SERVES_TRAFFIC_TO", + deployed_index=dep.get("id"), + ) + out.add(full_name(d.get("network"), "compute"), EdgeType.ATTACHED_TO) + psc = d.get("privateServiceConnectConfig") or {} + for project in _list(psc.get("projectAllowlist")): + out.add( + f"//cloudresourcemanager.googleapis.com/projects/{project}", + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + description="PSC project allowlist", + ) + if d.get("publicEndpointEnabled"): + out.expose("Vector Search public endpoint", protocol="tcp", ports=["443"]) + out.alias( + f"https://{d['publicEndpointDomainName']}" + if d.get("publicEndpointDomainName") + else None + ) + out.metadata.update( + public_endpoint=bool(d.get("publicEndpointEnabled")), + private_service_connect=bool(psc.get("enablePrivateServiceConnect")), + ) + + +@_extractor("discoveryengine.googleapis.com/DataStore") +def _de_data_store(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "dataStores"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + industry_vertical=d.get("industryVertical"), + content_config=d.get("contentConfig"), + solution_types=list(d.get("solutionTypes") or []), + ) + + +@_extractor("discoveryengine.googleapis.com/Engine") +def _de_engine(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + collection = parent_of(ctx.name, "engines") + for ds in _list(d.get("dataStoreIds")): + if collection and isinstance(ds, str): + out.add( + f"{collection}/dataStores/{ds}", + EdgeType.REFERENCES, + "READS_FROM", + description="engine data store", + ) + out.add( + full_name(dig(d, "chatEngineMetadata", "dialogflowAgent"), "dialogflow"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + out.metadata.update( + solution_type=d.get("solutionType"), industry_vertical=d.get("industryVertical") + ) + + +@_extractor("discoveryengine.googleapis.com/Assistant", "discoveryengine.googleapis.com/Collection") +def _de_child(ctx: GCPContext, out: Extracted) -> None: + out.add(parent_of(ctx.name, "assistants"), EdgeType.CONTAINS, reverse=True) + + +@_extractor("dialogflow.googleapis.com/Agent") +def _dialogflow_agent(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("securitySettings"), "dialogflow"), EdgeType.REFERENCES, "DEPENDS_ON") + logging_cfg = dig(d, "advancedSettings", "loggingSettings", default={}) or {} + out.metadata.update( + default_language=d.get("defaultLanguageCode"), + stackdriver_logging=logging_cfg.get("enableStackdriverLogging"), + interaction_logging=logging_cfg.get("enableInteractionLogging"), + locked=d.get("locked"), + ) + + +@_extractor("dialogflow.googleapis.com/KnowledgeBase") +def _dialogflow_kb(ctx: GCPContext, out: Extracted) -> None: + out.metadata["language"] = ctx.data.get("languageCode") + + +@_extractor("modelarmor.googleapis.com/Template") +def _model_armor(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + cfg = d.get("filterConfig") or {} + out.metadata.update( + filters=sorted(cfg.keys()), + log_operations=dig(d, "templateMetadata", "logTemplateOperations"), + log_sanitize_operations=dig(d, "templateMetadata", "logSanitizeOperations"), + ) + + +# --------------------------------------------------------------------------- +# Pipelines, training and prediction jobs +# --------------------------------------------------------------------------- + + +def _job_spec(ctx: GCPContext, out: Extracted, spec: Any) -> list[str]: + """Identity, network, images and output of a CustomJob-shaped spec.""" + if not isinstance(spec, dict): + return [] + out.sa(spec.get("serviceAccount"), ctx, "training job identity") + out.add(full_name(spec.get("network"), "compute"), EdgeType.ATTACHED_TO) + images = [] + for pool in _list(spec.get("workerPoolSpecs")): + if not isinstance(pool, dict): + continue + image = dig(pool, "containerSpec", "imageUri") or dig( + pool, "pythonPackageSpec", "executorImageUri" + ) + if image: + images.append(image) + out.add(image_repository(image), EdgeType.USES_IMAGE, "RUNS_ON") + for uri in _list(dig(pool, "pythonPackageSpec", "packageUris")): + out.add(_data_ref(uri), EdgeType.REFERENCES, "READS_FROM") + out.add( + _data_ref(dig(spec, "baseOutputDirectory", "outputUriPrefix")), + EdgeType.REFERENCES, + "WRITES_TO", + description="job output", + ) + out.add(full_name(spec.get("tensorboard"), _AI), EdgeType.LOGS_TO, "LOGS_TO") + return images + + +@_extractor("aiplatform.googleapis.com/CustomJob") +def _custom_job(ctx: GCPContext, out: Extracted) -> None: + images = _job_spec(ctx, out, ctx.data.get("jobSpec")) + out.metadata.update(images=images, job_state=ctx.data.get("state")) + + +@_extractor("aiplatform.googleapis.com/HyperparameterTuningJob", "aiplatform.googleapis.com/NasJob") +def _tuning_job(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + spec = d.get("trialJobSpec") or dig( + d, "nasJobSpec", "multiTrialAlgorithmSpec", "searchTrialSpec", "searchTrialJobSpec" + ) + out.metadata.update(images=_job_spec(ctx, out, spec), job_state=d.get("state")) + + +@_extractor("aiplatform.googleapis.com/TrainingPipeline") +def _training_pipeline(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + loc = parent_of(ctx.name, "trainingPipelines") + dataset = dig(d, "inputDataConfig", "datasetId") + if loc and dataset: + out.add( + f"{loc}/datasets/{dataset}", + EdgeType.REFERENCES, + "READS_FROM", + description="training data", + ) + model = dig(d, "modelToUpload", "name") or d.get("modelId") + if loc and model: + out.add( + model if "/" in str(model) else f"{loc}/models/{model}", + EdgeType.REFERENCES, + "WRITES_TO", + description="trained model", + ) + out.metadata.update( + training_task=str(d.get("trainingTaskDefinition") or "").rsplit("/", 1)[-1] or None, + pipeline_state=d.get("state"), + ) + + +@_extractor("aiplatform.googleapis.com/PipelineJob") +def _pipeline_job(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.sa(d.get("serviceAccount") or "default", ctx, "pipeline identity") + out.add(full_name(d.get("network"), "compute"), EdgeType.ATTACHED_TO) + out.add( + _data_ref(dig(d, "runtimeConfig", "gcsOutputDirectory")), + EdgeType.REFERENCES, + "WRITES_TO", + description="pipeline root", + ) + out.add(_data_ref(d.get("templateUri")), EdgeType.REFERENCES, "READS_FROM") + out.metadata.update(template_uri=d.get("templateUri"), pipeline_state=d.get("state")) + + +@_extractor("aiplatform.googleapis.com/BatchPredictionJob") +def _batch_prediction(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("model"), _AI), EdgeType.REFERENCES, "DEPENDS_ON") + src = d.get("inputConfig") or {} + for uri in _list(dig(src, "gcsSource", "uris")) + [dig(src, "bigquerySource", "inputUri")]: + out.add(_data_ref(uri), EdgeType.REFERENCES, "READS_FROM") + dst = d.get("outputConfig") or {} + for uri in ( + dig(dst, "gcsDestination", "outputUriPrefix"), + dig(dst, "bigqueryDestination", "outputUri"), + ): + out.add(_data_ref(uri), EdgeType.REFERENCES, "WRITES_TO") + out.sa(d.get("serviceAccount"), ctx, "prediction job identity") + + +@_extractor("aiplatform.googleapis.com/TuningJob") +def _model_tuning(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + spec = d.get("supervisedTuningSpec") or {} + for key in ("trainingDatasetUri", "validationDatasetUri"): + out.add(_data_ref(spec.get(key)), EdgeType.REFERENCES, "READS_FROM") + out.add(full_name(dig(d, "tunedModel", "model"), _AI), EdgeType.REFERENCES, "WRITES_TO") + out.sa(d.get("serviceAccount"), ctx, "tuning identity") + out.metadata["base_model"] = d.get("baseModel") + + +# --------------------------------------------------------------------------- +# Feature Store, datasets, Colab Enterprise +# --------------------------------------------------------------------------- + + +@_extractor("aiplatform.googleapis.com/FeatureGroup") +def _feature_group(ctx: GCPContext, out: Extracted) -> None: + out.add( + _data_ref(dig(ctx.data, "bigQuery", "bigQuerySource", "inputUri")), + EdgeType.REFERENCES, + "READS_FROM", + description="feature source table", + ) + + +@_extractor("aiplatform.googleapis.com/FeatureOnlineStore") +def _feature_online_store(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + endpoint = d.get("dedicatedServingEndpoint") or {} + psc = endpoint.get("privateServiceConnectConfig") or {} + for project in _list(psc.get("projectAllowlist")): + out.add( + f"//cloudresourcemanager.googleapis.com/projects/{project}", + EdgeType.GRANTS_ACCESS, + "CROSS_ACCOUNT_TRUST", + reverse=True, + ) + domain = endpoint.get("publicEndpointDomainName") + if domain and not psc.get("enablePrivateServiceConnect"): + out.expose("Feature online store public endpoint", protocol="tcp", ports=["443"]) + out.metadata.update( + storage="bigtable" if d.get("bigtable") else "optimized" if d.get("optimized") else None, + public_endpoint=domain, + ) + + +@_extractor("aiplatform.googleapis.com/Featurestore") +def _featurestore(ctx: GCPContext, out: Extracted) -> None: + out.metadata["online_serving_nodes"] = dig(ctx.data, "onlineServingConfig", "fixedNodeCount") + + +@_extractor("aiplatform.googleapis.com/Dataset") +def _vertex_dataset(ctx: GCPContext, out: Extracted) -> None: + md = ctx.data.get("metadata") or {} + src = md.get("inputConfig") or {} + for uri in _list(dig(src, "gcsSource", "uri")) + [dig(src, "bigquerySource", "uri")]: + out.add(_data_ref(uri), EdgeType.REFERENCES, "READS_FROM") + out.metadata["schema"] = str(ctx.data.get("metadataSchemaUri") or "").rsplit("/", 1)[-1] or None + + +@_extractor("aiplatform.googleapis.com/NotebookRuntimeTemplate") +def _runtime_template(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + net = d.get("networkSpec") or {} + out.in_subnet(full_name(net.get("subnetwork") or net.get("network"), "compute")) + out.sa(d.get("serviceAccount"), ctx, "runtime identity") + out.metadata.update( + internet_access=net.get("enableInternetAccess"), + machine_type=dig(d, "machineSpec", "machineType"), + ) + + +@_extractor("aiplatform.googleapis.com/NotebookRuntime") +def _runtime(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(dig(d, "notebookRuntimeTemplateRef", "notebookRuntimeTemplate"), _AI), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + out.sa(d.get("serviceAccount"), ctx, "runtime identity") + out.metadata.update(runtime_state=d.get("runtimeState"), health=d.get("healthState")) diff --git a/cloudg/inventory/gcp_relations/names.py b/cloudg/inventory/gcp_relations/names.py index 4fc8348..2d217e4 100644 --- a/cloudg/inventory/gcp_relations/names.py +++ b/cloudg/inventory/gcp_relations/names.py @@ -332,6 +332,10 @@ def _short(value: Any) -> Any: "password", "clientKey", "clientSecret", + # Eventarc channel activation, Monitoring channel labels (Slack, PagerDuty) + "activationToken", + "auth_token", + "service_key", } REDACTED = "" diff --git a/cloudg/inventory/gcp_relations/netsec.py b/cloudg/inventory/gcp_relations/netsec.py new file mode 100644 index 0000000..68a9e7a --- /dev/null +++ b/cloudg/inventory/gcp_relations/netsec.py @@ -0,0 +1,344 @@ +"""Network security and network services: Cloud NGFW endpoints, security +profiles, TLS policies, Cloud IDS, Secure Web Proxy / service mesh gateways +and routes, and Media CDN.""" + +from __future__ import annotations + +from cloudg.inventory.gcp_relations._refs import ( + backend_service_ref, + gcs_ref, + host_target, + secret_version_ref, +) +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import _list, dig, full_name, network_ref +from cloudg.schema.models import EdgeType + +_NS = "networksecurity" +_NSV = "networkservices" + + +# --------------------------------------------------------------------------- +# Cloud NGFW and Cloud IDS +# --------------------------------------------------------------------------- + + +@_extractor("networksecurity.googleapis.com/FirewallEndpoint") +def _firewall_endpoint(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for net in _list(d.get("associatedNetworks")): + out.add( + full_name(net, "compute"), + EdgeType.PROTECTS, + "PROTECTED_BY_NACL", + description="Cloud NGFW endpoint inspects the network", + ) + for assoc in _list(d.get("associations")): + if isinstance(assoc, dict): + out.add( + full_name(assoc.get("network"), "compute"), EdgeType.PROTECTS, "PROTECTED_BY_NACL" + ) + out.metadata.update( + billing_project=d.get("billingProjectId"), endpoint_type=d.get("type"), state=d.get("state") + ) + + +@_extractor("networksecurity.googleapis.com/FirewallEndpointAssociation") +def _firewall_endpoint_association(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("network"), "compute"), + EdgeType.PROTECTS, + "PROTECTED_BY_NACL", + description="network traffic inspected by Cloud NGFW", + disabled=d.get("disabled") or None, + ) + out.add(full_name(d.get("firewallEndpoint"), _NS), EdgeType.REFERENCES, "DEPENDS_ON") + out.add(full_name(d.get("tlsInspectionPolicy"), _NS), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata["disabled"] = bool(d.get("disabled")) + + +@_extractor("networksecurity.googleapis.com/SecurityProfileGroup") +def _security_profile_group(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for key in ( + "threatPreventionProfile", + "urlFilteringProfile", + "customMirroringProfile", + "customInterceptProfile", + ): + out.add(full_name(d.get(key), _NS), EdgeType.REFERENCES, "DEPENDS_ON", profile=key) + + +@_extractor("networksecurity.googleapis.com/SecurityProfile") +def _security_profile(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + tp = d.get("threatPreventionProfile") or {} + out.metadata.update( + profile_type=d.get("type"), + severity_overrides=len(_list(tp.get("severityOverrides"))), + threat_overrides=len(_list(tp.get("threatOverrides"))), + ) + + +@_extractor("networksecurity.googleapis.com/TlsInspectionPolicy") +def _tls_inspection(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(full_name(d.get("caPool"), "privateca"), EdgeType.REFERENCES, "DEPENDS_ON") + out.add( + full_name(d.get("trustConfig"), "certificatemanager"), EdgeType.REFERENCES, "DEPENDS_ON" + ) + out.metadata.update( + min_tls_version=d.get("minTlsVersion"), + exclude_public_ca_set=d.get("excludePublicCaSet"), + ) + + +@_extractor( + "networksecurity.googleapis.com/ServerTlsPolicy", + "networksecurity.googleapis.com/ClientTlsPolicy", +) +def _tls_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + mtls = d.get("mtlsPolicy") or {} + out.add( + full_name(mtls.get("clientValidationTrustConfig"), "certificatemanager"), + EdgeType.REFERENCES, + "DEPENDS_ON", + ) + out.metadata.update( + allow_open=d.get("allowOpen"), + mtls_mode=mtls.get("clientValidationMode"), + sni=d.get("sni"), + ) + + +@_extractor( + "networksecurity.googleapis.com/AuthorizationPolicy", + "networksecurity.googleapis.com/AuthzPolicy", +) +def _authorization_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for res in _list(dig(d, "target", "resources")): + out.add( + full_name(res, "compute"), + EdgeType.PROTECTS, + "PROTECTED_BY_WAF", + description="L7 authorization policy", + ) + ext = dig(d, "customProvider", "authzExtension", "resources", default=[]) + for res in _list(ext): + out.add(full_name(res, _NSV), EdgeType.INVOKES, "INVOKES", description="authz extension") + out.metadata.update( + action=d.get("action"), + rule_count=len(_list(d.get("rules")) or _list(d.get("httpRules"))), + load_balancing_scheme=dig(d, "target", "loadBalancingScheme"), + ) + + +@_extractor("networksecurity.googleapis.com/AddressGroup") +def _address_group(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.metadata.update( + group_type=d.get("type"), + item_count=len(_list(d.get("items"))), + capacity=d.get("capacity"), + purpose=list(d.get("purpose") or []), + ) + + +@_extractor( + "compute.googleapis.com/FirewallPolicy", + "compute.googleapis.com/NetworkFirewallPolicy", + "compute.googleapis.com/RegionNetworkFirewallPolicy", +) +def _firewall_policy_profiles(ctx: GCPContext, out: Extracted) -> None: + """Rules that hand traffic to Cloud NGFW name a security profile group.""" + for rule in _list(ctx.data.get("rules")): + if isinstance(rule, dict) and rule.get("securityProfileGroup"): + out.add( + full_name(rule["securityProfileGroup"], _NS), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="rule applies a security profile group", + priority=rule.get("priority"), + ) + + +@_extractor("ids.googleapis.com/Endpoint") +def _ids_endpoint(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("network"), "compute"), + EdgeType.PROTECTS, + "PROTECTED_BY_NACL", + description="Cloud IDS inspects mirrored traffic", + severity=d.get("severity"), + ) + # Packet mirroring policies send traffic to this forwarding rule + out.alias(full_name(d.get("endpointForwardingRule"), "compute")) + out.alias(d.get("endpointIp")) + out.metadata.update( + severity=d.get("severity"), + traffic_logs=bool(d.get("trafficLogs")), + threat_exceptions=list(d.get("threatExceptions") or []), + endpoint_ip=d.get("endpointIp"), + security_service="cloud-ids", + ) + + +# --------------------------------------------------------------------------- +# Network Services: gateways, meshes and routes +# --------------------------------------------------------------------------- + + +@_extractor("networkservices.googleapis.com/Gateway") +def _ns_gateway(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(network_ref(d.get("network"), ctx.project_id), EdgeType.ATTACHED_TO) + out.in_subnet(full_name(d.get("subnetwork"), "compute")) + for cert in _list(d.get("certificateUrls")): + out.add(full_name(cert, "certificatemanager"), EdgeType.REFERENCES, "CERTIFICATE_SECURES") + out.add(full_name(d.get("serverTlsPolicy"), _NS), EdgeType.REFERENCES, "DEPENDS_ON") + out.add( + full_name(d.get("gatewaySecurityPolicy"), _NS), + EdgeType.PROTECTS, + "PROTECTED_BY_WAF", + reverse=True, + description="Secure Web Proxy policy", + ) + out.alias(*[a for a in _list(d.get("addresses")) if isinstance(a, str)]) + out.metadata.update( + gateway_type=d.get("type"), + ports=list(d.get("ports") or []), + scope=d.get("scope"), + addresses=list(d.get("addresses") or []), + ) + + +@_extractor("networkservices.googleapis.com/Mesh") +def _ns_mesh(ctx: GCPContext, out: Extracted) -> None: + out.metadata["interception_port"] = ctx.data.get("interceptionPort") + + +@_extractor( + "networkservices.googleapis.com/HttpRoute", + "networkservices.googleapis.com/GrpcRoute", + "networkservices.googleapis.com/TcpRoute", + "networkservices.googleapis.com/TlsRoute", +) +def _ns_route(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for gw in _list(d.get("gateways")): + out.add(full_name(gw, _NSV), EdgeType.ATTACHED_TO, description="route on gateway") + for mesh in _list(d.get("meshes")): + out.add(full_name(mesh, _NSV), EdgeType.ATTACHED_TO, description="route on mesh") + for rule in _list(d.get("rules")): + if not isinstance(rule, dict): + continue + for dest in _list(dig(rule, "action", "destinations")): + if isinstance(dest, dict): + out.add( + backend_service_ref(dest.get("serviceName")), + EdgeType.ROUTE, + "SERVES_TRAFFIC_TO", + weight=dest.get("weight"), + ) + out.metadata["hostnames"] = list(d.get("hostnames") or []) + + +@_extractor("networkservices.googleapis.com/ServiceBinding") +def _ns_service_binding(ctx: GCPContext, out: Extracted) -> None: + out.add( + full_name(ctx.data.get("service"), "servicedirectory"), EdgeType.REFERENCES, "DEPENDS_ON" + ) + + +@_extractor("networkservices.googleapis.com/EndpointPolicy") +def _ns_endpoint_policy(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for key in ("authorizationPolicy", "serverTlsPolicy", "clientTlsPolicy"): + out.add(full_name(d.get(key), _NS), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata["policy_type"] = d.get("type") + + +# --------------------------------------------------------------------------- +# Media CDN +# --------------------------------------------------------------------------- + + +@_extractor("networkservices.googleapis.com/EdgeCacheService") +def _edge_cache_service(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + routing = d.get("routing") or {} + for matcher in _list(routing.get("pathMatchers")): + for rule in _list(dig(matcher, "routeRules")) if isinstance(matcher, dict) else []: + if isinstance(rule, dict): + out.add(full_name(rule.get("origin"), _NSV), EdgeType.ROUTE, "SERVES_TRAFFIC_TO") + hosts = [ + h + for hr in _list(routing.get("hostRules")) + if isinstance(hr, dict) + for h in hr.get("hosts", []) + ] + for host in hosts: + out.alias(host_target(host)) + out.add( + full_name(d.get("edgeSecurityPolicy"), "compute"), + EdgeType.PROTECTS, + "PROTECTED_BY_WAF", + reverse=True, + ) + for cert in _list(d.get("edgeSslCertificates")): + out.add(full_name(cert, "certificatemanager"), EdgeType.REFERENCES, "CERTIFICATE_SECURES") + out.alias(*[ip for ip in _list(d.get("ipv4Addresses")) + _list(d.get("ipv6Addresses")) if ip]) + out.expose("Media CDN edge cache service", protocol="tcp", ports=["80", "443"]) + out.metadata.update( + hosts=hosts, + logging=bool(dig(d, "logConfig", "enable")), + require_tls=d.get("requireTls"), + disable_quic=d.get("disableQuic"), + ) + + +@_extractor("networkservices.googleapis.com/EdgeCacheOrigin") +def _edge_cache_origin(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + addr = d.get("originAddress") + target = ( + gcs_ref(addr) if isinstance(addr, str) and addr.startswith("gs://") else host_target(addr) + ) + out.add(target, EdgeType.ROUTE, "SERVES_TRAFFIC_TO", description="origin address") + out.add(full_name(d.get("failoverOrigin"), _NSV), EdgeType.REFERENCES, "DEPENDS_ON") + out.add( + secret_version_ref(dig(d, "awsV4Authentication", "secretAccessKeyVersion")), + EdgeType.REFERENCES, + "READS_FROM", + description="origin credentials", + ) + out.metadata.update(origin_address=addr, protocol=d.get("protocol")) + + +@_extractor("networkservices.googleapis.com/EdgeCacheKeyset") +def _edge_cache_keyset(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for key in _list(d.get("validationSharedKeys")): + if isinstance(key, dict): + out.add(secret_version_ref(key.get("secretVersion")), EdgeType.REFERENCES, "READS_FROM") + out.metadata["public_key_count"] = len(_list(d.get("publicKeys"))) + + +@_extractor("networkservices.googleapis.com/ServiceLbPolicy") +def _service_lb_policy(ctx: GCPContext, out: Extracted) -> None: + out.metadata["load_balancing_algorithm"] = ctx.data.get("loadBalancingAlgorithm") + + +@_extractor("compute.googleapis.com/NetworkEdgeSecurityService") +def _edge_security_service(ctx: GCPContext, out: Extracted) -> None: + """Regional Cloud Armor network edge security for the project's external IPs.""" + out.add( + full_name(ctx.data.get("securityPolicy"), "compute"), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="advanced network DDoS protection policy", + ) diff --git a/cloudg/inventory/gcp_relations/ops.py b/cloudg/inventory/gcp_relations/ops.py new file mode 100644 index 0000000..47c7e25 --- /dev/null +++ b/cloudg/inventory/gcp_relations/ops.py @@ -0,0 +1,412 @@ +"""Operations: Cloud Monitoring alert policies, notification channels and +uptime checks, OS Config (VM Manager) patch deployments, OS policies and +vulnerability reports, and Backup and DR Service vaults, plans and +associations. + +The same extractors serve Cloud Asset Inventory records and resources read +from the service APIs by :mod:`cloudg.inventory.gcp_sources`. +""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.gcp_relations._refs import crm_ref, host_target, parent_of +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.names import ( + _list, + _num, + asset_type_service, + dig, + full_name, + url_alias, +) +from cloudg.schema.models import EdgeType + +# --------------------------------------------------------------------------- +# Cloud Monitoring +# --------------------------------------------------------------------------- + +_LABEL_RE = re.compile(r"resource\.labels?\.\"?([a-z_]+)\"?\s*(?:==|=|:)\s*\"([^\"]+)\"") +_TYPE_RE = re.compile(r"resource\.type\s*(?:==|=)\s*\"([a-z0-9_.]+)\"") + + +def _monitored(rtype: str, labels: dict[str, str], project: str | None) -> str | None: + """Full name of the one resource a monitored-resource type + labels pin down.""" + p = labels.get("project_id") or project + loc = labels.get("location") or labels.get("region") or labels.get("zone") + if rtype == "gce_instance" and labels.get("instance_id") and p: + zone = labels.get("zone") + if zone: + return f"//compute.googleapis.com/projects/{p}/zones/{zone}/instances/{labels['instance_id']}" + return f"gcp-instance-id:{p}/{labels['instance_id']}" + simple = { + "gcs_bucket": ("bucket_name", lambda v: f"//storage.googleapis.com/{v}"), + "pubsub_topic": ("topic_id", lambda v: f"//pubsub.googleapis.com/projects/{p}/topics/{v}"), + "pubsub_subscription": ( + "subscription_id", + lambda v: f"//pubsub.googleapis.com/projects/{p}/subscriptions/{v}", + ), + "bigquery_dataset": ( + "dataset_id", + lambda v: f"//bigquery.googleapis.com/projects/{p}/datasets/{v}", + ), + "spanner_instance": ( + "instance_id", + lambda v: f"//spanner.googleapis.com/projects/{p}/instances/{v}", + ), + "https_lb_rule": ( + "forwarding_rule_name", + lambda v: f"//compute.googleapis.com/projects/{p}/global/forwardingRules/{v}", + ), + } + if rtype in simple: + key, build = simple[rtype] + return build(labels[key]) if labels.get(key) and p else None + if rtype == "cloudsql_database" and ":" in labels.get("database_id", ""): + proj, inst = labels["database_id"].split(":", 1) + return f"//cloudsql.googleapis.com/projects/{proj}/instances/{inst}" + if rtype in ("cloud_run_revision", "cloud_run_job") and p and loc: + if labels.get("service_name"): + return f"//run.googleapis.com/projects/{p}/locations/{loc}/services/{labels['service_name']}" + if labels.get("job_name"): + return f"//run.googleapis.com/projects/{p}/locations/{loc}/jobs/{labels['job_name']}" + if ( + rtype in ("cloud_function", "cloud_run_function") + and p + and loc + and labels.get("function_name") + ): + return f"//cloudfunctions.googleapis.com/projects/{p}/locations/{loc}/functions/{labels['function_name']}" + if rtype.startswith("k8s_") and p and loc and labels.get("cluster_name"): + return f"//container.googleapis.com/projects/{p}/locations/{loc}/clusters/{labels['cluster_name']}" + if rtype == "redis_instance" and labels.get("instance_id"): + return full_name(labels["instance_id"], "redis") + return None + + +def filter_targets(text: Any, project: str | None) -> list[str]: + """Resources a Monitoring filter (``resource.type = ... AND resource.labels.x = ...``) names.""" + if not isinstance(text, str) or not text: + return [] + m = _TYPE_RE.search(text) + labels = {k: v for k, v in _LABEL_RE.findall(text)} + target = _monitored(m.group(1), labels, project) if m else None + return [target] if target else [] + + +def _condition_filters(cond: dict[str, Any]) -> list[str]: + out = [] + for key in ("conditionThreshold", "conditionAbsent", "conditionMatchedLog"): + flt = dig(cond, key, "filter") + if isinstance(flt, str): + out.append(flt) + for key, field in ( + ("conditionMonitoringQueryLanguage", "query"), + ("conditionPrometheusQueryLanguage", "query"), + ("conditionSql", "query"), + ): + q = dig(cond, key, field) + if isinstance(q, str): + out.append(q) + return out + + +@_extractor("monitoring.googleapis.com/AlertPolicy") +def _alert_policy(ctx: GCPContext, out: Extracted) -> None: + """MONITORS / MONITORED_BY the resources its conditions pin; INVOKES its channels.""" + d = ctx.data + pid = ctx.project_id + monitored: list[str] = [] + conditions = [c for c in _list(d.get("conditions")) if isinstance(c, dict)] + for cond in conditions: + for flt in _condition_filters(cond): + monitored += filter_targets(flt, pid) + for target in dict.fromkeys(monitored): + out.add(target, EdgeType.MONITORS, "MONITORED_BY", description="alert condition") + if conditions and not monitored and pid: + out.add( + crm_ref(f"projects/{pid}"), + EdgeType.MONITORS, + "MONITORED_BY", + description="project-wide alert condition", + ) + for channel in _list(d.get("notificationChannels")): + out.add( + full_name(channel, "monitoring"), + EdgeType.INVOKES, + "INVOKES", + description="alert notification", + ) + out.metadata.update( + enabled=d.get("enabled", True), + combiner=d.get("combiner"), + condition_count=len(conditions), + severity=d.get("severity"), + notification_channel_count=len(_list(d.get("notificationChannels"))), + ) + + +@_extractor("monitoring.googleapis.com/NotificationChannel") +def _notification_channel(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + labels = d.get("labels") or {} + pid = ctx.project_id + topic = labels.get("topic") + if topic: + out.add( + full_name(topic, "pubsub"), EdgeType.INVOKES, "INVOKES", description="Pub/Sub channel" + ) + url = labels.get("url") + if isinstance(url, str) and url.startswith("https://") and "REDACTED" not in url: + out.add(url_alias(url), EdgeType.INVOKES, "INVOKES", description="webhook channel") + out.metadata.update( + channel_type=d.get("type"), + enabled=d.get("enabled", True), + verification=d.get("verificationStatus"), + label_keys=sorted(labels.keys()), + project_id=pid, + ) + + +@_extractor("monitoring.googleapis.com/UptimeCheckConfig") +def _uptime_check(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + res = d.get("monitoredResource") or {} + labels = res.get("labels") or {} + targets = [] + if res.get("type") == "uptime_url": + targets.append(host_target(labels.get("host"))) + elif res.get("type"): + targets.append(_monitored(res["type"], labels, ctx.project_id)) + targets.append( + full_name(dig(d, "syntheticMonitor", "cloudFunctionV2", "name"), "cloudfunctions") + ) + for target in targets: + out.add(target, EdgeType.MONITORS, "MONITORED_BY", description="uptime check") + out.metadata.update( + checker_type=d.get("checkerType"), + period=d.get("period"), + protocol="http" if d.get("httpCheck") else "tcp" if d.get("tcpCheck") else None, + regions=list(d.get("selectedRegions") or []), + ) + + +# --------------------------------------------------------------------------- +# OS Config / VM Manager +# --------------------------------------------------------------------------- + + +def _instance_ref(value: Any, project: str | None) -> str | None: + """``zones/z/instances/i``, ``projects/p/zones/z/instances/i`` or a URL.""" + if not isinstance(value, str) or not value: + return None + if value.startswith("zones/") and project: + value = f"projects/{project}/{value}" + return full_name(value, "compute") + + +def instance_filter(f: Any, zone: str | None = None) -> dict[str, Any]: + """Normalised OS Config instance filter (both patch and OS policy shapes).""" + if not isinstance(f, dict): + return {} + labels = [g.get("labels") or {} for g in _list(f.get("groupLabels")) if isinstance(g, dict)] + labels += [ + g.get("labels") or {} for g in _list(f.get("inclusionLabels")) if isinstance(g, dict) + ] + zones = list(f.get("zones") or []) + if zone and not zones: + zones = [zone] + return { + "all": bool(f.get("all")), + "labels": labels, + "exclusion_labels": [ + g.get("labels") or {} for g in _list(f.get("exclusionLabels")) if isinstance(g, dict) + ], + "zones": zones, + "name_prefixes": list(f.get("instanceNamePrefixes") or []), + "os_short_names": [ + i.get("osShortName") for i in _list(f.get("inventories")) if isinstance(i, dict) + ], + } + + +@_extractor("osconfig.googleapis.com/PatchDeployment") +def _patch_deployment(ctx: GCPContext, out: Extracted) -> None: + """Explicit instances are linked here; label / zone filters by the deep collector.""" + d = ctx.data + flt = d.get("instanceFilter") or {} + for inst in _list(flt.get("instances")): + out.add( + _instance_ref(inst, ctx.project_id), + EdgeType.MANAGES, + "SCHEDULED_BY", + description="patch deployment target", + ) + if flt.get("all") and ctx.project_id: + out.add( + crm_ref(f"projects/{ctx.project_id}"), + EdgeType.MANAGES, + "SCHEDULED_BY", + description="patches every VM in the project", + ) + sched = d.get("recurringSchedule") or {} + out.metadata.update( + instance_filter=instance_filter(flt), + frequency=sched.get("frequency"), + time_zone=dig(sched, "timeZone", "id"), + one_time=bool(d.get("oneTimeSchedule")), + reboot=dig(d, "patchConfig", "rebootConfig"), + last_execute_time=d.get("lastExecuteTime"), + ) + + +@_extractor("osconfig.googleapis.com/OSPolicyAssignment") +def _os_policy_assignment(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + flt = d.get("instanceFilter") or {} + zone = ctx.location if ctx.location and ctx.location.count("-") == 2 else None + if flt.get("all") and not flt.get("inclusionLabels") and ctx.project_id and not zone: + out.add(crm_ref(f"projects/{ctx.project_id}"), EdgeType.MANAGES, "SCHEDULED_BY") + out.metadata.update( + instance_filter=instance_filter(flt, zone), + os_policies=[ + {"id": p.get("id"), "mode": p.get("mode")} + for p in _list(d.get("osPolicies")) + if isinstance(p, dict) + ], + rollout_state=d.get("rolloutState"), + baseline=d.get("baseline"), + ) + + +_VULN_REPORT_RE = re.compile( + r"projects/([^/]+)/locations/([^/]+)/instances/([^/]+)/vulnerabilityReport" +) + + +@_extractor("osconfig.googleapis.com/VulnerabilityReport") +def _vm_vulnerability_report(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + m = _VULN_REPORT_RE.search(ctx.name) + counts: dict[str, int] = {} + cves = [] + for v in _list(d.get("vulnerabilities")): + details = v.get("details") if isinstance(v, dict) else None + if not isinstance(details, dict): + continue + sev = str(details.get("severity") or "UNKNOWN").upper() + counts[sev] = counts.get(sev, 0) + 1 + if details.get("cve"): + cves.append(details["cve"]) + if m: + project, zone, inst = m.groups() + out.add( + f"//compute.googleapis.com/projects/{project}/zones/{zone}/instances/{inst}", + EdgeType.MONITORS, + "MONITORED_BY", + description="VM Manager vulnerability report", + severity_counts=counts, + ) + out.metadata.update( + severity_counts=counts, cves=cves[:50], vulnerability_count=sum(counts.values()) + ) + + +# --------------------------------------------------------------------------- +# Backup and DR Service +# --------------------------------------------------------------------------- + + +@_extractor("backupdr.googleapis.com/BackupVault") +def _backup_vault(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.sa(d.get("serviceAccount"), ctx, "backup vault service agent") + out.metadata.update( + min_retention=d.get("backupMinimumEnforcedRetentionDuration"), + access_restriction=d.get("accessRestriction"), + backup_count=_num(d.get("backupCount")), + locked_until=d.get("effectiveTime"), + vault_state=d.get("state"), + deletable=d.get("deletable"), + ) + + +@_extractor("backupdr.googleapis.com/BackupPlan") +def _backup_plan(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + full_name(d.get("backupVault"), "backupdr"), + EdgeType.REFERENCES, + "BACKUP_TO", + description="plan target vault", + ) + rules = [ + { + "id": r.get("ruleId"), + "retention_days": _num(r.get("backupRetentionDays")), + "recurrence": dig(r, "standardSchedule", "recurrenceType"), + } + for r in _list(d.get("backupRules")) + if isinstance(r, dict) + ] + out.metadata.update( + resource_type=d.get("resourceType"), + rules=rules, + vault_service_account=d.get("backupVaultServiceAccount"), + plan_state=d.get("state"), + ) + + +@_extractor("backupdr.googleapis.com/BackupPlanAssociation") +def _backup_plan_association(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + rtype = d.get("resourceType") + resource = full_name(d.get("resource"), asset_type_service(rtype)) + out.add( + resource, + EdgeType.MANAGES, + "BACKUP_TO", + description="resource protected by the backup plan", + ) + out.add(full_name(d.get("backupPlan"), "backupdr"), EdgeType.REFERENCES, "DEPENDS_ON") + out.add(full_name(d.get("dataSource"), "backupdr"), EdgeType.REFERENCES, "BACKUP_TO") + out.metadata.update(resource=resource, resource_type=rtype, association_state=d.get("state")) + + +@_extractor("backupdr.googleapis.com/DataSource") +def _backup_data_source(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "dataSources"), EdgeType.CONTAINS, reverse=True) + res = d.get("dataSourceGcpResource") or {} + out.add( + full_name(res.get("gcpResourcename"), asset_type_service(res.get("type"))), + EdgeType.REFERENCES, + "BACKUP_TO", + reverse=True, + description="resource backed up into the vault", + ) + out.metadata.update(backup_count=_num(d.get("backupCount")), config_state=d.get("configState")) + + +@_extractor("backupdr.googleapis.com/Backup") +def _backupdr_backup(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(parent_of(ctx.name, "backups"), EdgeType.CONTAINS, reverse=True) + out.metadata.update( + backup_type=d.get("backupType"), + expire_time=d.get("expireTime"), + enforced_retention_end=d.get("enforcedRetentionEndTime"), + ) + + +@_extractor("backupdr.googleapis.com/ManagementServer") +def _backup_management_server(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + for net in _list(d.get("networks")): + if isinstance(net, dict): + out.add(full_name(net.get("network"), "compute"), EdgeType.ATTACHED_TO) + uri = dig(d, "managementUri", "webUi") + out.alias(url_alias(uri)) + out.metadata.update(server_type=d.get("type"), server_state=d.get("state")) diff --git a/cloudg/inventory/gcp_relations/security_ext.py b/cloudg/inventory/gcp_relations/security_ext.py new file mode 100644 index 0000000..9dd0346 --- /dev/null +++ b/cloudg/inventory/gcp_relations/security_ext.py @@ -0,0 +1,356 @@ +"""Security services: Security Command Center (activation, services, sources, +notification and BigQuery exports, mute rules, per-service settings), +Sensitive Data Protection, Data Access audit logs, the IAM recommender and +Artifact Analysis vulnerability summaries. + +Security services carry a ``gcp-security::`` alias, the +GCP counterpart of the AWS ``aws-security:`` alias, so placeholders for +services that are not enabled and the configs that depend on them link +up. +""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.gcp_relations._refs import bq_ref, crm_ref, gcs_ref +from cloudg.inventory.gcp_relations.context import Extracted, GCPContext, _extractor +from cloudg.inventory.gcp_relations.identity_ext import principal_target +from cloudg.inventory.gcp_relations.names import ( + _list, + dig, + full_name, + image_repository, + relative_name, +) +from cloudg.schema.models import EdgeType + +SECURITY_ALIAS = "gcp-security:" +_PARENT_RE = re.compile(r"^((?:organizations|folders|projects)/[^/]+)") +# Settings resource type -> SCC service id +_SETTINGS = { + "securitycenter.googleapis.com/EventThreatDetectionSettings": "event-threat-detection", + "securitycenter.googleapis.com/ContainerThreatDetectionSettings": "container-threat-detection", + "securitycenter.googleapis.com/SecurityHealthAnalyticsSettings": "security-health-analytics", + "securitycenter.googleapis.com/VirtualMachineThreatDetectionSettings": "vm-threat-detection", + "securitycenter.googleapis.com/WebSecurityScannerSettings": "web-security-scanner", +} + + +def security_alias(service: str, parent: str | None) -> str | None: + """``gcp-security::``.""" + return f"{SECURITY_ALIAS}{service}:{parent}" if parent else None + + +def parent_resource(name: str) -> str | None: + """The project / folder / organization a resource name lives under.""" + m = _PARENT_RE.match(relative_name(name)) + return m.group(1) if m else None + + +# --------------------------------------------------------------------------- +# Security Command Center +# --------------------------------------------------------------------------- + + +@_extractor("securitycenter.googleapis.com/SecurityCenter") +def _scc(ctx: GCPContext, out: Extracted) -> None: + """SCC activation at a parent (built from the SCC API by the deep collector).""" + d = ctx.data + parent = d.get("parent") or parent_resource(ctx.name) + out.alias(security_alias("scc", parent)) + if d.get("enabled", True): + out.add( + crm_ref(parent), + EdgeType.MONITORS, + "MONITORED_BY", + description=f"Security Command Center {d.get('tier') or ''}".strip(), + ) + out.metadata.update( + security_service="scc", + enabled=bool(d.get("enabled", True)), + tier=d.get("tier"), + parent=parent, + ) + + +@_extractor("securitycentermanagement.googleapis.com/SecurityCenterService") +def _scc_service(ctx: GCPContext, out: Extracted) -> None: + """One SCC service (ETD, CTD, SHA, VMTD, WSS, ...) at a parent.""" + d = ctx.data + parent = parent_resource(ctx.name) + service = ctx.name.rsplit("/", 1)[-1] + effective = d.get("effectiveEnablementState") or d.get("intendedEnablementState") + enabled = effective == "ENABLED" + out.alias(security_alias(service, parent)) + if enabled: + out.add(crm_ref(parent), EdgeType.MONITORS, "MONITORED_BY", description=service) + out.add(security_alias("scc", parent), EdgeType.REFERENCES, "DEPENDS_ON") + modules = d.get("modules") or {} + out.metadata.update( + security_service=service, + enabled=enabled, + intended_state=d.get("intendedEnablementState"), + effective_state=effective, + module_count=len(modules), + disabled_modules=sorted( + k + for k, v in modules.items() + if isinstance(v, dict) and v.get("effectiveEnablementState") == "DISABLED" + ), + ) + + +@_extractor(*_SETTINGS) +def _scc_settings(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = parent_resource(ctx.name) + service = _SETTINGS.get(ctx.asset_type, "scc") + state = d.get("serviceEnablementState") + out.alias(security_alias(service, parent)) + if state in ("ENABLED", "INHERITED", None): + out.add(crm_ref(parent), EdgeType.MONITORS, "MONITORED_BY", description=service) + out.metadata.update( + security_service=service, enablement_state=state, enabled=state != "DISABLED" + ) + + +@_extractor("securitycenter.googleapis.com/Source") +def _scc_source(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = parent_resource(ctx.name) + out.add( + security_alias("scc", parent), + EdgeType.INVOKES, + "STREAMS_TO", + description="finding source", + ) + out.metadata.update(security_service="scc-source", canonical_name=d.get("canonicalName")) + + +@_extractor("securitycenter.googleapis.com/NotificationConfig") +def _scc_notification(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + parent = parent_resource(ctx.name) + out.add( + full_name(d.get("pubsubTopic"), "pubsub"), + EdgeType.INVOKES, + "STREAMS_TO", + description="findings exported to Pub/Sub", + ) + out.sa(d.get("serviceAccount"), ctx, "notification publisher") + out.add(security_alias("scc", parent), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata["filter"] = dig(d, "streamingConfig", "filter") + + +@_extractor("securitycenter.googleapis.com/BigQueryExport") +def _scc_bq_export(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add( + bq_ref(d.get("dataset")), + EdgeType.INVOKES, + "STREAMS_TO", + description="findings exported to BigQuery", + ) + out.sa(d.get("principal"), ctx, "export writer") + out.add(security_alias("scc", parent_resource(ctx.name)), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata["filter"] = d.get("filter") + + +@_extractor( + "securitycenter.googleapis.com/MuteConfig", "securitycenter.googleapis.com/ResourceValueConfig" +) +def _scc_config(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + out.add(security_alias("scc", parent_resource(ctx.name)), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata.update( + filter=d.get("filter"), config_type=d.get("type"), resource_value=d.get("resourceValue") + ) + + +# --------------------------------------------------------------------------- +# Sensitive Data Protection (Cloud DLP) +# --------------------------------------------------------------------------- + + +def _discovery_scope(ctx: GCPContext, d: dict[str, Any]) -> str | None: + loc = dig(d, "orgConfig", "location", default={}) or {} + if loc.get("folderId"): + return f"folders/{loc['folderId']}" + if loc.get("organizationId"): + return f"organizations/{loc['organizationId']}" + return parent_resource(ctx.name) + + +@_extractor("dlp.googleapis.com/DiscoveryConfig") +def _dlp_discovery(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + scope = _discovery_scope(ctx, d) + active = d.get("status") in (None, "RUNNING") + if active: + out.add( + crm_ref(scope), + EdgeType.MONITORS, + "MONITORED_BY", + description="sensitive data discovery", + ) + out.alias(security_alias("dlp-discovery", parent_resource(ctx.name))) + for action in _list(d.get("actions")): + if not isinstance(action, dict): + continue + table = dig(action, "exportData", "profileTable", default={}) or {} + if table.get("projectId") and table.get("datasetId"): + out.add( + bq_ref(f"{table['projectId']}.{table['datasetId']}"), + EdgeType.REFERENCES, + "WRITES_TO", + description="data profiles export", + ) + out.add( + full_name(dig(action, "pubSubNotification", "topic"), "pubsub"), + EdgeType.INVOKES, + "INVOKES", + ) + for tmpl in _list(d.get("inspectTemplates")): + out.add(full_name(tmpl, "dlp"), EdgeType.REFERENCES, "DEPENDS_ON") + targets = sorted({k for t in _list(d.get("targets")) if isinstance(t, dict) for k in t}) + out.metadata.update( + security_service="dlp-discovery", + enabled=active, + discovery_status=d.get("status"), + targets=targets, + scope=scope, + ) + + +@_extractor("dlp.googleapis.com/JobTrigger") +def _dlp_job_trigger(ctx: GCPContext, out: Extracted) -> None: + d = ctx.data + job = d.get("inspectJob") or {} + storage = job.get("storageConfig") or {} + url = dig(storage, "cloudStorageOptions", "fileSet", "url") + if isinstance(url, str): + out.add( + gcs_ref(url.removeprefix("gs://").split("/", 1)[0]), EdgeType.MONITORS, "MONITORED_BY" + ) + ref = dig(storage, "bigQueryOptions", "tableReference", default={}) or {} + if ref.get("projectId") and ref.get("datasetId"): + table = ref.get("tableId") + out.add( + bq_ref(f"{ref['projectId']}.{ref['datasetId']}" + (f".{table}" if table else "")), + EdgeType.MONITORS, + "MONITORED_BY", + ) + for action in _list(job.get("actions")): + if not isinstance(action, dict): + continue + out_table = dig(action, "saveFindings", "outputConfig", "table", default={}) or {} + if out_table.get("projectId") and out_table.get("datasetId"): + out.add( + bq_ref(f"{out_table['projectId']}.{out_table['datasetId']}"), + EdgeType.REFERENCES, + "WRITES_TO", + ) + out.add(full_name(dig(action, "pubSub", "topic"), "pubsub"), EdgeType.INVOKES, "INVOKES") + out.add(full_name(job.get("inspectTemplateName"), "dlp"), EdgeType.REFERENCES, "DEPENDS_ON") + out.metadata.update(trigger_status=d.get("status"), security_service="dlp-inspection") + + +@_extractor( + "dlp.googleapis.com/InspectTemplate", + "dlp.googleapis.com/DeidentifyTemplate", + "dlp.googleapis.com/StoredInfoType", +) +def _dlp_template(ctx: GCPContext, out: Extracted) -> None: + info_types = dig(ctx.data, "inspectConfig", "infoTypes", default=[]) or [] + out.metadata["info_type_count"] = len(_list(info_types)) or None + + +# --------------------------------------------------------------------------- +# Data Access audit logs (from IAM policies), IAM recommender, Artifact Analysis +# --------------------------------------------------------------------------- + + +@_extractor("logging.googleapis.com/AuditConfig") +def _audit_config(ctx: GCPContext, out: Extracted) -> None: + """Data Access audit logging configured on a project, folder or organization.""" + d = ctx.data + resource = d.get("resource") + out.add(resource, EdgeType.MONITORS, "MONITORED_BY", description="Data Access audit logs") + rel_res = relative_name(resource or "") + if rel_res.startswith("projects/") and d.get("enabled"): + out.add( + f"//logging.googleapis.com/{rel_res}/locations/global/buckets/_Default", + EdgeType.LOGS_TO, + "LOGS_TO", + description="Data Access logs land in the _Default bucket", + ) + out.alias(security_alias("audit-logs", rel_res or None)) + out.metadata.update( + security_service="audit-logs", + enabled=bool(d.get("enabled")), + services=d.get("services") or [], + all_services=d.get("allServices"), + exempted_members=d.get("exemptedMembers") or [], + ) + + +@_extractor("recommender.googleapis.com/IamPolicyRecommender") +def _iam_recommender(ctx: GCPContext, out: Extracted) -> None: + """IAM policy recommendations: over-granted roles, per principal (FINDING_AFFECTS).""" + d = ctx.data + out.add( + crm_ref(d.get("parent")), + EdgeType.MONITORS, + "MONITORED_BY", + description="IAM role recommendations", + ) + for rec in _list(d.get("recommendations")): + if not isinstance(rec, dict): + continue + out.add( + principal_target(rec.get("member")), + EdgeType.MONITORS, + "FINDING_AFFECTS", + description=rec.get("description"), + role=rec.get("role"), + action=rec.get("action"), + priority=rec.get("priority"), + resource=rec.get("resource"), + ) + out.metadata.update( + security_service="iam-recommender", + recommendation_count=len(_list(d.get("recommendations"))), + ) + + +@_extractor("containeranalysis.googleapis.com/VulnerabilitySummary") +def _vulnerability_summary(ctx: GCPContext, out: Extracted) -> None: + """Artifact Analysis findings per image (MONITORS / MONITORED_BY with counts).""" + d = ctx.data + parent = d.get("parent") + out.alias(security_alias("artifact-analysis", parent)) + for uri, summary in (d.get("images") or {}).items(): + if not isinstance(summary, dict): + continue + target = summary.get("target") or uri + out.add( + target, + EdgeType.MONITORS, + "MONITORED_BY", + description="Artifact Analysis vulnerability scan", + severity_counts=summary.get("severity_counts"), + fixable=summary.get("fixable"), + cves=(summary.get("cves") or [])[:20], + ) + repo = image_repository(str(uri).removeprefix("https://")) + if repo and repo != target: + out.add(repo, EdgeType.MONITORS, "MONITORED_BY", description="repository scanned") + out.metadata.update( + security_service="artifact-analysis", + enabled=d.get("enabled", True), + image_count=len(d.get("images") or {}), + occurrence_count=d.get("occurrence_count"), + truncated=d.get("truncated") or None, + ) diff --git a/cloudg/inventory/gcp_sources/__init__.py b/cloudg/inventory/gcp_sources/__init__.py new file mode 100644 index 0000000..e8d2d4a --- /dev/null +++ b/cloudg/inventory/gcp_sources/__init__.py @@ -0,0 +1,63 @@ +"""API-backed GCP sources for what Cloud Asset Inventory does not return. + +The deep GCP collector (:class:`~cloudg.inventory.gcp_deep.GCPDeepInventoryCollector`) +runs these from its ``_enrich()`` hook, each one best-effort with its own +coverage record (``gcp_``) and filtered by ``inventory.services`` / +``inventory.exclude_services`` like the AWS tasks: + +========================= =========== ================================================== +Source Family What it maps +========================= =========== ================================================== +cloud_identity identity Google groups and memberships (Cloud Identity) +iam_deny identity IAM deny policies (POLICY_DENIES_ACTION) +principal_access_boundary identity PAB policies and bindings (PERMISSION_BOUNDARY_LIMITS) +pam identity Privileged Access Manager entitlements +iam_roles identity Bound predefined roles and their permissions +scc security Security Command Center tier, services, sources, exports +artifact_analysis security Vulnerability occurrences per container image +iam_recommender security IAM role recommendations (FINDING_AFFECTS) +dlp_discovery security "Not enabled" placeholders for SDP discovery +audit_configs logging Data Access audit log configs (IAM policy auditConfigs) +monitoring logging Alert policies, channels, uptime checks, metrics scopes +dns_records dns Cloud DNS record sets (DNS_RESOLVED) +os_config ops Patch deployments, OS policies, VM vulnerability reports +cloud_scheduler integration Cloud Scheduler jobs (SCHEDULED_BY) +backup_dr backup Backup and DR vaults, plans, associations +firestore_backups backup Firestore backups and backup schedules +apigee serverless Apigee target servers and proxy deployments +infra_manager iac Resources Infrastructure Manager deployments manage +deployment_manager iac Deployment Manager deployments and their resources +========================= =========== ================================================== + +All REST calls go through :class:`~cloudg.inventory.gcp_sources.rest.GCPRestClient` +(google-auth ``AuthorizedSession``), which tests replace with a fake transport. +""" + +from __future__ import annotations + +# Source modules: imported for their registration side effect. +from cloudg.inventory.gcp_sources import identity, ops, platform, security # noqa: F401 +from cloudg.inventory.gcp_sources.base import ( + SOURCES, + GCPSourceOptions, + Source, + SourceContext, + SourceRun, + SourceSkipped, + run_sources, + source, +) +from cloudg.inventory.gcp_sources.rest import GCPApiError, GCPRestClient + +__all__ = [ + "GCPApiError", + "GCPRestClient", + "GCPSourceOptions", + "SOURCES", + "Source", + "SourceContext", + "SourceRun", + "SourceSkipped", + "run_sources", + "source", +] diff --git a/cloudg/inventory/gcp_sources/base.py b/cloudg/inventory/gcp_sources/base.py new file mode 100644 index 0000000..507ad2c --- /dev/null +++ b/cloudg/inventory/gcp_sources/base.py @@ -0,0 +1,434 @@ +"""Framework of the API-backed GCP sources: registry, run context, coverage. + +A source maps one family of resources that Cloud Asset Inventory does not +return (or returns without the detail AWS-depth mapping needs). It is a +function ``fn(ctx, run) -> list[CloudAsset]`` registered with +:func:`source`. Sources run in phases (identity first, so later phases see +the groups; roles last, so every bound role is known) and, inside a phase, +concurrently in worker threads. + +Each source is best-effort and gets its own coverage record +``gcp_``: + +- SUCCESS: every call answered (APIs that are not enabled in a project + count as "not in use"); +- PARTIAL: some calls failed, some answered; +- FAILED: nothing answered, or the source raised; +- SKIPPED: a prerequisite is missing (no REST transport, no organization, + no customer ID) or the API is not enabled anywhere it was asked. + +Resources a source reads go through the collector's own record pipeline +(:meth:`GCPCollector._build_asset`), so they get the same identifiers, +aliases, scrubbing and ``@_extractor`` relations as listed resources. +""" + +from __future__ import annotations + +import asyncio +import logging +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable, Iterable + +from cloudg.coverage import ServiceStatus +from cloudg.exceptions import CredentialError +from cloudg.inventory.gcp_sources.rest import GCPApiError +from cloudg.inventory.gcp_tasks import is_selected +from cloudg.resilience.errors import describe_error +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + +logger = logging.getLogger(__name__) + +_CRM = "//cloudresourcemanager.googleapis.com/" + + +@dataclass +class GCPSourceOptions: + """Limits and inputs of the API-backed sources (``gcp.*`` config keys).""" + + customer_id: str | None = None + max_dns_records_per_zone: int = 5000 + max_vulnerability_occurrences: int = 5000 + max_groups: int = 2000 + max_group_members: int = 5000 + expand_predefined_roles: bool = True + max_role_expansions: int = 500 + max_scheduler_locations: int = 25 + concurrency: int = 4 + + +SourceFn = Callable[["SourceContext", "SourceRun"], list[CloudAsset]] + + +@dataclass(frozen=True) +class Source: + name: str + family: str + fn: SourceFn + phase: int = 2 + needs_rest: bool = True + description: str = "" + + +SOURCES: dict[str, Source] = {} + + +def source( + name: str, family: str, *, phase: int = 2, needs_rest: bool = True, description: str = "" +) -> Callable[[SourceFn], SourceFn]: + """Register an API-backed source (coverage record ``gcp_``).""" + + def register(fn: SourceFn) -> SourceFn: + SOURCES[name] = Source( + name, + family, + fn, + phase, + needs_rest, + description or (fn.__doc__ or "").strip().split("\n")[0], + ) + return fn + + return register + + +class SourceSkipped(Exception): + """A prerequisite of the source is missing (recorded as SKIPPED).""" + + +class SourceRun: + """Outcome book-keeping of one source run.""" + + def __init__(self, name: str) -> None: + self.name = name + self.ok = 0 + self.errors: list[str] = [] + self.disabled: list[str] = [] + self.notes: list[str] = [] + + def call( + self, fn: Callable[..., Any], *args: Any, tolerate: bool = False, **kwargs: Any + ) -> Any: + """Run one API call; errors are recorded (not raised) and None returned. + + ``tolerate``: an error is expected for some setups and is not a failure. + API-not-enabled answers are recorded in ``disabled``; 404 counts as + an empty answer. + """ + try: + result = fn(*args, **kwargs) + except CredentialError: + raise + except GCPApiError as exc: + if exc.disabled or exc.precondition: + self.disabled.append(exc.url) + elif exc.not_found or tolerate: + self.ok += 1 + else: + self.errors.append(describe_error(exc)) + return None + except Exception as exc: + if not tolerate: + self.errors.append(f"{type(exc).__name__}: {describe_error(exc)}") + return None + self.ok += 1 + return result + + def status(self, produced: int) -> tuple[ServiceStatus, str | None]: + if self.errors: + status = ServiceStatus.PARTIAL if (produced or self.ok) else ServiceStatus.FAILED + more = f" (+{len(self.errors) - 1} more)" if len(self.errors) > 1 else "" + return status, self.errors[0][:400] + more + if not self.ok and self.disabled and not produced: + return ServiceStatus.SKIPPED, f"API not enabled ({len(self.disabled)} calls)" + return ServiceStatus.SUCCESS, "; ".join(self.notes) or None + + +class SourceContext: + """What a source sees: the collector, the assets so far and the REST client.""" + + def __init__( + self, + collector: Any, + assets: list[CloudAsset], + rest: Any, + options: GCPSourceOptions, + ) -> None: + self.collector = collector + self.rest = rest + self.options = options + self.state: dict[str, Any] = {} + self._assets = list(assets) + self._lock = threading.Lock() + self._index: dict[str, CloudAsset] | None = None + + # -- assets ----------------------------------------------------------- + + @property + def assets(self) -> list[CloudAsset]: + with self._lock: + return list(self._assets) + + def add(self, assets: Iterable[CloudAsset]) -> None: + with self._lock: + for a in assets: + self._assets.append(a) + if self._index is not None: + self._index_one(self._index, a) + + def _index_one(self, idx: dict[str, CloudAsset], a: CloudAsset) -> None: + for ident in [a.arn, *(a.metadata.get("aliases") or [])]: + if isinstance(ident, str) and ident: + idx.setdefault(ident, a) + idx.setdefault(ident.lower(), a) + + def lookup(self, identifier: Any) -> CloudAsset | None: + """Asset with this arn or alias (case-insensitive fallback).""" + if not isinstance(identifier, str) or not identifier: + return None + with self._lock: + if self._index is None: + self._index = {} + for a in self._assets: + self._index_one(self._index, a) + return self._index.get(identifier) or self._index.get(identifier.lower()) + + def by_type(self, *gcp_types: str) -> list[CloudAsset]: + wanted = set(gcp_types) + return [a for a in self.assets if a.metadata.get("gcp_asset_type") in wanted] + + # -- scope ------------------------------------------------------------ + + def projects(self) -> list[tuple[str, str | None]]: + """(project id, number) of every project in scope.""" + c = self.collector + if c._project_scope and c._project_id: + return [(c._project_id, c._id_to_number.get(c._project_id))] + out: dict[str, str | None] = {} + for a in self.assets: + if a.asset_type == AssetType.CLOUD_ACCOUNT and a.provider == CloudProvider.GCP: + pid = a.metadata.get("project_id") or a.account_id + if pid and not a.metadata.get("external"): + out.setdefault(pid, a.metadata.get("project_number")) + if c._project_filter: + out = {p: n for p, n in out.items() if {p, n} & c._project_filter} + return sorted(out.items()) + + def organization(self) -> str | None: + """Numeric organization ID (configured, scope, or the assets' ancestry).""" + c = self.collector + if c._organization_id: + return c._organization_id + if c._scope.startswith("organizations/"): + return c._scope.split("/", 1)[1] + for a in self.assets: + org = a.metadata.get("organization") + if isinstance(org, str) and org.startswith("organizations/"): + return org.split("/", 1)[1] + return None + + def folders(self) -> list[str]: + found: set[str] = set() + for a in self.assets: + for f in a.metadata.get("folders") or []: + if isinstance(f, str) and f.startswith("folders/"): + found.add(f) + if a.metadata.get("gcp_asset_type") == "cloudresourcemanager.googleapis.com/Folder": + found.add("folders/" + a.arn.rsplit("/", 1)[-1]) + return sorted(found) + + def parents(self, folders: bool = True) -> list[str]: + """Organization, folders and projects in scope (``organizations/1`` ...).""" + out = [f"projects/{p}" for p, _ in self.projects()] + if folders: + out = self.folders() + out + org = self.organization() + return ([f"organizations/{org}"] if org else []) + out + + def project_ancestors(self, project: str | None) -> list[str]: + """Ancestry of ``project`` as other assets in it record it.""" + if not project: + return [] + number = self.collector._id_to_number.get(project) + for a in self.assets: + if a.account_id == project and a.metadata.get("project_number"): + number = number or a.metadata.get("project_number") + rest = list(a.metadata.get("folders") or []) + if a.metadata.get("organization"): + rest.append(a.metadata["organization"]) + return [f"projects/{number}", *rest] + return [f"projects/{number or project}"] + + # -- REST helpers ----------------------------------------------------- + + def items( + self, run: "SourceRun", url: str, key: str, tolerate: bool = False, **kwargs: Any + ) -> list[dict[str, Any]]: + """Paged list through ``run.call`` (errors recorded, [] on failure).""" + res = run.call(self.rest.list, url, key, tolerate=tolerate, **kwargs) + if not res: + return [] + found, truncated = res + if truncated: + run.notes.append(f"{key} truncated at {len(found)}") + return found + + def get( + self, + run: "SourceRun", + url: str, + params: dict[str, Any] | None = None, + tolerate: bool = False, + ) -> dict[str, Any] | None: + return run.call(self.rest.get, url, params, tolerate=tolerate) + + # -- building --------------------------------------------------------- + + def build( + self, + name: str, + asset_type: str, + data: dict[str, Any], + *, + project: str | None = None, + location: str | None = None, + source_name: str = "", + global_resource: bool = False, + ) -> CloudAsset | None: + """A CloudAsset for an API resource, through the collector's record pipeline. + + Returns None when the resource is already in the inventory (listed + by Cloud Asset Inventory or built by another source). + """ + if not name.startswith("//"): + raise ValueError(f"full resource name expected, got {name!r}") + if self.lookup(name) is not None: + return None + parent = f"{_CRM}projects/{project}" if project else "" + rec = { + "name": name, + "asset_type": asset_type, + "data": data, + "location": location or "", + "parent": parent, + "ancestors": [] if global_resource else self.project_ancestors(project), + } + asset = self.collector._build_asset(rec) + if asset is None: + return None + for alias in asset.metadata.get("aliases") or []: + if self.lookup(alias) is not None and alias.startswith("//"): + return None + if global_resource: + asset.account_id = None + for key in ("project", "project_id", "project_number"): + asset.metadata[key] = None + asset.metadata["discovered_via"] = f"gcp-api:{source_name}" if source_name else "gcp-api" + return asset + + def placeholder( + self, + service: str, + parent: str, + asset_type: AssetType, + label: str, + reason: str = "not enabled", + gcp_type: str | None = None, + ) -> CloudAsset: + """A "not enabled" placeholder, like the AWS ``_disabled_asset``. + + ``arn`` is ``cloudg:gcp:::not-enabled`` and the + asset carries the service's ``gcp-security:`` alias, so configs + that depend on the service link to the gap. + """ + project = parent.split("/", 1)[1] if parent.startswith("projects/") else None + md: dict[str, Any] = { + "security_service": service, + "enabled": False, + "status": reason, + "parent": parent, + "gcp_asset_type": gcp_type, + "project_id": project, + "placeholder_for": "service", + "discovered_via": "gcp-api", + "aliases": [f"gcp-security:{service}:{parent}"], + } + if not project: + md["relations"] = [ + { + "target": f"{_CRM}{parent}", + "edge": "CONTAINS", + "reverse": True, + "description": f"{label} {reason}", + } + ] + return CloudAsset( + arn=f"cloudg:gcp:{service}:{parent}:not-enabled", + name=f"{label} ({reason})", + asset_type=asset_type, + provider=CloudProvider.GCP, + region="global", + account_id=project, + metadata=md, + ) + + +def _run_one(src: Source, ctx: SourceContext) -> list[CloudAsset]: + start = time.time() + record = ctx.collector._record + key = f"gcp_{src.name}" + if src.needs_rest and ctx.rest is None: + record(key, ServiceStatus.SKIPPED, 0, "no REST transport (no GCP credentials)", start=start) + return [] + run = SourceRun(src.name) + try: + assets = [a for a in (src.fn(ctx, run) or []) if a is not None] + except SourceSkipped as exc: + record(key, ServiceStatus.SKIPPED, 0, str(exc), start=start) + return [] + except Exception as exc: + logger.warning("GCP source %s failed: %s", src.name, exc) + logger.debug("GCP source %s failed", src.name, exc_info=True) + record(key, ServiceStatus.FAILED, 0, describe_error(exc), start=start) + return [] + status, error = run.status(len(assets)) + record(key, status, len(assets), error, start=start) + return assets + + +async def run_sources( + ctx: SourceContext, + include: Iterable[str] | None = None, + exclude: Iterable[str] | None = None, + phases: Iterable[int] | None = None, +) -> list[CloudAsset]: + """Run the selected sources phase by phase; returns every asset they built.""" + selected = [s for s in SOURCES.values() if is_selected(s.name, s.family, include, exclude)] + wanted = set(phases) if phases is not None else {s.phase for s in selected} + created: list[CloudAsset] = [] + sem = asyncio.Semaphore(max(1, ctx.options.concurrency)) + + async def one(src: Source) -> list[CloudAsset]: + async with sem: + return await asyncio.to_thread(_run_one, src, ctx) + + for phase in sorted(wanted): + batch = [s for s in selected if s.phase == phase] + results = await asyncio.gather(*(one(s) for s in batch)) + for assets in results: + ctx.add(assets) + created.extend(assets) + return created + + +__all__ = [ + "GCPSourceOptions", + "SOURCES", + "Source", + "SourceContext", + "SourceRun", + "SourceSkipped", + "run_sources", + "source", +] diff --git a/cloudg/inventory/gcp_sources/derived.py b/cloudg/inventory/gcp_sources/derived.py new file mode 100644 index 0000000..837ba55 --- /dev/null +++ b/cloudg/inventory/gcp_sources/derived.py @@ -0,0 +1,421 @@ +"""Cross-asset passes of the deep GCP collector (no API calls). + +Some GCP relations only exist between two other assets, or need the whole +inventory to resolve. These passes run once every asset is built: + +- tag bindings become ``TAGGED_WITH`` relations on the bound resource; +- resource policies on instances get the right verb by their kind + (snapshot schedule ``BACKUP_TO``, instance schedule ``SCHEDULED_BY``); +- a rotating secret ``ROTATES_SECRET`` the functions / services that + consume its rotation topic; +- ``goog-dm`` and ``managed-by-cnrm`` labels link Deployment Manager and + Config Connector (``MANAGES`` / ``OWNED_BY``); +- Principal Access Boundary bindings link the principal set to the policy; +- OS Config label / zone filters are matched to instances; +- API Gateway JWT providers become ``AUTHORIZER`` assets; +- ``gcp-principal:`` relation targets that are not assets get placeholders. +""" + +from __future__ import annotations + +import logging +from typing import Any, Callable + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.gcp_relations.apis import jwt_issuer_target +from cloudg.inventory.gcp_relations.iam import principal_asset +from cloudg.inventory.gcp_relations.names import PRINCIPAL_PREFIX, api_service +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType + +logger = logging.getLogger(__name__) + +_INSTANCE = "compute.googleapis.com/Instance" + + +def _index(assets: list[CloudAsset]) -> dict[str, CloudAsset]: + idx: dict[str, CloudAsset] = {} + for a in assets: + for ident in [a.arn, *(a.metadata.get("aliases") or [])]: + if isinstance(ident, str) and ident: + idx.setdefault(ident, a) + idx.setdefault(ident.lower(), a) + return idx + + +def _append(asset: CloudAsset, relation: dict[str, Any] | None) -> None: + if relation: + rels = asset.metadata.setdefault("relations", []) + if relation not in rels: + rels.append(relation) + + +def _gtype(a: CloudAsset) -> Any: + return a.metadata.get("gcp_asset_type") + + +# --------------------------------------------------------------------------- +# Tags +# --------------------------------------------------------------------------- + + +def fold_tag_bindings(assets: list[CloudAsset]) -> list[CloudAsset]: + """Replace each tag binding by a TAGGED_WITH relation on the resource it binds.""" + idx = _index(assets) + out: list[CloudAsset] = [] + for a in assets: + if _gtype(a) != "cloudresourcemanager.googleapis.com/TagBinding": + out.append(a) + continue + resource = idx.get(a.metadata.get("bound_resource") or "") + value = a.metadata.get("tag_value") + relation = rel( + value, + EdgeType.REFERENCES, + "TAGGED_WITH", + description=a.metadata.get("tag_value_namespaced_name"), + ) + if resource is None: + _append(a, relation) + out.append(a) + continue + _append(resource, relation) + tags = resource.metadata.setdefault("resource_manager_tags", []) + name = a.metadata.get("tag_value_namespaced_name") or value + if name and name not in tags: + tags.append(name) + return out + + +# --------------------------------------------------------------------------- +# Instance resource policies +# --------------------------------------------------------------------------- + +_POLICY_VERBS = { + "instance_schedule": (EdgeType.MANAGES, "SCHEDULED_BY", True, "instance start / stop schedule"), + "snapshot_schedule": (EdgeType.MANAGES, "BACKUP_TO", True, "snapshot schedule"), +} + + +def link_instance_policies(assets: list[CloudAsset]) -> None: + """Instance resource policies: schedules manage the VM, placement is a dependency.""" + idx = _index(assets) + for vm in assets: + if _gtype(vm) != _INSTANCE: + continue + for ref in vm.metadata.get("resource_policies") or []: + policy = idx.get(ref) + kind = policy.metadata.get("policy_kind") if policy is not None else None + edge, verb, reverse, desc = _POLICY_VERBS.get( + kind, (EdgeType.REFERENCES, "DEPENDS_ON", False, f"{kind or 'resource'} policy") + ) + _append(vm, rel(ref, edge, verb, reverse=reverse, description=desc)) + + +# --------------------------------------------------------------------------- +# Secret rotation +# --------------------------------------------------------------------------- + + +def _topic_consumers(assets: list[CloudAsset]) -> dict[str, list[str]]: + """Pub/Sub topic -> identifiers of what its messages invoke.""" + consumers: dict[str, list[str]] = {} + + def add(topic: str, target: str) -> None: + if target not in consumers.setdefault(topic, []): + consumers[topic].append(target) + + for a in assets: + rels = [r for r in a.metadata.get("relations") or [] if isinstance(r, dict)] + topics = [ + r["target"] + for r in rels + if r.get("reverse") + and api_service(r.get("target")) == "pubsub" + and "/topics/" in str(r.get("target")) + and r.get("edge") in (EdgeType.INVOKES.value,) + ] + if not topics or not a.arn: + continue + gtype = _gtype(a) or "" + if gtype.startswith(("cloudfunctions.", "run.")): + for t in topics: + add(t, a.arn) + continue + # Subscriptions and Eventarc triggers forward to what they invoke + forwards = [ + r["target"] + for r in rels + if not r.get("reverse") and r.get("edge") == EdgeType.INVOKES.value and r.get("target") + ] + for t in topics: + for f in forwards: + if api_service(f) != "pubsub": + add(t, f) + return consumers + + +def link_secret_rotation(assets: list[CloudAsset]) -> None: + """A rotating secret ROTATES_SECRET the handlers of its rotation topics (AWS: the rotation Lambda).""" + consumers: dict[str, list[str]] | None = None + for secret in assets: + if _gtype(secret) != "secretmanager.googleapis.com/Secret": + continue + topics = secret.metadata.get("rotation_topics") or [] + if not topics or not secret.metadata.get("rotation_period"): + continue + if consumers is None: + consumers = _topic_consumers(assets) + for topic in topics: + for handler in consumers.get(topic, []): + _append( + secret, + rel( + handler, + EdgeType.INVOKES, + "ROTATES_SECRET", + description="rotation handler (subscribes to the rotation topic)", + topic=topic, + ), + ) + + +# --------------------------------------------------------------------------- +# Deployment Manager / Config Connector labels +# --------------------------------------------------------------------------- + + +def _kcc_clusters(assets: list[CloudAsset]) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for a in assets: + if _gtype(a) != "container.googleapis.com/Cluster" or not a.account_id: + continue + data = (a.raw_data or {}).get("data") or {} + addons = data.get("addonsConfig") or {} + if (addons.get("configConnectorConfig") or {}).get("enabled"): + out.setdefault(a.account_id, []).append(a.arn) + return out + + +def link_managed_labels(assets: list[CloudAsset]) -> None: + """``goog-dm`` -> Deployment Manager deployment; ``managed-by-cnrm`` -> Config Connector.""" + kcc = _kcc_clusters(assets) + for a in assets: + if a.provider != CloudProvider.GCP or not a.tags: + continue + project = a.account_id + dm = a.tags.get("goog-dm") + if dm and project: + a.metadata["managed_by"] = "deployment-manager" + _append( + a, + rel( + f"//deploymentmanager.googleapis.com/projects/{project}/global/deployments/{dm}", + EdgeType.MANAGES, + "OWNED_BY", + reverse=True, + description="created by Deployment Manager", + ), + ) + if str(a.tags.get("managed-by-cnrm", "")).lower() == "true": + a.metadata["managed_by"] = "config-connector" + clusters = kcc.get(project or "", []) + if len(clusters) == 1: + _append( + a, + rel( + clusters[0], + EdgeType.MANAGES, + "OWNED_BY", + reverse=True, + description="managed by Config Connector", + ), + ) + if str(a.tags.get("goog-terraform-provisioned", "")).lower() == "true": + a.metadata.setdefault("managed_by", "terraform") + + +# --------------------------------------------------------------------------- +# Principal Access Boundary bindings +# --------------------------------------------------------------------------- + + +def fold_policy_bindings(assets: list[CloudAsset]) -> None: + """Principal set -> PERMISSION_BOUNDARY_LIMITS -> Principal Access Boundary policy.""" + idx = _index(assets) + for binding in assets: + if _gtype(binding) != "iam.googleapis.com/PolicyBinding": + continue + policy = idx.get(binding.metadata.get("policy") or "") + target = binding.metadata.get("target_principal_set") + if policy is not None and target: + _append( + policy, + rel( + target, + EdgeType.REFERENCES, + "PERMISSION_BOUNDARY_LIMITS", + reverse=True, + description="principals in this set are limited by the boundary", + binding=binding.arn, + ), + ) + + +# --------------------------------------------------------------------------- +# OS Config instance filters +# --------------------------------------------------------------------------- + + +def _matches(vm: CloudAsset, flt: dict[str, Any]) -> bool: + if flt.get("zones") and vm.region not in flt["zones"]: + return False + if flt.get("name_prefixes") and not any(vm.name.startswith(p) for p in flt["name_prefixes"]): + return False + labels = vm.tags or {} + groups = flt.get("labels") or [] + if groups and not any(all(labels.get(k) == v for k, v in g.items()) for g in groups): + return False + for g in flt.get("exclusion_labels") or []: + if g and all(labels.get(k) == v for k, v in g.items()): + return False + return bool(flt.get("zones") or flt.get("name_prefixes") or groups) + + +def link_os_config_filters(assets: list[CloudAsset]) -> None: + """Patch deployments and OS policy assignments MANAGES / SCHEDULED_BY matching VMs.""" + vms = [a for a in assets if _gtype(a) == _INSTANCE] + for cfg in assets: + if _gtype(cfg) not in ( + "osconfig.googleapis.com/PatchDeployment", + "osconfig.googleapis.com/OSPolicyAssignment", + ): + continue + flt = cfg.metadata.get("instance_filter") or {} + if not flt or flt.get("all"): + continue + for vm in vms: + if vm.account_id == cfg.account_id and _matches(vm, flt): + _append( + cfg, + rel( + vm.arn, + EdgeType.MANAGES, + "SCHEDULED_BY", + description="instance filter match", + ), + ) + + +# --------------------------------------------------------------------------- +# API Gateway authorizers +# --------------------------------------------------------------------------- + + +def authorizer_assets(assets: list[CloudAsset]) -> list[CloudAsset]: + """An AUTHORIZER per JWT provider declared in an API Gateway config.""" + out: list[CloudAsset] = [] + for cfg in assets: + for p in cfg.metadata.get("jwt_providers") or []: + if not isinstance(p, dict) or not p.get("name"): + continue + relations = [ + rel(cfg.arn, EdgeType.CONTAINS, reverse=True, description="API config authorizer"), + rel( + jwt_issuer_target(p.get("issuer")), + EdgeType.REFERENCES, + "DEPENDS_ON", + description="token issuer", + ), + ] + out.append( + CloudAsset( + arn=f"{cfg.arn}/authorizers/{p['name']}", + name=p["name"], + asset_type=AssetType.AUTHORIZER, + provider=CloudProvider.GCP, + region=cfg.region, + account_id=cfg.account_id, + metadata={ + "gcp_asset_type": "apigateway.googleapis.com/Authorizer", + "project_id": cfg.account_id, + "authorizer_type": "JWT", + "issuer": p.get("issuer"), + "jwks_uri": p.get("jwks_uri"), + "audiences": p.get("audiences") or [], + "relations": [r for r in relations if r], + "discovered_via": "api config document", + }, + ) + ) + return out + + +# --------------------------------------------------------------------------- +# Principal placeholders +# --------------------------------------------------------------------------- + + +def ensure_principal_placeholders(assets: list[CloudAsset]) -> list[CloudAsset]: + """Placeholders for ``gcp-principal:`` targets (deny rules, PAM, groups ...).""" + idx = _index(assets) + created: list[CloudAsset] = [] + for a in list(assets): + for r in a.metadata.get("relations") or []: + target = r.get("target") if isinstance(r, dict) else None + if not isinstance(target, str) or not target.startswith(PRINCIPAL_PREFIX): + continue + if target in idx or target.lower() in idx: + continue + placeholder = principal_asset( + target[len(PRINCIPAL_PREFIX) :], discovered_via="relation_reference" + ) + if placeholder is None: + continue + created.append(placeholder) + for ident in [placeholder.arn, *placeholder.metadata.get("aliases", [])]: + idx.setdefault(ident, placeholder) + idx.setdefault(ident.lower(), placeholder) + return created + + +def cap_images( + records: list[dict[str, Any]], max_per_repository: int | None +) -> list[dict[str, Any]]: + """Keep the newest ``max_per_repository`` Docker images per repository.""" + if not max_per_repository or max_per_repository <= 0: + return records + by_repo: dict[str, list[dict[str, Any]]] = {} + for rec in records: + if rec.get("asset_type") == "artifactregistry.googleapis.com/DockerImage": + repo = str(rec.get("name", "")).split("/dockerImages/", 1)[0] + by_repo.setdefault(repo, []).append(rec) + drop: set[int] = set() + for images in by_repo.values(): + if len(images) <= max_per_repository: + continue + images.sort(key=lambda r: str((r.get("data") or {}).get("uploadTime") or ""), reverse=True) + drop.update(id(r) for r in images[max_per_repository:]) + return [r for r in records if id(r) not in drop] + + +PASSES: tuple[tuple[str, Callable[[list[CloudAsset]], Any]], ...] = ( + ("instance_policies", link_instance_policies), + ("secret_rotation", link_secret_rotation), + ("managed_labels", link_managed_labels), + ("policy_bindings", fold_policy_bindings), + ("os_config_filters", link_os_config_filters), +) + + +__all__ = [ + "PASSES", + "authorizer_assets", + "cap_images", + "ensure_principal_placeholders", + "fold_policy_bindings", + "fold_tag_bindings", + "link_instance_policies", + "link_managed_labels", + "link_os_config_filters", + "link_secret_rotation", +] diff --git a/cloudg/inventory/gcp_sources/identity.py b/cloudg/inventory/gcp_sources/identity.py new file mode 100644 index 0000000..93c45b4 --- /dev/null +++ b/cloudg/inventory/gcp_sources/identity.py @@ -0,0 +1,213 @@ +"""Identity sources: Cloud Identity groups and memberships, IAM deny +policies, Principal Access Boundary policies and bindings, Privileged +Access Manager entitlements and predefined-role permissions.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import quote + +from cloudg.inventory.gcp_sources.base import SourceContext, SourceRun, SourceSkipped, source +from cloudg.schema.models import CloudAsset + +_CI = "https://cloudidentity.googleapis.com/v1" +_IAM = "https://iam.googleapis.com" +_PAM = "https://privilegedaccessmanager.googleapis.com/v1" +_CRM_API = "https://cloudresourcemanager.googleapis.com/v3" + + +def _customer_id(ctx: SourceContext, run: SourceRun) -> str | None: + if ctx.options.customer_id: + return ctx.options.customer_id.removeprefix("customers/") + for a in ctx.by_type("cloudresourcemanager.googleapis.com/Organization"): + cid = ((a.raw_data or {}).get("data") or {}).get("owner", {}).get("directoryCustomerId") + if cid: + return str(cid) + org = ctx.organization() + if org: + body = ctx.get(run, f"{_CRM_API}/organizations/{org}", tolerate=True) + if body and body.get("directoryCustomerId"): + return str(body["directoryCustomerId"]) + return None + + +@source("cloud_identity", "identity", phase=0) +def cloud_identity_groups(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Google groups (Cloud Identity) with their memberships.""" + customer = _customer_id(ctx, run) + if not customer: + raise SourceSkipped( + "no Cloud Identity customer ID: set gcp.cloud_identity_customer_id " + "or map at organization scope" + ) + opts = ctx.options + groups = ctx.items( + run, + f"{_CI}/groups", + "groups", + params={"parent": f"customers/{customer}", "view": "FULL"}, + page_size=500, + max_items=opts.max_groups, + ) + out: list[CloudAsset] = [] + for group in groups: + gname = group.get("name") + if not isinstance(gname, str) or not gname.startswith("groups/"): + continue + members = ctx.items( + run, + f"{_CI}/{gname}/memberships", + "memberships", + params={"view": "FULL"}, + page_size=1000, + max_items=opts.max_group_members, + ) + data = {**group, "memberships": members} + asset = ctx.build( + f"//cloudidentity.googleapis.com/{gname}", + "cloudidentity.googleapis.com/Group", + data, + source_name="cloud_identity", + global_resource=True, + ) + if asset is not None: + asset.name = str((group.get("groupKey") or {}).get("id") or asset.name) + out.append(asset) + return out + + +def _attachment(parent: str) -> str: + return quote(f"cloudresourcemanager.googleapis.com/{parent}", safe="") + + +@source("iam_deny", "identity") +def iam_deny_policies(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """IAM deny policies attached to the organization, folders and projects.""" + out: list[CloudAsset] = [] + for parent in ctx.parents(): + listed = ctx.items( + run, f"{_IAM}/v2/policies/{_attachment(parent)}/denypolicies", "policies" + ) + for pol in listed: + name = pol.get("name") + if not isinstance(name, str): + continue + # The list omits rules; fetch each policy + full = ctx.get(run, f"{_IAM}/v2/{name}") or pol + project = parent.split("/", 1)[1] if parent.startswith("projects/") else None + asset = ctx.build( + f"//iam.googleapis.com/{name}", + "iam.googleapis.com/PolicyV2", + {**full, "kind": full.get("kind") or "DenyPolicy"}, + project=project, + source_name="iam_deny", + global_resource=project is None, + ) + if asset is not None: + out.append(asset) + return out + + +@source("principal_access_boundary", "identity") +def principal_access_boundary(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Principal Access Boundary policies (organization) and their policy bindings.""" + out: list[CloudAsset] = [] + org = ctx.organization() + if org: + for pol in ctx.items( + run, + f"{_IAM}/v3/organizations/{org}/locations/global/principalAccessBoundaryPolicies", + "principalAccessBoundaryPolicies", + ): + name = pol.get("name") + if isinstance(name, str): + asset = ctx.build( + f"//iam.googleapis.com/{name}", + "iam.googleapis.com/PrincipalAccessBoundaryPolicy", + pol, + source_name="principal_access_boundary", + global_resource=True, + ) + if asset is not None: + out.append(asset) + for parent in ctx.parents(): + for binding in ctx.items( + run, f"{_IAM}/v3/{parent}/locations/global/policyBindings", "policyBindings" + ): + name = binding.get("name") + if not isinstance(name, str): + continue + project = parent.split("/", 1)[1] if parent.startswith("projects/") else None + asset = ctx.build( + f"//iam.googleapis.com/{name}", + "iam.googleapis.com/PolicyBinding", + binding, + project=project, + source_name="principal_access_boundary", + global_resource=project is None, + ) + if asset is not None: + out.append(asset) + return out + + +@source("pam", "identity") +def pam_entitlements(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Privileged Access Manager entitlements at every parent in scope.""" + out: list[CloudAsset] = [] + for parent in ctx.parents(): + for ent in ctx.items(run, f"{_PAM}/{parent}/locations/global/entitlements", "entitlements"): + name = ent.get("name") + if not isinstance(name, str): + continue + project = parent.split("/", 1)[1] if parent.startswith("projects/") else None + asset = ctx.build( + f"//privilegedaccessmanager.googleapis.com/{name}", + "privilegedaccessmanager.googleapis.com/Entitlement", + ent, + project=project, + source_name="pam", + global_resource=project is None, + ) + if asset is not None: + out.append(asset) + return out + + +def bound_roles(ctx: SourceContext) -> list[str]: + """Predefined roles bound anywhere: IAM bindings and PAM entitlements.""" + roles: list[str] = list(ctx.state.get("iam_roles") or []) + for a in ctx.by_type("privilegedaccessmanager.googleapis.com/Entitlement"): + roles += [r for r in a.metadata.get("roles") or [] if isinstance(r, str)] + return [r for r in dict.fromkeys(roles) if r.startswith("roles/")] + + +@source("iam_roles", "identity", phase=3, needs_rest=False) +def predefined_roles(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """An IAM_ROLE asset per bound predefined role, with its permissions (Roles API).""" + out: list[CloudAsset] = [] + expand = ctx.rest is not None and ctx.options.expand_predefined_roles + budget = ctx.options.max_role_expansions + for role in bound_roles(ctx): + data: dict[str, Any] = {"name": role, "title": role} + if expand and budget > 0: + budget -= 1 + body = ctx.get(run, f"{_IAM}/v1/{role}") + if body: + data = {**body, "name": role} + else: + run.ok += 1 + asset = ctx.build( + f"//iam.googleapis.com/{role}", + "iam.googleapis.com/Role", + data, + source_name="iam_roles", + global_resource=True, + ) + if asset is not None: + asset.name = role + asset.metadata["permissions_expanded"] = "includedPermissions" in data + out.append(asset) + if not expand: + run.notes.append("permissions not expanded (no REST transport or disabled)") + return out diff --git a/cloudg/inventory/gcp_sources/ops.py b/cloudg/inventory/gcp_sources/ops.py new file mode 100644 index 0000000..3470a1c --- /dev/null +++ b/cloudg/inventory/gcp_sources/ops.py @@ -0,0 +1,219 @@ +"""Operations sources: Cloud DNS record sets, Cloud Monitoring (alert +policies, notification channels, uptime checks, metrics scopes), OS Config +and Cloud Scheduler jobs.""" + +from __future__ import annotations + +import re +from typing import Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.gcp_sources.base import SourceContext, SourceRun, source +from cloudg.schema.models import CloudAsset, EdgeType + +_DNS = "https://dns.googleapis.com/dns/v1" +_MON3 = "https://monitoring.googleapis.com/v3" +_MON1 = "https://monitoring.googleapis.com/v1" +_OSC = "https://osconfig.googleapis.com/v1" +_SCHED = "https://cloudscheduler.googleapis.com/v1" +_CRM = "//cloudresourcemanager.googleapis.com/" +_REGION_RE = re.compile(r"^[a-z]+-[a-z]+\d+$") +_APPENGINE_REGIONS = {"us-central": "us-central1", "europe-west": "europe-west1"} + + +def _append(asset: CloudAsset, relation: dict[str, Any] | None) -> None: + if relation: + rels = asset.metadata.setdefault("relations", []) + if relation not in rels: + rels.append(relation) + + +# --------------------------------------------------------------------------- +# Cloud DNS record sets +# --------------------------------------------------------------------------- + + +@source("dns_records", "dns") +def dns_records(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Record sets of every collected Cloud DNS managed zone (DNS_RESOLVED).""" + out: list[CloudAsset] = [] + for zone in ctx.by_type("dns.googleapis.com/ManagedZone"): + data = (zone.raw_data or {}).get("data") or {} + project = zone.account_id + zone_name = data.get("name") or zone.arn.rsplit("/", 1)[-1] + if not project or not zone_name: + continue + records = ctx.items( + run, + f"{_DNS}/projects/{project}/managedZones/{zone_name}/rrsets", + "rrsets", + page_size=1000, + page_size_param="maxResults", + max_items=ctx.options.max_dns_records_per_zone, + ) + for rr in records: + name, rtype = rr.get("name"), rr.get("type") + if not name or not rtype: + continue + asset = ctx.build( + f"//dns.googleapis.com/projects/{project}/managedZones/{zone_name}/rrsets/{name}/{rtype}", + "dns.googleapis.com/ResourceRecordSet", + {**rr, "managedZone": zone.arn, "displayName": f"{name} {rtype}"}, + project=project, + source_name="dns_records", + ) + if asset is not None: + out.append(asset) + return out + + +# --------------------------------------------------------------------------- +# Cloud Monitoring +# --------------------------------------------------------------------------- + + +def _scoping_project_asset(ctx: SourceContext, project: str) -> CloudAsset | None: + return ctx.lookup(f"{_CRM}projects/{project}") + + +@source("monitoring", "logging") +def monitoring(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Alert policies, notification channels and uptime checks (MONITORED_BY / INVOKES), + and the projects each metrics scope monitors.""" + out: list[CloudAsset] = [] + kinds = ( + ("alertPolicies", "monitoring.googleapis.com/AlertPolicy"), + ("notificationChannels", "monitoring.googleapis.com/NotificationChannel"), + ("uptimeCheckConfigs", "monitoring.googleapis.com/UptimeCheckConfig"), + ) + for project, number in ctx.projects(): + for key, kind in kinds: + for item in ctx.items(run, f"{_MON3}/projects/{project}/{key}", key, page_size=500): + name = item.get("name") + if isinstance(name, str): + asset = ctx.build( + f"//monitoring.googleapis.com/{name}", + kind, + item, + project=project, + source_name="monitoring", + ) + if asset is not None: + out.append(asset) + scope = ctx.get(run, f"{_MON1}/locations/global/metricsScopes/{project}", tolerate=True) + holder = _scoping_project_asset(ctx, project) + for mp in (scope or {}).get("monitoredProjects") or []: + other = str(mp.get("name") or "").rsplit("/projects/", 1)[-1] + if holder is not None and other and other not in (project, number): + _append( + holder, + rel( + f"{_CRM}projects/{other}", + EdgeType.MONITORS, + "MONITORED_BY", + description="project in this metrics scope", + ), + ) + return out + + +# --------------------------------------------------------------------------- +# OS Config / VM Manager +# --------------------------------------------------------------------------- + + +@source("os_config", "ops") +def os_config(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Patch deployments, OS policy assignments and VM vulnerability reports.""" + zones: dict[str, set[str]] = {} + for vm in ctx.by_type("compute.googleapis.com/Instance"): + if vm.account_id and vm.region and vm.region != "global": + zones.setdefault(vm.account_id, set()).add(vm.region) + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + if project not in zones: + continue + for pd in ctx.items(run, f"{_OSC}/projects/{project}/patchDeployments", "patchDeployments"): + if isinstance(pd.get("name"), str): + out.append( + ctx.build( + f"//osconfig.googleapis.com/{pd['name']}", + "osconfig.googleapis.com/PatchDeployment", + pd, + project=project, + source_name="os_config", + ) + ) + for zone in sorted(zones[project]): + base = f"{_OSC}/projects/{project}/locations/{zone}" + for opa in ctx.items(run, f"{base}/osPolicyAssignments", "osPolicyAssignments"): + if isinstance(opa.get("name"), str): + out.append( + ctx.build( + f"//osconfig.googleapis.com/{opa['name']}", + "osconfig.googleapis.com/OSPolicyAssignment", + opa, + project=project, + location=zone, + source_name="os_config", + ) + ) + for report in ctx.items( + run, f"{base}/instances/-/vulnerabilityReports", "vulnerabilityReports" + ): + if isinstance(report.get("name"), str): + out.append( + ctx.build( + f"//osconfig.googleapis.com/{report['name']}", + "osconfig.googleapis.com/VulnerabilityReport", + report, + project=project, + location=zone, + source_name="os_config", + ) + ) + return [a for a in out if a is not None] + + +# --------------------------------------------------------------------------- +# Cloud Scheduler +# --------------------------------------------------------------------------- + + +def _scheduler_locations(ctx: SourceContext, project: str) -> list[str]: + """Regions worth asking: where the project has resources, plus App Engine's.""" + regions: set[str] = set() + for a in ctx.assets: + if a.account_id != project or not a.region: + continue + region = a.region + if re.fullmatch(r"[a-z]+-[a-z]+\d+-[a-z]", region): + region = region.rsplit("-", 1)[0] + if _REGION_RE.match(region): + regions.add(region) + if a.metadata.get("gcp_asset_type") == "appengine.googleapis.com/Application": + loc = ((a.raw_data or {}).get("data") or {}).get("locationId") + if isinstance(loc, str): + regions.add(_APPENGINE_REGIONS.get(loc, loc)) + return sorted(regions)[: ctx.options.max_scheduler_locations] + + +@source("cloud_scheduler", "integration") +def cloud_scheduler(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Cloud Scheduler jobs (INVOKES / SCHEDULED_BY their targets).""" + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + for loc in _scheduler_locations(ctx, project): + for job in ctx.items(run, f"{_SCHED}/projects/{project}/locations/{loc}/jobs", "jobs"): + if isinstance(job.get("name"), str): + out.append( + ctx.build( + f"//cloudscheduler.googleapis.com/{job['name']}", + "cloudscheduler.googleapis.com/Job", + job, + project=project, + location=loc, + source_name="cloud_scheduler", + ) + ) + return [a for a in out if a is not None] diff --git a/cloudg/inventory/gcp_sources/platform.py b/cloudg/inventory/gcp_sources/platform.py new file mode 100644 index 0000000..2b45cf4 --- /dev/null +++ b/cloudg/inventory/gcp_sources/platform.py @@ -0,0 +1,225 @@ +"""Backup and platform sources: Backup and DR Service, Firestore backups and +schedules, Apigee target servers and deployments, Infrastructure Manager +managed resources and Deployment Manager deployments.""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.aws_services._base import rel +from cloudg.inventory.gcp_relations.names import full_name +from cloudg.inventory.gcp_sources.base import SourceContext, SourceRun, source +from cloudg.schema.models import CloudAsset, EdgeType + +_BDR = "https://backupdr.googleapis.com/v1" +_FS = "https://firestore.googleapis.com/v1" +_APIGEE = "https://apigee.googleapis.com/v1" +_IM = "https://config.googleapis.com/v1" +_DM = "https://deploymentmanager.googleapis.com/deploymentmanager/v2" + + +def _append(asset: CloudAsset, relation: dict[str, Any] | None) -> None: + if relation: + rels = asset.metadata.setdefault("relations", []) + if relation not in rels: + rels.append(relation) + + +def _location(name: str) -> str | None: + parts = name.split("/") + return parts[parts.index("locations") + 1] if "locations" in parts[:-1] else None + + +# --------------------------------------------------------------------------- +# Backup and DR Service +# --------------------------------------------------------------------------- + + +@source("backup_dr", "backup") +def backup_dr(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Backup and DR vaults, plans, plan associations and management servers (BACKUP_TO).""" + kinds = ( + ("backupVaults", "backupdr.googleapis.com/BackupVault"), + ("backupPlans", "backupdr.googleapis.com/BackupPlan"), + ("backupPlanAssociations", "backupdr.googleapis.com/BackupPlanAssociation"), + ("managementServers", "backupdr.googleapis.com/ManagementServer"), + ) + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + for key, kind in kinds: + url = f"{_BDR}/projects/{project}/locations/-/{key}" + for item in ctx.items(run, url, key): + name = item.get("name") + if isinstance(name, str): + out.append( + ctx.build( + f"//backupdr.googleapis.com/{name}", + kind, + item, + project=project, + location=_location(name), + source_name="backup_dr", + ) + ) + return [a for a in out if a is not None] + + +@source("firestore_backups", "backup") +def firestore_backups(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Firestore backup schedules (per database) and backups (per project).""" + out: list[CloudAsset] = [] + databases = ctx.by_type("firestore.googleapis.com/Database") + projects = sorted({db.account_id for db in databases if db.account_id}) + for db in databases: + rel_name = db.arn.split("firestore.googleapis.com/", 1)[-1] + for sched in ctx.items(run, f"{_FS}/{rel_name}/backupSchedules", "backupSchedules"): + if isinstance(sched.get("name"), str): + out.append( + ctx.build( + f"//firestore.googleapis.com/{sched['name']}", + "firestore.googleapis.com/BackupSchedule", + sched, + project=db.account_id, + source_name="firestore_backups", + ) + ) + for project in projects: + for backup in ctx.items(run, f"{_FS}/projects/{project}/locations/-/backups", "backups"): + if isinstance(backup.get("name"), str): + out.append( + ctx.build( + f"//firestore.googleapis.com/{backup['name']}", + "firestore.googleapis.com/Backup", + backup, + project=project, + location=_location(backup["name"]), + source_name="firestore_backups", + ) + ) + return [a for a in out if a is not None] + + +# --------------------------------------------------------------------------- +# Apigee +# --------------------------------------------------------------------------- + + +def _names(body: Any, key: str) -> list[str]: + """Apigee lists come back as bare names or as ``{key: [{name: ...}]}``.""" + if isinstance(body, list): + return [str(n) for n in body] + if isinstance(body, dict): + items = body.get(key) or body.get("items") or [] + if isinstance(items, str): + items = [items] + return [str(i.get("name") if isinstance(i, dict) else i) for i in items] + return [] + + +@source("apigee", "serverless") +def apigee(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Apigee environments' target servers and proxy deployments.""" + out: list[CloudAsset] = [] + for org_asset in ctx.by_type("apigee.googleapis.com/Organization"): + org = org_asset.arn.rsplit("/", 1)[-1] + project = org_asset.account_id + envs = _names(ctx.get(run, f"{_APIGEE}/organizations/{org}/environments"), "environments") + for env in envs: + ts_body = ctx.get( + run, f"{_APIGEE}/organizations/{org}/environments/{env}/targetservers" + ) + for ts in _names(ts_body, "targetServers"): + detail = ctx.get( + run, f"{_APIGEE}/organizations/{org}/environments/{env}/targetservers/{ts}" + ) + if detail: + out.append( + ctx.build( + f"//apigee.googleapis.com/organizations/{org}/environments/{env}/targetservers/{ts}", + "apigee.googleapis.com/TargetServer", + detail, + project=project, + source_name="apigee", + ) + ) + body = ctx.get(run, f"{_APIGEE}/organizations/{org}/deployments") or {} + for dep in body.get("deployments") or []: + proxy = ctx.lookup( + f"//apigee.googleapis.com/organizations/{org}/apis/{dep.get('apiProxy')}" + ) + env = dep.get("environment") + if proxy is not None and env: + _append( + proxy, + rel( + f"//apigee.googleapis.com/organizations/{org}/environments/{env}", + EdgeType.ATTACHED_TO, + description=f"revision {dep.get('revision')} deployed", + revision=dep.get("revision"), + ), + ) + return [a for a in out if a is not None] + + +# --------------------------------------------------------------------------- +# Infrastructure Manager and Deployment Manager +# --------------------------------------------------------------------------- + + +@source("infra_manager", "iac") +def infra_manager(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Resources each Infrastructure Manager deployment's latest revision manages (MANAGES).""" + for dep in ctx.by_type("config.googleapis.com/Deployment"): + revision = ((dep.raw_data or {}).get("data") or {}).get("latestRevision") + if not isinstance(revision, str) or not revision: + continue + managed = [] + for res in ctx.items(run, f"{_IM}/{revision}/resources", "resources"): + for cai in (res.get("caiAssets") or {}).values(): + ref = full_name(cai.get("fullResourceName")) if isinstance(cai, dict) else None + if ref: + managed.append(ref) + _append( + dep, + rel( + ref, + EdgeType.MANAGES, + "OWNED_BY", + description="Terraform-managed resource", + ), + ) + dep.metadata["managed_resource_count"] = len(managed) + return [] + + +@source("deployment_manager", "iac") +def deployment_manager(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Deployment Manager deployments and the resources they created (MANAGES).""" + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + base = f"{_DM}/projects/{project}/global/deployments" + for dep in ctx.items(run, base, "deployments", page_size=500, page_size_param="maxResults"): + name = dep.get("name") + if not isinstance(name, str): + continue + resources = ctx.items( + run, + f"{base}/{name}/resources", + "resources", + page_size=500, + page_size_param="maxResults", + ) + data = { + **dep, + "resources": [{k: r.get(k) for k in ("name", "type", "url")} for r in resources], + } + out.append( + ctx.build( + f"//deploymentmanager.googleapis.com/projects/{project}/global/deployments/{name}", + "deploymentmanager.googleapis.com/Deployment", + data, + project=project, + source_name="deployment_manager", + ) + ) + return [a for a in out if a is not None] diff --git a/cloudg/inventory/gcp_sources/rest.py b/cloudg/inventory/gcp_sources/rest.py new file mode 100644 index 0000000..01ba687 --- /dev/null +++ b/cloudg/inventory/gcp_sources/rest.py @@ -0,0 +1,244 @@ +"""Minimal REST client for the Google APIs Cloud Asset Inventory does not cover. + +Built on google-auth's ``AuthorizedSession`` (requests), which the ``gcp`` +extra already installs. The session is injectable, so tests hand in a fake +transport and never touch the network. + +Every request goes through cloudg's resilience governor (rate limiter, +circuit breaker, retry budget) under the scope ``gcp / / ``, +and throttling (429) or unavailability (500, 502, 503, 504) is retried +with exponential backoff that honours ``Retry-After``. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable + +from cloudg.exceptions import CollectionError, CredentialError, ProviderSDKMissingError +from cloudg.resilience.gcp import gcp_scope +from cloudg.resilience.governor import get_governor + +logger = logging.getLogger(__name__) + +_RETRY_STATUSES = {429, 500, 502, 503, 504} +_DISABLED_REASONS = {"SERVICE_DISABLED", "accessNotConfigured", "API_DISABLED"} + + +class GCPApiError(CollectionError): + """A Google API answered with an error status. + + Attributes: + status_code: HTTP status. + error_code: ``error.status`` (``PERMISSION_DENIED``, ``NOT_FOUND`` ...) or + the first ``reason`` of the error details. + url: Request URL without the query string. + """ + + def __init__(self, status_code: int, error_code: str | None, message: str, url: str) -> None: + self.status_code = status_code + self.error_code = error_code + self.url = url + self.reasons: set[str] = set() + super().__init__(f"HTTP {status_code} {error_code or ''} from {url}: {message}".strip()) + + @property + def disabled(self) -> bool: + """The API is not enabled in the project (the service is not in use).""" + text = str(self) + return bool(self.reasons & _DISABLED_REASONS) or ( + self.status_code == 403 + and ("has not been used in project" in text or "it is disabled" in text) + ) + + @property + def not_found(self) -> bool: + return self.status_code == 404 + + @property + def precondition(self) -> bool: + """``FAILED_PRECONDITION``: the service exists but is not set up (not activated).""" + return self.error_code == "FAILED_PRECONDITION" or ( + self.status_code == 400 and "not activated" in str(self).lower() + ) + + +def _error_from(resp: Any, url: str) -> GCPApiError: + status = int(getattr(resp, "status_code", 0) or 0) + code, message, reasons = None, "", set() + try: + body = resp.json() + except Exception: + body = None + err = body.get("error") if isinstance(body, dict) else None + if isinstance(err, dict): + code = err.get("status") or None + message = str(err.get("message") or "") + for detail in err.get("details") or []: + if isinstance(detail, dict) and detail.get("reason"): + reasons.add(str(detail["reason"])) + for e in err.get("errors") or []: + if isinstance(e, dict) and e.get("reason"): + reasons.add(str(e["reason"])) + if not message: + message = str(getattr(resp, "text", "") or "")[:300] + exc = GCPApiError(status, code or (sorted(reasons)[0] if reasons else None), message[:300], url) + exc.reasons = reasons + return exc + + +def _service_of(url: str) -> str: + host = url.split("://", 1)[-1].split("/", 1)[0] + return host.split(".", 1)[0] or "googleapis" + + +def _parent_of(url: str) -> str | None: + for marker in ("/projects/", "/organizations/", "/folders/"): + if marker in url: + rest = url.split(marker, 1)[1].split("/", 1)[0] + return f"{marker.strip('/')}/{rest}" + return None + + +class GCPRestClient: + """GET / paged-list access to Google REST APIs. + + Args: + credentials: google-auth credentials (used to build an + ``AuthorizedSession`` when ``session`` is not given). + session: Injected transport with ``get(url, params=..., timeout=...)`` + returning a ``requests``-like response (tests). + timeout: Per-request timeout in seconds. + max_retries: Retries of throttled / unavailable requests. + sleep: Sleep function between retries (tests pass a no-op). + """ + + def __init__( + self, + credentials: Any = None, + *, + session: Any = None, + timeout: float = 60.0, + max_retries: int = 4, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._credentials = credentials + self._session = session + self._timeout = timeout + self._max_retries = max_retries + self._sleep = sleep + self.calls = 0 + + def _get_session(self) -> Any: + if self._session is None: + try: + from google.auth.transport.requests import AuthorizedSession + except ImportError as exc: + raise ProviderSDKMissingError("gcp", missing="google-auth[requests]") from exc + self._session = AuthorizedSession(self._credentials) + return self._session + + def get(self, url: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + """GET ``url`` and return the JSON body. + + Raises: + CredentialError: 401 (the credentials were refused). + GCPApiError: Any other error status after retries. + """ + return self._request("get", url, params, None) + + def post(self, url: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """POST a read-only RPC (``:getIamPolicy``) and return the JSON body.""" + return self._request("post", url, None, body or {}) + + def _request( + self, method: str, url: str, params: dict[str, Any] | None, body: dict[str, Any] | None + ) -> dict[str, Any]: + session = self._get_session() + gov = get_governor() + scope = gcp_scope(_parent_of(url), _service_of(url), f"rest.{method}") + enabled = gov.provider_enabled("gcp") + attempt = 0 + while True: + if enabled: + gov.check(scope) + gov.limiter.acquire_sync(scope) + gov.record_call(scope) + self.calls += 1 + if method == "post": + resp = session.post(url, json=body, timeout=self._timeout) + else: + resp = session.get(url, params=params or None, timeout=self._timeout) + status = int(getattr(resp, "status_code", 0) or 0) + if 200 <= status < 300: + gov.on_success(scope) + try: + body = resp.json() + except ValueError: + body = {} + return body if isinstance(body, dict) else {"items": body} + if status == 401: + raise CredentialError("gcp", f"request to {url} was refused (HTTP 401)") + err = _error_from(resp, url) + if ( + status in _RETRY_STATUSES + and attempt < self._max_retries + and gov.try_retry(scope, attempt + 1) + ): + attempt += 1 + retry_after = _retry_after(resp) + if status == 429: + gov.on_throttle(scope, retry_after, str(err)[:150]) + else: + gov.record_transient(scope, str(err)[:150]) + self._sleep(retry_after if retry_after is not None else min(2**attempt * 0.5, 30.0)) + continue + gov.on_final_error(scope, err, f"rest.{method}") + raise err + + def list( + self, + url: str, + key: str, + params: dict[str, Any] | None = None, + *, + max_items: int | None = None, + page_size: int | None = None, + page_size_param: str = "pageSize", + ) -> tuple[list[dict[str, Any]], bool]: + """Every item of a paged list (``nextPageToken`` / ``pageToken``). + + Returns: + (items, truncated): ``truncated`` is True when ``max_items`` cut the list. + """ + items: list[dict[str, Any]] = [] + query = dict(params or {}) + if page_size: + query[page_size_param] = page_size + token = None + for _ in range(10000): + if token: + query["pageToken"] = token + body = self.get(url, query) + page = body.get(key) or [] + for i, item in enumerate(page): + items.append(item if isinstance(item, dict) else {"name": item}) + if max_items is not None and len(items) >= max_items: + return items, i < len(page) - 1 or bool(body.get("nextPageToken")) + token = body.get("nextPageToken") + if not token: + break + return items, False + + +def _retry_after(resp: Any) -> float | None: + headers = getattr(resp, "headers", None) or {} + value = headers.get("Retry-After") if hasattr(headers, "get") else None + try: + return min(float(value), 60.0) if value is not None else None + except (TypeError, ValueError): + return None + + +__all__ = ["GCPApiError", "GCPRestClient"] diff --git a/cloudg/inventory/gcp_sources/security.py b/cloudg/inventory/gcp_sources/security.py new file mode 100644 index 0000000..cd15326 --- /dev/null +++ b/cloudg/inventory/gcp_sources/security.py @@ -0,0 +1,382 @@ +"""Security sources: Security Command Center, Artifact Analysis +vulnerability occurrences, Data Access audit log configs, the IAM +recommender and Sensitive Data Protection discovery coverage.""" + +from __future__ import annotations + +from typing import Any + +from cloudg.inventory.gcp_relations.names import image_repository +from cloudg.inventory.gcp_sources.base import SourceContext, SourceRun, source +from cloudg.schema.models import AssetType, CloudAsset + +_SCC = "https://securitycenter.googleapis.com" +_SCCM = "https://securitycentermanagement.googleapis.com/v1" +_CA = "https://containeranalysis.googleapis.com/v1" +_SU = "https://serviceusage.googleapis.com/v1" +_REC = "https://recommender.googleapis.com/v1" + +# SCC service id -> (asset type, label); the core services get placeholders +SCC_SERVICES: dict[str, tuple[AssetType, str]] = { + "event-threat-detection": (AssetType.THREAT_DETECTOR, "Event Threat Detection"), + "container-threat-detection": (AssetType.THREAT_DETECTOR, "Container Threat Detection"), + "vm-threat-detection": (AssetType.THREAT_DETECTOR, "Virtual Machine Threat Detection"), + "security-health-analytics": (AssetType.CONFIG_RECORDER, "Security Health Analytics"), + "web-security-scanner": (AssetType.VULNERABILITY_SCANNER, "Web Security Scanner"), +} +_SCC_TYPE = "securitycenter.googleapis.com/SecurityCenter" +_SVC_TYPE = "securitycentermanagement.googleapis.com/SecurityCenterService" + + +def _project(parent: str) -> str | None: + return parent.split("/", 1)[1] if parent.startswith("projects/") else None + + +def _scc_parent(ctx: SourceContext, run: SourceRun, parent: str) -> list[CloudAsset]: + """SCC activation, services, sources and exports at one parent.""" + project = _project(parent) + services_url = f"{_SCCM}/{parent}/locations/global/securityCenterServices" + before = (len(run.disabled), len(run.errors)) + services = ctx.items(run, services_url, "securityCenterServices", tolerate=False) + unreachable = (len(run.disabled), len(run.errors)) != before + active = [s for s in services if s.get("effectiveEnablementState") == "ENABLED"] + out: list[CloudAsset] = [] + if unreachable or not active: + if len(run.errors) > before[1]: + return out # permission problem: unknown, not "not enabled" + out.append( + ctx.placeholder( + "scc", parent, AssetType.SECURITY_HUB, "Security Command Center", gcp_type=_SCC_TYPE + ) + ) + for sid, (atype, label) in SCC_SERVICES.items(): + out.append(ctx.placeholder(sid, parent, atype, label, gcp_type=_SVC_TYPE)) + return out + tier = None + if parent.startswith("organizations/"): + sub = ctx.get(run, f"{_SCC}/v1beta2/{parent}/subscription", tolerate=True) or {} + tier = sub.get("tier") + hub = ctx.build( + f"//securitycenter.googleapis.com/{parent}/securityCenter", + _SCC_TYPE, + {"parent": parent, "tier": tier, "enabled": True, "displayName": "Security Command Center"}, + project=project, + source_name="scc", + global_resource=project is None, + ) + out.append(hub) + for svc in services: + name = svc.get("name") + if not isinstance(name, str): + continue + sid = name.rsplit("/", 1)[-1] + atype, label = SCC_SERVICES.get(sid, (AssetType.SECURITY_HUB, sid)) + enabled = svc.get("effectiveEnablementState") == "ENABLED" + asset = ctx.build( + f"//securitycentermanagement.googleapis.com/{name}", + _SVC_TYPE, + {**svc, "displayName": label if enabled else f"{label} (not enabled)"}, + project=project, + source_name="scc", + global_resource=project is None, + ) + if asset is not None: + asset.asset_type = atype + out.append(asset) + for key, kind in ( + ("sources", "securitycenter.googleapis.com/Source"), + ("notificationConfigs", "securitycenter.googleapis.com/NotificationConfig"), + ("bigQueryExports", "securitycenter.googleapis.com/BigQueryExport"), + ): + for item in ctx.items(run, f"{_SCC}/v1/{parent}/{key}", key, tolerate=key != "sources"): + name = item.get("name") + if isinstance(name, str): + out.append( + ctx.build( + f"//securitycenter.googleapis.com/{name}", + kind, + item, + project=project, + source_name="scc", + global_resource=project is None, + ) + ) + return [a for a in out if a is not None] + + +@source("scc", "security") +def security_command_center(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Security Command Center: tier, services (ETD, CTD, SHA, VMTD, WSS), sources, exports. + + Parents where SCC is not activated get "not enabled" placeholders. + """ + out: list[CloudAsset] = [] + for parent in ctx.parents(folders=False): + out += _scc_parent(ctx, run, parent) + return out + + +# --------------------------------------------------------------------------- +# Artifact Analysis +# --------------------------------------------------------------------------- + + +def _occurrence_summary(ctx: SourceContext, occurrences: list[dict[str, Any]]) -> dict[str, Any]: + images: dict[str, dict[str, Any]] = {} + for occ in occurrences: + uri = occ.get("resourceUri") + vuln = occ.get("vulnerability") or {} + if not isinstance(uri, str) or not uri: + continue + entry = images.setdefault(uri, {"severity_counts": {}, "fixable": 0, "cves": []}) + sev = str(vuln.get("effectiveSeverity") or vuln.get("severity") or "UNKNOWN").upper() + entry["severity_counts"][sev] = entry["severity_counts"].get(sev, 0) + 1 + if vuln.get("fixAvailable"): + entry["fixable"] += 1 + cve = vuln.get("shortDescription") or str(occ.get("noteName") or "").rsplit("/", 1)[-1] + if cve and cve not in entry["cves"] and len(entry["cves"]) < 50: + entry["cves"].append(cve) + for uri, entry in images.items(): + image = ctx.lookup(uri) or ctx.lookup(uri.removeprefix("https://")) + if image is not None: + entry["target"] = image.arn + image.metadata["vulnerabilities"] = dict(entry["severity_counts"]) + else: + entry["target"] = image_repository(uri.removeprefix("https://")) or uri + return images + + +@source("artifact_analysis", "security") +def artifact_analysis(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Artifact Analysis vulnerability occurrences per image, per project with registries.""" + registry_projects = { + a.account_id + for a in ctx.by_type( + "artifactregistry.googleapis.com/Repository", + "artifactregistry.googleapis.com/DockerImage", + ) + if a.account_id + } + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + if project not in registry_projects: + continue + svc = ctx.get( + run, + f"{_SU}/projects/{project}/services/containerscanning.googleapis.com", + tolerate=True, + ) + if svc and svc.get("state") == "DISABLED": + out.append( + ctx.placeholder( + "artifact-analysis", + f"projects/{project}", + AssetType.VULNERABILITY_SCANNER, + "Artifact Analysis vulnerability scanning", + gcp_type="containeranalysis.googleapis.com/VulnerabilitySummary", + ) + ) + continue + res = run.call( + ctx.rest.list, + f"{_CA}/projects/{project}/occurrences", + "occurrences", + {"filter": 'kind="VULNERABILITY"'}, + page_size=1000, + max_items=ctx.options.max_vulnerability_occurrences, + ) + if res is None: + continue + occurrences, truncated = res + data = { + "parent": f"projects/{project}", + "images": _occurrence_summary(ctx, occurrences), + "occurrence_count": len(occurrences), + "truncated": truncated, + "enabled": True, + "displayName": "Artifact Analysis vulnerabilities", + } + out.append( + ctx.build( + f"//containeranalysis.googleapis.com/projects/{project}/vulnerabilitySummary", + "containeranalysis.googleapis.com/VulnerabilitySummary", + data, + project=project, + source_name="artifact_analysis", + ) + ) + return [a for a in out if a is not None] + + +# --------------------------------------------------------------------------- +# Data Access audit logs (IAM policy auditConfigs, Resource Manager API) +# --------------------------------------------------------------------------- + +_LOG_TYPES = {0: "LOG_TYPE_UNSPECIFIED", 1: "ADMIN_READ", 2: "DATA_WRITE", 3: "DATA_READ"} +_CRM_V3 = "https://cloudresourcemanager.googleapis.com/v3" + + +def _g(d: Any, snake: str, camel: str) -> Any: + return d.get(snake, d.get(camel)) if isinstance(d, dict) else None + + +def audit_summary(policy: Any) -> dict[str, Any]: + """Services with Data Access logging, exemptions and whether ``allServices`` is on.""" + services: list[dict[str, Any]] = [] + exempt: set[str] = set() + for cfg in _g(policy, "audit_configs", "auditConfigs") or []: + if not isinstance(cfg, dict): + continue + types = [] + for lc in _g(cfg, "audit_log_configs", "auditLogConfigs") or []: + lt = _g(lc, "log_type", "logType") + lt = _LOG_TYPES.get(lt, str(lt)) if isinstance(lt, int) else str(lt) + types.append(lt) + exempt |= set(_g(lc, "exempted_members", "exemptedMembers") or []) + services.append({"service": cfg.get("service"), "log_types": types}) + data_access = [s for s in services if {"DATA_READ", "DATA_WRITE"} & set(s["log_types"])] + return { + "services": services, + "allServices": any(s["service"] == "allServices" for s in data_access), + "enabled": bool(data_access), + "exemptedMembers": sorted(exempt), + } + + +@source("audit_configs", "logging") +def audit_configs(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """Data Access audit log configuration of the organization, folders and projects. + + Admin Activity logs are always on; Data Access logs only where an IAM + policy ``auditConfigs`` entry turns them on. Parents without one get a + "not enabled" asset, like an AWS account without a CloudTrail trail. + """ + out: list[CloudAsset] = [] + for parent in ctx.parents(): + policy = run.call( + ctx.rest.post, + f"{_CRM_V3}/{parent}:getIamPolicy", + {"options": {"requestedPolicyVersion": 3}}, + ) + if policy is None: + continue + summary = audit_summary(policy) + name = f"//cloudresourcemanager.googleapis.com/{parent}" + target = ctx.lookup(name) + if target is not None: + target.metadata["data_access_audit_logs"] = summary["enabled"] + project = parent.split("/", 1)[1] if parent.startswith("projects/") else None + label = "Data Access audit logs" + data = { + **summary, + "resource": name, + "displayName": f"{label} @ {parent}" + ("" if summary["enabled"] else " (not enabled)"), + } + asset = ctx.build( + f"//logging.googleapis.com/{parent}/auditConfig", + "logging.googleapis.com/AuditConfig", + data, + project=project, + source_name="audit_configs", + global_resource=project is None, + ) + if asset is None: + continue + if not summary["enabled"]: + asset.arn = f"cloudg:gcp:audit-logs:{parent}:not-enabled" + asset.metadata["status"] = "not enabled" + out.append(asset) + return out + + +# --------------------------------------------------------------------------- +# IAM recommender +# --------------------------------------------------------------------------- + + +def _recommendation(rec: dict[str, Any]) -> list[dict[str, Any]]: + """(member, role, resource, action) of each operation of an IAM recommendation.""" + out = [] + for group in (rec.get("content") or {}).get("operationGroups") or []: + for op in (group or {}).get("operations") or []: + filters = op.get("pathFilters") or {} + member = filters.get("/iamPolicy/bindings/*/members/*") + role = filters.get("/iamPolicy/bindings/*/role") + if member or role: + out.append( + { + "member": member, + "role": role, + "resource": op.get("resource"), + "action": op.get("action"), + "priority": rec.get("priority"), + "description": rec.get("description"), + } + ) + return out + + +@source("iam_recommender", "security") +def iam_recommender(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """IAM role recommendations (excess permissions) per project.""" + out: list[CloudAsset] = [] + for project, _number in ctx.projects(): + recs = ctx.items( + run, + f"{_REC}/projects/{project}/locations/global/recommenders/google.iam.policy.Recommender/recommendations", + "recommendations", + page_size=500, + ) + active = [r for r in recs if (r.get("stateInfo") or {}).get("state") in (None, "ACTIVE")] + ops = [op for r in active for op in _recommendation(r)] + if not ops: + continue + out.append( + ctx.build( + f"//recommender.googleapis.com/projects/{project}/iamPolicyRecommender", + "recommender.googleapis.com/IamPolicyRecommender", + { + "parent": f"projects/{project}", + "recommendations": ops, + "displayName": "IAM recommender", + }, + project=project, + source_name="iam_recommender", + ) + ) + return [a for a in out if a is not None] + + +# --------------------------------------------------------------------------- +# Sensitive Data Protection discovery coverage +# --------------------------------------------------------------------------- + + +@source("dlp_discovery", "security", phase=3, needs_rest=False) +def dlp_discovery_coverage(ctx: SourceContext, run: SourceRun) -> list[CloudAsset]: + """A "not enabled" placeholder per project no Sensitive Data Protection discovery covers.""" + configs = ctx.by_type("dlp.googleapis.com/DiscoveryConfig") + covered: set[str] = set() + org_wide = False + for cfg in configs: + if not cfg.metadata.get("enabled", True): + continue + scope = str(cfg.metadata.get("scope") or "") + if scope.startswith(("organizations/", "folders/")): + org_wide = True + if cfg.account_id: + covered.add(cfg.account_id) + run.ok += 1 + if org_wide: + return [] + return [ + ctx.placeholder( + "dlp-discovery", + f"projects/{project}", + AssetType.DATA_SECURITY_SCANNER, + "Sensitive Data Protection discovery", + gcp_type="dlp.googleapis.com/DiscoveryConfig", + ) + for project, _ in ctx.projects() + if project not in covered + ] diff --git a/cloudg/inventory/gcp_tasks.py b/cloudg/inventory/gcp_tasks.py new file mode 100644 index 0000000..556f8a7 --- /dev/null +++ b/cloudg/inventory/gcp_tasks.py @@ -0,0 +1,96 @@ +"""Service families of the deep GCP inventory collector. + +GCP has no per-service collectors to pick from: Cloud Asset Inventory +returns every type in one listing. ``inventory.services`` / +``inventory.exclude_services`` (``--services`` / ``--exclude-services``) +therefore filter at two levels, the same way +:func:`cloudg.inventory.aws_deep_tasks.select_tasks` filters AWS tasks: + +- listed asset types, by their API service (``compute``, ``pubsub``, ...) + or by family (``network``, ``data``, ...; ``catalogs/gcp_asset_types.yaml`` + ``families`` and ``type_families``); +- the API-backed sources of :mod:`cloudg.inventory.gcp_sources`, by source + name (``dns_records``, ``scc``, ...) or by family. + +``include`` defaults to everything; ``exclude`` always wins. The +``hierarchy`` family (organization, folders, projects) is kept unless it is +excluded by name, so a filtered map still has its projects. +""" + +from __future__ import annotations + +from typing import Iterable + +from cloudg.inventory.catalogs import load_catalog + +_CATALOG = load_catalog("gcp_asset_types") +GCP_SERVICE_FAMILIES: dict[str, str] = { + str(k): str(v) for k, v in (_CATALOG.get("families") or {}).items() +} +GCP_TYPE_FAMILIES: dict[str, str] = { + str(k): str(v) for k, v in (_CATALOG.get("type_families") or {}).items() +} +ALWAYS_KEPT = frozenset({"hierarchy"}) + + +def gcp_service(asset_type: str) -> str: + """API service of a CAI asset type: ``compute.googleapis.com/Disk`` -> ``compute``, + ``k8s.io/Pod`` -> ``k8s.io``.""" + service = asset_type.split("/", 1)[0] + return service.removesuffix(".googleapis.com") + + +def gcp_family(asset_type: str) -> str: + """Service family of a CAI asset type (``other`` when the catalog has none).""" + return GCP_TYPE_FAMILIES.get(asset_type) or GCP_SERVICE_FAMILIES.get( + gcp_service(asset_type), "other" + ) + + +def is_selected( + name: str, + family: str, + include: Iterable[str] | None = None, + exclude: Iterable[str] | None = None, +) -> bool: + """True when ``name`` (a service or source) of ``family`` passes the filters.""" + inc = {i.lower() for i in (include or ["all"])} + exc = {e.lower() for e in (exclude or [])} + name, family = name.lower(), family.lower() + if name in exc or family in exc: + return False + return "all" in inc or name in inc or family in inc or family in ALWAYS_KEPT + + +def type_selected( + asset_type: str, include: Iterable[str] | None = None, exclude: Iterable[str] | None = None +) -> bool: + """Filter for one CAI asset type (by its service, family or the full type name).""" + exc = {e.lower() for e in (exclude or [])} + if asset_type.lower() in exc: + return False + inc = {i.lower() for i in (include or ["all"])} + if asset_type.lower() in inc: + return True + return is_selected(gcp_service(asset_type), gcp_family(asset_type), include, exclude) + + +def families() -> list[str]: + """Every family name the catalog and the API sources use.""" + from cloudg.inventory.gcp_sources import SOURCES + + names = set(GCP_SERVICE_FAMILIES.values()) | set(GCP_TYPE_FAMILIES.values()) + names |= {s.family for s in SOURCES.values()} + return sorted(names | {"other"}) + + +__all__ = [ + "ALWAYS_KEPT", + "GCP_SERVICE_FAMILIES", + "GCP_TYPE_FAMILIES", + "families", + "gcp_family", + "gcp_service", + "is_selected", + "type_selected", +] diff --git a/cloudg/inventory/kubernetes.py b/cloudg/inventory/kubernetes.py index f6f4ec5..a79e493 100644 --- a/cloudg/inventory/kubernetes.py +++ b/cloudg/inventory/kubernetes.py @@ -1,4 +1,4 @@ -"""Read-only Kubernetes workload mapping for EKS clusters. +"""Read-only Kubernetes workload mapping for EKS and AKS clusters. Authenticates exactly like ``aws eks get-token``: a presigned STS ``GetCallerIdentity`` URL carrying the ``x-k8s-aws-id`` header, sent as a @@ -18,6 +18,11 @@ ``LoadBalancer`` services -> the AWS load balancer by DNS name (ROUTE) - Ingresses -> backend services, and -> the ALB by DNS name - ServiceAccounts -> IAM roles through the IRSA annotation (ASSUMES_ROLE) + +AKS clusters go through the same mapper (``provider=CloudProvider.AZURE``, +see :mod:`cloudg.inventory.azure_kubernetes`): service accounts resolve to +user-assigned identities through the Entra Workload ID client-id +annotation, and load balancer IPs resolve to public IP assets. """ from __future__ import annotations @@ -43,6 +48,8 @@ "CronJob": "/apis/batch/v1/cronjobs", } _IRSA_ANNOTATION = "eks.amazonaws.com/role-arn" +_AZURE_WI_ANNOTATION = "azure.workload.identity/client-id" +_AZURE_WI_TENANT_ANNOTATION = "azure.workload.identity/tenant-id" _PAGE_LIMIT = 500 _MAX_OBJECTS = 5000 @@ -139,9 +146,13 @@ def _pod_labels(kind: str, obj: dict[str, Any]) -> dict[str, str]: return (tmpl.get("metadata") or {}).get("labels") or {} -def _lb_hostnames(obj: dict[str, Any]) -> list[str]: +def _lb_hostnames(obj: dict[str, Any], with_ips: bool = False) -> list[str]: + """Load balancer host names of a Service / Ingress status (plus IPs on AKS).""" ingress = ((obj.get("status") or {}).get("loadBalancer") or {}).get("ingress") or [] - return [i.get("hostname") for i in ingress if i.get("hostname")] + out = [i.get("hostname") for i in ingress if i.get("hostname")] + if with_ips: + out += [i.get("ip") for i in ingress if i.get("ip") and not i.get("hostname")] + return out def _ingress_backends(spec: dict[str, Any]) -> set[str]: @@ -162,12 +173,18 @@ class _ClusterMapper: """Maps one cluster's objects into assets, in listing order.""" def __init__( - self, reader: KubernetesReader, cluster_arn: str, region: str, account_id: str | None + self, + reader: KubernetesReader, + cluster_arn: str, + region: str, + account_id: str | None, + provider: CloudProvider = CloudProvider.AWS, ) -> None: self._reader = reader self._cluster_arn = cluster_arn self._region = region self._account_id = account_id + self._provider = provider self.assets: list[CloudAsset] = [] # ns, kind, name, pod labels, matched against service selectors self._workloads: list[tuple[str, str, str, dict[str, str]]] = [] @@ -193,7 +210,7 @@ def _add( arn=k8s_identifier(self._cluster_arn, namespace, kind, name), name=f"{namespace}/{name}" if namespace else name, asset_type=asset_type, - provider=CloudProvider.AWS, + provider=self._provider, region=self._region, account_id=self._account_id, tags={k: str(v) for k, v in (labels or {}).items()}, @@ -231,7 +248,11 @@ def map_namespaces(self) -> None: def map_service_accounts(self) -> None: for sa in self._reader.list("/api/v1/serviceaccounts"): meta = sa["metadata"] - role = (meta.get("annotations") or {}).get(_IRSA_ANNOTATION) + annotations = meta.get("annotations") or {} + if self._provider == CloudProvider.AZURE: + self._map_azure_service_account(meta, annotations) + continue + role = annotations.get(_IRSA_ANNOTATION) self._add( "ServiceAccount", (meta.get("namespace", ""), meta["name"]), @@ -243,6 +264,22 @@ def map_service_accounts(self) -> None: ], ) + def _map_azure_service_account(self, meta: dict[str, Any], annotations: dict) -> None: + """AKS service account; Entra Workload ID annotations name the identity it uses.""" + client_id = annotations.get(_AZURE_WI_ANNOTATION) + tenant = annotations.get(_AZURE_WI_TENANT_ANNOTATION) + ident = f"azure-client-id:{str(client_id).lower()}" if client_id else None + self._add( + "ServiceAccount", + (meta.get("namespace", ""), meta["name"]), + AssetType.K8S_SERVICE_ACCOUNT, + {"workload_identity_client_id": client_id, "workload_identity_tenant_id": tenant}, + [ + self._ns_rel(meta.get("namespace", "")), + rel(ident, EdgeType.ASSUMES_ROLE, "RUNS_ON", description="Entra Workload ID"), + ], + ) + def map_workloads(self) -> None: for kind, path in _WORKLOAD_KINDS.items(): try: @@ -320,7 +357,7 @@ def map_services(self) -> None: relations = [self._ns_rel(ns)] if selector: relations += self._selected_workloads(ns, selector) - hostnames = _lb_hostnames(svc) + hostnames = _lb_hostnames(svc, self._provider == CloudProvider.AZURE) relations += [ rel( host, @@ -332,9 +369,10 @@ def map_services(self) -> None: for host in hostnames ] svc_type = spec.get("type", "ClusterIP") - scheme = (meta.get("annotations") or {}).get( - "service.beta.kubernetes.io/aws-load-balancer-scheme" - ) + annotations = meta.get("annotations") or {} + scheme = annotations.get("service.beta.kubernetes.io/aws-load-balancer-scheme") + if annotations.get("service.beta.kubernetes.io/azure-load-balancer-internal") == "true": + scheme = "internal" self._add( "Service", (ns, meta["name"]), @@ -373,7 +411,7 @@ def _map_ingress(self, ing: dict[str, Any]) -> None: ) for b in backends ] - hostnames = _lb_hostnames(ing) + hostnames = _lb_hostnames(ing, self._provider == CloudProvider.AZURE) relations += [rel(h, EdgeType.ROUTE, "LOAD_BALANCED_BY", reverse=True) for h in hostnames] self._add( "Ingress", @@ -397,9 +435,10 @@ def map_cluster_objects( cluster_arn: str, region: str, account_id: str | None, + provider: CloudProvider = CloudProvider.AWS, ) -> list[CloudAsset]: """Read namespaces, workloads, services, ingresses and service accounts.""" - mapper = _ClusterMapper(reader, cluster_arn, region, account_id) + mapper = _ClusterMapper(reader, cluster_arn, region, account_id, provider) mapper.map_namespaces() mapper.map_service_accounts() mapper.map_workloads() diff --git a/cloudg/inventory/linker_index.py b/cloudg/inventory/linker_index.py index 3daf023..84c3391 100644 --- a/cloudg/inventory/linker_index.py +++ b/cloudg/inventory/linker_index.py @@ -34,6 +34,7 @@ "cloudg:", "aws-security:", "azure-security:", + "gcp-security:", # GCP parity "entra:", "loganalytics:", "gcp-principal:", diff --git a/cloudg/inventory/mapper.py b/cloudg/inventory/mapper.py index a48c9c2..6d8f936 100644 --- a/cloudg/inventory/mapper.py +++ b/cloudg/inventory/mapper.py @@ -292,12 +292,50 @@ def __init__( session, region, account_id, is_primary_region=is_primary_region, **options ) + # GCP parity: bind service filtering and the API-source options + from cloudg.inventory.gcp_deep import deep_options_from_config + + gcp_options = deep_options_from_config(inv, getattr(self._config, "gcp", None)) + + class _ConfiguredGCP(GCPDeepInventoryCollector): + def __init__( + self, project_id: str | None, credentials: Any = None, **kwargs: Any + ) -> None: + super().__init__(project_id, credentials, **{**gcp_options, **kwargs}) + + azure_options = self._azure_options() + + # Same idea for Azure: the (credential, subscription_id) signature + # with the depth options and the inventory.services filter bound. + class _ConfiguredAzure(AzureDeepInventoryCollector): + def __init__(self, credential: Any, subscription_id: str) -> None: + super().__init__(credential, subscription_id, options=azure_options) + return { "aws": _ConfiguredAWS, - "azure": AzureDeepInventoryCollector, - "gcp": GCPDeepInventoryCollector, + "azure": _ConfiguredAzure, + "gcp": _ConfiguredGCP, } + def _azure_options(self) -> Any: + """AzureDeepOptions from ``inventory`` and ``azure`` configuration.""" + from cloudg.inventory.azure_deep import AzureDeepOptions + + inv = self._inventory + az = getattr(self._config, "azure", None) + return AzureDeepOptions( + services=list(inv.services) if inv is not None else None, + exclude_services=list(inv.exclude_services) if inv is not None else None, + kubernetes=inv.kubernetes if inv is not None else True, + kubernetes_timeout=inv.kubernetes_timeout if inv is not None else 10, + child_resources=getattr(az, "child_resources", True), + max_child_parents=getattr(az, "max_child_parents", 500), + diagnostic_settings=getattr(az, "diagnostic_settings", True), + max_diagnostic_settings=getattr(az, "max_diagnostic_settings", 1000), + data_plane=getattr(az, "data_plane", True), + rest_concurrency=getattr(az, "rest_concurrency", 8), + ) + # ------------------------------------------------------------------ # Organizations / Control Tower # ------------------------------------------------------------------ @@ -373,6 +411,52 @@ async def _discover_azure_hierarchy( cov.record("management_groups", ServiceStatus.FAILED, error=str(exc)) return [] + async def _discover_entra( + self, cfg: CloudGConfig, coverage: list[CollectionCoverage] + ) -> list[CloudAsset]: + """Entra ID (Microsoft Graph), once per run; per-listing coverage records.""" + from cloudg.inventory.azure_graph.families import select_azure_tasks + + inv = self._inventory + cov = CollectionCoverage(provider="azure", region="global") + coverage.append(cov) + if not select_azure_tasks( + ["entra"], + list(inv.services) if inv is not None else None, + list(inv.exclude_services) if inv is not None else None, + ): + cov.record("entra", ServiceStatus.SKIPPED, error="not selected (inventory.services)") + return [] + t0 = time.time() + try: + from cloudg.credentials import build_azure_credential + from cloudg.inventory.azure_entra import discover_entra + + credential = build_azure_credential(cfg.azure) + found = await asyncio.to_thread( + discover_entra, + credential, + None, + coverage=cov, + max_objects=getattr(cfg.azure, "entra_max_objects", 10000), + max_expansions=getattr(cfg.azure, "entra_max_expansions", 1000), + ) + cov.record( + "entra", + ServiceStatus.PARTIAL + if any(s.status == ServiceStatus.FAILED for s in cov.services) + else ServiceStatus.SUCCESS, + asset_count=len(found), + duration_ms=int((time.time() - t0) * 1000), + ) + return found + except ImportError as exc: + cov.record("entra", ServiceStatus.SKIPPED, error=str(exc)) + except Exception as exc: + logger.error("Entra ID discovery failed: %s", exc) + cov.record("entra", ServiceStatus.FAILED, error=str(exc)) + return [] + async def _discover_gcp_hierarchy( self, cfg: CloudGConfig, coverage: list[CollectionCoverage] ) -> list[CloudAsset]: @@ -416,6 +500,8 @@ async def _discover_hierarchies( out: list[CloudAsset] = [] if "azure" in cfg.providers and getattr(cfg.azure, "map_management_groups", False): out.extend(await self._discover_azure_hierarchy(cfg, coverage)) + if "azure" in cfg.providers and getattr(cfg.azure, "map_entra", False): + out.extend(await self._discover_entra(cfg, coverage)) gcp = cfg.gcp if "gcp" in cfg.providers and gcp.organization_id and getattr(gcp, "map_hierarchy", False): out.extend(await self._discover_gcp_hierarchy(cfg, coverage)) diff --git a/cloudg/inventory/organization.py b/cloudg/inventory/organization.py index cafde4c..73484a9 100644 --- a/cloudg/inventory/organization.py +++ b/cloudg/inventory/organization.py @@ -25,6 +25,7 @@ from typing import Any # account_ref is re-exported: it is part of this module's public API. +from cloudg.exceptions import OrganizationScopeError from cloudg.inventory.organization_assets import account_ref, topology_assets from cloudg.schema.models import CloudAsset @@ -33,13 +34,6 @@ DEFAULT_MEMBER_ROLE = "AWSControlTowerExecution" -class OrganizationScopeError(ValueError): - """The requested organization scope (an OU filter) names nothing in the org. - - Raised instead of quietly mapping only the caller's account. - """ - - _POLICY_TYPES = [ "SERVICE_CONTROL_POLICY", "RESOURCE_CONTROL_POLICY", diff --git a/cloudg/renderers/html_report.py b/cloudg/renderers/html_report.py index b894d89..a64a8b0 100644 --- a/cloudg/renderers/html_report.py +++ b/cloudg/renderers/html_report.py @@ -10,6 +10,7 @@ from jinja2 import Environment, FileSystemLoader +from cloudg.renderers.iso_scene import build_scene from cloudg.schema.models import ScanResult logger = logging.getLogger(__name__) @@ -43,6 +44,24 @@ } +# The isometric map's drawing engines and the Hairline figures built on them, +# as paths under the templates directory. They have no CDN build, so they are +# always inlined; without them the report shows the 2D map only. +MAP_ENGINES: dict[str, str] = { + "isokit": "vendor/iso-kit.js", # Anatomy iso-kit (MIT): camera, solids, shading + "hairline": "vendor/hairline-kernel.js", # Hairline kernel (MIT): line figures, springs + "figures": "figures/empty-tray.js", # cloudg's empty-state tray, a Hairline figure +} + +# Figures end with hairline({...}); the report collects them instead of mounting them +_FIGURE_HOST = ( + "var hairline = function (spec) {" + " (window.HAIRLINE_FIGURES = window.HAIRLINE_FIGURES || {})[spec.name] = spec; };\n" +) + +_SEVERITY_RANK = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} + + def script_json(value: Any) -> str: """JSON for embedding in an HTML `` + + +{% endif %} - - -
- -
-
-

☁️ CloudG Security Report

-

Scan: {{ scan_id }} · Provider: {{ provider }} · Account: {{ account_id }} · {{ started_at }}

-
- -
+ - -
-
Total Assets
{{ total_assets }}
-
Total Findings
{{ total_findings }}
-
Critical
{{ severity_counts.CRITICAL }}
-
High
{{ severity_counts.HIGH }}
-
Medium
{{ severity_counts.MEDIUM }}
-
Low
{{ severity_counts.LOW }}
+
+
+ + cloudgSecurity report
- - -
- - - - -
- - -
-
-
- - - - +
+ + + + +
+ +
+
+
+
+
Security report · {{ provider }}
+

Estate posture

+
+

{{ total_findings }} findings across {{ total_assets }} assets.

+
+ Scan {{ scan_id }} + Account: {{ account_id }} + {% if summary.providers %}Providers {{ summary.providers|join(', ') }}{% endif %} + {% if summary.regions %}Regions {{ summary.regions|length }}{% endif %} + Started {{ started_at }}
-
-
Legend
-
Account
-
Region
-
VPC
-
Subnet
-
Compute
-
Storage
-
Security
-
Other
-
Has Findings
-
Internet Exposed
-
Ontology Entities
-
Security Finding (▲)
-
Compliance Framework (◆)
-
─ ─ FINDING_AFFECTS
-
─ ─ COMPLIANCE_GOVERNS
+
+
Assets
{{ total_assets }}
{{ summary.edges }} relations
+
Findings
{{ total_findings }}
{{ source_tools|length }} sources
+
Critical
{{ severity_counts.CRITICAL }}
High {{ severity_counts.HIGH }}
+
Exposed
{{ summary.exposed }}
reachable from the internet
+
+
+ +
-
+
+
+ + + + 0% + +
+

Assets with no open findings

+

{{ summary.clean }} of {{ total_assets }} assets have nothing open against them. The rest carry at least one finding.

+
+
+
+
Top risks
+
    +
    +
    +
    + + + + +
    +
    +
    +
    + + +
    + + + + + + + + + + + +
    +
    +
    +
    + Fig 1 + Estate map + Drag to pan · scroll to zoom · click an asset + rest + +
    +
    + +
    +
    +
    Each provider is a base plate. Accounts and regions stand on it as platforms, networks as slabs and subnets as tiles; every asset stands where it was found. Identity and governance assets sit on the control deck behind and above the plate. Trace lines follow each relation from its source port to its target port on a lane of their own, and a beacon above an asset shows its worst open finding.
    +
    +
    -
    -
    - + - + - + +
    - - - - - - - - - - - -
    SeverityTitleResourceSourceRisk Score
    -
    +
    + + + + + +
    SeverityFindingResourceSourceRisk
    + +
    + + + +
    +
    + + + +
    +
    + + + + + +
    AssetTypeProviderRegionFindings
    +
    +
    -
    +
    -
    -

    Severity Distribution

    - -
    -
    -

    Resource Types

    - -
    +

    Severity distribution {{ total_findings }} findings

    +

    Resource types top 12

    -
    -

    Findings by Source

    - -
    -
    -

    Compliance Overview

    - -
    +

    Findings by source {{ source_tools|length }} sources

    +

    Compliance overview pass and fail

    -
    + -
    +
    -
    -
    + + + +
    + Generated by cloudg {{ summary.version }} · scan {{ scan_id }} + Map drawn with Anatomy and Hairline (MIT) · charts by Chart.js (MIT) and D3 (ISC) +
    + - -
    - +
    + @@ -352,29 +528,32 @@ const graphData = {{ graph_json|safe }}; const hierarchyData = {{ hierarchy_json|safe }}; const findingsData = {{ findings_json|safe }}; + const assetsData = {{ assets_json|safe }}; const findingsPerResource = {{ findings_per_resource_json|safe }}; const severityCounts = {{ severity_counts_json|safe }}; const resourceTypes = {{ resource_types_json|safe }}; const sourceTools = {{ source_tools_json|safe }}; const complianceSummary = {{ compliance_summary_json|safe }}; + const reportSummary = {{ summary_json|safe }}; + const topRisks = {{ top_risks_json|safe }}; + const isoScene = {{ scene_json|safe }}; let currentLayout = 'tree'; // 'tree' or 'force' // ── Tab Switching ── function switchTab(tabId) { document.querySelectorAll('.tab-content').forEach(el => el.classList.remove('active')); - document.querySelectorAll('.tab').forEach(el => el.classList.remove('active')); + document.querySelectorAll('.tab').forEach(el => el.classList.toggle('active', el.dataset.tab === tabId)); document.getElementById('tab-' + tabId).classList.add('active'); - event.target.classList.add('active'); - if (tabId === 'topology') { - setTimeout(() => renderTopology(), 50); - } + if (tabId === 'topology') setTimeout(() => renderTopology(), 30); + if (tabId === 'charts') setTimeout(() => renderCharts(), 30); } // ── HTML escaping ── // Finding, asset and compliance values come from scanner output and // cloud metadata. Markup is only ever built with the html`` tag below, - // which escapes every interpolated value. + // which escapes every interpolated value. The map builds its SVG with + // createElementNS and textContent, so it never parses markup at all. function escapeHtml(value) { if (value === null || value === undefined) return ''; return String(value) @@ -395,6 +574,16 @@ function severityClass(severity) { return 'severity-' + (SEVERITIES.includes(severity) ? severity : 'INFO'); } + function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + } + + // Asset names for resource columns, by id and by ARN + const assetById = new Map(); + assetsData.forEach(a => { assetById.set(a.id, a); if (a.arn) assetById.set(a.arn, a); }); + function assetOf(f) { + return assetById.get(f.resource_id) || assetById.get(f.resource_arn) || null; + } // ── Findings Table ── // Rows keep the finding's position in findingsData, so the detail panel @@ -403,22 +592,24 @@ function renderFindings(data) { const tbody = document.getElementById('findings-body'); - tbody.innerHTML = data.map(f => html` + tbody.innerHTML = data.map(f => { + const asset = assetOf(f); + return html` ${f.severity} - ${f.title} - ${f.resource_arn || f.resource_id || ''} - ${f.source_tool} - ${f.risk_score || '-'} +
    ${f.title}
    + ${asset ? asset.name : ''}
    ${f.resource_arn || f.resource_id || ''}
    + ${f.source_tool} + ${f.risk_score || '-'} - `).join(''); + `; + }).join(''); + const count = document.getElementById('findings-count'); + count.textContent = data.length + ' of ' + findingsData.length; + const empty = document.getElementById('findings-empty'); + empty.hidden = data.length > 0; } - document.getElementById('findings-body').addEventListener('click', (e) => { - const row = e.target.closest('tr[data-index]'); - if (row) showDetail(Number(row.dataset.index)); - }); - function filterFindings() { const severity = document.getElementById('filter-severity').value; const source = document.getElementById('filter-source').value; @@ -429,219 +620,840 @@ if (search) filtered = filtered.filter(f => String(f.title || '').toLowerCase().includes(search) || String(f.resource_arn || '').toLowerCase().includes(search) || + String(f.resource_id || '').toLowerCase().includes(search) || String(f.description || '').toLowerCase().includes(search) ); renderFindings(filtered); } - // Populate source filter - const sourceSelect = document.getElementById('filter-source'); - Object.keys(sourceTools).forEach(tool => { - const opt = document.createElement('option'); - opt.value = tool; opt.textContent = tool; - sourceSelect.appendChild(opt); - }); - // ── Detail Panel ── function showDetail(index) { const f = findingsData[index]; if (!f) return; const panel = document.getElementById('detail-panel'); const content = document.getElementById('detail-content'); + const asset = assetOf(f); content.innerHTML = html` + ${f.severity}

    ${f.title}

    -
    Severity
    ${f.severity}
    -
    Resource
    ${f.resource_arn || f.resource_id || 'N/A'}
    +
    Resource
    ${asset ? asset.name : ''} ${f.resource_arn || f.resource_id || 'N/A'}
    Description
    ${f.description || 'N/A'}
    Evidence
    ${f.evidence || 'N/A'}
    Remediation
    ${f.remediation || 'N/A'}
    -
    Source Tool
    ${f.source_tool}
    -
    CVSS Score
    ${f.cvss_score || 'N/A'}
    -
    Compliance Frameworks
    ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
    -
    Risk Score
    ${f.risk_score || 'N/A'}
    +
    Source tool
    ${f.source_tool}
    +
    CVSS score
    ${f.cvss_score || 'N/A'}
    +
    Compliance frameworks
    ${(f.compliance_frameworks || []).join(', ') || 'N/A'}
    +
    Risk score
    ${f.risk_score || 'N/A'}
    `; panel.classList.add('open'); + document.getElementById('scrim').classList.add('open'); + } + function closeDetail() { + document.getElementById('detail-panel').classList.remove('open'); + document.getElementById('scrim').classList.remove('open'); + } + + // ── Compliance Grid ── + function renderCompliance() { + const grid = document.getElementById('compliance-grid'); + const entries = Object.entries(complianceSummary); + grid.innerHTML = entries.map(([fw, data]) => { + const pass = Number(data.pass) || 0; + const fail = Number(data.fail) || 0; + const total = pass + fail; + const pct = total > 0 ? Math.round((pass / total) * 100) : 0; + return html` +
    +

    ${fw}

    +
    ${pct}%
    +
    +
    ${pass} pass${fail} fail${total} controls
    +
    + `; + }).join(''); + document.getElementById('compliance-empty').hidden = entries.length > 0; + } + + // ── Overview ── + function renderOverview() { + const total = SEVERITIES.reduce((n, s) => n + (severityCounts[s] || 0), 0); + const bar = document.getElementById('sevbar'); + bar.innerHTML = SEVERITIES.filter(s => severityCounts[s]).map(s => + html``).join(''); + document.getElementById('sevlegend').innerHTML = SEVERITIES.map(s => + html`${s.charAt(0) + s.slice(1).toLowerCase()} ${severityCounts[s] || 0}`).join(''); + const assets = assetsData.length; + const pct = assets ? Math.round((reportSummary.clean / assets) * 100) : 100; + const ring = document.getElementById('ring-val'); + ring.style.strokeDashoffset = String(251.3 * (1 - pct / 100)); + if (pct < 50) ring.style.stroke = cssVar('--high'); + document.getElementById('ring-text').textContent = pct + '%'; + const worst = SEVERITIES.find(s => severityCounts[s]); + const providers = reportSummary.providers.length; + document.getElementById('lede').innerHTML = html`${total} findings across ${assets} assets in ${providers} ${providers === 1 ? 'provider' : 'providers'}. ${worst ? 'The worst open finding is ' + worst.toLowerCase() + '.' : 'Nothing is open.'} ${reportSummary.exposed} ${reportSummary.exposed === 1 ? 'asset is' : 'assets are'} reachable from the internet.`; + const list = document.getElementById('top-risks'); + list.innerHTML = topRisks.length ? topRisks.map(i => { + const f = findingsData[i]; + const asset = assetOf(f); + return html`
    • `; + }).join('') : html`
  • No open findings.
    • `; + } + + // ── Assets Table ── + function renderAssets() { + const provider = document.getElementById('asset-provider').value; + const search = document.getElementById('asset-search').value.toLowerCase(); + let rows = assetsData; + if (provider) rows = rows.filter(a => a.provider === provider); + if (search) rows = rows.filter(a => [a.name, a.asset_type, a.region, a.arn, a.id, a.account_id] + .some(v => String(v || '').toLowerCase().includes(search))); + const shown = rows.slice(0, 2000); + document.getElementById('assets-body').innerHTML = shown.map(a => { + const counts = findingsPerResource[a.id] || findingsPerResource[a.arn || ''] || null; + const worst = counts ? SEVERITIES.find(s => counts[s]) : null; + // Escaped fragments are joined, never nested: html`` would escape them again + const exposed = a.is_internet_exposed ? html` exposed` : ''; + const badge = worst ? html`${counts.total}` : html`none`; + return html` +
    ${a.name}
    ${a.arn || a.id}
    + ${a.asset_type}` + exposed + html` + ${a.provider} + ${a.region} + ` + badge + html` + `; + }).join(''); + document.getElementById('assets-count').textContent = rows.length + ' of ' + assetsData.length + (rows.length > shown.length ? ' (first 2000 shown)' : ''); } - function closeDetail() { document.getElementById('detail-panel').classList.remove('open'); } - - // ── Hierarchical Topology ── - const typeColors = { - account: '#6366F1', region: '#22C55E', vpc: '#4A90D9', vnet: '#4A90D9', - subnet: '#7EC8E3', global: '#8B5CF6', - EC2: '#FF9900', VIRTUAL_MACHINE: '#FF9900', GCE_INSTANCE: '#FF9900', - VPC: '#4A90D9', VNET: '#4A90D9', SUBNET: '#7EC8E3', - RDS_INSTANCE: '#3B48CC', AURORA_CLUSTER: '#3B48CC', AZURE_SQL: '#3B48CC', CLOUD_SQL: '#3B48CC', - S3_BUCKET: '#569A31', BLOB_STORAGE: '#569A31', GCS_BUCKET: '#569A31', - SECURITY_GROUP: '#DD4B39', NSG: '#DD4B39', NACL: '#DD4B39', - LOAD_BALANCER: '#8C4FFF', CLOUDFRONT: '#8C4FFF', - IAM_USER: '#F472B6', IAM_ROLE: '#F472B6', IAM_POLICY: '#F472B6', - LAMBDA_FUNCTION: '#FF9900', CLOUD_FUNCTION: '#FF9900', - KMS_KEY: '#14B8A6', SECRET: '#14B8A6', - EBS_VOLUME: '#569A31', DYNAMODB_TABLE: '#3B48CC', - ECS_CLUSTER: '#FF9900', EKS_CLUSTER: '#FF9900', - SECURITY_FINDING: '#EF4444', COMPLIANCE_FRAMEWORK: '#14B8A6', - }; - const typeRadius = { - account: 22, region: 18, vpc: 16, vnet: 16, subnet: 14, global: 18, + // ── Isometric map ── + // The scene (cloudg/renderers/iso_scene.py) holds every position in world + // units. This code projects it with the Anatomy iso-kit camera, paints it + // back to front, and animates it. Nothing here parses markup: elements are + // made with createElementNS and names go in with textContent. + const SVGNS = 'http://www.w3.org/2000/svg'; + const FAMILIES = [ + ['exposure', 'Internet exposure'], ['network', 'Network path'], ['identity', 'Identity and access'], + ['data', 'Data and invocation'], ['governance', 'Governance and protection'], + ]; + const MAP = { + az: 45, el: 30, P: null, view: null, base: null, labels: false, + families: new Set(FAMILIES.map(f => f[0])), selected: null, focusTrace: null, + traceEls: new Map(), nodeEls: new Map(), byId: new Map(), pulses: [], raf: 0, + built: false, flat: false, turning: false, }; + // The drawing box: 1200 wide, as tall as the stage's aspect asks (set in buildMap) + let W = 1200, H = 760; + + function svgEl(tag, attrs, parent) { + const el = document.createElementNS(SVGNS, tag); + if (attrs) for (const k in attrs) if (attrs[k] !== undefined && attrs[k] !== null && attrs[k] !== '') el.setAttribute(k, attrs[k]); + if (parent) parent.appendChild(el); + return el; + } + function reducedMotion() { + return window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + } - let zoomBehavior, svgG; + // A kit solid as DOM: halo, fill, four side tones, top, crease, bevel, edge + function solid(parent, paths, opts) { + const o = opts || {}; + const g = svgEl('g', { class: 'iso-solid' + (o.cls ? ' ' + o.cls : '') }, parent); + if (o.lit) g.setAttribute('data-lit', ''); + if (o.halo) svgEl('path', { class: 'iso-halo', d: paths.fill }, g); + svgEl('path', { class: 'iso-fill', d: paths.fill }, g); + if (!o.flat) { + paths.shades.forEach((d, i) => { if (d) svgEl('path', { class: 'iso-shade', 'data-shade': i, d }, g); }); + svgEl('path', { class: 'iso-top', d: paths.top }, g); + } + if (paths.crease) svgEl('path', { class: 'iso-line iso-crease', 'data-tone': o.crease || 'faint', d: paths.crease }, g); + if (paths.bevel) svgEl('path', { class: 'iso-line iso-bevel', 'data-tone': o.crease || 'faint', d: paths.bevel }, g); + if (o.inner) o.inner(g); + svgEl('path', { class: 'iso-line iso-edge', 'data-tone': o.tone || 'hi', d: paths.outline }, g); + return g; + } + function line(parent, d, tone, cls) { + if (!d) return null; + return svgEl('path', { class: 'iso-line' + (cls ? ' ' + cls : ''), 'data-tone': tone || 'lo', d }, parent); + } + + // A matrix that lays text flat on a top face and keeps it reading left to right + function faceMatrix(at, z, P) { + const o = IsoKit.iso([at[0], at[1], z], P); + const vx = IsoKit.iso([at[0] + 1, at[1], z], P), vy = IsoKit.iso([at[0], at[1] + 1, z], P); + const ex = [vx[0] - o[0], vx[1] - o[1]], ey = [vy[0] - o[0], vy[1] - o[1]]; + const cands = [[ex, ey], [[-ex[0], -ex[1]], ey], [ey, ex], [[-ey[0], -ey[1]], ex]]; + let best = cands[0]; + for (const c of cands) if (c[0][0] > best[0][0]) best = c; + let [u, v] = best; + if (v[1] < 0) v = [-v[0], -v[1]]; + const f = n => Math.round(n * 1000) / 1000; + return 'matrix(' + [u[0], u[1], v[0], v[1], o[0], o[1]].map(f).join(' ') + ')'; + } + + // Draw order for siblings with disjoint footprints: a before b when b is + // nearer along the one axis that separates them (a topological sort, with + // centre depth breaking ties), so each platform's contents paint in order. + function orderSiblings(items, P) { + const [vx, vy] = IsoKit.towardViewer(P); + const n = items.length; + if (n < 2) return items.slice(); + const depth = it => (it.cx * vx + it.cy * vy); + if (n > 60) return items.slice().sort((a, b) => depth(a) - depth(b)); + const after = items.map(() => []), deg = new Array(n).fill(0); + for (let i = 0; i < n; i++) for (let j = i + 1; j < n; j++) { + const a = items[i], b = items[j]; + const sx = a.x1 <= b.x0 + 1e-6 ? 1 : (b.x1 <= a.x0 + 1e-6 ? -1 : 0); + const sy = a.y1 <= b.y0 + 1e-6 ? 1 : (b.y1 <= a.y0 + 1e-6 ? -1 : 0); + let aFirst = null; + if (sx && !sy) aFirst = sx * vx > 0; + else if (sy && !sx) aFirst = sy * vy > 0; + if (aFirst === null) continue; + if (aFirst) { after[i].push(j); deg[j]++; } else { after[j].push(i); deg[i]++; } + } + const out = [], ready = []; + for (let i = 0; i < n; i++) if (!deg[i]) ready.push(i); + while (ready.length) { + ready.sort((p, q) => depth(items[q]) - depth(items[p])); + const i = ready.pop(); + out.push(items[i]); + for (const j of after[i]) if (!--deg[j]) ready.push(j); + } + if (out.length < n) return items.slice().sort((a, b) => depth(a) - depth(b)); + return out; + } + function rectOf(x, y, w, d) { return { x0: x, y0: y, x1: x + w, y1: y + d, cx: x + w / 2, cy: y + d / 2 }; } + + // ── Parts ── + function drawPlatform(parent, p, P) { + const K = IsoKit; + const radius = { provider: 3.2, region: 2.4, network: 1.8, subnet: 1.2, zone: 1.2, deck: 2.4 }[p.kind] || 1.2; + const plan = { x: p.x, y: p.y, w: p.w, d: p.d, r: radius }; + const g = svgEl('g', { class: 'platform pl-' + p.kind }, parent); + if (p.kind === 'provider') { + // Feet under the plate, and a soft halo on the bench + for (const [fx, fy] of K.corners(plan, 7)) solid(g, K.cylinder(fx, fy, 2.6, -3, 3, P, 18), { tone: 'mid' }); + } + const top = p.z + p.h; + const tones = { provider: 'mid', region: 'mid', network: 'mid', subnet: 'lo', zone: 'lo', deck: 'mid' }; + solid(g, K.slabOf(plan, p.z, p.h, P, 8, p.kind === 'provider' ? 1.4 : 0.8), { + tone: tones[p.kind] || 'lo', halo: p.kind === 'provider' || p.kind === 'deck', + inner: s => { + const inset = K.insetPlan(plan, Math.min(3.5, Math.min(p.w, p.d) / 6)); + line(s, K.planOutline(inset, top, P), 'faint'); + if (p.kind === 'provider' || p.kind === 'deck') { + // Ruler ticks along the front edge and a screw at each corner + const t = K.topTicks(p.x + 6, p.x + p.w - 6, 4, 5, p.y + p.d - 2.2, top, [1.0, 1.9], P, 'x'); + line(s, t.minor, 'faint'); line(s, t.major, 'lo'); + for (const [sx, sy] of K.corners(plan, 3.2)) { + const c = K.iso([sx, sy, top], P); + svgEl('circle', { class: 'screw', cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 1.1 }, s); + } + } + if (p.kind === 'network') line(s, K.sideSeam(plan, p.z + p.h * 0.5, P), 'faint'); + }, + }); + if (p.kind !== 'zone' && p.w > 30) { + const label = svgEl('text', { + class: 'pl-label' + (p.kind === 'provider' || p.kind === 'deck' ? ' major' : ''), + transform: faceMatrix([p.x + 4.5, p.y + 3.2], top, P), 'font-size': p.kind === 'provider' ? 4.2 : 3.2, + 'dominant-baseline': 'hanging', + }, g); + const max = Math.max(6, Math.floor((p.w - 10) / (p.kind === 'provider' ? 2.6 : 2.0))); + const text = p.label.length > max ? p.label.slice(0, max - 1) + '…' : p.label; + label.textContent = text; + } + return g; + } + + function drawNode(parent, n, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node c-' + n.category, 'data-node': n.id, tabindex: '-1' }, parent); + const title = svgEl('title', null, g); + title.textContent = n.name + ' · ' + n.type + (n.region ? ' · ' + n.region : ''); + const { x, y, z, w, d, h } = n; + const plan = { x: x - w / 2, y: y - d / 2, w, d, r: 1.2 }; + const top = z + h; + if (n.exposed) line(g, K.ring(x, y, Math.max(w, d) / 2 + 2.4, z + 0.05, P, 36), null, 'exposed-ring'); + const box = (pl, z0, hh, opt) => solid(g, K.slabOf(pl, z0, hh, P, 6, 0.5), opt || {}); + const cyl = (r, z0, hh, steps, opt) => solid(g, K.cylinder(x, y, r, z0, hh, P, steps || 28, 0.4), opt || {}); + switch (n.shape) { + case 'server': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.sideSeam(plan, z + (h * i) / 4, P), 'faint'); + for (const dot of K.dotGrid(x - 3, y - 3, 3, 3, 3, top, P)) svgEl('circle', { class: 'iso-dot-vent', cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .45, fill: 'var(--anatomy-lo)' }, s); + } }); + break; + case 'cluster': { + box({ x: plan.x, y: plan.y, w, d, r: 1.4 }, z, 3); + const q = w / 2 - 0.8; + for (const [ox, oy] of [[-1, -1], [1, -1], [-1, 1], [1, 1]].sort((a, b) => (a[0] + a[1]) - (b[0] + b[1]))) + box({ x: x + (ox < 0 ? -q - 0.4 : 0.4), y: y + (oy < 0 ? -q - 0.4 : 0.4), w: q, d: q, r: 0.8 }, z + 3, h - 3); + break; + } + case 'function': + box(plan, z, h, { inner: s => line(s, K.sideSeam(plan, z + h * 0.7, P), 'faint') }); + box({ x: x - w / 2 - 0.6, y: y - d / 2 - 0.6, w: w + 1.2, d: d + 1.2, r: 1 }, top, 1.2); + break; + case 'database': + cyl(w / 2, z, h, 30, { inner: s => { + line(s, K.sideArc(x, y, w / 2, z + h / 3, P), 'faint'); + line(s, K.sideArc(x, y, w / 2, z + (2 * h) / 3, P), 'faint'); + } }); + break; + case 'bucket': + cyl(w / 2 - 0.6, z, h, 30); + line(g, K.ring(x, y, w / 2 - 2, top, P, 30), 'lo'); + break; + case 'queue': { + box(plan, z, h, { inner: s => { + for (let i = 1; i < 4; i++) line(s, K.lineOnTop([plan.x + (w * i) / 4, plan.y + 1], [plan.x + (w * i) / 4, plan.y + d - 1], top, P), 'faint'); + } }); + break; + } + case 'gateway': + box(plan, z, h, { inner: s => { + for (const dot of K.dotGrid(x - 4, y, 3, 1, 4, top, P)) svgEl('circle', { class: 'led' + (n.exposed ? ' exposed' : ''), cx: dot[0].toFixed(1), cy: dot[1].toFixed(1), r: .9 }, s); + } }); + break; + case 'shield': + cyl(w / 2, z, h, 8, { inner: s => line(s, K.ring(x, y, w / 2 - 2, top, P, 8), 'faint') }); + break; + case 'key': + cyl(w / 2, z, h * 0.6, 24); + cyl(w / 4, z + h * 0.6, h * 0.4, 18); + break; + case 'log': + for (let i = 0; i < 3; i++) box({ x: plan.x + i * 0.4, y: plan.y + i * 0.4, w: w - i * 0.8, d: d - i * 0.8, r: 1 }, z + i * (h / 3), h / 3 - 0.3); + break; + case 'core': + cyl(w / 2, z, h * 0.7, 6); + cyl(w / 4, z + h * 0.7, h * 0.3, 20, { lit: true }); + break; + case 'principal': + cyl(w / 2 - 1, z, h * 0.65, 22); + cyl(w / 2, z + h * 0.65, h * 0.35, 22); + break; + case 'plate': + box(plan, z, h, { inner: s => { + for (let i = 1; i <= 3; i++) line(s, K.lineOnTop([plan.x + 2.5, plan.y + (d * i) / 4.4], [plan.x + w - 2.5, plan.y + (d * i) / 4.4], top, P), 'faint'); + } }); + break; + default: + box(plan, z, h); + } + return g; + } + + function drawBeacon(parent, n, P) { + if (!n.worst) return null; + const K = IsoKit; + const rank = SEVERITIES.indexOf(n.worst); + const mastTop = n.z + n.h + 7 + (4 - rank) * 2.6; + const g = svgEl('g', { class: 'beacon-group sev-' + n.worst, 'data-node': n.id }, parent); + svgEl('path', { class: 'mast', d: K.segment([n.x, n.y, n.z + n.h], [n.x, n.y, mastTop], P) }, g); + solid(g, K.cylinder(n.x, n.y, 1.9, mastTop, 2.4, P, 18), { cls: 'beacon', tone: 'hi' }); + const c = K.iso([n.x, n.y, mastTop + 2.4], P); + const halo = svgEl('circle', { class: 'beacon-halo' + (rank <= 1 ? ' pulse' : ''), cx: c[0].toFixed(1), cy: c[1].toFixed(1), r: 3.2 }, g); + halo.style.animationDelay = ((n.x * 7 + n.y * 13) % 2400) + 'ms'; + return g; + } + + function drawExternal(parent, e, P) { + const K = IsoKit; + const g = svgEl('g', { class: 'node external ' + (e.kind === 'internet' ? 'internet' : ''), 'data-node': e.id }, parent); + const t = svgEl('title', null, g); + t.textContent = e.kind === 'internet' ? 'The internet (' + e.name + ')' : 'Outside the estate: ' + e.name; + if (e.kind === 'internet') { + // An uplink mast: a weighted foot, a tapering column, three dish rings + solid(g, K.cylinder(e.x, e.y, 9, 0, 3, P, 36, 0.6), { tone: 'mid', halo: true }); + solid(g, K.cylinder(e.x, e.y, 3.4, 3, e.h - 9, P, 24), { tone: 'hi' }); + for (let i = 0; i < 3; i++) line(g, K.sideArc(e.x, e.y, 3.4, 8 + i * 6, P), 'faint'); + solid(g, K.cylinder(e.x, e.y, 6.5, e.h - 6, 2, P, 32, 0.4), { tone: 'hi' }); + solid(g, K.cylinder(e.x, e.y, 2.2, e.h - 4, 4, P, 20), { tone: 'hi', lit: true }); + } else { + solid(g, K.slabOf({ x: e.x - e.w / 2, y: e.y - e.d / 2, w: e.w, d: e.d, r: 1.4 }, 0, e.h, P, 6, 0.5), { tone: 'mid' }); + } + return g; + } + + function roundedPath(pts, r) { + if (pts.length < 3) return 'M' + pts.map(p => p[0].toFixed(1) + ' ' + p[1].toFixed(1)).join('L'); + let d = 'M' + pts[0][0].toFixed(1) + ' ' + pts[0][1].toFixed(1); + for (let i = 1; i < pts.length - 1; i++) { + const [ax, ay] = pts[i - 1], [bx, by] = pts[i], [cx, cy] = pts[i + 1]; + const l1 = Math.hypot(bx - ax, by - ay), l2 = Math.hypot(cx - bx, cy - by); + const rr = Math.min(r, l1 / 2, l2 / 2); + if (rr < 0.3) { d += 'L' + bx.toFixed(1) + ' ' + by.toFixed(1); continue; } + const p1 = [bx - ((bx - ax) / l1) * rr, by - ((by - ay) / l1) * rr]; + const p2 = [bx + ((cx - bx) / l2) * rr, by + ((cy - by) / l2) * rr]; + d += 'L' + p1[0].toFixed(1) + ' ' + p1[1].toFixed(1) + 'Q' + bx.toFixed(1) + ' ' + by.toFixed(1) + ' ' + p2[0].toFixed(1) + ' ' + p2[1].toFixed(1); + } + const last = pts[pts.length - 1]; + return d + 'L' + last[0].toFixed(1) + ' ' + last[1].toFixed(1); + } + + function drawTrace(parent, t, P) { + const g = svgEl('g', { class: 'trace f-' + t.family + (MAP.families.has(t.family) ? '' : ' hidden-family'), 'data-trace': t.id }, parent); + const pts = t.points.map(p => IsoKit.iso(p, P)).filter((p, i, a) => i === 0 || Math.hypot(p[0] - a[i - 1][0], p[1] - a[i - 1][1]) > 0.05); + const d = roundedPath(pts, 4); + const title = svgEl('title', null, g); + title.textContent = nameOf(t.source) + ' → ' + nameOf(t.target) + ' · ' + t.label + (t.ports ? ' · ' + t.ports : ''); + svgEl('path', { class: 'trace-hit', d }, g); + const lineEl = svgEl('path', { class: 'trace-line', d }, g); + svgEl('path', { class: 'trace-flow', d }, g); + const a = pts[0], b = pts[pts.length - 1]; + svgEl('circle', { class: 'trace-end', cx: a[0].toFixed(1), cy: a[1].toFixed(1), r: 1.5 }, g); + svgEl('circle', { class: 'trace-end to', cx: b[0].toFixed(1), cy: b[1].toFixed(1), r: 1.7 }, g); + MAP.traceEls.set(t.id, { g, line: lineEl, t }); + return g; + } + + function nameOf(id) { + const n = MAP.byId.get(id); + return n ? n.name : id; + } + + // ── Build ── + function buildMap() { + const svg = document.getElementById('iso-svg'); + while (svg.firstChild) svg.removeChild(svg.firstChild); + MAP.traceEls.clear(); MAP.nodeEls.clear(); + const s = isoScene; + const b = s.bounds; + const stageEl = document.getElementById('stage'); + const sw = stageEl.clientWidth || 1200, sh = stageEl.clientHeight || 760; + const nextH = Math.round(W * Math.min(1.6, Math.max(0.45, sh / sw))); + if (nextH !== H) { H = nextH; MAP.view = null; } + const corners = []; + for (const x of b.x) for (const y of b.y) for (const z of b.z) corners.push([x, y, z]); + corners.push([b.x[0], b.y[0], -3]); + const P = IsoKit.fitProjection(corners, W, H, { pad: 46, azimuth: MAP.az, elevation: MAP.el }); + MAP.P = P; + if (!MAP.view) MAP.view = [0, 0, W, H]; + svg.setAttribute('viewBox', MAP.view.join(' ')); + svg.setAttribute('preserveAspectRatio', 'xMidYMid meet'); + svg.classList.add('iso-svg'); + + const kids = new Map(); + const push = (key, item) => { if (!kids.has(key)) kids.set(key, []); kids.get(key).push(item); }; + s.platforms.forEach(p => push(p.parent || '', { kind: 'platform', p, ...rectOf(p.x, p.y, p.w, p.d) })); + s.nodes.forEach(n => push(n.parent, { kind: 'node', n, ...rectOf(n.x - 9, n.y - 9, 18, 18) })); + + const layers = { decks: svgEl('g', { class: 'layer-decks' }), ground: svgEl('g', { class: 'layer-ground' }), + groundTraces: svgEl('g', { class: 'layer-traces' }), deckTraces: svgEl('g', { class: 'layer-traces deck' }), + labels: svgEl('g', { class: 'layer-labels' }) }; + const rising = []; + const paint = (parent, item, depthIndex) => { + if (item.kind === 'platform') { + const p = item.p; + const g = svgEl('g', { class: 'stack' }, parent); + if (p.kind === 'deck') { + // Four posts carry the control deck above the plate + const posts = IsoKit.corners({ x: p.x, y: p.y, w: p.w, d: p.d }, 4).map(([px, py]) => ({ px, py, ...rectOf(px - 1, py - 1, 2, 2) })); + for (const post of orderSiblings(posts, P)) solid(g, IsoKit.cylinder(post.px, post.py, 1.5, 0, p.z, P, 16), { cls: 'post', tone: 'lo' }); + } + drawPlatform(g, p, P); + rising.push(g); + for (const child of orderSiblings(kids.get(p.id) || [], P)) paint(g, child, depthIndex + 1); + } else { + const n = item.n; + const g = drawNode(parent, n, P); + MAP.nodeEls.set(n.id, g); + drawBeacon(parent, n, P); + } + }; + const roots = orderSiblings(kids.get('') || [], P); + for (const root of roots) paint(root.p.kind === 'deck' ? layers.decks : layers.ground, root, 0); + const externals = s.externals.map(e => ({ e, ...rectOf(e.x - e.w, e.y - e.d, e.w * 2, e.d * 2) })); + for (const ext of orderSiblings(externals, P)) { const g = drawExternal(layers.ground, ext.e, P); MAP.nodeEls.set(ext.e.id, g); rising.push(g); } + + const deckIds = new Set(s.nodes.filter(n => n.parent && n.parent.startsWith('d:')).map(n => n.id)); + for (const t of s.traces) drawTrace(deckIds.has(t.source) || deckIds.has(t.target) ? layers.deckTraces : layers.groundTraces, t, P); + + // The deck is behind the plate (toward -y): paint it first unless the camera looks from behind + const deckInFront = IsoKit.towardViewer(P)[1] < 0; + const order = deckInFront + ? [layers.ground, layers.groundTraces, layers.decks, layers.deckTraces, layers.labels] + : [layers.decks, layers.ground, layers.groundTraces, layers.deckTraces, layers.labels]; + order.forEach(l => svg.appendChild(l)); + MAP.layers = layers; + MAP.rising = rising; + renderLabels(); + applyFocus(); + MAP.built = true; + } + + function renderLabels() { + const layer = MAP.layers && MAP.layers.labels; + if (!layer) return; + while (layer.firstChild) layer.removeChild(layer.firstChild); + const ids = new Set(); + if (MAP.labels) isoScene.nodes.forEach(n => ids.add(n.id)); + if (MAP.selected) { + ids.add(MAP.selected); + for (const t of isoScene.traces) if (t.source === MAP.selected || t.target === MAP.selected) { ids.add(t.source); ids.add(t.target); } + } + if (MAP.hover) ids.add(MAP.hover); + // Labels are placed top to bottom; one that would overlap a placed label + // moves up in steps, and a leader then ties it to its asset's top + const anchors = [...ids].map(id => MAP.byId.get(id)).filter(Boolean).map(n => { + const top = (n.z || 0) + (n.h || 0) + (n.worst ? 18 : 5); + return { n, a: IsoKit.iso([n.x, n.y, top], MAP.P) }; + }).sort((p, q) => p.a[1] - q.a[1]); + const placed = []; + const hits = (r) => placed.some(q => r.x0 < q.x1 && q.x0 < r.x1 && r.y0 < q.y1 && q.y0 < r.y1); + for (const { n, a } of anchors) { + const name = n.name.length > 34 ? n.name.slice(0, 33) + '…' : n.name; + const half = Math.max(name.length * 3.1, String(n.type || '').length * 2.7) + 4; + let dy = 0, rect; + for (let k = 0; k < 14; k++) { + rect = { x0: a[0] - half, x1: a[0] + half, y0: a[1] - 16 - dy, y1: a[1] + 9 - dy }; + if (!hits(rect)) break; + dy += 13; + } + placed.push(rect); + if (dy) svgEl('path', { class: 'iso-line', 'data-tone': 'mid', d: 'M' + a[0].toFixed(1) + ' ' + (a[1] + 2).toFixed(1) + 'V' + (a[1] - dy + 10).toFixed(1) }, layer); + if (dy) svgEl('circle', { cx: a[0].toFixed(1), cy: (a[1] + 2).toFixed(1), r: 1.2, fill: 'var(--anatomy-mid)' }, layer); + const text = svgEl('text', { class: 'node-label', x: a[0].toFixed(1), y: (a[1] - 4 - dy).toFixed(1), 'text-anchor': 'middle' }, layer); + text.appendChild(document.createTextNode(name)); + const sub = svgEl('tspan', { x: a[0].toFixed(1), dy: '11' }, text); + sub.textContent = n.type || n.kind || ''; + } + } + + // ── Intro: plates rise, assets settle, traces draw in, then flow ── + function playIntro() { + const svg = document.getElementById('iso-svg'); + svg.classList.remove('flowing'); + if (reducedMotion()) { svg.classList.add('flowing'); return; } + const groups = MAP.rising || []; + groups.forEach(g => { g.classList.remove('risen'); g.classList.add('rise-in'); }); + const traces = [...MAP.traceEls.values()]; + traces.forEach(({ line: l }) => { + const len = l.getTotalLength ? l.getTotalLength() : 0; + l.style.transition = 'none'; + l.style.strokeDasharray = len + ' ' + len; + l.style.strokeDashoffset = String(len); + }); + svg.getBoundingClientRect(); + const step = Math.min(70, 900 / Math.max(1, groups.length)); + groups.forEach((g, i) => setTimeout(() => { g.classList.remove('rise-in'); g.classList.add('risen'); }, 80 + i * step)); + const start = 260 + groups.length * step; + const famDelay = { network: 0, data: 220, identity: 440, governance: 660, exposure: 880 }; + traces.forEach(({ line: l, t }, i) => { + const delay = start + (famDelay[t.family] || 0) + (i % 12) * 30; + setTimeout(() => { + l.style.transition = 'stroke-dashoffset 1.1s cubic-bezier(.32,.72,0,1)'; + l.style.strokeDashoffset = '0'; + }, delay); + }); + clearTimeout(MAP.flowTimer); + MAP.flowTimer = setTimeout(() => { + traces.forEach(({ line: l }) => { l.style.strokeDasharray = ''; l.style.strokeDashoffset = ''; l.style.transition = ''; }); + svg.classList.add('flowing'); + }, start + 2200); + } + + // ── Focus: a selected asset, a traced path ── + function connectedTraces(id) { + return isoScene.traces.filter(t => t.source === id || t.target === id); + } + function applyFocus(traceIds) { + const svg = document.getElementById('iso-svg'); + const on = new Set(), onNodes = new Set(); + if (traceIds) traceIds.forEach(id => on.add(id)); + else if (MAP.selected) connectedTraces(MAP.selected).forEach(t => on.add(t.id)); + for (const id of on) { const t = MAP.traceEls.get(id); if (t) { onNodes.add(t.t.source); onNodes.add(t.t.target); } } + if (MAP.selected) onNodes.add(MAP.selected); + svg.classList.toggle('focus', on.size > 0 || !!MAP.selected); + MAP.traceEls.forEach(({ g }, id) => g.classList.toggle('on', on.has(id))); + svg.querySelectorAll('[data-node]').forEach(el => { + const id = el.getAttribute('data-node'); + el.classList.toggle('on', onNodes.has(id)); + el.classList.toggle('sel', id === MAP.selected); + }); + startPulses([...on]); + } + + // A dot runs along each lit trace from source to target, so the eye can follow it + function startPulses(ids, hops) { + cancelAnimationFrame(MAP.raf); + MAP.pulses.forEach(p => p.el.remove()); + MAP.pulses = []; + if (!ids.length || reducedMotion()) return; + const svg = document.getElementById('iso-svg'); + ids.slice(0, 40).forEach((id, i) => { + const t = MAP.traceEls.get(id); + if (!t || t.g.classList.contains('hidden-family')) return; + const el = svgEl('circle', { class: 'pulse-dot', r: 2.3 }, t.g); + MAP.pulses.push({ el, path: t.line, len: t.line.getTotalLength(), delay: hops ? (hops.get(id) || 0) * 420 : i * 90 }); + }); + const t0 = performance.now(); + const tick = now => { + for (const p of MAP.pulses) { + const e = now - t0 - p.delay; + if (e < 0) { p.el.setAttribute('opacity', '0'); continue; } + const dur = Math.max(900, p.len * 9); + const f = (e % (dur + 500)) / dur; + if (f > 1) { p.el.setAttribute('opacity', '0'); continue; } + const eased = f < .5 ? 2 * f * f : 1 - Math.pow(-2 * f + 2, 2) / 2; + const pt = p.path.getPointAtLength(eased * p.len); + p.el.setAttribute('cx', pt.x.toFixed(1)); + p.el.setAttribute('cy', pt.y.toFixed(1)); + p.el.setAttribute('opacity', String(Math.min(1, Math.sin(f * Math.PI) * 1.6))); + } + MAP.raf = requestAnimationFrame(tick); + }; + MAP.raf = requestAnimationFrame(tick); + } + + function selectNode(id, opts) { + MAP.selected = id && MAP.byId.has(id) ? id : null; + id = MAP.selected; + MAP.focusTrace = null; + applyFocus(); + renderLabels(); + renderInspector(); + if (id && opts && opts.center) centerOn(id); + const n = MAP.byId.get(id); + setReadout(n ? n.name + ' · ' + (n.type || n.kind) + ' · ' + connectedTraces(id).length + ' relations' : null); + } + + function focusTrace(traceId) { + const t = MAP.traceEls.get(traceId); + if (!t) return; + MAP.focusTrace = traceId; + applyFocus([traceId]); + document.querySelectorAll('.conn-item').forEach(el => el.classList.toggle('on', el.dataset.trace === traceId)); + setReadout(nameOf(t.t.source) + ' → ' + nameOf(t.t.target) + ' · ' + t.t.label); + } + + function traceFromInternet() { + const starts = isoScene.externals.filter(e => e.kind === 'internet').map(e => e.id); + if (!starts.length) { setReadout('nothing in this scan is reachable from the internet'); return; } + const out = new Map(); + isoScene.traces.forEach(t => { if (!out.has(t.source)) out.set(t.source, []); out.get(t.source).push(t); }); + const hops = new Map(), seen = new Set(starts); + let frontier = starts.slice(), depth = 0; + while (frontier.length && depth < 5) { + const next = []; + for (const id of frontier) for (const t of out.get(id) || []) { + if (!MAP.families.has(t.family)) continue; + if (!hops.has(t.id)) hops.set(t.id, depth); + if (!seen.has(t.target)) { seen.add(t.target); next.push(t.target); } + } + frontier = next; depth++; + } + const maxHop = hops.size ? Math.max(...hops.values()) + 1 : 0; + MAP.selected = null; + applyFocus([...hops.keys()]); + startPulses([...hops.keys()], hops); + renderLabels(); + renderInspector({ internet: { reached: seen.size - starts.length, hops: maxHop, traces: hops.size } }); + setReadout('the internet reaches ' + (seen.size - starts.length) + ' assets in up to ' + maxHop + (maxHop === 1 ? ' hop' : ' hops')); + } + + function setReadout(text) { + document.getElementById('readout').textContent = text || (isoScene.nodes ? isoScene.nodes.length + ' assets · ' + isoScene.traces.length + ' traces' : 'rest'); + } + + function renderInspector(extra) { + const box = document.getElementById('inspector'); + const n = MAP.selected ? MAP.byId.get(MAP.selected) : null; + if (extra && extra.internet) { + const x = extra.internet; + box.innerHTML = html`
    Attack surface

    Reachable from the internet

    +
    Assets${x.reached}Hops${x.hops}Traces${x.traces}
    +

    The lit traces start at the internet mast and follow each relation's direction, one hop after another. Click an asset to see its own connections.

    `; + return; + } + if (!n) { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + box.innerHTML = html`
    Inspector

    Pick an asset

    +

    Click any asset to light its relations; each lit trace carries a dot from its source to its target. Click a relation in this panel to follow it alone.

    +
    Assets${(isoScene.nodes || []).length}Platforms${(isoScene.platforms || []).length}Traces${(isoScene.traces || []).length}
    +

    [ ] turn · + - zoom · 0 fit · Esc clear

    `; + return; + } + const conns = connectedTraces(n.id); + const connItems = conns.map(t => html` + `).join(''); + const f = n.findings || {}; + const fsum = SEVERITIES.filter(s => f[s]).map(s => s.charAt(0) + s.slice(1).toLowerCase() + ' ' + f[s]).join(', '); + box.innerHTML = html`
    ${n.type || n.kind}

    ${n.name}

    +
    + Provider${n.provider || ''} + Region${n.region || ''} + Account${n.account || ''} + Identifier${n.arn || n.id} + Exposure${n.exposed ? 'Reachable from the internet' : 'Not exposed'} + Findings${fsum || 'None open'} +
    +
    Relations ${conns.length}
    ` + connItems + html`
    `; + } + + // ── Pan, zoom, turn ── + function setView(v) { + MAP.view = v; + document.getElementById('iso-svg').setAttribute('viewBox', v.map(n => n.toFixed(1)).join(' ')); + } + function zoomBy(factor, cx, cy) { + const [x, y, w, h] = MAP.view; + const nw = Math.min(W * 4, Math.max(W / 14, w / factor)), nh = nw * (h / w); + const px = cx === undefined ? x + w / 2 : cx, py = cy === undefined ? y + h / 2 : cy; + setView([px - (px - x) * (nw / w), py - (py - y) * (nh / h), nw, nh]); + } + function fitView() { setView([0, 0, W, H]); } + function centerOn(id) { + const n = MAP.byId.get(id); + if (!n || !MAP.P) return; + const p = IsoKit.iso([n.x, n.y, (n.z || 0) + (n.h || 0)], MAP.P); + const [, , w, h] = MAP.view; + const nw = Math.min(w, W / 2.2), nh = nw * (h / w); + animateView([p[0] - nw / 2, p[1] - nh / 2, nw, nh]); + } + function animateView(target) { + if (reducedMotion()) { setView(target); return; } + const from = MAP.view.slice(), t0 = performance.now(), dur = 520; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = 1 - Math.pow(1 - k, 3); + setView(from.map((v, i) => v + (target[i] - v) * e)); + if (k < 1) requestAnimationFrame(step); + }; + requestAnimationFrame(step); + } + function turn(by) { + if (MAP.turning) return; + const from = MAP.az, to = MAP.az + by; + const heavy = (isoScene.nodes || []).length > 260 || reducedMotion(); + const finish = () => { MAP.az = ((to % 360) + 360) % 360; MAP.turning = false; buildMap(); document.getElementById('iso-svg').classList.add('flowing'); }; + if (heavy) { finish(); return; } + MAP.turning = true; + const t0 = performance.now(), dur = 640; + const step = now => { + const k = Math.min(1, (now - t0) / dur), e = k < .5 ? 4 * k * k * k : 1 - Math.pow(-2 * k + 2, 3) / 2; + MAP.az = from + (to - from) * e; + buildMap(); + if (k < 1) requestAnimationFrame(step); else finish(); + }; + requestAnimationFrame(step); + } + + function setFlat(flat) { + MAP.flat = flat; + document.getElementById('stage').classList.toggle('flat', flat); + document.getElementById('view-3d').setAttribute('aria-pressed', String(!flat)); + document.getElementById('view-2d').setAttribute('aria-pressed', String(flat)); + document.getElementById('layout-2d').hidden = !flat; + ['rot-left', 'rot-right', 'labels-btn', 'trace-internet', 'replay-btn'].forEach(id => { document.getElementById(id).disabled = flat; }); + renderTopology(); + } + + function renderFamilies() { + const counts = {}; + (isoScene.traces || []).forEach(t => { counts[t.family] = (counts[t.family] || 0) + 1; }); + document.getElementById('families').innerHTML = FAMILIES.map(([key, label]) => + html``).join(''); + } + + function renderLegend() { + const legend = document.getElementById('legend'); + legend.innerHTML = FAMILIES.map(([key, label]) => html`${label}`).join('') + + SEVERITIES.slice(0, 4).map(s => html`${s.charAt(0) + s.slice(1).toLowerCase()} beacon`).join('') + + html`Internet exposed`; + } + // ── Topology entry point: the 3D map, or the 2D tree and force views ── function renderTopology() { const container = document.getElementById('topology-container'); - // Remove existing SVG only - container.querySelectorAll('svg').forEach(s => s.remove()); + const stage = document.getElementById('stage'); + const can3d = typeof IsoKit !== 'undefined' && isoScene && isoScene.nodes && !isoScene.too_large && !isoScene.error && isoScene.nodes.length > 0; + if (!can3d && !MAP.flat) { + MAP.flat = true; + stage.classList.add('flat'); + document.getElementById('view-3d').disabled = true; + document.getElementById('view-3d').setAttribute('aria-pressed', 'false'); + document.getElementById('view-2d').setAttribute('aria-pressed', 'true'); + document.getElementById('layout-2d').hidden = false; + const hint = document.getElementById('map-hint'); + if (isoScene && isoScene.too_large) hint.textContent = 'The 3D map is drawn for up to ' + isoScene.limit + ' assets; this scan has ' + isoScene.asset_count; + else hint.textContent = 'Scroll to zoom · drag to pan'; + } + if (!MAP.flat) { + if (!MAP.built) { buildMap(); setReadout(); renderInspector(); playIntro(); } + return; + } + container.querySelectorAll('svg, p').forEach(s => s.remove()); const width = container.clientWidth || 1200; - const height = 600; + const height = container.clientHeight || 640; if (!hierarchyData || !hierarchyData.children || hierarchyData.children.length === 0) { - // Fallback to force-directed if no hierarchy if (graphData.nodes && graphData.nodes.length > 0) { renderForceTopology(container, width, height); } else { const msg = document.createElement('p'); - msg.style.cssText = 'padding:40px;text-align:center;color:var(--text-muted)'; - msg.textContent = 'No topology data available. Run a collection first.'; + msg.className = 'map-note'; + msg.textContent = 'No topology data in this scan. Run cloudg collect or cloudg map first.'; container.appendChild(msg); } return; } + if (currentLayout === 'force') renderForceTopology(container, width, height); + else renderTreeTopology(container, width, height); + } - if (currentLayout === 'force') { - renderForceTopology(container, width, height); - } else { - renderTreeTopology(container, width, height); - } + // ── 2D views (D3) ── + const CATEGORY_COLORS = { + account: '--accent', region: '--f-governance', vpc: '--f-network', vnet: '--f-network', subnet: '--f-network', global: '--f-data', + }; + function typeColor(type) { + if (CATEGORY_COLORS[type]) return cssVar(CATEGORY_COLORS[type]); + if (type === 'SECURITY_FINDING') return cssVar('--critical'); + if (type === 'COMPLIANCE_FRAMEWORK') return cssVar('--f-governance'); + return cssVar('--muted'); } + let zoomBehavior, svgG; function renderTreeTopology(container, width, height) { const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); - const g = svg.append('g').attr('transform', 'translate(80, 20)'); + const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.1, 4]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - const root = d3.hierarchy(hierarchyData); - root.descendants().forEach(d => { d._children = d.children; }); - - const treeLayout = d3.tree().nodeSize([28, 220]); - treeLayout(root); - - // Center vertically + d3.tree().nodeSize([26, 230])(root); const yExtent = d3.extent(root.descendants(), d => d.x); - const yOffset = height / 2 - (yExtent[0] + yExtent[1]) / 2; - g.attr('transform', `translate(80, ${yOffset})`); - - // Links + svg.call(zoomBehavior.transform, d3.zoomIdentity.translate(80, height / 2 - (yExtent[0] + yExtent[1]) / 2)); g.selectAll('.link').data(root.links()).join('path') - .attr('class', 'link') .attr('d', d3.linkHorizontal().x(d => d.y).y(d => d.x)) - .attr('fill', 'none') - .attr('stroke', '#475569') - .attr('stroke-width', 1.5) - .attr('stroke-opacity', 0.6); - - // Nodes + .attr('fill', 'none').attr('stroke', cssVar('--line-strong')).attr('stroke-width', 1.2); const node = g.selectAll('.node').data(root.descendants()).join('g') - .attr('class', 'node') - .attr('transform', d => `translate(${d.y},${d.x})`) - .style('cursor', d => d.children || d._children ? 'pointer' : 'default') - .on('click', (e, d) => { - if (d.children) { d._children = d.children; d.children = null; } - else if (d._children) { d.children = d._children; d._children = null; } - renderTopology(); - }); - - // Node circles + .attr('transform', d => `translate(${d.y},${d.x})`); node.append('circle') - .attr('r', d => { - const nodeData = d.data; - return typeRadius[nodeData.type] || 10; - }) - .attr('fill', d => typeColors[d.data.type] || '#95A5A6') + .attr('r', d => d.children ? 7 : 5) + .attr('fill', d => d.children ? cssVar('--surface') : typeColor(d.data.type)) .attr('stroke', d => { - const nodeData = d.data; - if (nodeData.findings && nodeData.findings.total > 0) { - if (nodeData.findings.CRITICAL > 0) return '#EF4444'; - if (nodeData.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - } - if (nodeData.internet_exposed) return '#FF9900'; - return 'rgba(255,255,255,0.3)'; + const f = d.data.findings; + if (f && f.total > 0) return cssVar(f.CRITICAL ? '--critical' : f.HIGH ? '--high' : '--medium'); + if (d.data.internet_exposed) return cssVar('--f-exposure'); + return typeColor(d.data.type); }) - .attr('stroke-width', d => { - const nodeData = d.data; - if ((nodeData.findings && nodeData.findings.total > 0) || nodeData.internet_exposed) return 3; - return 1.5; - }); - - // Findings badge - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('circle') - .attr('cx', d => (typeRadius[d.data.type] || 10) - 2) - .attr('cy', d => -(typeRadius[d.data.type] || 10) + 2) - .attr('r', 8) - .attr('fill', d => { - if (d.data.findings.CRITICAL > 0) return '#EF4444'; - if (d.data.findings.HIGH > 0) return '#F97316'; - return '#EAB308'; - }); - - node.filter(d => d.data.findings && d.data.findings.total > 0) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) - 2) - .attr('y', d => -(typeRadius[d.data.type] || 10) + 6) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '9px') - .attr('font-weight', '700') - .text(d => d.data.findings.total); - - // Collapse indicator - node.filter(d => d._children && !d.children) - .append('text') - .attr('x', 0).attr('y', 4) - .attr('text-anchor', 'middle') - .attr('fill', 'white') - .attr('font-size', '14px') - .attr('font-weight', '700') - .text('+'); - - // Labels + .attr('stroke-width', d => (d.data.findings && d.data.findings.total) || d.data.internet_exposed ? 2.5 : 1.4); node.append('text') - .attr('x', d => (d.children || d._children ? -(typeRadius[d.data.type] || 10) - 6 : (typeRadius[d.data.type] || 10) + 6)) - .attr('dy', '0.35em') - .attr('text-anchor', d => d.children || d._children ? 'end' : 'start') - .attr('fill', '#CBD5E1') - .attr('font-size', d => ['account','region','vpc'].includes(d.data.type) ? '13px' : '11px') - .attr('font-weight', d => ['account','region','vpc'].includes(d.data.type) ? '600' : '400') - .text(d => { - const name = d.data.name || d.data.id || ''; - return name.length > 35 ? name.substring(0, 32) + '…' : name; - }); - - // Type badge for leaf nodes - node.filter(d => d.data.type && !['account','region','vpc','subnet','global','vnet'].includes(d.data.type)) - .append('text') - .attr('x', d => (typeRadius[d.data.type] || 10) + 6) - .attr('dy', '1.8em') - .attr('fill', '#64748B') - .attr('font-size', '9px') - .text(d => d.data.type); - - // Tooltips + .attr('x', d => d.children ? -12 : 10).attr('dy', '0.32em') + .attr('text-anchor', d => d.children ? 'end' : 'start') + .attr('fill', cssVar('--text')).attr('font-size', d => d.children ? '12px' : '11px') + .attr('font-weight', d => d.children ? 600 : 400) + .text(d => { const n = d.data.name || d.data.id || ''; return n.length > 36 ? n.slice(0, 33) + '…' : n; }); + node.filter(d => !d.children).append('text') + .attr('x', 10).attr('dy', '1.7em').attr('fill', cssVar('--muted')).attr('font-size', '9px') + .attr('font-family', 'ui-monospace, monospace').text(d => d.data.type); node.append('title').text(d => { let tip = `${d.data.type}: ${d.data.name || d.data.id || ''}`; if (d.data.arn) tip += `\nARN: ${d.data.arn}`; - if (d.data.findings && d.data.findings.total > 0) { - tip += `\n\nFindings: ${d.data.findings.total}`; - if (d.data.findings.CRITICAL) tip += `\n Critical: ${d.data.findings.CRITICAL}`; - if (d.data.findings.HIGH) tip += `\n High: ${d.data.findings.HIGH}`; - if (d.data.findings.MEDIUM) tip += `\n Medium: ${d.data.findings.MEDIUM}`; - } - if (d.data.internet_exposed) tip += '\n\n⚠ INTERNET EXPOSED'; + if (d.data.findings && d.data.findings.total > 0) tip += `\nFindings: ${d.data.findings.total}`; + if (d.data.internet_exposed) tip += '\nReachable from the internet'; return tip; }); } @@ -650,225 +1462,296 @@ const svg = d3.select(container).append('svg').attr('width', width).attr('height', height); const g = svg.append('g'); svgG = g; - zoomBehavior = d3.zoom().scaleExtent([0.2, 5]).on('zoom', (e) => g.attr('transform', e.transform)); svg.call(zoomBehavior); - if (!graphData.nodes || graphData.nodes.length === 0) return; - - const sim = d3.forceSimulation(graphData.nodes) - .force('link', d3.forceLink(graphData.links).id(d => d.id).distance(100)) - .force('charge', d3.forceManyBody().strength(-200)) + const nodes = graphData.nodes.map(n => Object.assign({}, n)); + const links = graphData.links.map(l => Object.assign({}, l)); + const sim = d3.forceSimulation(nodes) + .force('link', d3.forceLink(links).id(d => d.id).distance(80)) + .force('charge', d3.forceManyBody().strength(-180)) .force('center', d3.forceCenter(width / 2, height / 2)) - .force('collision', d3.forceCollide().radius(30)); - - const link = g.append('g').selectAll('line').data(graphData.links).join('line') - .attr('stroke', d => { - if (d.relation === 'FINDING_AFFECTS') { - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB'}; - return sevColors[d.severity] || '#EF4444'; - } - if (d.relation === 'COMPLIANCE_GOVERNS') return '#14B8A6'; - return d.cidr === '0.0.0.0/0' ? '#E74C3C' : '#555'; - }) - .attr('stroke-width', d => { - if (d.relation === 'FINDING_AFFECTS') return 2; - if (d.relation === 'COMPLIANCE_GOVERNS') return 1.5; - return d.cidr === '0.0.0.0/0' ? 2 : 1; - }) - .attr('stroke-dasharray', d => { - if (d.relation === 'FINDING_AFFECTS' || d.relation === 'COMPLIANCE_GOVERNS') return '6,3'; - return null; - }) - .attr('stroke-opacity', d => d.relation ? 0.7 : 0.5); - - const node = g.append('g').selectAll('g').data(graphData.nodes).join('g') + .force('collision', d3.forceCollide().radius(22)); + const sevVar = { CRITICAL: '--critical', HIGH: '--high', MEDIUM: '--medium', LOW: '--low' }; + const link = g.append('g').selectAll('line').data(links).join('line') + .attr('stroke', d => d.relation === 'FINDING_AFFECTS' ? cssVar(sevVar[d.severity] || '--critical') + : d.relation === 'COMPLIANCE_GOVERNS' ? cssVar('--f-governance') : cssVar('--line-strong')) + .attr('stroke-width', d => d.relation ? 1.4 : 1) + .attr('stroke-dasharray', d => d.relation ? '5,3' : null); + const node = g.append('g').selectAll('g').data(nodes).join('g') .call(d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended)); - - // Node shape: findings get triangles, compliance gets diamonds, rest get circles - node.each(function(d) { - const el = d3.select(this); - const r = d.is_external ? 8 : 16; - if (d.type === 'SECURITY_FINDING') { - // Triangle for findings - const sevColors = {CRITICAL:'#EF4444',HIGH:'#F97316',MEDIUM:'#EAB308',LOW:'#3498DB',INFO:'#64748B'}; - el.append('polygon') - .attr('points', `0,${-r} ${r},${r} ${-r},${r}`) - .attr('fill', sevColors[d.severity] || '#EF4444') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else if (d.type === 'COMPLIANCE_FRAMEWORK') { - // Diamond for compliance - el.append('polygon') - .attr('points', `0,${-r} ${r},0 0,${r} ${-r},0`) - .attr('fill', '#14B8A6') - .attr('stroke', '#fff') - .attr('stroke-width', 1.5); - } else { - el.append('circle').attr('r', r) - .attr('fill', typeColors[d.type] || '#95A5A6') - .attr('stroke', d.is_internet_exposed ? '#FF0000' : '#fff') - .attr('stroke-width', d.is_internet_exposed ? 3 : 1.5); - } - }); - - node.append('text').text(d => d.name ? d.name.substring(0, 15) : d.id.substring(0, 10)) - .attr('dy', 28).attr('text-anchor', 'middle') - .attr('fill', '#CBD5E1').attr('font-size', '10px'); - - node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region}\nARN: ${d.arn || 'N/A'}`); - + node.append('circle').attr('r', d => d.is_external ? 6 : d.type === 'SECURITY_FINDING' ? 5 : 9) + .attr('fill', d => d.type === 'SECURITY_FINDING' ? cssVar(sevVar[d.severity] || '--critical') : typeColor(d.type)) + .attr('stroke', d => d.is_internet_exposed ? cssVar('--f-exposure') : cssVar('--surface')) + .attr('stroke-width', d => d.is_internet_exposed ? 2.5 : 1.5); + node.append('text').text(d => (d.name || d.id || '').slice(0, 18)) + .attr('dy', 20).attr('text-anchor', 'middle').attr('fill', cssVar('--muted')).attr('font-size', '10px'); + node.append('title').text(d => `${d.type}: ${d.name}\nRegion: ${d.region || ''}\nARN: ${d.arn || 'N/A'}`); sim.on('tick', () => { - link.attr('x1', d => d.source.x).attr('y1', d => d.source.y) - .attr('x2', d => d.target.x).attr('y2', d => d.target.y); + link.attr('x1', d => d.source.x).attr('y1', d => d.source.y).attr('x2', d => d.target.x).attr('y2', d => d.target.y); node.attr('transform', d => `translate(${d.x},${d.y})`); }); - function dragstarted(e, d) { if (!e.active) sim.alphaTarget(0.3).restart(); d.fx = d.x; d.fy = d.y; } function dragged(e, d) { d.fx = e.x; d.fy = e.y; } function dragended(e, d) { if (!e.active) sim.alphaTarget(0); d.fx = null; d.fy = null; } } - function toggleLayout() { - currentLayout = currentLayout === 'tree' ? 'force' : 'tree'; - renderTopology(); - } - function resetZoom() { + if (!MAP.flat) { animateView([0, 0, W, H]); return; } const svg = d3.select('#topology-container svg'); - if (svg.node() && zoomBehavior) { - svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity); - } - } - - function expandAll() { - function expand(d) { if (d._children) { d.children = d._children; d._children = null; } if (d.children) d.children.forEach(expand); } - if (hierarchyData) { const root = d3.hierarchy(hierarchyData); expand(root); } - renderTopology(); - } - - function collapseAll() { - // Re-render with collapsed tree (only top 2 levels open) - renderTopology(); + if (svg.node() && zoomBehavior) svg.transition().duration(500).call(zoomBehavior.transform, d3.zoomIdentity.translate(80, 20)); } // ── Chart.js Charts ── + let chartsBuilt = false; + const charts = []; function renderCharts() { - // Severity Donut - new Chart(document.getElementById('chart-severity'), { + if (chartsBuilt || typeof Chart === 'undefined') return; + chartsBuilt = true; + charts.splice(0).forEach(c => c.destroy()); + const text = cssVar('--text'), muted = cssVar('--muted'), grid = cssVar('--line'); + const sev = SEVERITIES.map(s => cssVar('--' + s.toLowerCase())); + Chart.defaults.font.family = getComputedStyle(document.body).fontFamily; + Chart.defaults.color = muted; + const legend = { position: 'bottom', labels: { color: text, usePointStyle: true, pointStyle: 'circle', boxWidth: 8, padding: 14 } }; + charts.push(new Chart(document.getElementById('chart-severity'), { type: 'doughnut', - data: { - labels: Object.keys(severityCounts), - datasets: [{ - data: Object.values(severityCounts), - backgroundColor: ['#EF4444', '#F97316', '#EAB308', '#3498DB', '#64748B'], - borderWidth: 0, hoverOffset: 8, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Resource Types Bar - new Chart(document.getElementById('chart-resources'), { + data: { labels: SEVERITIES, datasets: [{ data: SEVERITIES.map(s => severityCounts[s] || 0), backgroundColor: sev, borderColor: cssVar('--surface'), borderWidth: 3, hoverOffset: 6 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); + const types = Object.entries(resourceTypes).sort((a, b) => b[1] - a[1]).slice(0, 12); + charts.push(new Chart(document.getElementById('chart-resources'), { type: 'bar', - data: { - labels: Object.keys(resourceTypes), - datasets: [{ - data: Object.values(resourceTypes), - backgroundColor: '#6366F1', borderRadius: 6, - }] - }, - options: { - responsive: true, indexAxis: 'y', - plugins: { legend: { display: false } }, - scales: { - x: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' } }, - y: { ticks: { color: '#CBD5E1' }, grid: { display: false } }, - }, - } - }); - - // Sources Donut - new Chart(document.getElementById('chart-sources'), { + data: { labels: types.map(t => t[0]), datasets: [{ data: types.map(t => t[1]), backgroundColor: cssVar('--accent'), borderRadius: 4, barThickness: 12 }] }, + options: { responsive: true, maintainAspectRatio: false, indexAxis: 'y', plugins: { legend: { display: false } }, + scales: { x: { ticks: { color: muted }, grid: { color: grid } }, y: { ticks: { color: text, font: { family: 'ui-monospace, monospace', size: 10 } }, grid: { display: false } } } }, + })); + const fam = ['--f-network', '--f-data', '--f-identity', '--f-governance', '--accent', '--high'].map(cssVar); + charts.push(new Chart(document.getElementById('chart-sources'), { type: 'doughnut', - data: { - labels: Object.keys(sourceTools), - datasets: [{ - data: Object.values(sourceTools), - backgroundColor: ['#6366F1', '#8B5CF6', '#EC4899', '#14B8A6', '#F97316', '#EAB308'], - borderWidth: 0, - }] - }, - options: { - responsive: true, plugins: { legend: { position: 'bottom', labels: { color: '#CBD5E1' } } }, - cutout: '65%', - } - }); - - // Compliance + data: { labels: Object.keys(sourceTools), datasets: [{ data: Object.values(sourceTools), backgroundColor: fam, borderColor: cssVar('--surface'), borderWidth: 3 }] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, cutout: '68%' }, + })); const fws = Object.keys(complianceSummary); - const passData = fws.map(fw => complianceSummary[fw]?.pass || 0); - const failData = fws.map(fw => complianceSummary[fw]?.fail || 0); - new Chart(document.getElementById('chart-compliance'), { + charts.push(new Chart(document.getElementById('chart-compliance'), { type: 'bar', - data: { - labels: fws, - datasets: [ - { label: 'Pass', data: passData, backgroundColor: '#22C55E', borderRadius: 4 }, - { label: 'Fail', data: failData, backgroundColor: '#EF4444', borderRadius: 4 }, - ] - }, - options: { - responsive: true, - plugins: { legend: { labels: { color: '#CBD5E1' } } }, - scales: { - x: { ticks: { color: '#CBD5E1' }, grid: { display: false }, stacked: true }, - y: { ticks: { color: '#94A3B8' }, grid: { color: '#1E293B' }, stacked: true }, - }, - } - }); + data: { labels: fws, datasets: [ + { label: 'Pass', data: fws.map(fw => complianceSummary[fw]?.pass || 0), backgroundColor: cssVar('--pass'), borderRadius: 3 }, + { label: 'Fail', data: fws.map(fw => complianceSummary[fw]?.fail || 0), backgroundColor: cssVar('--fail'), borderRadius: 3 }, + ] }, + options: { responsive: true, maintainAspectRatio: false, plugins: { legend }, + scales: { x: { ticks: { color: text }, grid: { display: false }, stacked: true }, y: { ticks: { color: muted }, grid: { color: grid }, stacked: true } } }, + })); } - // ── Compliance Grid ── - function renderCompliance() { - const grid = document.getElementById('compliance-grid'); - grid.innerHTML = Object.entries(complianceSummary).map(([fw, data]) => { - const pass = Number(data.pass) || 0; - const fail = Number(data.fail) || 0; - const total = pass + fail; - const pct = total > 0 ? Math.round((pass / total) * 100) : 0; - const barColor = pct >= 80 ? 'var(--success)' : pct >= 50 ? 'var(--medium)' : 'var(--critical)'; - return html` -
    -

    ${fw}

    -
    - Pass: ${pass}Fail: ${fail}${pct}% -
    -
    -
    -
    -
    - `; - }).join(''); + // ── Empty states: Hairline figures ── + // cloudg/templates/figures/empty-tray.js is a figure in Hairline's own + // format; this is the small host that mounts it (the stage, its svg and a + // read-out the figure writes to) in place of Hairline's bench page. + function mountEmptyTray(host) { + const spec = window.HAIRLINE_FIGURES && window.HAIRLINE_FIGURES.tray; + if (typeof HL === 'undefined' || !spec || !host || host.dataset.mounted) return; + host.dataset.mounted = '1'; + HL.inject(document); + const svg = HL.mk('svg', { viewBox: '0 0 400 320', 'aria-hidden': 'true' }, host); + const read = document.createElement('span'); + read.setAttribute('data-hairline-live', ''); + host.appendChild(read); + spec.mount({ stage: host, svg, read }, spec.range[1]); } // ── Export ── function exportJSON() { - const blob = new Blob([JSON.stringify({findings: findingsData, graph: graphData, hierarchy: hierarchyData}, null, 2)], {type: 'application/json'}); + const blob = new Blob([JSON.stringify({ findings: findingsData, assets: assetsData, graph: graphData, hierarchy: hierarchyData, scene: isoScene }, null, 2)], { type: 'application/json' }); const a = document.createElement('a'); a.href = URL.createObjectURL(blob); a.download = 'findings.json'; a.click(); } + function exportCSV() { + const cols = ['severity', 'title', 'resource_id', 'resource_arn', 'source_tool', 'risk_score', 'cvss_score', 'compliance_frameworks', 'remediation']; + const cell = v => { + let s = Array.isArray(v) ? v.join('; ') : (v === null || v === undefined ? '' : String(v)); + if (/^[=+\-@\t\r]/.test(s)) s = "'" + s; // keep spreadsheets from evaluating it + return '"' + s.replace(/"/g, '""') + '"'; + }; + const rows = [cols.join(',')].concat(findingsData.map(f => cols.map(c => cell(f[c])).join(','))); + const blob = new Blob([rows.join('\r\n')], { type: 'text/csv' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = 'cloudg-findings.csv'; + a.click(); + } // ── Init ── (tests/test_html_report.py runs the script up to this line) - renderFindings(findingsData); - renderTopology(); - renderCharts(); - renderCompliance(); + (function init() { + const root = document.documentElement; + let theme = null; + try { theme = localStorage.getItem('cloudg-report-theme'); } catch (e) { theme = null; } + if (!theme) theme = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; + root.dataset.theme = theme; + document.body.dataset.theme = theme; + + // The kit's tones, scoped to the report's body.iso + if (typeof IsoKit !== 'undefined') { + const style = document.createElement('style'); + style.textContent = IsoKit.ISO_CSS; + document.head.insertBefore(style, document.head.firstChild); + } + (isoScene.nodes || []).forEach(n => MAP.byId.set(n.id, n)); + (isoScene.externals || []).forEach(e => MAP.byId.set(e.id, e)); + + document.getElementById('main-tabs').addEventListener('click', e => { + const tab = e.target.closest('[data-tab]'); + if (tab) switchTab(tab.dataset.tab); + }); + document.getElementById('theme-toggle').addEventListener('click', () => { + const next = root.dataset.theme === 'light' ? 'dark' : 'light'; + root.dataset.theme = next; document.body.dataset.theme = next; + try { localStorage.setItem('cloudg-report-theme', next); } catch (e) { /* private mode */ } + if (chartsBuilt) { chartsBuilt = false; if (document.getElementById('tab-charts').classList.contains('active')) renderCharts(); } + if (MAP.flat) renderTopology(); + }); + document.getElementById('print-btn').addEventListener('click', () => window.print()); + document.getElementById('csv-btn').addEventListener('click', exportCSV); + document.getElementById('scrim').addEventListener('click', closeDetail); + document.addEventListener('keydown', e => { if (e.key === 'Escape') closeDetail(); }); + + document.getElementById('findings-body').addEventListener('click', (e) => { + const row = e.target.closest('tr[data-index]'); + if (row) showDetail(Number(row.dataset.index)); + }); + document.getElementById('top-risks').addEventListener('click', e => { + const b = e.target.closest('[data-index]'); + if (b) showDetail(Number(b.dataset.index)); + }); + ['filter-severity', 'filter-source'].forEach(id => document.getElementById(id).addEventListener('change', filterFindings)); + document.getElementById('filter-search').addEventListener('input', filterFindings); + const sourceSelect = document.getElementById('filter-source'); + Object.keys(sourceTools).forEach(tool => { + const opt = document.createElement('option'); + opt.value = tool; opt.textContent = tool; + sourceSelect.appendChild(opt); + }); + const providerSelect = document.getElementById('asset-provider'); + [...new Set(assetsData.map(a => a.provider))].sort().forEach(p => { + const opt = document.createElement('option'); + opt.value = p; opt.textContent = p; + providerSelect.appendChild(opt); + }); + providerSelect.addEventListener('change', renderAssets); + document.getElementById('asset-search').addEventListener('input', renderAssets); + document.getElementById('assets-body').addEventListener('click', e => { + const row = e.target.closest('tr[data-asset]'); + if (!row || !MAP.byId.has(row.dataset.asset)) return; + switchTab('topology'); + if (MAP.flat && !document.getElementById('view-3d').disabled) setFlat(false); + setTimeout(() => selectNode(row.dataset.asset, { center: true }), 80); + }); + + // Map controls + const stage = document.getElementById('stage'); + const svg = document.getElementById('iso-svg'); + document.getElementById('view-3d').addEventListener('click', () => setFlat(false)); + document.getElementById('view-2d').addEventListener('click', () => setFlat(true)); + document.getElementById('layout-tree').addEventListener('click', () => { currentLayout = 'tree'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'true'); document.getElementById('layout-force').setAttribute('aria-pressed', 'false'); renderTopology(); }); + document.getElementById('layout-force').addEventListener('click', () => { currentLayout = 'force'; document.getElementById('layout-tree').setAttribute('aria-pressed', 'false'); document.getElementById('layout-force').setAttribute('aria-pressed', 'true'); renderTopology(); }); + document.getElementById('rot-left').addEventListener('click', () => turn(-90)); + document.getElementById('rot-right').addEventListener('click', () => turn(90)); + document.getElementById('zoom-in').addEventListener('click', () => MAP.flat ? null : zoomBy(1.35)); + document.getElementById('zoom-out').addEventListener('click', () => MAP.flat ? null : zoomBy(1 / 1.35)); + document.getElementById('zoom-fit').addEventListener('click', resetZoom); + document.getElementById('labels-btn').addEventListener('click', e => { MAP.labels = !MAP.labels; e.currentTarget.setAttribute('aria-pressed', String(MAP.labels)); renderLabels(); }); + document.getElementById('trace-internet').addEventListener('click', traceFromInternet); + document.getElementById('replay-btn').addEventListener('click', () => { MAP.selected = null; applyFocus(); renderInspector(); playIntro(); }); + document.getElementById('families').addEventListener('click', e => { + const b = e.target.closest('[data-family]'); + if (!b) return; + const f = b.dataset.family; + if (MAP.families.has(f)) MAP.families.delete(f); else MAP.families.add(f); + b.setAttribute('aria-pressed', String(MAP.families.has(f))); + MAP.traceEls.forEach(({ g, t }) => g.classList.toggle('hidden-family', !MAP.families.has(t.family))); + applyFocus(MAP.focusTrace ? [MAP.focusTrace] : undefined); + }); + document.getElementById('inspector').addEventListener('click', e => { + const b = e.target.closest('[data-trace]'); + if (b) focusTrace(b.dataset.trace); + }); + svg.addEventListener('click', e => { + if (MAP.dragMoved) return; + const tr = e.target.closest('[data-trace]'); + if (tr && !e.target.closest('[data-node]')) { const t = MAP.traceEls.get(tr.dataset.trace); if (t) { MAP.selected = t.t.source; renderInspector(); focusTrace(tr.dataset.trace); renderLabels(); } return; } + const node = e.target.closest('[data-node]'); + selectNode(node ? node.getAttribute('data-node') : null); + }); + svg.addEventListener('pointerover', e => { + const node = e.target.closest('[data-node]'); + const id = node ? node.getAttribute('data-node') : null; + if (id === MAP.hover) return; + MAP.hover = id; renderLabels(); + const n = id && MAP.byId.get(id); + if (n) setReadout(n.name + ' · ' + (n.type || n.kind)); else if (!MAP.selected) setReadout(); + }); + // Pan with a drag, zoom with the wheel about the pointer + const toView = (cx, cy) => { + const r = svg.getBoundingClientRect(), [x, y, w, h] = MAP.view; + const s = Math.min(r.width / w, r.height / h); + const ox = (r.width - w * s) / 2, oy = (r.height - h * s) / 2; + return [x + (cx - r.left - ox) / s, y + (cy - r.top - oy) / s, s]; + }; + stage.addEventListener('wheel', e => { + if (MAP.flat) return; + e.preventDefault(); + const [px, py] = toView(e.clientX, e.clientY); + zoomBy(Math.exp(-e.deltaY * 0.0015), px, py); + }, { passive: false }); + let drag = null; + stage.addEventListener('pointerdown', e => { + if (MAP.flat || e.button !== 0) return; + drag = { x: e.clientX, y: e.clientY, view: MAP.view.slice(), s: toView(e.clientX, e.clientY)[2] }; + MAP.dragMoved = false; + }); + window.addEventListener('pointermove', e => { + if (!drag) return; + const dx = e.clientX - drag.x, dy = e.clientY - drag.y; + if (!MAP.dragMoved && Math.hypot(dx, dy) < 4) return; + MAP.dragMoved = true; stage.classList.add('dragging'); + setView([drag.view[0] - dx / drag.s, drag.view[1] - dy / drag.s, drag.view[2], drag.view[3]]); + }); + window.addEventListener('pointerup', () => { drag = null; stage.classList.remove('dragging'); setTimeout(() => { MAP.dragMoved = false; }, 0); }); + stage.addEventListener('keydown', e => { + if (MAP.flat) return; + const ids = (isoScene.nodes || []).map(n => n.id); + if (e.key === '[') turn(-90); + else if (e.key === ']') turn(90); + else if (e.key === '+' || e.key === '=') zoomBy(1.35); + else if (e.key === '-') zoomBy(1 / 1.35); + else if (e.key === '0') resetZoom(); + else if (e.key === 'Escape') selectNode(null); + else if (e.key === 'ArrowRight' || e.key === 'ArrowLeft' || e.key === 'ArrowDown' || e.key === 'ArrowUp') { + const i = ids.indexOf(MAP.selected), d = e.key === 'ArrowRight' || e.key === 'ArrowDown' ? 1 : -1; + selectNode(ids[(i + d + ids.length) % ids.length], { center: true }); + } else return; + e.preventDefault(); + }); + const list = document.getElementById('map-assets'); + (isoScene.nodes || []).slice(0, 3000).forEach(n => { const o = document.createElement('option'); o.value = n.name; list.appendChild(o); }); + document.getElementById('map-search').addEventListener('change', e => { + const q = e.target.value.trim().toLowerCase(); + const n = (isoScene.nodes || []).find(x => x.name.toLowerCase() === q) || (isoScene.nodes || []).find(x => x.name.toLowerCase().includes(q) || (x.arn || '').toLowerCase().includes(q)); + if (n) { if (MAP.flat) setFlat(false); selectNode(n.id, { center: true }); } + }); + + renderOverview(); + renderFindings(findingsData); + renderAssets(); + renderCompliance(); + renderFamilies(); + renderLegend(); + renderTopology(); + mountEmptyTray(document.getElementById('findings-empty-figure')); + mountEmptyTray(document.getElementById('compliance-empty-figure')); + })(); diff --git a/cloudg/templates/vendor/LICENSE-anatomy.txt b/cloudg/templates/vendor/LICENSE-anatomy.txt new file mode 100644 index 0000000..aa20733 --- /dev/null +++ b/cloudg/templates/vendor/LICENSE-anatomy.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ryan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cloudg/templates/vendor/LICENSE-hairline.txt b/cloudg/templates/vendor/LICENSE-hairline.txt new file mode 100644 index 0000000..dec0685 --- /dev/null +++ b/cloudg/templates/vendor/LICENSE-hairline.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Lucas Marques + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cloudg/templates/vendor/NOTICE b/cloudg/templates/vendor/NOTICE index c7da8ab..b14bb49 100644 --- a/cloudg/templates/vendor/NOTICE +++ b/cloudg/templates/vendor/NOTICE @@ -1,10 +1,11 @@ Third-party JavaScript bundled with cloudg ========================================== -The HTML report (report.html) inlines these two libraries when +The HTML report (report.html) inlines Chart.js and D3 when report.inline_js is true (the default), so the report works without -network access. The files are the unmodified builds published on npm; -their own licence headers are kept. +network access. Those two files are the unmodified builds published on +npm; their own licence headers are kept. The isometric map's two drawing +engines below have no CDN build and are always inlined. chart.umd.js Chart.js 4.4.0, file dist/chart.umd.js of the npm package chart.js@4.4.0 @@ -20,6 +21,24 @@ d3.min.js ISC License, see LICENSE-d3.txt sha256 f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539 +iso-kit.js + Anatomy iso-kit, file kit/iso-kit.mjs of https://github.com/wheresryan22/anatomy + Copyright (c) 2026 Ryan + MIT License, see LICENSE-anatomy.txt + The module is wrapped as IsoKit for a classic script element: its + "export" keywords are removed, and the two script tags inside one + string are written "\x3cscript" and "<\/script>", which are the same + characters at run time. Source sha256 + 6d3ecc725d30c3b0346da309cf61dfa5a65017c23620bbd4093009d4fb104018 + +hairline-kernel.js + Hairline kernel, the src/core build of @lucasmarkes/hairline 0.5.1 + https://hairline.lucasmarkes.com + Copyright (c) 2026 Lucas Marques + MIT License, see LICENSE-hairline.txt + Unmodified. sha256 + 47ebec7647174e04e4f59c0305277c2a1dab83a763676946b17fa1694e7952fe + To update a library, replace the file with the build from the new npm release, update this notice, and update the version, CDN URL and SRI hash in cloudg/renderers/html_report.py (tests/test_html_report.py checks that diff --git a/cloudg/templates/vendor/hairline-kernel.js b/cloudg/templates/vendor/hairline-kernel.js new file mode 100644 index 0000000..336d4b6 --- /dev/null +++ b/cloudg/templates/vendor/hairline-kernel.js @@ -0,0 +1,714 @@ +/* hairline kernel sha256:346b48bdef2cdfafd5343eba01494255cf75f9c078e6fcd1aafc62bab62980a1 */ +/* + * HL: everything a figure may call. Read this index; the code under it is the + * package's src/core, unchanged, and a figure should not need to read it. + * + * Every figure is drawn in a 400 × 320 viewBox. World space is x/y on the + * ground and z up. Plates are filled with the ground colour and painted back + * to front, so a nearer one covers a farther one: append in that order. At + * the default camera, Cam(45, 0.5, S), +x runs down to the right and +y down + * to the left, so the corner with the largest x and y is nearest the viewer: + * append by ascending x + y, from the far corner (smallest x and y) to it. + * + * Camera + * Cam(azDeg, k, S) a camera: azimuth in degrees, k = sin(elevation) (0.5 is the 2:1 view), S = scale + * fit(C, points, cx, cy) centres the box of [x, y, z] points on (cx, cy); call it once, before proj + * it only centres, it never scales: choose S by trying values, with the most extreme pose in points. + * The six figures use S 1.42 to 2.12; the boxes they fit come out 230 to 310 wide and 180 to 245 tall + * proj(C) returns P(x, y, z), which gives [sx, sy] + * unproj(C, sx, sy, z) the world [x, y] under a screen point, on the plane at height z + * facing(C) returns front(sample): whether a ring sample faces the camera + * Rounded solids + * rrect(u0, v0, u1, v1, r, n) a rounded rectangle, as a ring of samples {u, v, nu, nv} + * circ(R, n) a circle, as a ring + * rings(x0, y0, x1, y1, r, b) [ring, inner]: a rounded footprint and its crease ring, inset by b + * r, the corner radius, is cut to half the shorter side. b, the crease's inset in world units + * (0.6 to 2.2 in the figures), must stay under half the shorter side or the crease turns inside out + * prism(P, front, ring, inner, z0, z1) {sil, crease}: a solid standing from z0 to z1, as two path strings + * ringAt(P, ring, z) the ring's points, projected at height z + * run(ring, keep) the one cyclic run of samples that pass keep + * hull(points) the convex hull of screen points + * extremes(P, ring) [left, right, nearest] samples: where dashed drops fall from + * fillet(points, radii, n) rounds every vertex of a closed polygon; returns the new points + * radii is an array, one radius per vertex, each cut to half its shorter edge; a single number gives NaN. + * n is the steps round each corner, default 4: each vertex becomes n + 1 points + * ghost(P, front, ring, z0, depth) a reflection's path {d, y0, y1}; reflect() draws it for you + * Paths and numbers + * poly(points) a closed path string + * open(points) an open polyline string + * seg(a, b) one segment between two screen points [sx, sy], as its own subpath; project world points with P first + * clamp(v, a, b) + * lerp(a, b, t) + * rad(deg) + * r2(n) two decimals + * The continuous clock: one spring per moving number + * spring(x, opts) at rest on x; write .t to retarget; opts {k, c, m, eps}, default k 100, c 18, m 1 + * stepS(sp, dt) advances by dt seconds; returns whether it is still moving + * The discrete clock: a 700ms tween on (.32, .72, 0, 1) + * tween(v, dur) at rest on v; dur defaults to 700 (ms) + * tset(tw, to, now, delay) retargets from where it is, after delay ms: the stagger + * the same target again does nothing, so calling it on every pointer move is safe + * tval(tw, now) its value at now + * tdone(tw, now) whether it has landed + * bezier(x1, y1, x2, y2) a CSS cubic-bezier, as a function of progress + * EASE_LIFT the lift curve itself + * reducedMotion() true when the reader asked for less motion; springs and tweens already land at once + * setReducedMotion(on) the loop's business, not a figure's + * Drawing + * mk(tag, attrs, parent) one svg element: the only way a figure makes a node + * solid(parent) {g, sil, cr}: a group holding a silhouette path and a crease path + * put(solid, paths) writes prism()'s {sil, crease} into solid()'s {sil, cr}: sil into sil, crease into cr + * flatDot(parent, C, r, cls) a dot lying on the ground plane; cls is "dot", "dot m" or "dot off" + * place(el, point) moves a dot or a circle to [sx, sy] + * reflect(svg, parent, P, front, ring, z0, depth) a fading mirror under a solid + * fade(svg, y0, y1, a0) a vertical fade, as a mask; returns the value for a mask attribute + * Life + * register(stage, tick) joins the one frame loop; tick(dt in seconds, now in ms) returns true to ask for another frame; gives {wake, unregister} + * pointer(stage, handlers) {move(point), down(point), leave()}, points in viewBox units; returns its disposer + * disposer() {add, on, dispose}: collects tear-down, so destroy is bag.dispose + * The bench's business, not a figure's + * css(lightDark) + * inject(root) + * tour(stage, stops, onStop) an unseen pointer that walks the stops, [x, y] in viewBox units or null to leave, and gives way to a real one; onStop(i) on each arrival; returns {stop} + * LAP the default stops: a diamond around the centre, then a leave + * + * Classes, on path, polygon, ellipse and line. They are the whole palette; a + * figure sets no colour, width or fill of its own. + * (none) filled with the ground colour, medium stroke + * sil the silhouette's stroke hi the bright stroke: the only highlight + * lo the dim stroke nf no fill fo fill only, no stroke + * dash a dashed guide + * dot a bright dot dot m a medium dot dot off a dim dot + */ +var HL = (() => { + var __defProp = Object.defineProperty; + var __getOwnPropDesc = Object.getOwnPropertyDescriptor; + var __getOwnPropNames = Object.getOwnPropertyNames; + var __hasOwnProp = Object.prototype.hasOwnProperty; + var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); + }; + var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; + }; + var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + + // packages/hairline/src/core/kernel.ts + var kernel_exports = {}; + __export(kernel_exports, { + Cam: () => Cam, + EASE_LIFT: () => EASE_LIFT, + LAP: () => LAP, + bezier: () => bezier, + circ: () => circ, + clamp: () => clamp, + css: () => css, + disposer: () => disposer, + extremes: () => extremes, + facing: () => facing, + fade: () => fade, + fillet: () => fillet, + fit: () => fit, + flatDot: () => flatDot, + ghost: () => ghost, + hull: () => hull, + inject: () => inject, + lerp: () => lerp, + mk: () => mk, + open: () => open, + place: () => place, + pointer: () => pointer, + poly: () => poly, + prism: () => prism, + proj: () => proj, + put: () => put, + r2: () => r2, + rad: () => rad, + reducedMotion: () => reducedMotion, + reflect: () => reflect, + register: () => register, + ringAt: () => ringAt, + rings: () => rings, + rrect: () => rrect, + run: () => run, + seg: () => seg, + setReducedMotion: () => setReducedMotion, + solid: () => solid, + spring: () => spring, + stepS: () => stepS, + tdone: () => tdone, + tour: () => tour, + tset: () => tset, + tval: () => tval, + tween: () => tween, + unproj: () => unproj + }); + + // packages/hairline/src/core/iso.ts + var clamp = (v, a, b) => Math.max(a, Math.min(b, v)); + var lerp = (a, b, t) => a + (b - a) * t; + var rad = (d) => d * Math.PI / 180; + var r2 = (n) => Math.round(n * 100) / 100; + var poly = (pts) => "M" + pts.map((p) => r2(p[0]) + " " + r2(p[1])).join("L") + "Z"; + var seg = (a, b) => `M${r2(a[0])} ${r2(a[1])}L${r2(b[0])} ${r2(b[1])}`; + var open = (pts) => pts.length < 2 ? "" : "M" + pts.map((p) => r2(p[0]) + " " + r2(p[1])).join("L"); + var Cam = (azDeg, k, S) => ({ az: rad(azDeg), k, S, ox: 0, oy: 0 }); + function proj(C) { + const c = Math.cos(C.az), s = Math.sin(C.az), zf = Math.sqrt(1 - C.k * C.k); + return (x, y, z) => { + const X = x * c - y * s, Y = x * s + y * c; + return [C.ox + C.S * X, C.oy + C.S * (Y * C.k - z * zf)]; + }; + } + function unproj(C, sx, sy, z) { + const c = Math.cos(C.az), s = Math.sin(C.az), zf = Math.sqrt(1 - C.k * C.k); + const X = (sx - C.ox) / C.S, Y = ((sy - C.oy) / C.S + z * zf) / C.k; + return [X * c + Y * s, -X * s + Y * c]; + } + function fit(C, pts, cx, cy) { + C.ox = 0; + C.oy = 0; + const P = proj(C); + let a = 1e9, b = -1e9, c = 1e9, d = -1e9; + for (const p of pts) { + const q = P(p[0], p[1], p[2]); + a = Math.min(a, q[0]); + b = Math.max(b, q[0]); + c = Math.min(c, q[1]); + d = Math.max(d, q[1]); + } + C.ox = cx - (a + b) / 2; + C.oy = cy - (c + d) / 2; + } + function rrect(u0, v0, u1, v1, r, n = 4) { + r = Math.max(0, Math.min(r, (u1 - u0) / 2, (v1 - v0) / 2)); + const out = []; + for (const [cu, cv, a0] of [[u1 - r, v1 - r, 0], [u0 + r, v1 - r, 90], [u0 + r, v0 + r, 180], [u1 - r, v0 + r, 270]]) + for (let k = 0; k <= n; k++) { + const a = rad(a0 + 90 * k / n), ca = Math.cos(a), sa = Math.sin(a); + out.push({ u: cu + r * ca, v: cv + r * sa, nu: ca, nv: sa }); + } + return out; + } + function circ(R, n = 96) { + const out = []; + for (let k = 0; k < n; k++) { + const a = k / n * Math.PI * 2, ca = Math.cos(a), sa = Math.sin(a); + out.push({ u: R * ca, v: R * sa, nu: ca, nv: sa }); + } + return out; + } + function hull(input) { + const pts = input.slice().sort((a, b) => a[0] - b[0] || a[1] - b[1]); + const x = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lo = [], up = []; + for (const p of pts) { + while (lo.length > 1 && x(lo[lo.length - 2], lo[lo.length - 1], p) <= 0) lo.pop(); + lo.push(p); + } + for (let i = pts.length - 1; i >= 0; i--) { + const p = pts[i]; + while (up.length > 1 && x(up[up.length - 2], up[up.length - 1], p) <= 0) up.pop(); + up.push(p); + } + lo.pop(); + up.pop(); + return lo.concat(up); + } + var ringAt = (P, ring, z) => ring.map((q) => P(q.u, q.v, z)); + var facing = (C) => { + const s = Math.sin(C.az), c = Math.cos(C.az); + return (q) => q.nu * s + q.nv * c >= -1e-6; + }; + function run(ring, keep) { + const n = ring.length; + let s = -1; + for (let i = 0; i < n; i++) if (keep(ring[i]) && !keep(ring[(i + n - 1) % n])) { + s = i; + break; + } + if (s < 0) return keep(ring[0]) ? ring.slice() : []; + const out = []; + for (let k = 0; k < n && keep(ring[(s + k) % n]); k++) out.push(ring[(s + k) % n]); + return out; + } + function prism(P, front, ring, inner, z0, z1) { + return { + sil: poly(hull(ringAt(P, ring, z1).concat(ringAt(P, ring, z0)))), + crease: inner ? open(ringAt(P, run(inner, front), z1)) : "" + }; + } + var rings = (x0, y0, x1, y1, r, b) => [ + rrect(x0, y0, x1, y1, r), + rrect(x0 + b, y0 + b, x1 - b, y1 - b, Math.max(0.3, r - b)) + ]; + function extremes(P, ring) { + const pr = ring.map((q) => P(q.u, q.v, 0)); + let a = 0, b = 0, c = 0; + pr.forEach((p, k) => { + if (p[0] < pr[a][0]) a = k; + if (p[0] > pr[b][0]) b = k; + if (p[1] > pr[c][1]) c = k; + }); + return [ring[a], ring[b], ring[c]]; + } + function fillet(pts, rs, n = 4) { + const m = pts.length, out = []; + for (let i = 0; i < m; i++) { + const a = pts[(i + m - 1) % m], p = pts[i], b = pts[(i + 1) % m]; + const la = Math.hypot(a[0] - p[0], a[1] - p[1]), lb = Math.hypot(b[0] - p[0], b[1] - p[1]); + const t = Math.min(rs[i], la / 2, lb / 2); + const p1 = [p[0] + (a[0] - p[0]) / la * t, p[1] + (a[1] - p[1]) / la * t]; + const p2 = [p[0] + (b[0] - p[0]) / lb * t, p[1] + (b[1] - p[1]) / lb * t]; + for (let k = 0; k <= n; k++) { + const s = k / n, w = 1 - s; + out.push([w * w * p1[0] + 2 * w * s * p[0] + s * s * p2[0], w * w * p1[1] + 2 * w * s * p[1] + s * s * p2[1]]); + } + } + return out; + } + function ghost(P, front, ring, z0, depth) { + const f = run(ring, front), lowP = ringAt(P, f, z0 - depth); + return { + d: open(lowP) + [f[0], f[f.length - 1]].map((q) => seg(P(q.u, q.v, z0), P(q.u, q.v, z0 - depth))).join(""), + y0: Math.min(...ringAt(P, f, z0).map((p) => p[1])), + y1: Math.max(...lowP.map((p) => p[1])) + 2 + }; + } + + // packages/hairline/src/core/motion.ts + var reduced = false; + var setReducedMotion = (on) => { + reduced = on; + }; + var reducedMotion = () => reduced; + function spring(x, o = {}) { + return { x, v: 0, t: x, k: o.k ?? 100, c: o.c ?? 18, m: o.m ?? 1, eps: o.eps ?? 0.01 }; + } + function stepS(sp, dt) { + if (reduced) { + sp.x = sp.t; + sp.v = 0; + return false; + } + const n = Math.max(1, Math.ceil(dt * 240)), h = dt / n; + for (let i = 0; i < n; i++) { + const a = (-sp.k * (sp.x - sp.t) - sp.c * sp.v) / sp.m; + sp.v += a * h; + sp.x += sp.v * h; + } + if (Math.abs(sp.x - sp.t) < sp.eps && Math.abs(sp.v) < sp.eps * 10) { + sp.x = sp.t; + sp.v = 0; + return false; + } + return true; + } + function bezier(x1, y1, x2, y2) { + const cx = 3 * x1, bx = 3 * (x2 - x1) - cx, ax = 1 - cx - bx; + const cy = 3 * y1, by = 3 * (y2 - y1) - cy, ay = 1 - cy - by; + const X = (u) => ((ax * u + bx) * u + cx) * u; + const Y = (u) => ((ay * u + by) * u + cy) * u; + const dX = (u) => (3 * ax * u + 2 * bx) * u + cx; + return (t) => { + if (t <= 0) return 0; + if (t >= 1) return 1; + let u = t; + for (let i = 0; i < 8; i++) { + const e = X(u) - t; + if (Math.abs(e) < 1e-5) break; + const d = dX(u); + if (Math.abs(d) < 1e-6) break; + u -= e / d; + } + if (!(u >= 0 && u <= 1) || Math.abs(X(u) - t) > 1e-4) { + let lo = 0, hi = 1; + u = t; + for (let i = 0; i < 24; i++) { + if (X(u) < t) lo = u; + else hi = u; + u = (lo + hi) / 2; + } + } + return Y(u); + }; + } + var EASE_LIFT = bezier(0.32, 0.72, 0, 1); + var tween = (v, dur = 700) => ({ from: v, to: v, t0: -1e9, dur }); + var tval = (tw, now) => { + const p = clamp((now - tw.t0) / tw.dur, 0, 1); + return tw.from + (tw.to - tw.from) * (reduced ? 1 : EASE_LIFT(p)); + }; + var tset = (tw, to, now, delay) => { + if (tw.to === to) return; + tw.from = tval(tw, now); + tw.to = to; + tw.t0 = now + delay; + }; + var tdone = (tw, now) => reduced || now >= tw.t0 + tw.dur; + + // packages/hairline/src/core/stage.ts + var NS = "http://www.w3.org/2000/svg"; + function mk(tag, attrs, parent) { + const e = document.createElementNS(NS, tag); + if (attrs) for (const k in attrs) e.setAttribute(k, String(attrs[k])); + if (parent) parent.appendChild(e); + return e; + } + function solid(parent) { + const g = mk("g", {}, parent); + return { g, sil: mk("path", { class: "sil" }, g), cr: mk("path", { class: "nf lo" }, g) }; + } + var put = (el, s) => { + el.sil.setAttribute("d", s.sil); + el.cr.setAttribute("d", s.crease); + }; + var flatDot = (parent, C, r, cls) => mk("ellipse", { rx: r2(r * C.S), ry: r2(r * C.S * C.k), class: cls }, parent); + var place = (el, q) => { + el.setAttribute("cx", String(r2(q[0]))); + el.setAttribute("cy", String(r2(q[1]))); + }; + var fid = 0; + function fade(svg, y0, y1, a0 = 0.7) { + const id = "hl-fd" + ++fid, defs = mk("defs", {}, svg); + const lg = mk("linearGradient", { id: id + "g", gradientUnits: "userSpaceOnUse", x1: 0, y1: r2(y0), x2: 0, y2: r2(y1) }, defs); + mk("stop", { offset: 0, "stop-color": "#fff", "stop-opacity": a0 }, lg); + mk("stop", { offset: 1, "stop-color": "#fff", "stop-opacity": 0 }, lg); + const m = mk("mask", { id, maskUnits: "userSpaceOnUse", x: 0, y: 0, width: 400, height: 320 }, defs); + mk("rect", { x: 0, y: 0, width: 400, height: 320, fill: `url(#${id}g)` }, m); + return `url(#${id})`; + } + function reflect(svg, parent, P, front, ring, z0, depth) { + const r = ghost(P, front, ring, z0, depth); + const gh = mk("g", { class: "ghost", mask: fade(svg, r.y0, r.y1) }, parent); + mk("path", { d: r.d }, gh); + } + var boards = []; + var byStage = /* @__PURE__ */ new Map(); + var raf = 0; + var last = 0; + var io = null; + var rm = null; + function frame() { + const now = performance.now(); + const dt = Math.min(0.05, Math.max(0, (now - last) / 1e3)); + last = now; + let any = false; + for (const b of boards.slice()) if (b.vis && b.awake) { + try { + b.awake = !!b.tick(dt, now); + } catch (err) { + b.awake = false; + setTimeout(() => { + throw err; + }); + } + any = any || b.awake; + } + raf = any ? requestAnimationFrame(frame) : 0; + } + function wake(b) { + b.awake = true; + if (!raf) { + last = performance.now(); + raf = requestAnimationFrame(frame); + } + } + var onMotion = () => { + setReducedMotion(!!rm?.matches); + boards.forEach(wake); + }; + function start() { + if (io) return; + io = new IntersectionObserver((es) => { + for (const e of es) { + const set = byStage.get(e.target); + if (!set) continue; + for (const b of set) { + b.vis = e.isIntersecting; + if (b.vis) wake(b); + } + } + }, { rootMargin: "80px" }); + rm = matchMedia("(prefers-reduced-motion: reduce)"); + setReducedMotion(rm.matches); + rm.addEventListener("change", onMotion); + } + function stop() { + if (raf) cancelAnimationFrame(raf); + raf = 0; + io?.disconnect(); + io = null; + rm?.removeEventListener("change", onMotion); + rm = null; + } + function register(stage, tick) { + start(); + const b = { stage, tick, vis: false, awake: true }; + boards.push(b); + const peers = byStage.get(stage); + if (peers) { + for (const p of peers) b.vis = p.vis; + peers.add(b); + if (b.vis) wake(b); + } else { + byStage.set(stage, /* @__PURE__ */ new Set([b])); + io.observe(stage); + } + tick(0, performance.now()); + let gone = false; + return { + wake: () => { + if (!gone) wake(b); + }, + unregister: () => { + if (gone) return; + gone = true; + boards = boards.filter((x) => x !== b); + const set = byStage.get(stage); + if (set) { + set.delete(b); + if (!set.size) { + byStage.delete(stage); + io?.unobserve(stage); + } + } + if (!boards.length) stop(); + } + }; + } + var handlers = /* @__PURE__ */ new WeakMap(); + var touring = /* @__PURE__ */ new WeakMap(); + function pointer(stage, on) { + handlers.set(stage, on); + let tm = 0; + const pt = (e) => { + const r = stage.getBoundingClientRect(); + return [(e.clientX - r.left) / r.width * 400, (e.clientY - r.top) / r.height * 320]; + }; + const move = (e) => { + clearTimeout(tm); + touring.get(stage)?.hold(); + on.move(pt(e), e); + }; + const down = (e) => { + clearTimeout(tm); + touring.get(stage)?.hold(); + if (e.pointerType !== "mouse") stage.releasePointerCapture?.(e.pointerId); + if (on.down) on.down(pt(e), e); + else on.move(pt(e), e); + }; + const leave = (e) => { + clearTimeout(tm); + tm = window.setTimeout(() => { + on.leave(e); + touring.get(stage)?.release(); + }, e.pointerType === "mouse" ? 0 : 1400); + }; + stage.addEventListener("pointermove", move); + stage.addEventListener("pointerdown", down); + stage.addEventListener("pointerleave", leave); + return () => { + clearTimeout(tm); + if (handlers.get(stage) === on) handlers.delete(stage); + stage.removeEventListener("pointermove", move); + stage.removeEventListener("pointerdown", down); + stage.removeEventListener("pointerleave", leave); + }; + } + var LAP = [[128, 150], [200, 118], [272, 150], [200, 206], null]; + var TRAVEL = 900; + var DWELL = 1200; + var REST = 1800; + var RESUME = 1200; + var STAGGER = 450; + var GHOST = { pointerType: "ghost" }; + var started = 0; + function entry([x, y]) { + const dx = x - 200, dy = y - 160; + if (!dx && !dy) return [200, 320]; + const k = Math.min(dx ? (dx > 0 ? 200 : -200) / dx : Infinity, dy ? (dy > 0 ? 160 : -160) / dy : Infinity); + return [200 + dx * k, 160 + dy * k]; + } + function tour(stage, stops, onStop) { + let i = 0; + let wait = RESUME + STAGGER * (started++ % 4); + let t = -1; + let at = null; + let origin = [200, 320]; + let hand = false, keys = stage.contains(stage.ownerDocument.activeElement); + let gone = false; + const leave = () => { + at = null; + t = -1; + handlers.get(stage)?.leave(GHOST); + }; + const tick = (dt) => { + if (hand || keys || !stops.length) return false; + if (reducedMotion()) { + if (at) leave(); + return false; + } + if (wait > 0) { + wait -= dt * 1e3; + return true; + } + const stop2 = stops[i]; + if (stop2 === null) { + if (at) leave(); + onStop?.(i); + i = (i + 1) % stops.length; + wait = REST; + return true; + } + const h = handlers.get(stage); + if (!h) return true; + if (t < 0) { + origin = at ?? entry(stop2); + t = 0; + } + t = Math.min(TRAVEL, t + dt * 1e3); + const k = EASE_LIFT(t / TRAVEL); + at = [origin[0] + (stop2[0] - origin[0]) * k, origin[1] + (stop2[1] - origin[1]) * k]; + h.move(at, GHOST); + if (t < TRAVEL) return true; + onStop?.(i); + i = (i + 1) % stops.length; + t = -1; + wait = DWELL; + return true; + }; + const take = () => { + at = null; + t = -1; + }; + const give = () => { + if (hand || keys) return; + i = 0; + t = -1; + wait = RESUME; + board.wake(); + }; + const me = { + hold: () => { + hand = true; + take(); + }, + release: () => { + if (!hand) return; + hand = false; + give(); + } + }; + const focusIn = () => { + keys = true; + take(); + }; + const focusOut = (e) => { + if (keys && !stage.contains(e.relatedTarget)) { + keys = false; + give(); + } + }; + stage.addEventListener("focusin", focusIn); + stage.addEventListener("focusout", focusOut); + touring.set(stage, me); + const board = register(stage, tick); + return { + stop: () => { + if (gone) return; + gone = true; + board.unregister(); + stage.removeEventListener("focusin", focusIn); + stage.removeEventListener("focusout", focusOut); + if (touring.get(stage) === me) touring.delete(stage); + if (at) leave(); + } + }; + } + function disposer() { + let fns = []; + return { + add: (fn) => { + fns.push(fn); + }, + on: (target, type, fn, opts) => { + const h = fn; + target.addEventListener(type, h, opts); + fns.push(() => target.removeEventListener(type, h, opts)); + }, + dispose: () => { + const run2 = fns; + fns = []; + for (let i = run2.length - 1; i >= 0; i--) run2[i](); + } + }; + } + + // packages/hairline/src/core/styles.ts + var LIGHT = { plate: "#ffffff", hi: "#232327", edge: "#a4a4ac", mid: "#c3c3c9", lo: "#e0e0e4" }; + var DARK = { plate: "#08090a", hi: "#d0d6e0", edge: "#5b5d64", mid: "#3e3e44", lo: "#29292d" }; + var KEYS = ["plate", "hi", "edge", "mid", "lo"]; + var vars = (p) => KEYS.map((k) => `--hl-${k}:var(--hairline-${k},${p[k]});`).join(""); + var EASE = "cubic-bezier(0.5,0,0.1,1)"; + var SVG = ":where([data-hairline]>svg)"; + function css(lightDark) { + const both = Object.fromEntries(KEYS.map((k) => [k, `light-dark(${LIGHT[k]},${DARK[k]})`])); + return [ + // the box, and the palette: light unless something below says otherwise + `:where([data-hairline]){display:block;position:relative;aspect-ratio:5/4;touch-action:pan-y;user-select:none;-webkit-user-select:none;--hl-sw:var(--hairline-stroke,0.9);${vars(LIGHT)}}`, + // the page's color-scheme + lightDark ? `:where([data-hairline]){${vars(both)}}` : "", + // an ancestor that says dark + `:where(.dark,[data-theme="dark"]) :where([data-hairline]){${vars(DARK)}}`, + // the figure's own theme option + `:where([data-hairline][data-hairline-theme="light"]){${vars(LIGHT)}}`, + `:where([data-hairline][data-hairline-theme="dark"]){${vars(DARK)}}`, + `:where([data-hairline]:focus-visible){outline:1.5px solid var(--hl-hi);outline-offset:2px}`, + `${SVG}{position:absolute;inset:0;width:100%;height:100%;display:block}`, + // Riffle's live region: read, not seen + `:where([data-hairline]>[data-hairline-live]){position:absolute;width:1px;height:1px;margin:-1px;padding:0;border:0;overflow:hidden;clip-path:inset(50%);white-space:nowrap}`, + // the drawing: plates are filled with the plate colour and painted back to front + `${SVG} :where(path,polygon,ellipse,line){fill:var(--hl-plate);stroke:var(--hl-mid);stroke-width:var(--hl-sw);vector-effect:non-scaling-stroke;stroke-linejoin:round;stroke-linecap:round;transition:stroke 260ms ${EASE}}`, + `${SVG} :where(.nf){fill:none}`, + `${SVG} :where(.fo){stroke:none}`, + `${SVG} :where(.sil){stroke:var(--hl-edge)}`, + `${SVG} :where(.hi){stroke:var(--hl-hi)}`, + `${SVG} :where(.lo){stroke:var(--hl-lo)}`, + `${SVG} :where(.dash){stroke-dasharray:1 3}`, + `${SVG} :where(.dot){stroke:none;fill:var(--hl-hi);transition:fill 260ms ${EASE}}`, + `${SVG} :where(.dot.m){fill:var(--hl-edge)}`, + `${SVG} :where(.dot.off){fill:var(--hl-lo)}`, + `${SVG} :where(.ghost path){fill:none;stroke:var(--hl-mid)}` + ].join(""); + } + var done = /* @__PURE__ */ new WeakSet(); + function inject(root) { + if (done.has(root)) return; + done.add(root); + const doc = root.nodeType === 9 ? root : root.ownerDocument; + const win = doc.defaultView; + const text = css(!!win?.CSS?.supports?.("color", "light-dark(#000,#fff)")); + if (win && "adoptedStyleSheets" in root) { + try { + const sheet = new win.CSSStyleSheet(); + sheet.replaceSync(text); + root.adoptedStyleSheets = [...root.adoptedStyleSheets, sheet]; + return; + } catch { + } + } + const style = doc.createElement("style"); + style.setAttribute("data-hairline-style", ""); + style.textContent = text; + (root.nodeType === 9 ? doc.head ?? doc.documentElement : root).appendChild(style); + } + return __toCommonJS(kernel_exports); +})(); +/* /hairline kernel */ diff --git a/cloudg/templates/vendor/iso-kit.js b/cloudg/templates/vendor/iso-kit.js new file mode 100644 index 0000000..45c297b --- /dev/null +++ b/cloudg/templates/vendor/iso-kit.js @@ -0,0 +1,684 @@ +/* Anatomy iso-kit (kit/iso-kit.mjs), MIT License, Copyright (c) 2026 Ryan. + https://github.com/wheresryan22/anatomy . The module source is unchanged except that + its 'export' keywords are removed, the two script tags inside one string are written + '\x3cscript' and '<\/script' (the same characters at run time) so the file can sit + inside a script element, and it is wrapped so its names live on IsoKit. */ +var IsoKit = (function () { +"use strict"; +const AZIMUTH = 45; +const ELEVATION = 30; +const SHADES = 4; +const RADIANS = Math.PI / 180; +const UNIT = Math.sqrt(1.6); +function cameraOf({ scale, azimuth = AZIMUTH, elevation = ELEVATION }) { + const a = azimuth * RADIANS; + const e = elevation * RADIANS; + return { sinA: Math.sin(a), cosA: Math.cos(a), sinE: Math.sin(e), cosE: Math.cos(e), k: scale * UNIT }; +} +function iso([x, y, z], projection) { + const { origin } = projection; + const { sinA, cosA, sinE, cosE, k } = cameraOf(projection); + return [origin[0] + (x * sinA - y * cosA) * k, origin[1] + ((x * cosA + y * sinA) * sinE - z * cosE) * k]; +} +function towardViewer(projection) { + const { sinA, cosA } = cameraOf(projection); + return [cosA, sinA]; +} +function depthOf([x, y, z], projection) { + const { sinA, cosA, sinE, cosE } = cameraOf(projection); + return (x * cosA + y * sinA) * cosE + z * sinE; +} +const tenth = (value) => { + const text = (Math.round(value * 10) / 10).toFixed(1).replace(/\.0$/, ""); + return text === "-0" ? "0" : text; +}; +function pathOf(points, closed = false) { + if (!points.length) + return ""; + return points.map(([x, y], at) => `${at ? "L" : "M"}${tenth(x)} ${tenth(y)}`).join("") + (closed ? "Z" : ""); +} +const joinPaths = (paths) => paths.filter(Boolean).join(""); +function roundedPlan({ x, y, w, d, r }, steps = 8) { + const radius = Math.max(0, Math.min(r, w / 2, d / 2)); + const corners = [ + [x + w - radius, y + radius, -90], + [x + w - radius, y + d - radius, 0], + [x + radius, y + d - radius, 90], + [x + radius, y + radius, 180], + ]; + const out = []; + for (const [cx, cy, start] of corners) { + const count = radius > 0 ? steps : 0; + for (let step = 0; step <= count; step++) { + const angle = (start + (count ? (90 * step) / count : 45)) * RADIANS; + out.push([cx + radius * Math.cos(angle), cy + radius * Math.sin(angle)]); + } + } + return out; +} +function signedArea(points) { + let area = 0; + for (let index = 0; index < points.length; index++) { + const [ax, ay] = points[index]; + const [bx, by] = points[(index + 1) % points.length]; + area += ax * by - bx * ay; + } + return area / 2; +} +const oriented = (points) => (signedArea(points) < 0 ? [...points].reverse() : points); +function convexHull(points) { + const sorted = [...points].sort((a, b) => a[0] - b[0] || a[1] - b[1]); + if (sorted.length < 3) + return sorted; + const cross = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lower = []; + for (const point of sorted) { + while (lower.length >= 2 && cross(lower[lower.length - 2], lower[lower.length - 1], point) <= 0) + lower.pop(); + lower.push(point); + } + const upper = []; + for (let index = sorted.length - 1; index >= 0; index--) { + const point = sorted[index]; + while (upper.length >= 2 && cross(upper[upper.length - 2], upper[upper.length - 1], point) <= 0) + upper.pop(); + upper.push(point); + } + return [...lower.slice(0, -1), ...upper.slice(0, -1)]; +} +function runsOf(flags, want) { + const count = flags.length; + const runs = []; + const start = flags.findIndex((flag, index) => flag !== flags[(index + count - 1) % count]); + if (start < 0) + return flags[0] === want ? [Array.from({ length: count + 1 }, (_, index) => index % count)] : []; + let run = null; + for (let step = 0; step < count; step++) { + const index = (start + step) % count; + if (flags[index] === want) { + if (!run) + run = [index]; + run.push((index + 1) % count); + } + else if (run) { + runs.push(run); + run = null; + } + } + if (run) + runs.push(run); + return runs; +} +function extrude(ring, z, height, projection, options = {}) { + const convex = options.convex ?? true; + const count = ring.length; + const counter = signedArea(ring) > 0; + const [vx, vy] = towardViewer(projection); + const { sinA, cosA } = cameraOf(projection); + const normals = ring.map(([ax, ay], index) => { + const [bx, by] = ring[(index + 1) % count]; + const dx = bx - ax; + const dy = by - ay; + const length = Math.hypot(dx, dy) || 1; + return (counter ? [dy / length, -dx / length] : [-dy / length, dx / length]); + }); + const facing = normals.map(([nx, ny]) => nx * vx + ny * vy > 1e-9); + const top = ring.map(([x, y]) => iso([x, y, z + height], projection)); + const bottom = ring.map(([x, y]) => iso([x, y, z], projection)); + const outline = []; + const crease = []; + for (const run of runsOf(facing, false)) + outline.push(pathOf(run.map((index) => top[index]))); + for (const run of runsOf(facing, true)) { + crease.push(pathOf(run.map((index) => top[index]))); + if (height > 0) + outline.push(pathOf(run.map((index) => bottom[index]))); + } + if (height > 0) { + for (let index = 0; index < count; index++) { + if (facing[(index + count - 1) % count] !== facing[index]) + outline.push(pathOf([top[index], bottom[index]])); + } + } + let fill; + if (convex) { + fill = pathOf(convexHull([...top, ...bottom]), true); + } + else { + const parts = [pathOf(oriented(top), true), pathOf(oriented(bottom), true)]; + for (let index = 0; index < count; index++) { + if (!facing[index]) + continue; + const next = (index + 1) % count; + parts.push(pathOf(oriented([top[index], top[next], bottom[next], bottom[index]]), true)); + } + fill = parts.join(""); + } + const buckets = Array.from({ length: SHADES }, () => []); + if (height > 0) { + const [lx, ly] = [-sinA, cosA]; + for (let index = 0; index < count; index++) { + if (!facing[index]) + continue; + const [nx, ny] = normals[index]; + const lit = (nx * lx + ny * ly + 1) / 2; + const shade = Math.min(SHADES - 1, Math.max(0, Math.floor(lit * SHADES))); + const next = (index + 1) % count; + buckets[shade].push(pathOf(oriented([top[index], top[next], bottom[next], bottom[index]]), true)); + } + } + let bevel; + if (options.bevel && options.bevel > 0) { + const by = options.bevel; + const inset = ring.map(([x, y], index) => { + const [ax, ay] = normals[(index + count - 1) % count]; + const [bx, by2] = normals[index]; + const mx = ax + bx; + const my = ay + by2; + const length = Math.hypot(mx, my) || 1; + return [x - (mx / length) * by, y - (my / length) * by]; + }); + const insetTop = inset.map(([x, y]) => iso([x, y, z + height], projection)); + bevel = runsOf(facing, true) + .map((run) => pathOf([top[run[0]], ...run.slice(1, -1).map((index) => insetTop[index]), top[run[run.length - 1]]])) + .join(""); + } + return { + fill, + outline: outline.join(""), + crease: crease.join(""), + top: pathOf(oriented(top), true), + shades: buckets.map((paths) => paths.join("")), + bevel, + }; +} +const ringOf = (plan, steps = 8) => roundedPlan(plan, steps); +function slabOf(plan, z, height, projection, steps = 8, bevel = 0) { + return extrude(ringOf(plan, steps), z, height, projection, { convex: true, bevel }); +} +function circleRing(cx, cy, radius, steps = 40) { + return Array.from({ length: steps }, (_, step) => { + const angle = (step / steps) * Math.PI * 2; + return [cx + radius * Math.cos(angle), cy + radius * Math.sin(angle)]; + }); +} +function cylinder(cx, cy, radius, z, height, projection, steps = 32, bevel = 0) { + return extrude(circleRing(cx, cy, radius, steps), z, height, projection, { convex: true, bevel }); +} +const insetPlan = ({ x, y, w, d, r }, by) => ({ x: x + by, y: y + by, w: w - by * 2, d: d - by * 2, r: Math.max(0, r - by) }); +function onTop(points, z, projection, closed = false) { + return pathOf(points.map(([x, y]) => iso([x, y, z], projection)), closed); +} +const planOutline = (plan, z, projection, steps = 8) => onTop(ringOf(plan, steps), z, projection, true); +const lineOnTop = (from, to, z, projection) => onTop([from, to], z, projection); +const segment = (from, to, projection) => pathOf([iso(from, projection), iso(to, projection)]); +const rise = (at, from, to, projection) => segment([at[0], at[1], from], [at[0], at[1], to], projection); +const ring = (cx, cy, radius, z, projection, steps = 40) => onTop(circleRing(cx, cy, radius, steps), z, projection, true); +const pointsAt = (points, projection) => points.map((point) => iso(point, projection)); +function dotGrid(x, y, columns, rows, gap, z, projection) { + const dots = []; + for (let row = 0; row < rows; row++) { + for (let column = 0; column < columns; column++) + dots.push(iso([x + column * gap, y + row * gap, z], projection)); + } + return dots; +} +function corners({ x, y, w, d }, inset = 0) { + return [ + [x + inset, y + inset], + [x + w - inset, y + inset], + [x + inset, y + d - inset], + [x + w - inset, y + d - inset], + ]; +} +function boxCorners({ x, y, w, d }, z0, z1) { + const out = []; + for (const px of [x, x + w]) + for (const py of [y, y + d]) + for (const z of [z0, z1]) + out.push([px, py, z]); + return out; +} +function knurl(cx, cy, radius, z0, z1, count, projection, inset = 0.4) { + const [vx, vy] = towardViewer(projection); + const lines = []; + for (let index = 0; index < count; index++) { + const angle = (index / count) * Math.PI * 2; + const nx = Math.cos(angle); + const ny = Math.sin(angle); + if (nx * vx + ny * vy < 0.15) + continue; + lines.push(segment([cx + radius * nx, cy + radius * ny, z0 + inset], [cx + radius * nx, cy + radius * ny, z1 - inset], projection)); + } + return lines.join(""); +} +function sideArc(cx, cy, radius, z, projection, steps = 24) { + const [vx, vy] = towardViewer(projection); + const facing = Math.atan2(vy, vx); + const points = []; + for (let step = 0; step <= steps; step++) { + const angle = facing - Math.PI / 2 + (step / steps) * Math.PI; + points.push(iso([cx + radius * Math.cos(angle), cy + radius * Math.sin(angle), z], projection)); + } + return pathOf(points); +} +function ringSeam(ring, z, projection) { + const count = ring.length; + const counter = signedArea(ring) > 0; + const [vx, vy] = towardViewer(projection); + const facing = ring.map(([ax, ay], index) => { + const [bx, by] = ring[(index + 1) % count]; + const dx = bx - ax; + const dy = by - ay; + const [nx, ny] = counter ? [dy, -dx] : [-dy, dx]; + return nx * vx + ny * vy > 1e-9; + }); + return runsOf(facing, true).map((run) => pathOf(run.map((index) => iso([ring[index][0], ring[index][1], z], projection)))).join(""); +} +const sideSeam = (plan, z, projection, steps = 8) => ringSeam(ringOf(plan, steps), z, projection); +function sideRing(center, radius, axis, projection, steps = 20) { + const [px, py, pz] = center; + const points = []; + for (let step = 0; step < steps; step++) { + const angle = (step / steps) * Math.PI * 2; + const u = radius * Math.cos(angle); + const v = radius * Math.sin(angle); + points.push(iso(axis === "x" ? [px, py + u, pz + v] : [px + u, py, pz + v], projection)); + } + return pathOf(points, true); +} +function coil(from, to, center, radius, turns, axis, projection, perTurn = 16) { + const count = Math.max(1, Math.round(turns * perTurn)); + const back = []; + const front = []; + let run = []; + let near = null; + for (let index = 0; index <= count; index++) { + const share = index / count; + const angle = share * turns * Math.PI * 2; + const u = radius * Math.cos(angle); + const v = radius * Math.sin(angle); + const along = from + (to - from) * share; + const point = axis === "x" ? [along, center[0] + u, center[1] + v] : [center[0] + u, along, center[1] + v]; + const normal = axis === "x" ? [0, Math.cos(angle), Math.sin(angle)] : [Math.cos(angle), 0, Math.sin(angle)]; + const facing = depthOf(normal, projection) > 0; + const flat = iso(point, projection); + if (near === null) + near = facing; + if (facing !== near) { + run.push(flat); + (near ? front : back).push(pathOf(run)); + run = [flat]; + near = facing; + continue; + } + run.push(flat); + } + if (run.length > 1) + (near ? front : back).push(pathOf(run)); + return { back: back.join(""), front: front.join("") }; +} +function topTicks(from, to, step, every, edge, z, lengths, projection, axis = "x") { + const minor = []; + const major = []; + const first = Math.ceil(from / step) * step; + for (let at = first, index = Math.round(first / step); at <= to + 1e-9; at += step, index++) { + const big = index % every === 0; + const length = big ? lengths[1] : lengths[0]; + const a = axis === "x" ? [at, edge] : [edge, at]; + const b = axis === "x" ? [at, edge + length] : [edge + length, at]; + (big ? major : minor).push(lineOnTop(a, b, z, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +function sideTicks(from, to, step, every, face, top, lengths, projection, axis = "x") { + const minor = []; + const major = []; + const first = Math.ceil(from / step) * step; + for (let at = first, index = Math.round(first / step); at <= to + 1e-9; at += step, index++) { + const big = index % every === 0; + const length = big ? lengths[1] : lengths[0]; + const a = axis === "x" ? [at, face, top] : [face, at, top]; + const b = axis === "x" ? [at, face, top - length] : [face, at, top - length]; + (big ? major : minor).push(segment(a, b, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +function radialTicks(cx, cy, radius, count, every, z, lengths, projection, from = 0, span = Math.PI * 2) { + const minor = []; + const major = []; + const closed = Math.abs(span - Math.PI * 2) < 1e-9; + const steps = closed ? count : count + 1; + for (let index = 0; index < steps; index++) { + const angle = from + (index / count) * span; + const big = index % every === 0; + const inner = radius - (big ? lengths[1] : lengths[0]); + const sx = Math.sin(angle); + const sy = -Math.cos(angle); + (big ? major : minor).push(lineOnTop([cx + sx * inner, cy + sy * inner], [cx + sx * radius, cy + sy * radius], z, projection)); + } + return { minor: minor.join(""), major: major.join("") }; +} +const haloOf = (plan, z, height, projection) => slabOf(plan, z, height, projection).fill; +function axisVector(projection, [dx, dy, dz]) { + const origin = iso([0, 0, 0], projection); + const moved = iso([dx, dy, dz], projection); + return [moved[0] - origin[0], moved[1] - origin[1]]; +} +const fine = (value) => Math.round(value * 100) / 100; +function translateAlong(projection, by) { + const [x, y] = axisVector(projection, by); + return `translate(${fine(x)} ${fine(y)})`; +} +const thousandth = (value) => Math.round(value * 1000) / 1000; +function axesAt(at, z, projection) { + const origin = iso([at[0], at[1], z], projection); + const along = (point) => { + const [px, py] = iso(point, projection); + return [px - origin[0], py - origin[1]]; + }; + return { origin, x: along([at[0] + 1, at[1], z]), y: along([at[0], at[1] + 1, z]), z: along([at[0], at[1], z + 1]) }; +} +const matrixOf = (u, v, origin) => `matrix(${thousandth(u[0])} ${thousandth(u[1])} ${thousandth(v[0])} ${thousandth(v[1])} ${thousandth(origin[0])} ${thousandth(origin[1])})`; +function holdStill(element, [dx, dy]) { + const text = dx || dy ? `translate(${-dx} ${-dy})` : ""; + element.querySelectorAll("[data-hold]").forEach((node) => (text ? node.setAttribute("transform", text) : node.removeAttribute("transform"))); +} +function topMatrix(at, z, projection, along = "x") { + const { origin, x, y } = axesAt(at, z, projection); + return along === "x" ? matrixOf(x, y, origin) : matrixOf([-y[0], -y[1]], x, origin); +} +function sideMatrix(at, z, projection, face) { + const { origin, x, y, z: up } = axesAt(at, z, projection); + const down = [-up[0], -up[1]]; + return face === "left" ? matrixOf(x, down, origin) : matrixOf([-y[0], -y[1]], down, origin); +} +function screenRise(height, projection) { + const { cosE, k } = cameraOf(projection); + return height * cosE * k; +} +function fitProjection(points, width, height, { pad = 24, headroom = 0, azimuth = AZIMUTH, elevation = ELEVATION } = {}) { + const unit = { origin: [0, 0], scale: 1, azimuth, elevation }; + const flat = points.map((point) => iso(point, unit)); + const xs = flat.map(([x]) => x); + const ys = flat.map(([, y]) => y); + const left = Math.min(...xs); + const right = Math.max(...xs); + const up = Math.min(...ys); + const down = Math.max(...ys); + const scale = Math.min((width - pad * 2) / (right - left), (height - pad * 2 - headroom) / (down - up)); + return { + origin: [width / 2 - ((left + right) / 2) * scale, headroom + (height - headroom) / 2 - ((up + down) / 2) * scale], + scale, + azimuth, + elevation, + }; +} +const escape = (text) => text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +const attrs = (pairs) => Object.entries(pairs) + .filter(([, value]) => value !== undefined && value !== false) + .map(([key, value]) => ` ${key}="${escape(String(value))}"`) + .join(""); +function solidSvg(paths, { tone = "hi", crease = "faint", halo = false, lit = false, flat = false, className, inner = "" } = {}) { + const parts = []; + if (halo) + parts.push(``); + parts.push(``); + if (!flat) { + paths.shades.forEach((d, shade) => { + if (d) + parts.push(``); + }); + parts.push(``); + } + if (crease !== "none" && paths.crease) + parts.push(``); + if (crease !== "none" && paths.bevel) + parts.push(``); + if (inner) + parts.push(inner); + parts.push(``); + return `${parts.join("")}`; +} +function lineSvg(d, { tone = "lo", dotted = false, dashed = false, flow = false, className, free } = {}) { + if (!d) + return ""; + return ``; +} +function dotsSvg(points, { size = 0.5, tone = "mid", pulse = false } = {}) { + const dots = points + .map(([x, y, alpha], index) => ``) + .join(""); + return `${dots}`; +} +function fadedSvg(list, style = {}) { + const groups = new Map(); + for (const entry of list) { + const [d, alpha] = Array.isArray(entry) ? entry : [entry.d, entry.alpha]; + if (!d || !(alpha > 0.02)) + continue; + const level = Math.min(1, Math.round(alpha * 20) / 20); + groups.set(level, (groups.get(level) ?? "") + d); + } + return [...groups].map(([alpha, d]) => (alpha >= 1 ? lineSvg(d, style) : `${lineSvg(d, style)}`)).join(""); +} +function fadeLineSvg(points, { fade = [0, 0], tone = "lo", className, stops = 6 } = {}) { + if (points.length < 2) + return ""; + const ease = (t) => t * t * (3 - 2 * t); + const hashOf = (text) => { + let hash = 2166136261; + for (let index = 0; index < text.length; index++) + hash = Math.imul(hash ^ text.charCodeAt(index), 16777619); + return (hash >>> 0).toString(36); + }; + const lengths = [0]; + for (let index = 1; index < points.length; index++) + lengths.push(lengths[index - 1] + Math.hypot(points[index][0] - points[index - 1][0], points[index][1] - points[index - 1][1])); + const at = (length) => { + let index = 1; + while (index < points.length - 1 && lengths[index] < length) + index++; + const span = lengths[index] - lengths[index - 1] || 1; + const t = Math.max(0, Math.min(1, (length - lengths[index - 1]) / span)); + return [points[index - 1][0] + (points[index][0] - points[index - 1][0]) * t, points[index - 1][1] + (points[index][1] - points[index - 1][1]) * t]; + }; + const pieceOf = (from, to) => [at(from), ...points.filter((_, index) => lengths[index] > from + 1e-6 && lengths[index] < to - 1e-6), at(to)]; + const total = lengths[lengths.length - 1]; + if (total < 1e-6) + return ""; + let [start, end] = Array.isArray(fade) ? fade : [fade, fade]; + if (start + end > total) { + const share = total / (start + end); + start *= share; + end *= share; + } + const paint = `var(--anatomy-${tone})`; + const ramp = (piece, rising) => { + const d = pathOf(piece); + const [x1, y1] = piece[0]; + const [x2, y2] = piece[piece.length - 1]; + const id = `iso-fade-${hashOf(`${d}|${tone}|${rising}`)}`; + const marks = Array.from({ length: stops + 1 }, (_, index) => { + const t = index / stops; + const alpha = ease(rising ? t : 1 - t); + return ``; + }).join(""); + const gradient = `${marks}`; + return `${gradient}`; + }; + const parts = []; + if (start > 0.05) + parts.push(ramp(pieceOf(0, start), true)); + if (total - end - start > 0.05) + parts.push(lineSvg(pathOf(pieceOf(start, total - end)), { tone, className })); + if (end > 0.05) + parts.push(ramp(pieceOf(total - end, total), false)); + return `${parts.join("")}`; +} +function wireSvg(from, to, { via = [], tone = "lo", dotted = false, dashed = false, flow = false, ends = "both", size = 1.2 } = {}) { + const d = pathOf([from, ...via, to]); + const end = ([x, y]) => ``; + return `${lineSvg(d, { tone, dotted, dashed, flow })}${ends === "both" || ends === "from" ? end(from) : ""}${ends === "both" || ends === "to" ? end(to) : ""}`; +} +function faceTextSvg(transform, text, { size = 3, tone = "mid", anchor = "start" } = {}) { + return `${escape(text)}`; +} +function groupSvg(children, extra = {}) { + return `${Array.isArray(children) ? children.join("") : children}`; +} +function figureSvg({ width, height, label, body, standalone = false, className }) { + const style = standalone ? `` : ""; + return `${style}${Array.isArray(body) ? body.join("") : body}`; +} +function swatchSvg(mark) { + if (mark === "raised" || mark === "flat" || mark === "lit") { + const body = mark === "flat" ? "M2 6.5 8 3.5 14 6.5 8 9.5Z" : "M2 4.5 8 1.5 14 4.5 14 6 8 9 2 6Z"; + const crease = mark === "flat" ? "" : ``; + return ``; + } + return ``; +} +function plateHtml({ fig, title, hint, readout = "", keys = [], caption, body }) { + const corner = (name, text, extra = "") => `${escape(text)}`; + const legend = keys.length || caption + ? `
    ${keys.length ? `${keys.map((key) => `${swatchSvg(key.mark)}${escape(key.label)}`).join("")}` : ""}${caption ? `${escape(caption)}` : ""}
    ` + : ""; + return `
    ${body}
    ${legend}
    `; +} +function pageHtml({ title, theme = "dark", body, script = "", width = 650 }) { + return `${escape(title)}
    ${body}
    ${script ? `\x3cscript type="module">${script.replace(/<\/(script)/gi, "<\\/$1")}<\/script>` : ""}`; +} +const PAGE_CSS = ` +*{box-sizing:border-box} +html{color-scheme:dark} +body[data-theme="light"]{color-scheme:light} +body{margin:0;background:var(--anatomy-page);color:var(--anatomy-text);font:14px/1.6 ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;-webkit-font-smoothing:antialiased} +.iso-page{width:min(__WIDTH__px,100% - 32px);margin:64px auto 96px} +`; +const ISO_CSS = ` +.iso{ + --anatomy-page:#0b0b0b;--anatomy-card:#131313;--anatomy-text:#a3a3a3;--anatomy-strong:#e6e6e6;--anatomy-muted:#6e6e6e; + --anatomy-paper:#131313;--anatomy-face:#1b1b1b;--anatomy-top:#1c1c1c; + --anatomy-shade-0:#0f0f0f;--anatomy-shade-1:#121212;--anatomy-shade-2:#151515;--anatomy-shade-3:#181818; + --anatomy-lit-top:#262626;--anatomy-lit-shade:#1d1d1d; + --anatomy-hi:#d4d4d4;--anatomy-mid:#6a6a6a;--anatomy-lo:#3e3e3e;--anatomy-faint:#2a2a2a;--anatomy-lit:#ffffff; + --anatomy-halo:#050505;--anatomy-halo-opacity:.9;--anatomy-dot:#e6e6e6; + --anatomy-red:#ff6b6b;--anatomy-green:#6be0a0;--anatomy-blue:#6b9bff; + --anatomy-weight:.6px;--anatomy-ease:cubic-bezier(.32,.72,0,1); + --anatomy-mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace; + --anatomy-lift:inset 0 1px 0 0 #ffffff0a,inset 0 0 0 1px #ffffff06,0 0 0 1px #00000026,0 1px 1px -.5px #00000014,0 3px 3px -1.5px #00000014,0 6px 6px -3px #00000014,0 12px 12px -6px #00000014; +} +.iso[data-theme="light"]{ + --anatomy-page:#f2f2ef;--anatomy-card:#fbfbfa;--anatomy-text:#57575a;--anatomy-strong:#18181a;--anatomy-muted:#8c8c90; + --anatomy-paper:#fbfbfa;--anatomy-face:#ffffff;--anatomy-top:#ffffff; + --anatomy-shade-0:#dcdcd9;--anatomy-shade-1:#e5e5e2;--anatomy-shade-2:#ededea;--anatomy-shade-3:#f4f4f1; + --anatomy-lit-top:#ffffff;--anatomy-lit-shade:#eeeeeb; + --anatomy-hi:#26262a;--anatomy-mid:#8a8a8f;--anatomy-lo:#b4b4b8;--anatomy-faint:#d4d4d6;--anatomy-lit:#000000; + --anatomy-halo:#8a8a80;--anatomy-halo-opacity:.32;--anatomy-dot:#3a3a3e; + --anatomy-red:#e5484d;--anatomy-green:#30a46c;--anatomy-blue:#3e63dd; + --anatomy-lift:inset 0 1px 0 0 #ffffff,0 0 0 1px #0000000f,0 1px 1px -.5px #0000000a,0 3px 3px -1.5px #0000000a,0 6px 6px -3px #0000000a,0 12px 12px -6px #0000000a; +} +.iso-svg{display:block;width:100%;height:auto;overflow:visible} +.iso-svg path,.iso-svg circle{vector-effect:non-scaling-stroke} +.iso-fill{fill:var(--anatomy-paper);stroke:none} +.iso-solid[data-lit]>.iso-fill{fill:var(--anatomy-face)} +.iso-shade,.iso-top{stroke:none} +.iso-shade[data-shade="0"]{fill:var(--anatomy-shade-0)} +.iso-shade[data-shade="1"]{fill:var(--anatomy-shade-1)} +.iso-shade[data-shade="2"]{fill:var(--anatomy-shade-2)} +.iso-shade[data-shade="3"]{fill:var(--anatomy-shade-3)} +.iso-top{fill:var(--anatomy-top)} +.iso-solid[data-lit]>.iso-top{fill:var(--anatomy-lit-top)} +.iso-solid[data-lit]>.iso-shade{fill:var(--anatomy-lit-shade)} +.iso-solid[data-seamless]>.iso-shade,.iso-solid[data-seamless]>.iso-top{stroke-width:.4;stroke-linejoin:round} +.iso-solid[data-seamless]>.iso-shade[data-shade="0"]{stroke:var(--anatomy-shade-0)} +.iso-solid[data-seamless]>.iso-shade[data-shade="1"]{stroke:var(--anatomy-shade-1)} +.iso-solid[data-seamless]>.iso-shade[data-shade="2"]{stroke:var(--anatomy-shade-2)} +.iso-solid[data-seamless]>.iso-shade[data-shade="3"]{stroke:var(--anatomy-shade-3)} +.iso-solid[data-seamless]>.iso-top{stroke:var(--anatomy-top)} +.iso-solid[data-seamless][data-lit]>.iso-top{stroke:var(--anatomy-lit-top)} +.iso-solid[data-seamless][data-lit]>.iso-shade{stroke:var(--anatomy-lit-shade)} +.iso-solid[data-lit]>.iso-edge{stroke:var(--anatomy-lit)} +.iso-halo{fill:var(--anatomy-halo);stroke:none;filter:blur(9px);opacity:var(--anatomy-halo-opacity);transform:translateY(3px)} +.iso-line{fill:none;stroke:var(--anatomy-lo);stroke-width:var(--anatomy-weight);stroke-linecap:round;stroke-linejoin:round} +.iso-line[data-tone="hi"]{stroke:var(--anatomy-hi)} +.iso-line[data-tone="mid"]{stroke:var(--anatomy-mid)} +.iso-line[data-tone="lo"]{stroke:var(--anatomy-lo)} +.iso-line[data-tone="faint"]{stroke:var(--anatomy-faint)} +.iso-line[data-tone="lit"]{stroke:var(--anatomy-lit)} +.iso-line[data-tone="red"]{stroke:var(--anatomy-red)} +.iso-line[data-tone="green"]{stroke:var(--anatomy-green)} +.iso-line[data-tone="blue"]{stroke:var(--anatomy-blue)} +.iso-line[data-dotted]{stroke-dasharray:.01 3.5;stroke-width:1} +.iso-line[data-dashed]{stroke-dasharray:3 3} +.iso-line[data-flow]{stroke-dasharray:2 6} +.iso-wire-end{fill:var(--anatomy-paper);stroke:var(--anatomy-lo);stroke-width:var(--anatomy-weight)} +.iso-wire[data-tone="hi"] .iso-wire-end{stroke:var(--anatomy-hi)} +.iso-wire[data-tone="mid"] .iso-wire-end{stroke:var(--anatomy-mid)} +.iso-wire[data-tone="faint"] .iso-wire-end{stroke:var(--anatomy-faint)} +.iso-wire[data-tone="lit"] .iso-wire-end{stroke:var(--anatomy-lit)} +.iso-dots circle{fill:var(--anatomy-dot);stroke:none;opacity:.32} +.iso-dots[data-tone="hi"] circle{opacity:.85} +.iso-dots[data-tone="mid"] circle{opacity:.45} +.iso-dots[data-tone="lo"] circle{opacity:.22} +.iso-dots[data-tone="faint"] circle{opacity:.12} +.iso-face-text{fill:var(--anatomy-mid);font-family:var(--anatomy-mono);letter-spacing:.04em} +.iso-face-text[data-tone="hi"]{fill:var(--anatomy-hi)} +.iso-face-text[data-tone="lo"]{fill:var(--anatomy-lo)} +.iso-face-text[data-tone="faint"]{fill:var(--anatomy-faint)} +.tb-body{fill:var(--anatomy-shade-2);stroke:none} +.tb-shine{fill:var(--anatomy-top);stroke:none} +.tb-shade{fill:var(--anatomy-shade-0);stroke:none} +.tb-ring{fill:none;stroke:var(--anatomy-lo);stroke-width:.45;stroke-linecap:butt;stroke-linejoin:round} +.tb-ring[data-tone="mid"]{stroke:var(--anatomy-mid)} +.tb-ring[data-tone="faint"]{stroke:var(--anatomy-faint)} +.tb-ring[data-tone="hi"]{stroke:var(--anatomy-hi)} +@keyframes iso-flow{to{stroke-dashoffset:-32}} +@keyframes iso-pulse{0%,100%{opacity:.18}12%{opacity:1}40%{opacity:.18}} +.iso-line[data-flow]{animation:iso-flow 1600ms linear infinite} +.iso-dots circle[data-pulse]{animation:iso-pulse 1600ms steps(1,end) infinite;animation-delay:var(--pulse-at,0ms)} +.iso-figure{margin:32px 0 36px} +.iso-plate{position:relative;padding:52px 24px;border-radius:0;background:var(--anatomy-card);box-shadow:0 0 0 1px var(--anatomy-mid);--anatomy-paper:var(--anatomy-card)} +.iso-plate-corners{position:absolute;inset:18px 20px;pointer-events:none} +.iso-plate-corner{position:absolute;color:var(--anatomy-muted);font-family:var(--anatomy-mono);font-size:10.5px;line-height:1;letter-spacing:.04em;white-space:nowrap} +.iso-plate-corner[data-corner="fig"]{top:0;left:0;color:var(--anatomy-text)} +.iso-plate-corner[data-corner="title"]{top:0;right:0;text-transform:uppercase;letter-spacing:.1em} +.iso-plate-corner[data-corner="hint"]{bottom:0;left:0;text-transform:uppercase;letter-spacing:.1em} +.iso-plate-corner[data-corner="readout"]{right:0;bottom:0;color:var(--anatomy-text);font-variant-numeric:tabular-nums} +.iso-plate .iso-svg{margin:0 auto} +.iso-legend{display:grid;gap:8px;margin-top:12px;color:var(--anatomy-muted);font-size:13px;line-height:1.55} +.iso-keys{display:flex;flex-wrap:wrap;gap:6px 18px} +.iso-key{display:inline-flex;align-items:center;gap:7px} +.iso-swatch{width:16px;height:10px;flex:none;overflow:visible} +.iso-swatch path{fill:none;stroke:var(--anatomy-hi);stroke-width:1;vector-effect:non-scaling-stroke;stroke-linejoin:round;stroke-linecap:round} +.iso-swatch[data-mark="raised"] path,.iso-swatch[data-mark="flat"] path{stroke:var(--anatomy-mid)} +.iso-swatch[data-mark="lit"] path{stroke:var(--anatomy-lit)} +.iso-swatch[data-mark="dim"] path{stroke:var(--anatomy-lo)} +.iso-swatch[data-mark="dotted"] path{stroke-dasharray:.01 3;stroke-width:1.4} +.iso-swatch[data-mark="dashed"] path{stroke-dasharray:3 2.5} +.iso-swatch[data-mark="red"] path{stroke:var(--anatomy-red)} +.iso-swatch[data-mark="green"] path{stroke:var(--anatomy-green)} +.iso-swatch[data-mark="blue"] path{stroke:var(--anatomy-blue)} +.iso-swatch-crease{stroke:var(--anatomy-lo)!important} +@media (max-width:560px){.iso-plate{padding:48px 12px 70px}.iso-plate-corner[data-corner="hint"]{bottom:18px}} +@media (prefers-reduced-motion:reduce){.iso-line[data-flow],.iso-dots circle[data-pulse]{animation:none}} +`; + +return { AZIMUTH, ELEVATION, SHADES, cameraOf, iso, towardViewer, depthOf, pathOf, joinPaths, roundedPlan, extrude, ringOf, slabOf, circleRing, cylinder, insetPlan, onTop, planOutline, lineOnTop, segment, rise, ring, pointsAt, dotGrid, corners, boxCorners, knurl, sideArc, ringSeam, sideSeam, sideRing, coil, topTicks, sideTicks, radialTicks, haloOf, axisVector, translateAlong, holdStill, topMatrix, sideMatrix, screenRise, fitProjection, solidSvg, lineSvg, dotsSvg, fadedSvg, fadeLineSvg, wireSvg, faceTextSvg, groupSvg, figureSvg, plateHtml, pageHtml, ISO_CSS }; +})(); diff --git a/docs/DOCUMENTATION.md b/docs/DOCUMENTATION.md index 89bc985..35953cf 100644 --- a/docs/DOCUMENTATION.md +++ b/docs/DOCUMENTATION.md @@ -8,6 +8,8 @@ cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, run This is the complete reference: every CLI command, the exact input and output formats, the data models, the full Python API with integration examples, configuration, plugins and authentication. Release history lives in the [changelog](https://github.com/morpheuslord/cloudg/blob/main/CHANGELOG.md). +The same material, split into guides, a generated CLI and Python API reference and runnable examples, is on the documentation site at [morpheuslord.github.io/cloudg](https://morpheuslord.github.io/cloudg/). To see what a report looks like before running anything, open the [sample report](https://morpheuslord.github.io/cloudg/sample-report.html): the `report.html` of a synthetic three-cloud estate. + The inventory mapper (`cloudg map`, `cloudg deps`, `cloudg.inventory`) has three deeper companion documents: - [Inventory reference](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_REFERENCE.md): the structure of every return value and exported file, field by field, with annotated examples (`CloudAsset`, `NetworkEdge`, relation objects, `InventoryResult`, `summary`, coverage records, dependency trees, blast radius, security coverage, `inventory-map.json`, `inventory-graph.json`, GraphML, `inventory-organization.json`, `asset-map.json`, `compliance-map.json`, `cloudg deps --json`). diff --git a/docs/INVENTORY_CATALOG.md b/docs/INVENTORY_CATALOG.md index 314de6f..f8661da 100644 --- a/docs/INVENTORY_CATALOG.md +++ b/docs/INVENTORY_CATALOG.md @@ -26,6 +26,8 @@ several types (`null | str`) was observed with each. 4. [Relationship matrix](#relationship-matrix) 5. [Relationship vocabulary](#relationship-vocabulary) +6. [Azure parity additions](#azure-parity-additions) + ## Common metadata keys Keys that mean the same thing on every asset type that carries them. @@ -90,6 +92,14 @@ Keys that mean the same thing on every asset type that carries them. | Platforms and data processing | [`MESSAGE_BROKER`](#message_broker), [`BATCH_ENVIRONMENT`](#batch_environment), [`JOB_QUEUE`](#job_queue), [`JOB_DEFINITION`](#job_definition), [`DATABASE_PROXY`](#database_proxy), [`DATA_CATALOG`](#data_catalog), [`ETL_JOB`](#etl_job), [`BIG_DATA_CLUSTER`](#big_data_cluster), [`QUERY_WORKGROUP`](#query_workgroup), [`DATA_TRANSFER`](#data_transfer), [`ML_WORKSPACE`](#ml_workspace), [`ML_ENDPOINT`](#ml_endpoint), [`ML_MODEL`](#ml_model), [`AI_AGENT`](#ai_agent), [`KNOWLEDGE_BASE`](#knowledge_base), [`AI_GUARDRAIL`](#ai_guardrail) | | Generic | [`OTHER`](#other) | + +The GCP depth mapping adds four values in a separate block of `models.py`: [`CONTAINER_IMAGE`](#container_image), +[`DATA_TABLE`](#data_table), [`TAG`](#tag) and [`FLEET`](#fleet), described under +[GCP parity types](#gcp-parity-types). It also produces many existing types on GCP for the first +time; `cloudg/inventory/catalogs/gcp_asset_types.yaml` maps 346 Cloud Asset Inventory and API +types. + + ## Per-type reference For each type: what it represents, the providers that produced it in the observations, the @@ -2299,6 +2309,52 @@ Catalog entries that classify native resources as this type: - **Declared relations:** `MANAGES` - **Edges out:**
    `MANAGES` → `EC2`, `VIRTUAL_MACHINE`
    `REFERENCES` → `MESSAGE_QUEUE` + +### GCP parity types + +Four `AssetType` values were added for the GCP depth mapping, where no existing value fit. The +GCP mapping also fills many existing types for the first time on GCP; the full type table is +`cloudg/inventory/catalogs/gcp_asset_types.yaml`. + +#### CONTAINER_IMAGE + +- An Artifact Registry Docker image (one digest). +- GCP Cloud Asset Inventory: `artifactregistry.googleapis.com/DockerImage` +- **Providers:** GCP +- **Metadata keys:** `create_time`, `description`, `folders`, `gcp_asset_type`, `location`, `network_tags`, `organization`, `parent_asset_type`, `parent_full_resource_name`, `project`, `project_id`, `project_number`, `state`, `tags` (list), `upload_time` (str), `uri` (str), `vulnerabilities` (object: severity -> count, from Artifact Analysis) +- **Declared relations:** `CONTAINS` ← `CONTAINER_REGISTRY` +- **Edges in:** `CONTAINS` ← `CONTAINER_REGISTRY`; `MONITORS / MONITORED_BY` ← `VULNERABILITY_SCANNER` +- Aliases: the image URI (`us-docker.pkg.dev/p/repo/img@sha256:...`) with and without `https://`, which is how Artifact Analysis names it. At most `inventory.max_images_per_repository` newest images per repository are kept. + +#### DATA_TABLE + +- A table or view: BigQuery table / view / materialized view, Bigtable table and authorized view. +- GCP Cloud Asset Inventory: `bigquery.googleapis.com/Table` (was `OTHER`), `bigtableadmin.googleapis.com/Table`, `bigtableadmin.googleapis.com/AuthorizedView`, `bigtableadmin.googleapis.com/LogicalView`, `bigtableadmin.googleapis.com/MaterializedView` +- **Providers:** GCP +- **Metadata keys:** `change_stream` (bool), `deletion_protection` (bool), `partitioned` (bool), `table_type` (str), `num_rows`, `num_bytes`, `require_partition_filter`, `expiration_time`, plus the common GCP keys +- **Declared relations:** `CONTAINS` ← `DATA_WAREHOUSE` / `DYNAMODB_TABLE`; `REFERENCES / READS_FROM` → `DATA_TABLE` (view query sources), `GCS_BUCKET` (external tables); `REFERENCES / DEPENDS_ON` → `SOURCE_CONNECTION`; `REFERENCES / BACKUP_TO` ← (snapshot base table) +- **Edges in:** `GOVERNS / COMPLIANCE_GOVERNS` ← `GUARDRAIL` (row access policy); `MONITORS / MONITORED_BY` ← `ETL_JOB` (Dataplex scan); `REFERENCES / READS_FROM` ← `DATA_CATALOG`, `JOB_DEFINITION`, `ML_ENDPOINT` + +#### TAG + +- A Resource Manager tag key or tag value. +- GCP Cloud Asset Inventory: `cloudresourcemanager.googleapis.com/TagKey`, `cloudresourcemanager.googleapis.com/TagValue` (tag bindings are folded into relations, see INVENTORY_INTERNALS.md 9.7) +- **Providers:** GCP +- **Metadata keys:** `namespaced_name` (str), `short_name` (str), `purpose` (str), plus the common GCP keys +- **Declared relations:** `CONTAINS` ← `ORGANIZATION` / `CLOUD_ACCOUNT` (key), ← `TAG` (value); `ATTACHED_TO` → `VPC` (firewall-purpose keys) +- **Edges in:** `REFERENCES / TAGGED_WITH` ← any tagged resource +- Aliases: `tagKeys/` / `tagValues/` and the namespaced name (`999/env/prod`). + +#### FLEET + +- A GKE fleet, fleet membership, membership binding or fleet scope. +- GCP Cloud Asset Inventory: `gkehub.googleapis.com/Fleet`, `gkehub.googleapis.com/Membership`, `gkehub.googleapis.com/MembershipBinding`, `gkehub.googleapis.com/Scope` +- **Providers:** GCP +- **Metadata keys:** `workload_identity_pool`, `issuer`, `unique_id` (memberships), `display_name`, plus the common GCP keys +- **Declared relations:** `MANAGES / OWNED_BY` → `GKE_CLUSTER` (membership); `CONTAINS` → `FLEET` (membership binding); `REFERENCES / DEPENDS_ON` → `FLEET` (binding to scope) +- **Edges in:** `MANAGES / OWNED_BY` ← `CLUSTER_ADDON` (fleet feature) + + ## Relationship matrix Every `(source type, edge type, relationship, target type)` combination the linker produced in @@ -2844,6 +2900,409 @@ edges, and the account `CONTAINS` hierarchy) are described in INVENTORY_REFERENC | `TASK_DEFINITION` | none | `CONTAINER_REGISTRY` | | `TASK_DEFINITION` | `RUNS_ON` | `CONTAINER_REGISTRY` | + +### GCP parity relationship matrix + +Combinations the linker produced from the GCP parity fixtures (`tests/test_gcp_parity_extractors.py` +and `tests/test_gcp_sources.py`): every new extractor and API-backed source, plus the older GCP +relations those fixtures also trigger. Recorded the same way as the matrix above, so it lists +observed shapes, not every possible one. + + +#### GCP parity: ASSUMES_ROLE + +| Source type | Relationship | Target type | +|---|---|---| +| `K8S_SERVICE_ACCOUNT` | `RUNS_ON` | `SERVICE_PRINCIPAL` | +| `LOG_SINK` | `CROSS_ACCOUNT_TRUST` | `CLOUD_ACCOUNT` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `RESOURCE_SHARE` | + +#### GCP parity: ATTACHED_TO + +| Source type | Relationship | Target type | +|---|---|---| +| `ELASTIC_IP` | `none` | `LOAD_BALANCER` | +| `VPN_CONNECTION` | `TRANSIT_ROUTED` | `CUSTOMER_GATEWAY` | +| `VPN_CONNECTION` | `TRANSIT_ROUTED` | `VPN_GATEWAY` | + +#### GCP parity: CONTAINS + +| Source type | Relationship | Target type | +|---|---|---| +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `CLUSTER_ADDON` | +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `DEPLOYMENT_GROUP` | +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `K8S_NAMESPACE` | +| `API_GATEWAY` | `none` | `API_GATEWAY` | +| `API_GATEWAY` | `none` | `AUTHORIZER` | +| `API_GATEWAY` | `none` | `PARAMETER` | +| `API_GATEWAY` | `none` | `PRODUCT_PORTFOLIO` | +| `API_GATEWAY` | `none` | `TARGET_GROUP` | +| `AZURE_SQL` | `none` | `AZURE_SQL` | +| `BACKUP_VAULT` | `none` | `BACKUP_PLAN` | +| `BATCH_ENVIRONMENT` | `none` | `BATCH_ENVIRONMENT` | +| `BLOB_STORAGE` | `none` | `EVENT_RULE` | +| `BLOB_STORAGE` | `none` | `IDENTITY_USER` | +| `CDN` | `none` | `CDN` | +| `CDN` | `none` | `CERTIFICATE` | +| `CDN` | `none` | `LB_LISTENER` | +| `CDN` | `none` | `TARGET_GROUP` | +| `CLOUD_ACCOUNT` | `ACCOUNT_CONTAINS_REGION` | `RESOURCE_GROUP` | +| `CLOUD_ACCOUNT` | `none` | `GUARDRAIL` | +| `CLOUD_ACCOUNT` | `none` | `LOG_SINK` | +| `CLOUD_ACCOUNT` | `none` | `ORG_POLICY` | +| `CLOUD_ACCOUNT` | `none` | `RESOURCE_SHARE` | +| `CLOUD_FUNCTION` | `CLUSTER_CONTAINS_SERVICE` | `CLOUD_FUNCTION` | +| `CLOUD_FUNCTION` | `none` | `SCHEDULE` | +| `CONTAINER_REGISTRY` | `none` | `ACCESS_KEY` | +| `CONTAINER_REGISTRY` | `none` | `BUILD_PROJECT` | +| `CONTAINER_REGISTRY` | `none` | `CONTAINER_REGISTRY` | +| `CONTAINER_REGISTRY` | `none` | `EVENT_RULE` | +| `CONTAINER_REGISTRY` | `none` | `IAM_POLICY` | +| `DATA_STREAM` | `none` | `DATA_STREAM` | +| `DATA_WAREHOUSE` | `none` | `DATA_CATALOG` | +| `DATA_WAREHOUSE` | `none` | `DATA_WAREHOUSE` | +| `DATA_WAREHOUSE` | `none` | `ETL_JOB` | +| `DATA_WAREHOUSE` | `none` | `SOURCE_CONNECTION` | +| `DEPLOYMENT_GROUP` | `none` | `DEPLOYMENT_GROUP` | +| `DNS_RESOLVER` | `none` | `DNS_RESOLVER` | +| `DNS_ZONE` | `none` | `DNS_RECORD` | +| `DYNAMODB_TABLE` | `none` | `DYNAMODB_TABLE` | +| `ECS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `CONTAINER_SERVICE` | +| `ETL_JOB` | `none` | `BATCH_ENVIRONMENT` | +| `ETL_JOB` | `none` | `DATA_CATALOG` | +| `ETL_JOB` | `none` | `ETL_JOB` | +| `ETL_JOB` | `none` | `SCHEDULE` | +| `ETL_JOB` | `none` | `SOURCE_CONNECTION` | +| `IDENTITY_GROUP` | `none` | `IDENTITY_USER` | +| `IDENTITY_GROUP` | `none` | `SERVICE_PRINCIPAL` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_SERVICE` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_SERVICE_ACCOUNT` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_WORKLOAD` | +| `KEY_VAULT` | `none` | `GUARDRAIL` | +| `KEY_VAULT` | `none` | `KMS_KEY` | +| `KEY_VAULT` | `none` | `LOG_SINK` | +| `KEY_VAULT` | `none` | `SECRET` | +| `LOAD_BALANCER` | `none` | `LB_LISTENER` | +| `LOG_GROUP` | `none` | `ALARM` | +| `LOG_GROUP` | `none` | `EVENT_RULE` | +| `LOG_GROUP` | `none` | `LOG_SINK` | +| `LOG_GROUP` | `none` | `THREAT_DETECTOR` | +| `MESSAGE_QUEUE` | `none` | `MESSAGE_QUEUE` | +| `MESSAGE_QUEUE` | `none` | `NOTIFICATION_TOPIC` | +| `ML_ENDPOINT` | `CLUSTER_CONTAINS_SERVICE` | `ML_MODEL` | +| `ML_ENDPOINT` | `none` | `AI_GUARDRAIL` | +| `ML_ENDPOINT` | `none` | `LOG_SINK` | +| `ML_ENDPOINT` | `none` | `ML_MODEL` | +| `ML_WORKSPACE` | `none` | `BATCH_ENVIRONMENT` | +| `ML_WORKSPACE` | `none` | `ML_ENDPOINT` | +| `ML_WORKSPACE` | `none` | `ML_WORKSPACE` | +| `ML_WORKSPACE` | `none` | `SOURCE_CONNECTION` | +| `NACL` | `none` | `NACL` | +| `NODE_GROUP` | `CLUSTER_CONTAINS_SERVICE` | `VIRTUAL_MACHINE` | +| `NOTIFICATION_TOPIC` | `none` | `EVENT_RULE` | +| `NOTIFICATION_TOPIC` | `none` | `MESSAGE_QUEUE` | +| `PARAMETER` | `none` | `PARAMETER` | +| `PLACEMENT_GROUP` | `none` | `PLACEMENT_GROUP` | +| `PLACEMENT_GROUP` | `none` | `VIRTUAL_MACHINE` | +| `PREFIX_LIST` | `none` | `ELASTIC_IP` | +| `PRODUCT_PORTFOLIO` | `none` | `DEPLOYMENT_GROUP` | +| `PRODUCT_PORTFOLIO` | `none` | `PRODUCT_PORTFOLIO` | +| `PRODUCT_PORTFOLIO` | `none` | `SOURCE_CONNECTION` | +| `RESOURCE_GROUP` | `none` | `AKS_CLUSTER` | +| `RESOURCE_GROUP` | `none` | `BACKUP_VAULT` | +| `RESOURCE_GROUP` | `none` | `BLOB_STORAGE` | +| `RESOURCE_GROUP` | `none` | `CONTAINER_REGISTRY` | +| `RESOURCE_GROUP` | `none` | `GUARDRAIL` | +| `RESOURCE_GROUP` | `none` | `KEY_VAULT` | +| `RESOURCE_GROUP` | `none` | `LOG_GROUP` | +| `RESOURCE_GROUP` | `none` | `LOG_SINK` | +| `RESOURCE_GROUP` | `none` | `SCHEDULE` | +| `RESOURCE_GROUP` | `none` | `SERVICE_NETWORK` | +| `RESOURCE_GROUP` | `none` | `VIRTUAL_MACHINE` | +| `RESOURCE_GROUP` | `none` | `VNET` | +| `SCHEDULE` | `none` | `NODE_GROUP` | +| `SCHEDULE` | `none` | `RUNBOOK` | +| `SCHEDULE` | `none` | `SCHEDULE` | +| `SEARCH_DOMAIN` | `none` | `ETL_JOB` | +| `SEARCH_DOMAIN` | `none` | `KNOWLEDGE_BASE` | +| `SEARCH_DOMAIN` | `none` | `SOURCE_CONNECTION` | +| `SERVICE_NETWORK` | `none` | `CUSTOMER_GATEWAY` | +| `SERVICE_NETWORK` | `none` | `NACL` | +| `SERVICE_NETWORK` | `none` | `SERVICE_NETWORK` | +| `SERVICE_NETWORK` | `none` | `TRANSIT_GATEWAY` | +| `SERVICE_NETWORK` | `none` | `VNET` | +| `SERVICE_PRINCIPAL` | `none` | `IDENTITY_PROVIDER` | +| `STATE_MACHINE` | `none` | `SCHEDULE` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `AZURE_SQL` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `BATCH_ENVIRONMENT` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `BIG_DATA_CLUSTER` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `DATA_WAREHOUSE` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `DNS_RESOLVER` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `FILE_SYSTEM` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `NETWORK_INTERFACE` | +| `TARGET_GROUP` | `none` | `TARGET_GROUP` | +| `TRANSIT_GATEWAY` | `none` | `DIRECT_CONNECT` | +| `TRANSIT_GATEWAY` | `none` | `NETWORK_FIREWALL` | +| `TRANSIT_GATEWAY` | `none` | `NETWORK_INTERFACE` | +| `TRANSIT_GATEWAY` | `none` | `PEERING_CONNECTION` | +| `TRANSIT_GATEWAY` | `none` | `ROUTE_TABLE` | +| `TRANSIT_GATEWAY` | `none` | `VPN_GATEWAY` | +| `VIRTUAL_MACHINE` | `none` | `GUARDRAIL` | +| `VIRTUAL_MACHINE` | `none` | `OTHER` | +| `VNET` | `VPC_CONTAINS_SUBNET` | `SUBNET` | + +#### GCP parity: GOVERNS + +| Source type | Relationship | Target type | +|---|---|---| +| `GUARDRAIL` | `COMPLIANCE_GOVERNS` | `CLOUD_ACCOUNT` | +| `GUARDRAIL` | `COMPLIANCE_GOVERNS` | `RESOURCE_GROUP` | +| `GUARDRAIL` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `GUARDRAIL` | `POLICY_DENIES_ACTION` | `KEY_VAULT` | +| `ORG_POLICY` | `SCP_RESTRICTS` | `CLOUD_ACCOUNT` | +| `SERVICE_NETWORK` | `COMPLIANCE_GOVERNS` | `CLOUD_ACCOUNT` | + +#### GCP parity: GRANTS_ACCESS + +| Source type | Relationship | Target type | +|---|---|---| +| `ACCESS_KEY` | `POLICY_ALLOWS_ACTION` | `CONTAINER_REGISTRY` | +| `BACKUP_VAULT` | `POLICY_ALLOWS_ACTION` | `AKS_CLUSTER` | +| `CLOUD_ACCOUNT` | `CROSS_ACCOUNT_TRUST` | `MACHINE_IMAGE` | +| `CLOUD_ACCOUNT` | `POLICY_ALLOWS_ACTION` | `ENDPOINT_SERVICE` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `AKS_CLUSTER` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `DATA_WAREHOUSE` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `ORGANIZATION` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `AZURE_SQL` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `BLOB_STORAGE` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `CLOUD_ACCOUNT` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `DEPLOYMENT_GROUP` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `DYNAMODB_TABLE` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `KEY_VAULT` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `ORGANIZATION` | +| `RESOURCE_SHARE` | `CROSS_ACCOUNT_TRUST` | `CLOUD_ACCOUNT` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `PRODUCT_PORTFOLIO` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `PROVISIONED_PRODUCT` | +| `SERVICE_PRINCIPAL` | `POLICY_ALLOWS_ACTION` | `BLOB_STORAGE` | +| `SERVICE_PRINCIPAL` | `POLICY_ALLOWS_ACTION` | `SERVICE_PRINCIPAL` | +| `SUBNET` | `POLICY_ALLOWS_ACTION` | `APP_SERVICE` | + +#### GCP parity: IAM_POLICY_ATTACHMENT + +| Source type | Relationship | Target type | +|---|---|---| +| `IDENTITY_GROUP` | `GROUP_HAS_POLICY` | `IAM_POLICY` | +| `IDENTITY_USER` | `USER_HAS_POLICY` | `IAM_POLICY` | +| `SERVICE_PRINCIPAL` | `ROLE_HAS_POLICY` | `IAM_POLICY` | + +#### GCP parity: IAM_TRUST + +| Source type | Relationship | Target type | +|---|---|---| +| `AKS_CLUSTER` | `ROLE_ASSUMES_ROLE` | `SERVICE_PRINCIPAL` | +| `IDENTITY_PROVIDER` | `CROSS_ACCOUNT_TRUST` | `SERVICE_PRINCIPAL` | +| `IDENTITY_PROVIDER` | `ROLE_ASSUMES_ROLE` | `SERVICE_PRINCIPAL` | + +#### GCP parity: INVOKES + +| Source type | Relationship | Target type | +|---|---|---| +| `ALARM` | `INVOKES` | `NOTIFICATION_TOPIC` | +| `BLOB_STORAGE` | `INVOKES` | `CLOUD_FUNCTION` | +| `ETL_JOB` | `INVOKES` | `CLOUD_FUNCTION` | +| `EVENT_RULE` | `INVOKES` | `CLOUD_FUNCTION` | +| `EVENT_RULE` | `INVOKES` | `STATE_MACHINE` | +| `KEY_VAULT` | `ROTATES_SECRET` | `CLOUD_FUNCTION` | +| `KEY_VAULT` | `TRIGGERED_BY` | `NOTIFICATION_TOPIC` | +| `KNOWLEDGE_BASE` | `INVOKES` | `ML_ENDPOINT` | +| `MESSAGE_QUEUE` | `TRIGGERED_BY` | `CLOUD_FUNCTION` | +| `NOTIFICATION_TOPIC` | `INVOKES` | `CLOUD_FUNCTION` | +| `NOTIFICATION_TOPIC` | `STREAMS_TO` | `DATA_STREAM` | +| `STATE_MACHINE` | `INVOKES` | `CLOUD_FUNCTION` | +| `STATE_MACHINE` | `INVOKES` | `SOURCE_CONNECTION` | + +#### GCP parity: LOAD_BALANCER_TARGET + +| Source type | Relationship | Target type | +|---|---|---| +| `API_GATEWAY` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | +| `API_GATEWAY` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `APP_SERVICE` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | +| `CDN` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `EDGE_FUNCTION` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `K8S_SERVICE` | `SERVES_TRAFFIC_TO` | `K8S_WORKLOAD` | +| `LB_LISTENER` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `LB_LISTENER` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `LOAD_BALANCER` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `TARGET_GROUP` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `TARGET_GROUP` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | + +#### GCP parity: LOGS_TO + +| Source type | Relationship | Target type | +|---|---|---| +| `AZURE_SQL` | `LOGS_TO` | `BLOB_STORAGE` | +| `DATA_STREAM` | `STREAMS_TO` | `BLOB_STORAGE` | +| `EVENT_RULE` | `STREAMS_TO` | `DATA_STREAM` | +| `KEY_VAULT` | `LOGS_TO` | `LOG_GROUP` | +| `KEY_VAULT` | `STREAMS_TO` | `DATA_STREAM` | +| `LOG_SINK` | `LOGS_TO` | `BLOB_STORAGE` | +| `LOG_SINK` | `LOGS_TO` | `LOG_GROUP` | +| `LOG_SINK` | `STREAMS_TO` | `DATA_STREAM` | +| `ML_ENDPOINT` | `LOGS_TO` | `BLOB_STORAGE` | +| `OTHER` | `LOGS_TO` | `LOG_GROUP` | +| `VIRTUAL_MACHINE` | `LOGS_TO` | `LOG_SINK` | + +#### GCP parity: MANAGES + +| Source type | Relationship | Target type | +|---|---|---| +| `APP_REGISTRATION` | `OWNED_BY` | `SERVICE_PRINCIPAL` | +| `BACKUP_PLAN` | `BACKUP_TO` | `VIRTUAL_MACHINE` | +| `BACKUP_PLAN` | `SCHEDULED_BY` | `FILE_SYSTEM` | +| `BACKUP_VAULT` | `REPLICATES_TO` | `VIRTUAL_MACHINE` | +| `DATA_CATALOG` | `OWNED_BY` | `BLOB_STORAGE` | +| `DEPLOYMENT_GROUP` | `OWNED_BY` | `AKS_CLUSTER` | +| `DEPLOYMENT_GROUP` | `OWNED_BY` | `CLOUD_ACCOUNT` | +| `IAC_STACK` | `OWNED_BY` | `VIRTUAL_MACHINE` | +| `IDENTITY_USER` | `OWNED_BY` | `APP_REGISTRATION` | +| `IDENTITY_USER` | `OWNED_BY` | `SERVICE_PRINCIPAL` | +| `SCHEDULE` | `SCHEDULED_BY` | `CLOUD_FUNCTION` | +| `SCHEDULE` | `SCHEDULED_BY` | `ETL_JOB` | +| `SCHEDULE` | `SCHEDULED_BY` | `RUNBOOK` | +| `SCHEDULE` | `SCHEDULED_BY` | `STATE_MACHINE` | +| `SCHEDULE` | `SCHEDULED_BY` | `VIRTUAL_MACHINE` | + +#### GCP parity: MONITORS + +| Source type | Relationship | Target type | +|---|---|---| +| `ALARM` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `ALARM` | `MONITORED_BY` | `CLOUD_FUNCTION` | +| `ALARM` | `MONITORED_BY` | `LOG_GROUP` | +| `ALARM` | `MONITORED_BY` | `VIRTUAL_MACHINE` | +| `LOG_SINK` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `LOG_SINK` | `MONITORED_BY` | `VIRTUAL_MACHINE` | +| `SECURITY_HUB` | `FINDING_AFFECTS` | `CONTAINER_REGISTRY` | +| `SECURITY_HUB` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `SECURITY_HUB` | `MONITORED_BY` | `BLOB_STORAGE` | +| `SECURITY_HUB` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `THREAT_DETECTOR` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `BLOB_STORAGE` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `VIRTUAL_MACHINE` | + +#### GCP parity: PROTECTS + +| Source type | Relationship | Target type | +|---|---|---| +| `AI_GUARDRAIL` | `none` | `ML_MODEL` | +| `NACL` | `PROTECTED_BY_NACL` | `SERVICE_NETWORK` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `CDN` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `LB_LISTENER` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `LOAD_BALANCER` | + +#### GCP parity: REFERENCES + +| Source type | Relationship | Target type | +|---|---|---| +| `ACCESS_KEY` | `DEPENDS_ON` | `IAM_POLICY` | +| `API_GATEWAY` | `DEPENDS_ON` | `AUTHORIZER` | +| `APP_SERVICE` | `READS_FROM` | `SOURCE_CONNECTION` | +| `AZURE_SQL` | `REPLICATES_TO` | `AZURE_SQL` | +| `BACKUP_PLAN` | `BACKUP_TO` | `BACKUP_VAULT` | +| `BIG_DATA_CLUSTER` | `READS_FROM` | `BLOB_STORAGE` | +| `BLOB_STORAGE` | `BACKUP_TO` | `BACKUP_VAULT` | +| `BLOB_STORAGE` | `REPLICATES_TO` | `BLOB_STORAGE` | +| `BUILD_PROJECT` | `READS_FROM` | `SOURCE_CONNECTION` | +| `BUILD_PROJECT` | `WRITES_TO` | `CONTAINER_REGISTRY` | +| `CERTIFICATE` | `CERTIFICATE_SECURES` | `CUSTOM_DOMAIN` | +| `CERTIFICATE` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `CLOUD_FUNCTION` | `WRITES_TO` | `DATA_STREAM` | +| `CONTAINER_REGISTRY` | `REPLICATES_TO` | `CONTAINER_REGISTRY` | +| `DATA_CATALOG` | `DEPENDS_ON` | `SOURCE_CONNECTION` | +| `DATA_WAREHOUSE` | `READS_FROM` | `BLOB_STORAGE` | +| `DATA_WAREHOUSE` | `READS_FROM` | `DATA_STREAM` | +| `DEPLOYMENT_GROUP` | `READS_FROM` | `SOURCE_CONNECTION` | +| `DNS_RESOLVER` | `DEPENDS_ON` | `DNS_RESOLVER` | +| `DNS_RESOLVER` | `DNS_RESOLVED` | `VNET` | +| `DYNAMODB_TABLE` | `REPLICATES_TO` | `DYNAMODB_TABLE` | +| `ENDPOINT_SERVICE` | `DEPENDS_ON` | `LOG_GROUP` | +| `ETL_JOB` | `DEPENDS_ON` | `ETL_JOB` | +| `ETL_JOB` | `READS_FROM` | `DATA_CATALOG` | +| `ETL_JOB` | `READS_FROM` | `DATA_STREAM` | +| `ETL_JOB` | `READS_FROM` | `SOURCE_CONNECTION` | +| `ETL_JOB` | `WRITES_TO` | `AZURE_SQL` | +| `ETL_JOB` | `WRITES_TO` | `DATA_CATALOG` | +| `ETL_JOB` | `WRITES_TO` | `KNOWLEDGE_BASE` | +| `FILE_SYSTEM` | `BACKUP_TO` | `BACKUP_VAULT` | +| `FILE_SYSTEM` | `REPLICATES_TO` | `FILE_SYSTEM` | +| `GUARDRAIL` | `DEPENDS_ON` | `GUARDRAIL` | +| `GUARDRAIL` | `DEPENDS_ON` | `ORG_POLICY` | +| `IAC_STACK` | `DEPENDS_ON` | `PRODUCT_PORTFOLIO` | +| `IAM_POLICY` | `none` | `CLOUD_ACCOUNT` | +| `IDENTITY_USER` | `PERMISSION_BOUNDARY_LIMITS` | `GUARDRAIL` | +| `K8S_WORKLOAD` | `RUNS_ON` | `K8S_SERVICE_ACCOUNT` | +| `LB_LISTENER` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `LOAD_BALANCER` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `MACHINE_IMAGE` | `DEPENDS_ON` | `VIRTUAL_MACHINE` | +| `MESSAGE_QUEUE` | `WRITES_TO` | `MESSAGE_QUEUE` | +| `ML_WORKSPACE` | `READS_FROM` | `BLOB_STORAGE` | +| `NETWORK_FIREWALL` | `DEPENDS_ON` | `PREFIX_LIST` | +| `ORG_POLICY` | `DEPENDS_ON` | `ORG_POLICY` | +| `OTHER` | `DEPENDS_ON` | `LOG_GROUP` | +| `PARAMETER` | `READS_FROM` | `KEY_VAULT` | +| `PARAMETER` | `REPLICATES_TO` | `PARAMETER` | +| `PRODUCT_PORTFOLIO` | `DEPENDS_ON` | `API_GATEWAY` | +| `PROVISIONED_PRODUCT` | `DEPENDS_ON` | `PRODUCT_PORTFOLIO` | +| `RUNBOOK` | `RUNS_ON` | `NODE_GROUP` | +| `SEARCH_DOMAIN` | `DEPENDS_ON` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `AZURE_SQL` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `MESSAGE_QUEUE` | +| `SOURCE_CONNECTION` | `READS_FROM` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `READS_FROM` | `SOURCE_CONNECTION` | +| `SOURCE_CONNECTION` | `RUNS_ON` | `BATCH_ENVIRONMENT` | +| `STATE_MACHINE` | `DEPENDS_ON` | `SOURCE_CONNECTION` | +| `THREAT_DETECTOR` | `DEPENDS_ON` | `LOG_GROUP` | +| `VIRTUAL_MACHINE` | `BACKUP_TO` | `BACKUP_VAULT` | +| `VIRTUAL_MACHINE` | `REPLICATES_TO` | `VIRTUAL_MACHINE` | +| `VPC_ENDPOINT` | `DEPENDS_ON` | `BLOB_STORAGE` | + +#### GCP parity: ROUTE + +| Source type | Relationship | Target type | +|---|---|---| +| `DIRECT_CONNECT` | `TRANSIT_ROUTED` | `DIRECT_CONNECT` | +| `DNS_RECORD` | `DNS_RESOLVED` | `API_GATEWAY` | +| `DNS_RECORD` | `DNS_RESOLVED` | `ELASTIC_IP` | +| `DNS_RECORD` | `DNS_RESOLVED` | `VPC_ENDPOINT` | +| `DNS_RESOLVER` | `DNS_RESOLVED` | `DNS_RESOLVER` | +| `ELASTIC_IP` | `LOAD_BALANCED_BY` | `K8S_SERVICE` | +| `PEERING_CONNECTION` | `TRANSIT_ROUTED` | `NETWORK_INTERFACE` | +| `PEERING_CONNECTION` | `TRANSIT_ROUTED` | `VNET` | +| `ROUTE_TABLE` | `TRANSIT_ROUTED` | `NETWORK_FIREWALL` | +| `SERVICE_NETWORK` | `TRANSIT_ROUTED` | `SERVICE_NETWORK` | +| `SERVICE_NETWORK` | `TRANSIT_ROUTED` | `VNET` | +| `TRANSIT_GATEWAY` | `TRANSIT_ROUTED` | `NETWORK_FIREWALL` | +| `TRANSIT_GATEWAY` | `TRANSIT_ROUTED` | `VNET` | +| `VPN_GATEWAY` | `TRANSIT_ROUTED` | `CUSTOMER_GATEWAY` | + +#### GCP parity: USES_IMAGE + +| Source type | Relationship | Target type | +|---|---|---| +| `APP_SERVICE` | `RUNS_ON` | `CONTAINER_REGISTRY` | +| `K8S_WORKLOAD` | `RUNS_ON` | `CONTAINER_REGISTRY` | + +New data sources behind these edges (API-backed, see INVENTORY_INTERNALS.md 9.6): Cloud DNS +record sets, Security Command Center, Cloud Identity groups, IAM deny and Principal Access +Boundary policies, PAM entitlements, predefined role permissions, Data Access audit configs, +Cloud Monitoring, OS Config, Artifact Analysis, Backup and DR, Firestore backups, Cloud +Scheduler, Apigee, Infrastructure Manager, Deployment Manager and the IAM recommender. Their +`gcp.*` configuration keys are listed in INVENTORY_INTERNALS.md 9.9. + + ## Relationship vocabulary `relationship` values are `RelationType` names from `cloudg/graph/ontology_rules.py` (64 in @@ -2893,3 +3352,502 @@ inferred relations are added after it. The inventory uses: | `VPC_CONTAINS_SUBNET` | VPC / VNet contains a subnet. | | `VPC_PEERED` | VPC peering. | | `WRITES_TO` | Writes data to the target (dead-letter queues, destinations, output locations). | + + +## Azure parity additions + +The Azure inventory reads child resources, more Resource Graph tables, subscription-level ARM +listings, two data planes, the Kubernetes API of AKS clusters and Entra ID (see +[INVENTORY_INTERNALS.md 8.4 to 8.10](INVENTORY_INTERNALS.md#84-depth-beyond-the-breadth-query)). +This block lists what that added: two `AssetType` values, 153 ARM types in +`catalogs/azure_arm_types.yaml` (278 in total), and the relationship combinations observed +in the Azure parity tests (`tests/test_azure_parity_*.py`, `test_azure_entra.py`, +`test_azure_kubernetes.py`), recorded the same way as the rest of this catalog. + +### New asset types + +#### PLACEMENT_GROUP + +A placement construct that groups machines: availability set, proximity placement group, +dedicated host group or host. + +- Azure ARM: `microsoft.compute/availabilitysets`, `microsoft.compute/proximityplacementgroups`, `microsoft.compute/hostgroups`, `microsoft.compute/hostgroups/hosts` +- **Providers:** AZURE +- **Metadata keys:** `member_count` (int), `fault_domains` (int), `update_domains` (null) +- **Declared relations:** `CONTAINS` → member VMs and scale sets, `CONTAINS` ← proximity placement group, `CONTAINS` → dedicated hosts + +#### APP_REGISTRATION + +An Entra application object (app registration), read from Microsoft Graph. Its service +principal is a separate `SERVICE_PRINCIPAL`. + +- Identifier: `entra:application/`, alias `entra:app/` +- **Providers:** AZURE +- **Metadata keys:** `object_id` (str), `app_id` (str), `sign_in_audience` (str), `multi_tenant` (bool), `secrets` (object: `count`, `latest_expiry`), `certificates` (object), `requested_permissions` (list), `redirect_uris` (list), `federated_credentials` (list), `collected_via` (str), `tenant_id` (str) +- **Declared relations:** `MANAGES / OWNED_BY` → its service principal, `MANAGES / OWNED_BY` ← owners, `IAM_TRUST / CROSS_ACCOUNT_TRUST` or `ROLE_ASSUMES_ROLE` ← federated credential issuers + +### ARM types added to the catalog + +| Asset type | ARM types | +|---|---| +| `ACCESS_KEY` | `microsoft.containerregistry/registries/tokens` | +| `AI_GUARDRAIL` | `microsoft.cognitiveservices/accounts/raipolicies` | +| `AKS_CLUSTER` | `microsoft.kubernetes/connectedclusters` | +| `ALARM` | `microsoft.alertsmanagement/smartdetectoralertrules`, `microsoft.securityinsights/alertrules` | +| `API_GATEWAY` | `microsoft.apimanagement/service/apis`, `microsoft.apimanagement/service/gateways` | +| `AUTHORIZER` | `microsoft.apimanagement/service/apis/authorizers`, `microsoft.apimanagement/service/authorizationservers` | +| `AZURE_SQL` | `microsoft.sql/managedinstances/databases` | +| `BACKUP_PLAN` | `microsoft.dataprotection/backupvaults/backuppolicies`, `microsoft.netapp/netappaccounts/backuppolicies`, `microsoft.netapp/netappaccounts/snapshotpolicies`, `microsoft.recoveryservices/vaults/backuppolicies`, `microsoft.recoveryservices/vaults/replicationpolicies` | +| `BACKUP_VAULT` | `microsoft.netapp/netappaccounts/backupvaults` | +| `BATCH_ENVIRONMENT` | `microsoft.batch/batchaccounts/pools`, `microsoft.datafactory/factories/integrationruntimes`, `microsoft.machinelearningservices/workspaces/computes`, `microsoft.synapse/workspaces/integrationruntimes` | +| `BIG_DATA_CLUSTER` | `microsoft.synapse/workspaces/bigdatapools` | +| `BUILD_PROJECT` | `microsoft.containerregistry/registries/tasks` | +| `CERTIFICATE` | `microsoft.cdn/profiles/secrets` | +| `CLUSTER_ADDON` | `microsoft.kubernetesconfiguration/extensions` | +| `CONTAINER_REGISTRY` | `microsoft.containerregistry/registries/replications` | +| `CONTAINER_SERVICE` | `microsoft.appplatform/spring/apps` | +| `CUSTOMER_GATEWAY` | `microsoft.network/vpnsites` | +| `DATA_CATALOG` | `microsoft.datafactory/factories/datasets`, `microsoft.synapse/workspaces/datasets` | +| `DATA_STREAM` | `microsoft.eventhub/namespaces/eventhubs` | +| `DATA_WAREHOUSE` | `microsoft.kusto/clusters/databases`, `microsoft.synapse/workspaces/sqlpools` | +| `DEPLOYMENT_GROUP` | `microsoft.devcenter/projects`, `microsoft.devcenter/projects/environmenttypes`, `microsoft.kubernetesconfiguration/fluxconfigurations`, `microsoft.kubernetesconfiguration/sourcecontrolconfigurations` | +| `DIRECT_CONNECT` | `microsoft.network/expressrouteports` | +| `DNS_RECORD` | `microsoft.network/dnszones/a`, `microsoft.network/dnszones/aaaa`, `microsoft.network/dnszones/caa`, `microsoft.network/dnszones/cname`, `microsoft.network/dnszones/ds`, `microsoft.network/dnszones/mx`, `microsoft.network/dnszones/ns`, `microsoft.network/dnszones/ptr`, `microsoft.network/dnszones/soa`, `microsoft.network/dnszones/srv`, `microsoft.network/dnszones/txt`, `microsoft.network/privatednszones/a`, `microsoft.network/privatednszones/aaaa`, `microsoft.network/privatednszones/cname`, `microsoft.network/privatednszones/mx`, `microsoft.network/privatednszones/ptr`, `microsoft.network/privatednszones/soa`, `microsoft.network/privatednszones/srv`, `microsoft.network/privatednszones/txt` | +| `DNS_RESOLVER` | `microsoft.network/dnsforwardingrulesets`, `microsoft.network/dnsforwardingrulesets/forwardingrules`, `microsoft.network/dnsresolvers/inboundendpoints`, `microsoft.network/dnsresolvers/outboundendpoints` | +| `DYNAMODB_TABLE` | `microsoft.documentdb/databaseaccounts/regions` | +| `ECS_CLUSTER` | `microsoft.appplatform/spring` | +| `EDGE_FUNCTION` | `microsoft.cdn/profiles/rulesets`, `microsoft.cdn/profiles/rulesets/rules` | +| `ENDPOINT_SERVICE` | `microsoft.insights/privatelinkscopes` | +| `ETL_JOB` | `microsoft.datafactory/factories/pipelines`, `microsoft.search/searchservices/indexers`, `microsoft.search/searchservices/skillsets`, `microsoft.synapse/workspaces/pipelines` | +| `EVENT_RULE` | `microsoft.containerregistry/registries/webhooks`, `microsoft.eventgrid/eventsubscriptions`, `microsoft.securityinsights/automationrules` | +| `FILE_SYSTEM` | `microsoft.netapp/netappaccounts/capacitypools`, `microsoft.netapp/netappaccounts/capacitypools/volumes` | +| `GUARDRAIL` | `microsoft.authorization/denyassignments`, `microsoft.authorization/locks`, `microsoft.guestconfiguration/guestconfigurationassignments` | +| `IAC_STACK` | `microsoft.resources/deployments`, `microsoft.resources/deploymentstacks` | +| `IAM_POLICY` | `microsoft.authorization/roledefinitions`, `microsoft.containerregistry/registries/scopemaps` | +| `IDENTITY_PROVIDER` | `microsoft.apimanagement/service/openidconnectproviders`, `microsoft.managedidentity/userassignedidentities/federatedidentitycredentials` | +| `IDENTITY_USER` | `microsoft.storage/storageaccounts/localusers` | +| `KMS_KEY` | `microsoft.keyvault/vaults/keys` | +| `KNOWLEDGE_BASE` | `microsoft.search/searchservices/indexes` | +| `LB_LISTENER` | `microsoft.cdn/profiles/afdendpoints/routes`, `microsoft.network/applicationgateways/httplisteners` | +| `LOG_SINK` | `microsoft.insights/datacollectionendpoints`, `microsoft.securityinsights/dataconnectors` | +| `MACHINE_IMAGE` | `microsoft.compute/galleries` | +| `MESSAGE_QUEUE` | `microsoft.servicebus/namespaces/queues`, `microsoft.servicebus/namespaces/topics/subscriptions` | +| `ML_ENDPOINT` | `microsoft.machinelearningservices/workspaces/serverlessendpoints` | +| `ML_MODEL` | `microsoft.cognitiveservices/accounts/deployments`, `microsoft.machinelearningservices/workspaces/onlineendpoints/deployments` | +| `ML_WORKSPACE` | `microsoft.cognitiveservices/accounts/projects` | +| `NACL` | `microsoft.network/networkmanagers/securityadminconfigurations`, `microsoft.network/networkmanagers/securityadminconfigurations/rulecollections` | +| `NETWORK_INTERFACE` | `microsoft.network/virtualhubs/ipconfigurations` | +| `NODE_GROUP` | `microsoft.automation/automationaccounts/hybridrunbookworkergroups` | +| `NOTIFICATION_TOPIC` | `microsoft.servicebus/namespaces/topics` | +| `ORG_POLICY` | `microsoft.authorization/policydefinitions`, `microsoft.authorization/policysetdefinitions` | +| `PARAMETER` | `microsoft.apimanagement/service/namedvalues`, `microsoft.appconfiguration/configurationstores/replicas` | +| `PEERING_CONNECTION` | `microsoft.network/virtualhubs/bgpconnections`, `microsoft.network/virtualhubs/hubvirtualnetworkconnections` | +| `PLACEMENT_GROUP` | `microsoft.compute/availabilitysets`, `microsoft.compute/hostgroups`, `microsoft.compute/hostgroups/hosts`, `microsoft.compute/proximityplacementgroups` | +| `PRODUCT_PORTFOLIO` | `microsoft.apimanagement/service/products`, `microsoft.devcenter/devcenters`, `microsoft.resources/templatespecs`, `microsoft.resources/templatespecs/versions`, `microsoft.solutions/applicationdefinitions` | +| `PROVISIONED_PRODUCT` | `microsoft.solutions/applications`, `microsoft.subscription/aliases` | +| `RESOURCE_SHARE` | `microsoft.managedservices/registrationassignments`, `microsoft.managedservices/registrationdefinitions` | +| `ROUTE_TABLE` | `microsoft.network/networkmanagers/routingconfigurations`, `microsoft.network/virtualhubs/hubroutetables`, `microsoft.network/virtualhubs/routingintent` | +| `RUNBOOK` | `microsoft.automation/automationaccounts/runbooks` | +| `SCHEDULE` | `microsoft.automation/automationaccounts/schedules`, `microsoft.automation/automationaccounts/softwareupdateconfigurations`, `microsoft.datafactory/factories/triggers`, `microsoft.logic/workflows/triggers`, `microsoft.maintenance/maintenanceconfigurations`, `microsoft.synapse/workspaces/triggers`, `microsoft.web/sites/functions/timers` | +| `SECRET` | `microsoft.keyvault/vaults/secrets` | +| `SERVICE_NETWORK` | `microsoft.network/networkmanagers/connectivityconfigurations`, `microsoft.network/networkmanagers/networkgroups` | +| `SOURCE_CONNECTION` | `microsoft.cognitiveservices/accounts/connections`, `microsoft.cognitiveservices/accounts/projects/connections`, `microsoft.datafactory/factories/linkedservices`, `microsoft.devcenter/devcenters/catalogs`, `microsoft.devcenter/projects/catalogs`, `microsoft.machinelearningservices/workspaces/connections`, `microsoft.machinelearningservices/workspaces/datastores`, `microsoft.search/searchservices/datasources`, `microsoft.synapse/workspaces/linkedservices`, `microsoft.web/connections` | +| `TARGET_GROUP` | `microsoft.apimanagement/service/backends`, `microsoft.cdn/profiles/origingroups`, `microsoft.cdn/profiles/origingroups/origins` | +| `THREAT_DETECTOR` | `microsoft.securityinsights/onboardingstates` | +| `USER_POOL` | `microsoft.azureactivedirectory/b2cdirectories`, `microsoft.azureactivedirectory/ciamdirectories` | +| `VPN_GATEWAY` | `microsoft.network/p2svpngateways` | +| `WAF_WEB_ACL` | `microsoft.cdn/profiles/securitypolicies` | + +### Azure parity relationship matrix + +307 `(source type, edge type, relationship, target type)` combinations the linker produced +from the Azure parity fixtures, grouped by edge type. `none` means the edge carries no +fine-grained relationship. + +#### ASSUMES_ROLE + +| Source type | Relationship | Target type | +|---|---|---| +| `K8S_SERVICE_ACCOUNT` | `RUNS_ON` | `SERVICE_PRINCIPAL` | +| `LOG_SINK` | `CROSS_ACCOUNT_TRUST` | `CLOUD_ACCOUNT` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `RESOURCE_SHARE` | + +#### ATTACHED_TO + +| Source type | Relationship | Target type | +|---|---|---| +| `ELASTIC_IP` | `none` | `LOAD_BALANCER` | +| `VPN_CONNECTION` | `TRANSIT_ROUTED` | `CUSTOMER_GATEWAY` | +| `VPN_CONNECTION` | `TRANSIT_ROUTED` | `VPN_GATEWAY` | + +#### CONTAINS + +| Source type | Relationship | Target type | +|---|---|---| +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `CLUSTER_ADDON` | +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `DEPLOYMENT_GROUP` | +| `AKS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `K8S_NAMESPACE` | +| `API_GATEWAY` | `none` | `API_GATEWAY` | +| `API_GATEWAY` | `none` | `AUTHORIZER` | +| `API_GATEWAY` | `none` | `PARAMETER` | +| `API_GATEWAY` | `none` | `PRODUCT_PORTFOLIO` | +| `API_GATEWAY` | `none` | `TARGET_GROUP` | +| `AZURE_SQL` | `none` | `AZURE_SQL` | +| `BACKUP_VAULT` | `none` | `BACKUP_PLAN` | +| `BATCH_ENVIRONMENT` | `none` | `BATCH_ENVIRONMENT` | +| `BLOB_STORAGE` | `none` | `EVENT_RULE` | +| `BLOB_STORAGE` | `none` | `IDENTITY_USER` | +| `CDN` | `none` | `CDN` | +| `CDN` | `none` | `CERTIFICATE` | +| `CDN` | `none` | `LB_LISTENER` | +| `CDN` | `none` | `TARGET_GROUP` | +| `CLOUD_ACCOUNT` | `ACCOUNT_CONTAINS_REGION` | `RESOURCE_GROUP` | +| `CLOUD_ACCOUNT` | `none` | `GUARDRAIL` | +| `CLOUD_ACCOUNT` | `none` | `LOG_SINK` | +| `CLOUD_ACCOUNT` | `none` | `ORG_POLICY` | +| `CLOUD_ACCOUNT` | `none` | `RESOURCE_SHARE` | +| `CLOUD_FUNCTION` | `CLUSTER_CONTAINS_SERVICE` | `CLOUD_FUNCTION` | +| `CLOUD_FUNCTION` | `none` | `SCHEDULE` | +| `CONTAINER_REGISTRY` | `none` | `ACCESS_KEY` | +| `CONTAINER_REGISTRY` | `none` | `BUILD_PROJECT` | +| `CONTAINER_REGISTRY` | `none` | `CONTAINER_REGISTRY` | +| `CONTAINER_REGISTRY` | `none` | `EVENT_RULE` | +| `CONTAINER_REGISTRY` | `none` | `IAM_POLICY` | +| `DATA_STREAM` | `none` | `DATA_STREAM` | +| `DATA_WAREHOUSE` | `none` | `DATA_CATALOG` | +| `DATA_WAREHOUSE` | `none` | `DATA_WAREHOUSE` | +| `DATA_WAREHOUSE` | `none` | `ETL_JOB` | +| `DATA_WAREHOUSE` | `none` | `SOURCE_CONNECTION` | +| `DEPLOYMENT_GROUP` | `none` | `DEPLOYMENT_GROUP` | +| `DNS_RESOLVER` | `none` | `DNS_RESOLVER` | +| `DNS_ZONE` | `none` | `DNS_RECORD` | +| `DYNAMODB_TABLE` | `none` | `DYNAMODB_TABLE` | +| `ECS_CLUSTER` | `CLUSTER_CONTAINS_SERVICE` | `CONTAINER_SERVICE` | +| `ETL_JOB` | `none` | `BATCH_ENVIRONMENT` | +| `ETL_JOB` | `none` | `DATA_CATALOG` | +| `ETL_JOB` | `none` | `ETL_JOB` | +| `ETL_JOB` | `none` | `SCHEDULE` | +| `ETL_JOB` | `none` | `SOURCE_CONNECTION` | +| `IDENTITY_GROUP` | `none` | `IDENTITY_USER` | +| `IDENTITY_GROUP` | `none` | `SERVICE_PRINCIPAL` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_SERVICE` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_SERVICE_ACCOUNT` | +| `K8S_NAMESPACE` | `CLUSTER_CONTAINS_SERVICE` | `K8S_WORKLOAD` | +| `KEY_VAULT` | `none` | `GUARDRAIL` | +| `KEY_VAULT` | `none` | `KMS_KEY` | +| `KEY_VAULT` | `none` | `LOG_SINK` | +| `KEY_VAULT` | `none` | `SECRET` | +| `LOAD_BALANCER` | `none` | `LB_LISTENER` | +| `LOG_GROUP` | `none` | `ALARM` | +| `LOG_GROUP` | `none` | `EVENT_RULE` | +| `LOG_GROUP` | `none` | `LOG_SINK` | +| `LOG_GROUP` | `none` | `THREAT_DETECTOR` | +| `MESSAGE_QUEUE` | `none` | `MESSAGE_QUEUE` | +| `MESSAGE_QUEUE` | `none` | `NOTIFICATION_TOPIC` | +| `ML_ENDPOINT` | `CLUSTER_CONTAINS_SERVICE` | `ML_MODEL` | +| `ML_ENDPOINT` | `none` | `AI_GUARDRAIL` | +| `ML_ENDPOINT` | `none` | `LOG_SINK` | +| `ML_ENDPOINT` | `none` | `ML_MODEL` | +| `ML_WORKSPACE` | `none` | `BATCH_ENVIRONMENT` | +| `ML_WORKSPACE` | `none` | `ML_ENDPOINT` | +| `ML_WORKSPACE` | `none` | `ML_WORKSPACE` | +| `ML_WORKSPACE` | `none` | `SOURCE_CONNECTION` | +| `NACL` | `none` | `NACL` | +| `NODE_GROUP` | `CLUSTER_CONTAINS_SERVICE` | `VIRTUAL_MACHINE` | +| `NOTIFICATION_TOPIC` | `none` | `EVENT_RULE` | +| `NOTIFICATION_TOPIC` | `none` | `MESSAGE_QUEUE` | +| `PARAMETER` | `none` | `PARAMETER` | +| `PLACEMENT_GROUP` | `none` | `PLACEMENT_GROUP` | +| `PLACEMENT_GROUP` | `none` | `VIRTUAL_MACHINE` | +| `PREFIX_LIST` | `none` | `ELASTIC_IP` | +| `PRODUCT_PORTFOLIO` | `none` | `DEPLOYMENT_GROUP` | +| `PRODUCT_PORTFOLIO` | `none` | `PRODUCT_PORTFOLIO` | +| `PRODUCT_PORTFOLIO` | `none` | `SOURCE_CONNECTION` | +| `RESOURCE_GROUP` | `none` | `AKS_CLUSTER` | +| `RESOURCE_GROUP` | `none` | `BACKUP_VAULT` | +| `RESOURCE_GROUP` | `none` | `BLOB_STORAGE` | +| `RESOURCE_GROUP` | `none` | `CONTAINER_REGISTRY` | +| `RESOURCE_GROUP` | `none` | `GUARDRAIL` | +| `RESOURCE_GROUP` | `none` | `KEY_VAULT` | +| `RESOURCE_GROUP` | `none` | `LOG_GROUP` | +| `RESOURCE_GROUP` | `none` | `LOG_SINK` | +| `RESOURCE_GROUP` | `none` | `SCHEDULE` | +| `RESOURCE_GROUP` | `none` | `SERVICE_NETWORK` | +| `RESOURCE_GROUP` | `none` | `VIRTUAL_MACHINE` | +| `RESOURCE_GROUP` | `none` | `VNET` | +| `SCHEDULE` | `none` | `NODE_GROUP` | +| `SCHEDULE` | `none` | `RUNBOOK` | +| `SCHEDULE` | `none` | `SCHEDULE` | +| `SEARCH_DOMAIN` | `none` | `ETL_JOB` | +| `SEARCH_DOMAIN` | `none` | `KNOWLEDGE_BASE` | +| `SEARCH_DOMAIN` | `none` | `SOURCE_CONNECTION` | +| `SERVICE_NETWORK` | `none` | `CUSTOMER_GATEWAY` | +| `SERVICE_NETWORK` | `none` | `NACL` | +| `SERVICE_NETWORK` | `none` | `SERVICE_NETWORK` | +| `SERVICE_NETWORK` | `none` | `TRANSIT_GATEWAY` | +| `SERVICE_NETWORK` | `none` | `VNET` | +| `SERVICE_PRINCIPAL` | `none` | `IDENTITY_PROVIDER` | +| `STATE_MACHINE` | `none` | `SCHEDULE` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `AZURE_SQL` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `BATCH_ENVIRONMENT` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `BIG_DATA_CLUSTER` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `DATA_WAREHOUSE` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `DNS_RESOLVER` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `FILE_SYSTEM` | +| `SUBNET` | `SUBNET_CONTAINS_INSTANCE` | `NETWORK_INTERFACE` | +| `TARGET_GROUP` | `none` | `TARGET_GROUP` | +| `TRANSIT_GATEWAY` | `none` | `DIRECT_CONNECT` | +| `TRANSIT_GATEWAY` | `none` | `NETWORK_FIREWALL` | +| `TRANSIT_GATEWAY` | `none` | `NETWORK_INTERFACE` | +| `TRANSIT_GATEWAY` | `none` | `PEERING_CONNECTION` | +| `TRANSIT_GATEWAY` | `none` | `ROUTE_TABLE` | +| `TRANSIT_GATEWAY` | `none` | `VPN_GATEWAY` | +| `VIRTUAL_MACHINE` | `none` | `GUARDRAIL` | +| `VIRTUAL_MACHINE` | `none` | `OTHER` | +| `VNET` | `VPC_CONTAINS_SUBNET` | `SUBNET` | + +#### GOVERNS + +| Source type | Relationship | Target type | +|---|---|---| +| `GUARDRAIL` | `COMPLIANCE_GOVERNS` | `CLOUD_ACCOUNT` | +| `GUARDRAIL` | `COMPLIANCE_GOVERNS` | `RESOURCE_GROUP` | +| `GUARDRAIL` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `GUARDRAIL` | `POLICY_DENIES_ACTION` | `KEY_VAULT` | +| `ORG_POLICY` | `SCP_RESTRICTS` | `CLOUD_ACCOUNT` | +| `SERVICE_NETWORK` | `COMPLIANCE_GOVERNS` | `CLOUD_ACCOUNT` | + +#### GRANTS_ACCESS + +| Source type | Relationship | Target type | +|---|---|---| +| `ACCESS_KEY` | `POLICY_ALLOWS_ACTION` | `CONTAINER_REGISTRY` | +| `BACKUP_VAULT` | `POLICY_ALLOWS_ACTION` | `AKS_CLUSTER` | +| `CLOUD_ACCOUNT` | `CROSS_ACCOUNT_TRUST` | `MACHINE_IMAGE` | +| `CLOUD_ACCOUNT` | `POLICY_ALLOWS_ACTION` | `ENDPOINT_SERVICE` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `AKS_CLUSTER` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `DATA_WAREHOUSE` | +| `IDENTITY_GROUP` | `POLICY_ALLOWS_ACTION` | `ORGANIZATION` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `AZURE_SQL` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `BLOB_STORAGE` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `CLOUD_ACCOUNT` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `DEPLOYMENT_GROUP` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `DYNAMODB_TABLE` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `KEY_VAULT` | +| `IDENTITY_USER` | `POLICY_ALLOWS_ACTION` | `ORGANIZATION` | +| `RESOURCE_SHARE` | `CROSS_ACCOUNT_TRUST` | `CLOUD_ACCOUNT` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `PRODUCT_PORTFOLIO` | +| `SERVICE_PRINCIPAL` | `CROSS_ACCOUNT_TRUST` | `PROVISIONED_PRODUCT` | +| `SERVICE_PRINCIPAL` | `POLICY_ALLOWS_ACTION` | `BLOB_STORAGE` | +| `SERVICE_PRINCIPAL` | `POLICY_ALLOWS_ACTION` | `SERVICE_PRINCIPAL` | +| `SUBNET` | `POLICY_ALLOWS_ACTION` | `APP_SERVICE` | + +#### IAM_POLICY_ATTACHMENT + +| Source type | Relationship | Target type | +|---|---|---| +| `IDENTITY_GROUP` | `GROUP_HAS_POLICY` | `IAM_POLICY` | +| `IDENTITY_USER` | `USER_HAS_POLICY` | `IAM_POLICY` | +| `SERVICE_PRINCIPAL` | `ROLE_HAS_POLICY` | `IAM_POLICY` | + +#### IAM_TRUST + +| Source type | Relationship | Target type | +|---|---|---| +| `AKS_CLUSTER` | `ROLE_ASSUMES_ROLE` | `SERVICE_PRINCIPAL` | +| `IDENTITY_PROVIDER` | `CROSS_ACCOUNT_TRUST` | `SERVICE_PRINCIPAL` | +| `IDENTITY_PROVIDER` | `ROLE_ASSUMES_ROLE` | `SERVICE_PRINCIPAL` | + +#### INVOKES + +| Source type | Relationship | Target type | +|---|---|---| +| `ALARM` | `INVOKES` | `NOTIFICATION_TOPIC` | +| `BLOB_STORAGE` | `INVOKES` | `CLOUD_FUNCTION` | +| `ETL_JOB` | `INVOKES` | `CLOUD_FUNCTION` | +| `EVENT_RULE` | `INVOKES` | `CLOUD_FUNCTION` | +| `EVENT_RULE` | `INVOKES` | `STATE_MACHINE` | +| `KEY_VAULT` | `ROTATES_SECRET` | `CLOUD_FUNCTION` | +| `KEY_VAULT` | `TRIGGERED_BY` | `NOTIFICATION_TOPIC` | +| `KNOWLEDGE_BASE` | `INVOKES` | `ML_ENDPOINT` | +| `MESSAGE_QUEUE` | `TRIGGERED_BY` | `CLOUD_FUNCTION` | +| `NOTIFICATION_TOPIC` | `INVOKES` | `CLOUD_FUNCTION` | +| `NOTIFICATION_TOPIC` | `STREAMS_TO` | `DATA_STREAM` | +| `STATE_MACHINE` | `INVOKES` | `CLOUD_FUNCTION` | +| `STATE_MACHINE` | `INVOKES` | `SOURCE_CONNECTION` | + +#### LOAD_BALANCER_TARGET + +| Source type | Relationship | Target type | +|---|---|---| +| `API_GATEWAY` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | +| `API_GATEWAY` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `APP_SERVICE` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | +| `CDN` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `EDGE_FUNCTION` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `K8S_SERVICE` | `SERVES_TRAFFIC_TO` | `K8S_WORKLOAD` | +| `LB_LISTENER` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `LB_LISTENER` | `SERVES_TRAFFIC_TO` | `TARGET_GROUP` | +| `LOAD_BALANCER` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `TARGET_GROUP` | `SERVES_TRAFFIC_TO` | `APP_SERVICE` | +| `TARGET_GROUP` | `SERVES_TRAFFIC_TO` | `CLOUD_FUNCTION` | + +#### LOGS_TO + +| Source type | Relationship | Target type | +|---|---|---| +| `AZURE_SQL` | `LOGS_TO` | `BLOB_STORAGE` | +| `DATA_STREAM` | `STREAMS_TO` | `BLOB_STORAGE` | +| `EVENT_RULE` | `STREAMS_TO` | `DATA_STREAM` | +| `KEY_VAULT` | `LOGS_TO` | `LOG_GROUP` | +| `KEY_VAULT` | `STREAMS_TO` | `DATA_STREAM` | +| `LOG_SINK` | `LOGS_TO` | `BLOB_STORAGE` | +| `LOG_SINK` | `LOGS_TO` | `LOG_GROUP` | +| `LOG_SINK` | `STREAMS_TO` | `DATA_STREAM` | +| `ML_ENDPOINT` | `LOGS_TO` | `BLOB_STORAGE` | +| `OTHER` | `LOGS_TO` | `LOG_GROUP` | +| `VIRTUAL_MACHINE` | `LOGS_TO` | `LOG_SINK` | + +#### MANAGES + +| Source type | Relationship | Target type | +|---|---|---| +| `APP_REGISTRATION` | `OWNED_BY` | `SERVICE_PRINCIPAL` | +| `BACKUP_PLAN` | `BACKUP_TO` | `VIRTUAL_MACHINE` | +| `BACKUP_PLAN` | `SCHEDULED_BY` | `FILE_SYSTEM` | +| `BACKUP_VAULT` | `REPLICATES_TO` | `VIRTUAL_MACHINE` | +| `DATA_CATALOG` | `OWNED_BY` | `BLOB_STORAGE` | +| `DEPLOYMENT_GROUP` | `OWNED_BY` | `AKS_CLUSTER` | +| `DEPLOYMENT_GROUP` | `OWNED_BY` | `CLOUD_ACCOUNT` | +| `IAC_STACK` | `OWNED_BY` | `VIRTUAL_MACHINE` | +| `IDENTITY_USER` | `OWNED_BY` | `APP_REGISTRATION` | +| `IDENTITY_USER` | `OWNED_BY` | `SERVICE_PRINCIPAL` | +| `SCHEDULE` | `SCHEDULED_BY` | `CLOUD_FUNCTION` | +| `SCHEDULE` | `SCHEDULED_BY` | `ETL_JOB` | +| `SCHEDULE` | `SCHEDULED_BY` | `RUNBOOK` | +| `SCHEDULE` | `SCHEDULED_BY` | `STATE_MACHINE` | +| `SCHEDULE` | `SCHEDULED_BY` | `VIRTUAL_MACHINE` | + +#### MONITORS + +| Source type | Relationship | Target type | +|---|---|---| +| `ALARM` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `ALARM` | `MONITORED_BY` | `CLOUD_FUNCTION` | +| `ALARM` | `MONITORED_BY` | `LOG_GROUP` | +| `ALARM` | `MONITORED_BY` | `VIRTUAL_MACHINE` | +| `LOG_SINK` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `LOG_SINK` | `MONITORED_BY` | `VIRTUAL_MACHINE` | +| `SECURITY_HUB` | `FINDING_AFFECTS` | `CONTAINER_REGISTRY` | +| `SECURITY_HUB` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `SECURITY_HUB` | `MONITORED_BY` | `BLOB_STORAGE` | +| `SECURITY_HUB` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `THREAT_DETECTOR` | `FINDING_AFFECTS` | `VIRTUAL_MACHINE` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `BLOB_STORAGE` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `CLOUD_ACCOUNT` | +| `THREAT_DETECTOR` | `MONITORED_BY` | `VIRTUAL_MACHINE` | + +#### PROTECTS + +| Source type | Relationship | Target type | +|---|---|---| +| `AI_GUARDRAIL` | `none` | `ML_MODEL` | +| `NACL` | `PROTECTED_BY_NACL` | `SERVICE_NETWORK` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `CDN` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `LB_LISTENER` | +| `WAF_WEB_ACL` | `PROTECTED_BY_WAF` | `LOAD_BALANCER` | + +#### REFERENCES + +| Source type | Relationship | Target type | +|---|---|---| +| `ACCESS_KEY` | `DEPENDS_ON` | `IAM_POLICY` | +| `API_GATEWAY` | `DEPENDS_ON` | `AUTHORIZER` | +| `APP_SERVICE` | `READS_FROM` | `SOURCE_CONNECTION` | +| `AZURE_SQL` | `REPLICATES_TO` | `AZURE_SQL` | +| `BACKUP_PLAN` | `BACKUP_TO` | `BACKUP_VAULT` | +| `BIG_DATA_CLUSTER` | `READS_FROM` | `BLOB_STORAGE` | +| `BLOB_STORAGE` | `BACKUP_TO` | `BACKUP_VAULT` | +| `BLOB_STORAGE` | `REPLICATES_TO` | `BLOB_STORAGE` | +| `BUILD_PROJECT` | `READS_FROM` | `SOURCE_CONNECTION` | +| `BUILD_PROJECT` | `WRITES_TO` | `CONTAINER_REGISTRY` | +| `CERTIFICATE` | `CERTIFICATE_SECURES` | `CUSTOM_DOMAIN` | +| `CERTIFICATE` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `CLOUD_FUNCTION` | `WRITES_TO` | `DATA_STREAM` | +| `CONTAINER_REGISTRY` | `REPLICATES_TO` | `CONTAINER_REGISTRY` | +| `DATA_CATALOG` | `DEPENDS_ON` | `SOURCE_CONNECTION` | +| `DATA_WAREHOUSE` | `READS_FROM` | `BLOB_STORAGE` | +| `DATA_WAREHOUSE` | `READS_FROM` | `DATA_STREAM` | +| `DEPLOYMENT_GROUP` | `READS_FROM` | `SOURCE_CONNECTION` | +| `DNS_RESOLVER` | `DEPENDS_ON` | `DNS_RESOLVER` | +| `DNS_RESOLVER` | `DNS_RESOLVED` | `VNET` | +| `DYNAMODB_TABLE` | `REPLICATES_TO` | `DYNAMODB_TABLE` | +| `EBS_VOLUME` | `BACKUP_TO` | `SNAPSHOT` | +| `ENDPOINT_SERVICE` | `DEPENDS_ON` | `LOG_GROUP` | +| `ETL_JOB` | `DEPENDS_ON` | `ETL_JOB` | +| `ETL_JOB` | `READS_FROM` | `DATA_CATALOG` | +| `ETL_JOB` | `READS_FROM` | `DATA_STREAM` | +| `ETL_JOB` | `READS_FROM` | `SOURCE_CONNECTION` | +| `ETL_JOB` | `WRITES_TO` | `AZURE_SQL` | +| `ETL_JOB` | `WRITES_TO` | `DATA_CATALOG` | +| `ETL_JOB` | `WRITES_TO` | `KNOWLEDGE_BASE` | +| `FILE_SYSTEM` | `BACKUP_TO` | `BACKUP_VAULT` | +| `FILE_SYSTEM` | `REPLICATES_TO` | `FILE_SYSTEM` | +| `GUARDRAIL` | `DEPENDS_ON` | `GUARDRAIL` | +| `GUARDRAIL` | `DEPENDS_ON` | `ORG_POLICY` | +| `IAC_STACK` | `DEPENDS_ON` | `PRODUCT_PORTFOLIO` | +| `IAM_POLICY` | `none` | `CLOUD_ACCOUNT` | +| `IDENTITY_USER` | `PERMISSION_BOUNDARY_LIMITS` | `GUARDRAIL` | +| `K8S_WORKLOAD` | `RUNS_ON` | `K8S_SERVICE_ACCOUNT` | +| `LB_LISTENER` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `LOAD_BALANCER` | `CERTIFICATE_SECURES` | `KEY_VAULT` | +| `MACHINE_IMAGE` | `DEPENDS_ON` | `VIRTUAL_MACHINE` | +| `MESSAGE_QUEUE` | `WRITES_TO` | `MESSAGE_QUEUE` | +| `ML_WORKSPACE` | `READS_FROM` | `BLOB_STORAGE` | +| `NETWORK_FIREWALL` | `DEPENDS_ON` | `PREFIX_LIST` | +| `ORG_POLICY` | `DEPENDS_ON` | `ORG_POLICY` | +| `OTHER` | `DEPENDS_ON` | `LOG_GROUP` | +| `PARAMETER` | `READS_FROM` | `KEY_VAULT` | +| `PARAMETER` | `REPLICATES_TO` | `PARAMETER` | +| `PRODUCT_PORTFOLIO` | `DEPENDS_ON` | `API_GATEWAY` | +| `PROVISIONED_PRODUCT` | `DEPENDS_ON` | `PRODUCT_PORTFOLIO` | +| `RUNBOOK` | `RUNS_ON` | `NODE_GROUP` | +| `SEARCH_DOMAIN` | `DEPENDS_ON` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `AZURE_SQL` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `DEPENDS_ON` | `MESSAGE_QUEUE` | +| `SOURCE_CONNECTION` | `READS_FROM` | `BLOB_STORAGE` | +| `SOURCE_CONNECTION` | `READS_FROM` | `SOURCE_CONNECTION` | +| `SOURCE_CONNECTION` | `RUNS_ON` | `BATCH_ENVIRONMENT` | +| `STATE_MACHINE` | `DEPENDS_ON` | `SOURCE_CONNECTION` | +| `THREAT_DETECTOR` | `DEPENDS_ON` | `LOG_GROUP` | +| `VIRTUAL_MACHINE` | `BACKUP_TO` | `BACKUP_VAULT` | +| `VIRTUAL_MACHINE` | `REPLICATES_TO` | `VIRTUAL_MACHINE` | +| `VPC_ENDPOINT` | `DEPENDS_ON` | `BLOB_STORAGE` | + +#### ROUTE + +| Source type | Relationship | Target type | +|---|---|---| +| `DIRECT_CONNECT` | `TRANSIT_ROUTED` | `DIRECT_CONNECT` | +| `DNS_RECORD` | `DNS_RESOLVED` | `API_GATEWAY` | +| `DNS_RECORD` | `DNS_RESOLVED` | `ELASTIC_IP` | +| `DNS_RECORD` | `DNS_RESOLVED` | `VPC_ENDPOINT` | +| `DNS_RESOLVER` | `DNS_RESOLVED` | `DNS_RESOLVER` | +| `ELASTIC_IP` | `LOAD_BALANCED_BY` | `K8S_SERVICE` | +| `PEERING_CONNECTION` | `TRANSIT_ROUTED` | `NETWORK_INTERFACE` | +| `PEERING_CONNECTION` | `TRANSIT_ROUTED` | `VNET` | +| `ROUTE_TABLE` | `TRANSIT_ROUTED` | `NETWORK_FIREWALL` | +| `SERVICE_NETWORK` | `TRANSIT_ROUTED` | `SERVICE_NETWORK` | +| `SERVICE_NETWORK` | `TRANSIT_ROUTED` | `VNET` | +| `TRANSIT_GATEWAY` | `TRANSIT_ROUTED` | `NETWORK_FIREWALL` | +| `TRANSIT_GATEWAY` | `TRANSIT_ROUTED` | `VNET` | +| `VPN_GATEWAY` | `TRANSIT_ROUTED` | `CUSTOMER_GATEWAY` | + +#### USES_IMAGE + +| Source type | Relationship | Target type | +|---|---|---| +| `APP_SERVICE` | `RUNS_ON` | `CONTAINER_REGISTRY` | +| `K8S_WORKLOAD` | `RUNS_ON` | `CONTAINER_REGISTRY` | + diff --git a/docs/INVENTORY_INTERNALS.md b/docs/INVENTORY_INTERNALS.md index 1bb4507..2a9f544 100644 --- a/docs/INVENTORY_INTERNALS.md +++ b/docs/INVENTORY_INTERNALS.md @@ -116,6 +116,13 @@ Companion documents: | `catalogs/` | YAML reference data and its loader ([13](#13-reference-catalogs)). | | `_util.py` | `image_repository(image)`, `walk_strings(value, max_depth=12)`. | +GCP parity modules: + +| Module | Responsibility | +|---|---| +| `gcp_sources/` | API-backed GCP sources (Cloud DNS, SCC, Cloud Identity, deny / boundary policies, PAM, Monitoring, OS Config, Backup and DR ...), `GCPRestClient`, cross-asset passes ([9.6](#96-api-backed-sources-gcp_sources)). | +| `gcp_tasks.py` | GCP service families and `--services` selection (`gcp_family`, `is_selected`, `type_selected`). | + ### 3.2 Outside the package | Module | Responsibility for the inventory | @@ -888,6 +895,253 @@ resources; others become `entra:principal/` placeholders. An AKS kubelet ide Defender for Cloud: one `THREAT_DETECTOR` per plan, `security_service: "defender-"`, `enabled` when the tier is Standard, `MONITORS` the subscription when enabled. + +### 8.4 Depth beyond the breadth query + +The `resources` query gives breadth. Depth comes from four more sources, each split into named +tasks the way the AWS deep collector splits its work. Every task writes one `ServiceCoverage` +record to `AzureDeepInventoryCollector.coverage`, and a failing task never stops the others. + +```mermaid +flowchart LR + ARG[Resource Graph: resources] --> B[AzureAssetBuilder] + T[Resource Graph tables: policy, security, backup, ...] --> B + R[ARM REST: child resources and subscription listings] --> B + D[Data planes: AI Search, Synapse] --> B + K[Kubernetes API of AKS clusters] --> B + B --> A[CloudAssets with declared relations] + G[Microsoft Graph: Entra ID, once per tenant] --> M[InventoryMapper] + A --> M + M --> L[RelationshipLinker] +``` + +| Source | Module | Tasks | +|---|---|---| +| More Resource Graph tables | `azure_graph/tables.py` | `management_group_role_assignments`, `role_definitions`, `policy`, `policy_states`, `backup_items`, `kubernetes_configuration`, `network_resources`, `guest_configuration`, `maintenance`, `insights`, `security_assessments`, `secure_score`, `security_alerts` | +| ARM REST | `azure_graph/children.py` | `deny_assignments`, `pim_eligibility`, `lighthouse`, `locks`, `deployments`, `deployment_stacks`, `activity_log`, `managed_identity_credentials`, `key_vault_items`, `apim`, `functions`, `web_apps`, `static_sites`, `service_bus`, `event_hubs`, `stream_analytics`, `data_factory`, `synapse`, `kusto`, `sql`, `cosmos`, `storage`, `dns_records`, `dns_resolver`, `virtual_wan`, `network_manager`, `front_door`, `acr`, `aks`, `cognitive_services`, `machine_learning`, `search`, `automation`, `app_configuration`, `monitor_private_link`, `batch`, `spring_apps`, `data_protection`, `dev_center`, `diagnostic_settings`, `sentinel` | +| Data planes | `azure_graph/dataplane.py` | `search_data_plane`, `synapse_data_plane` | +| Kubernetes API | `inventory/azure_kubernetes.py` | `aks_kubernetes`, recorded per cluster as `kubernetes:` | +| Microsoft Graph | `inventory/azure_entra.py` | `entra`, recorded per listing as `entra_users`, `entra_groups`, ... (see 8.8) | + +The Resource Graph tasks run only on the Resource Graph path. On the SDK fallback they are +recorded `SKIPPED` with "Resource Graph unavailable", and the REST, data-plane and Kubernetes +tasks still run. + +Coverage status of a task: + +- `SUCCESS`: every call answered. A 404, 400, 405 or 409 counts as "this parent has none of + these" (for example gateways on a Consumption APIM), not as a failure. +- `PARTIAL`: some calls were refused (403) or failed, the others returned data. +- `FAILED`: every call failed, or the Resource Graph query raised. +- `SKIPPED`: not selected by `inventory.services`, turned off in configuration, or the credential + could not issue a token for the audience the task needs. + +`azure_full` becomes `PARTIAL` when any task record is `FAILED` or `PARTIAL`, the same rule +`aws_full` follows. + +#### Service filtering + +`inventory.services` and `inventory.exclude_services` (and `--services` / `--exclude-services`) +apply to Azure as well. `families.select_azure_tasks()` mirrors `aws_deep_tasks.select_tasks()`: +a name matches a task or its family, `exclude` wins, and `kubernetes` is a pseudo-family that only +toggles in-cluster AKS mapping (selected with `containers` too). The families are the AWS names: +`compute`, `network`, `storage`, `data`, `analytics`, `ml`, `identity`, `containers`, `serverless`, +`integration`, `security`, `logging`, `operations`, `dns`, `hybrid`, `iac`, `governance`, `backup`, +`cicd`, `kubernetes`, `sweep`. + +`resources` and `resource_containers` always run. With a filter set, the breadth rows are kept +only for the selected families (`family_of_type()` maps ARM namespaces and a few full types), +while resource groups and the subscription always stay. `--services network` therefore maps the +VNets, gateways and load balancers of a subscription and nothing else. + +#### Options and configuration keys + +```python +import asyncio + +from cloudg.credentials import build_azure_credential +from cloudg.config import AzureConfig +from cloudg.inventory.azure_deep import AzureDeepInventoryCollector + +credential = build_azure_credential(AzureConfig()) +collector = AzureDeepInventoryCollector( + credential, + "00000000-0000-0000-0000-000000000000", + services=["network", "dns"], + exclude_services=["front_door"], + max_child_parents=200, +) +assets = asyncio.run(collector.collect()) +for record in collector.coverage.services: + print(record.service, record.status.value, record.asset_count, record.error or "") +``` + +The keyword options are the fields of `AzureDeepOptions`; an unknown one raises `TypeError`. +`InventoryMapper` builds them from these keys: + +| Key | Default | Effect | +|---|---|---| +| `inventory.services` / `inventory.exclude_services` | `["all"]` / `[]` | Task and family filter (above). | +| `inventory.kubernetes`, `inventory.kubernetes_timeout` | `true`, `10` | In-cluster AKS mapping and its API timeout. | +| `azure.child_resources` | `true` | ARM REST child resources and subscription listings. | +| `azure.max_child_parents` | `500` | Most parents queried per child kind (APIs per APIM service, ...). | +| `azure.diagnostic_settings` | `true` | Read diagnostic settings of the logged resource types (`children.DIAGNOSTIC_TYPES`). | +| `azure.max_diagnostic_settings` | `1000` | Most resources whose diagnostic settings are read. | +| `azure.data_plane` | `true` | AI Search and Synapse data-plane reads. | +| `azure.rest_concurrency` | `8` | Concurrent ARM REST calls per subscription. | +| `azure.map_entra` | `true` | Map Entra ID through Microsoft Graph (the opt-out). | +| `azure.entra_max_objects` | `10000` | Most objects per Graph listing. | +| `azure.entra_max_expansions` | `1000` | Most groups / apps / service principals expanded one by one. | + +### 8.5 More Resource Graph tables + +`tables.collect_tables()` runs after the four original queries. ARM-shaped rows go through +`add_rows()`, so they meet the extractors registered for their type; the aggregate tables have +handlers instead, because one asset per row would flood the map. + +| Task | Table and types | What it adds | +|---|---|---| +| `management_group_role_assignments` | `authorizationresources`, scope a management group or `/`, run tenant-wide | Assignments on the subscription's ancestor groups (from `managementGroupAncestorsChain`) as `GRANTS_ACCESS` grants with `inherited_from: "management group"`. Assignments on other groups are dropped, and so are all group assignments when the chain is unknown. | +| `role_definitions` | `authorizationresources` role definitions, tenant-wide | `IAM_POLICY` per assigned role and per custom role assignable here, with `actions`, `data_actions`, `privileged`; each principal holding it gets `IAM_POLICY_ATTACHMENT` with `USER_HAS_POLICY`, `GROUP_HAS_POLICY` or `ROLE_HAS_POLICY`. | +| `policy` | `policyresources` assignments and exemptions, then the definitions they use | `GUARDRAIL` assignments and exemptions (same shape and ID as the tenant hierarchy builds), `ORG_POLICY` definitions and initiatives. A definition whose effective effect is `deny` or `denyAction`, assigned with enforcement on, `GOVERNS / SCP_RESTRICTS` the assignment scope. The effect is resolved through the assignment parameters, then the definition defaults; an initiative restricts when any member does. | +| `policy_states` | `policyresources` non-compliant `policystates` | Assignment `GOVERNS / FINDING_AFFECTS` each non-compliant resource (500 per assignment), and `non_compliant_resources` on the assignment. | +| `security_assessments` | `securityresources` assessments and sub-assessments | One `SECURITY_HUB` asset, `cloudg:azure:defender-cspm:`, `MONITORS` each assessed resource: `FINDING_AFFECTS` with `unhealthy`, `high`, `titles`, `vulnerabilities` and image names when something is unhealthy, `MONITORED_BY` otherwise. | +| `secure_score` | `securityresources` secure score and controls | `secure_score` and the 30 worst `secure_score_controls` on the same asset. | +| `security_alerts` | `securityresources` active alerts | `THREAT_DETECTOR` `cloudg:azure:defender-alerts:` `MONITORS / FINDING_AFFECTS` each resource an alert names. | +| `backup_items` | `recoveryservicesresources` | Protected items and backup instances: resource `BACKUP_TO` vault, policy (or vault) `MANAGES / BACKUP_TO` resource. Site Recovery items: source `REPLICATES_TO` recovery VM, network or resource group. Backup policies as `BACKUP_PLAN`. | +| `kubernetes_configuration` | `kubernetesconfigurationresources` | Cluster extensions (`CLUSTER_ADDON`) and Flux configurations (`DEPLOYMENT_GROUP`) contained by their cluster; Flux `READS_FROM` its Git repository and `MANAGES` the cluster. | +| `network_resources` | `networkresources` | Virtual Network Manager rows (see the REST task `network_manager`). | +| `guest_configuration` | `guestconfigurationresources` | `GUARDRAIL` per assignment, `GOVERNS` the machine (`FINDING_AFFECTS` when non-compliant). | +| `maintenance` | `maintenanceresources` assignments | Maintenance configuration `MANAGES / SCHEDULED_BY` the resource. | +| `insights` | `insightsresources` data collection rule associations | Resource `LOGS_TO` the rule, rule `MONITORS` the resource. | + +Defender plans gained two things in the builder. An enabled plan now also `MONITORS / MONITORED_BY` +every resource of the types it covers (`security.PLAN_RESOURCE_TYPES`), except resources whose own +resource-level pricing is Free. When the pricing query answered, every plan in +`security.STANDARD_PLANS` that the subscription does not list gets a "not enabled" placeholder, +`cloudg:azure:defender-::not-enabled`, built like the AWS `_disabled_asset`. + +### 8.6 ARM REST child resources + +ARM answers child listings with the same ARM-shaped JSON Resource Graph returns, so the rows feed +`add_row()` unchanged. `children.CHILD_SPECS` is the plan, one `ChildSpec` per kind: + +```python +ChildSpec( + "service_bus", # task + ("microsoft.servicebus/namespaces",), # parent types, or SUBSCRIPTION + "topics", # path under the parent ID + "2021-11-01", # api-version + nested=(ChildSpec("service_bus", (), "subscriptions", "2021-11-01"),), +) +``` + +`fetch_children()` picks the parent drafts (filtered by the spec's `when`, capped at +`max_child_parents`), issues the GETs in a thread pool and applies the rows on the calling thread, +then runs nested specs against the children it just created. Rows that only link other assets +(static members of a network group, Service Bus product APIs, VNet links of a forwarding +ruleset, job schedules, Stream Analytics inputs and outputs, function site config, ...) are +registered with `@relation_row(type)`: `add_row()` hands them to the handler and creates no asset. + +The REST client (`rest.AzureRestClient`) uses the azure-identity credential cloudg already holds, +issues only GET (and POST for the read-only `listClusterUserCredential`), follows `nextLink` and +`@odata.nextLink` only on its own host, retries 429 and 503 after `Retry-After`, and paces every +call through the shared rate limiter under the `arm`, `msgraph` and `dataplane` buckets. It is +injected through `arm_client_factory`, so tests never touch the network. + +Secrets stay out of the map. Key Vault secret values, APIM backend credentials and named-value +values, API connection parameters, Data Factory `typeProperties`, AML datastore credentials, AI +Search data source credentials, storage local-user keys and action group e-mail addresses and +phone numbers are scanned for endpoints where useful and then dropped from `properties`. + +### 8.7 Data planes + +`dataplane.fetch_data_planes()` reads AI Search data sources, indexers, indexes and skillsets +(`https://.search.windows.net`, token audience `https://search.azure.com/.default`) and +Synapse linked services, datasets, pipelines and triggers (the workspace `dev` endpoint, audience +`https://dev.azuresynapse.net/.default`). Rows get synthetic ARM-style IDs under their service and +go through the same extractors as Data Factory. The calls need a data-plane reader role; without +one the task is `PARTIAL` or `FAILED` and the ARM view of the service is unchanged. + +### 8.8 Entra ID + +`azure_entra.discover_entra(credential, client_factory=None, *, coverage, max_objects, +max_expansions)` runs once per mapping run from `InventoryMapper._discover_hierarchies()` when +`azure.map_entra` is on and the `entra` task is selected. Every listing is its own coverage record: + +| Record | Graph call | Mapped as | +|---|---|---| +| `entra_tenant` | `/organization` | `ORGANIZATION` `entra:tenant/` | +| `entra_users` | `/users` | `IDENTITY_USER` `entra:principal/` (`guest`, `user_principal_name`, ...) | +| `entra_groups` | `/groups` | `IDENTITY_GROUP`; `role_assignable`, `dynamic` | +| `entra_group_members` | `/groups/{id}/members` | group `CONTAINS` member | +| `entra_applications` | `/applications` | `APP_REGISTRATION` `entra:application/`; secret and certificate counts and latest expiry, requested permissions, `multi_tenant` | +| `entra_application_owners` | `/applications/{id}/owners` | owner `MANAGES / OWNED_BY` app | +| `entra_federated_credentials` | `/applications/{id}/federatedIdentityCredentials` | issuer `IAM_TRUST` app: `ROLE_ASSUMES_ROLE` for AKS issuers, `CROSS_ACCOUNT_TRUST` for the rest | +| `entra_service_principals` | `/servicePrincipals` | `SERVICE_PRINCIPAL` `entra:principal/`, `external_application` when another tenant owns the app; app `MANAGES / OWNED_BY` its service principal. Managed identities are skipped: the resource or user-assigned identity owning them already carries that principal. | +| `entra_service_principal_owners` | `/servicePrincipals/{id}/owners` | owner `MANAGES / OWNED_BY` service principal | +| `entra_app_role_assignments` | `/servicePrincipals/{id}/appRoleAssignedTo`, Microsoft Graph first | principal `GRANTS_ACCESS / POLICY_ALLOWS_ACTION` resource service principal with `role` (for example `RoleManagement.ReadWrite.Directory`) and `privileged` | +| `entra_oauth2_grants` | `/oauth2PermissionGrants` | client `GRANTS_ACCESS` resource with `delegated`, `scopes`, `consent_type`; `privileged` for admin consent to high scopes | +| `entra_directory_roles` | `/roleManagement/directory/roleDefinitions`, `roleAssignments` | `IAM_POLICY` `entra:directoryRole/`; principal `IAM_POLICY_ATTACHMENT` role and `GRANTS_ACCESS` tenant | +| `entra_pim_eligibility` | `/roleManagement/directory/roleEligibilitySchedules` | the same relations with `eligible: true` | + +Entra objects carry no `discovered_via`, so in `deduplicate()` they win over the +`entra:principal/` placeholders the subscription collectors create for role assignments, +and the placeholders' grants merge into them. A 403 is recorded as "Graph permission missing"; +no token at all raises `CredentialError` and the `entra` record is `FAILED`. + +Subscription vending aliases (`Microsoft.Subscription/aliases`) are read once per run by +`discover_azure_hierarchy()` through ARM REST: each alias is a `PROVISIONED_PRODUCT` that +`MANAGES / OWNED_BY` its subscription. A credential without vending rights maps none and +reports nothing. + +Azure RBAC PIM eligibility is a subscription task (`pim_eligibility`): it reads +`roleEligibilityScheduleInstances` and adds the grants with `eligible: true`. + +### 8.9 In-cluster AKS + +`azure_kubernetes.collect_aks_workloads()` maps an AKS cluster through the same +`kubernetes.map_cluster_objects()` EKS uses, with `provider=CloudProvider.AZURE`. The API server +and CA come from the kubeconfig of `listClusterUserCredential` (only the `clusters` section is +parsed), and the bearer token is an Entra token for the AKS server application +(`6dae42f8-4368-4678-94ff-3960e28e3630/.default`), as `kubelogin` gets it. Service accounts with +the `azure.workload.identity/client-id` annotation `ASSUMES_ROLE / RUNS_ON` the user-assigned +identity with that client ID (alias `azure-client-id:`), and `LoadBalancer` services resolve +their ingress IP to the public IP asset. + +Clusters without Microsoft Entra integration are `SKIPPED`: their user credentials are secrets. +Private clusters are tried and come out `FAILED` when cloudg cannot reach the API server. The +AKS run-command (`az aks command invoke`) is not used, because it starts a pod in the cluster. + +Everything AKS exposes through ARM is mapped as well: OIDC issuer (an alias, so federated +credentials resolve to the cluster), workload identity, Entra admin groups (`GRANTS_ACCESS` with +`role: "cluster-admin"`), trusted access role bindings, extensions and Flux. + +### 8.10 Relations, AWS to Azure + +The Azure extractors reuse the relationship names AWS uses for the same idea. No new +`RelationType` members were needed. + +| Concept | AWS source | Azure source | Relation | +|---|---|---|---| +| Backup protection | Backup plan, selection | Recovery Services / Backup vault protected items, NetApp backup | `BACKUP_TO` | +| Replication | RDS global DB, DynamoDB replicas, S3 replication | SQL geo-replicas and failover groups, Cosmos regions, storage object replication, ACR geo-replication, App Configuration replicas, NetApp, Site Recovery | `REPLICATES_TO` | +| Streaming export | Firehose, log subscription filters | Event Hubs Capture, diagnostic settings to Event Hubs, data collection rules, Defender continuous export | `STREAMS_TO` | +| Data movement | CodePipeline, Glue | Data Factory and Synapse pipelines, Stream Analytics, function bindings, AML datastores, AI Search indexers | `READS_FROM` / `WRITES_TO` | +| Secret rotation | Secrets Manager rotation Lambda | Event Grid near-expiry subscription on a vault | `ROTATES_SECRET` | +| Schedules | SSM associations, maintenance windows | Function timers, Logic Apps recurrence, Automation schedules and update deployments, ADF triggers, maintenance assignments, NetApp snapshot policies | `MANAGES / SCHEDULED_BY` | +| Org guardrails | SCP | Deny-effect policy definition | `GOVERNS / SCP_RESTRICTS` | +| Permission limits | Permissions boundary | Deny assignment | `PERMISSION_BOUNDARY_LIMITS` | +| Policy attachments | IAM managed policies | RBAC role definitions, Entra directory roles | `ROLE_HAS_POLICY`, `USER_HAS_POLICY`, `GROUP_HAS_POLICY` | +| Cross-tenant trust | Role trust of another account, RAM | Lighthouse, managed application publishers, gallery sharing, external federated credentials, Sentinel AWS connectors | `CROSS_ACCOUNT_TRUST` | +| Monitoring | CloudWatch alarms | Metric, log and activity log alerts, Defender plans and assessments, Sentinel connectors | `MONITORS / MONITORED_BY` | +| Findings | (none) | Defender recommendations and alerts, non-compliant policy states, guest configuration | `FINDING_AFFECTS` | +| Invocation | EventBridge targets, alarm actions | Action groups, alert actions, Logic Apps actions, APIM `send-request`, skillsets | `INVOKES` | +| Fronting traffic | ALB, CloudFront, API Gateway | APIM APIs and backends, Front Door routes and rules, Application Gateway listeners, Traffic Manager, Static Web App backends | `SERVES_TRAFFIC_TO` | +| Name resolution | Route 53 records, resolver rules | DNS record sets, Private Resolver forwarding rules and VNet links | `DNS_RESOLVED` | +| Transit | Transit Gateway, VPN, Direct Connect | Virtual WAN hubs, routing intent, ExpressRoute, VPN sites, Route Server BGP peers, Virtual Network Manager | `TRANSIT_ROUTED` | + + ## 9. GCP ### 9.1 Collection @@ -913,6 +1167,14 @@ Defender for Cloud: one `THREAT_DETECTOR` per plan, `security_service: "defender `GCPDeepInventoryCollector` adds the deep type map (`gcp_asset_types.yaml` `deep` over `base`) and IAM, and does not link locally: the mapper's linker does it for every provider at once. + +The deep collector also takes `GCPDeepOptions`: `services`, `exclude_services`, `rest_client` +(an injected REST client for tests), `deep_sources`, `rest_timeout`, `max_images_per_repository` +and `source_options` (`GCPSourceOptions` or a dict of its fields). The mapper binds them from +`inventory.*` and `gcp.*` with `deep_options_from_config()`. Its `_enrich()` runs the API-backed +sources and the IAM policy search (9.6), and its `_finalize()` the cross-asset passes (9.7). + + ### 9.2 The extraction framework (`gcp_relations/context.py`) ```python @@ -968,6 +1230,31 @@ queries can carry tokens. 123 asset types have dedicated extractors; every other type still gets the generic CMEK scan, parent containment and a catalog classification. + +GCP parity modules (added for depth parity with AWS). Several of them register a second +extractor for a type an older module already handles, and the registry runs both (it keeps a +list per type): + +| Module | Area | Cloud Asset Inventory and API types | +|---|---|---| +| `gcp_relations/apis.py` | Apigee organizations, instances, environments, proxies, target servers; API Gateway JWT providers and backends | `apigateway.googleapis.com/ApiConfig`, `apigateway.googleapis.com/Authorizer`, `apigee.googleapis.com/ApiProxy`, `apigee.googleapis.com/ApiProxyRevision`, `apigee.googleapis.com/Environment`, `apigee.googleapis.com/Instance`, `apigee.googleapis.com/Organization`, `apigee.googleapis.com/TargetServer` | +| `gcp_relations/cicd.py` | Cloud Build worker pools, 2nd gen connections and repositories, Developer Connect, trigger sources and pools | `cloudbuild.googleapis.com/BitbucketServerConfig`, `cloudbuild.googleapis.com/BuildTrigger`, `cloudbuild.googleapis.com/Connection`, `cloudbuild.googleapis.com/GithubEnterpriseConfig`, `cloudbuild.googleapis.com/Repository`, `cloudbuild.googleapis.com/WorkerPool`, `developerconnect.googleapis.com/Connection`, `developerconnect.googleapis.com/GitRepositoryLink`, `developerconnect.googleapis.com/InsightsConfig` | +| `gcp_relations/containers_ext.py` | Artifact Registry images and upstreams, Binary Authorization, GKE fleets, Backup for GKE, GKE namespaces and RBAC roles | `artifactregistry.googleapis.com/DockerImage`, `artifactregistry.googleapis.com/Repository`, `binaryauthorization.googleapis.com/Attestor`, `binaryauthorization.googleapis.com/PlatformPolicy`, `binaryauthorization.googleapis.com/Policy`, `gkebackup.googleapis.com/Backup`, `gkebackup.googleapis.com/BackupPlan`, `gkebackup.googleapis.com/RestorePlan`, `gkehub.googleapis.com/Feature`, `gkehub.googleapis.com/Fleet`, `gkehub.googleapis.com/Membership`, `gkehub.googleapis.com/MembershipBinding`, `gkehub.googleapis.com/Namespace`, `gkehub.googleapis.com/RBACRoleBinding`, `gkehub.googleapis.com/Scope`, `k8s.io/Namespace`, `rbac.authorization.k8s.io/ClusterRole`, `rbac.authorization.k8s.io/Role` | +| `gcp_relations/data_ext.py` | Spanner, Bigtable, Firestore, BigQuery tables / routines / connections / transfers, Dataplex, Data Catalog, Metastore, Data Fusion, Datastream, Storage Transfer, NetApp Volumes, Filestore backups, secret rotation, Parameter Manager, key rings | `bigquery.googleapis.com/Model`, `bigquery.googleapis.com/Routine`, `bigquery.googleapis.com/RowAccessPolicy`, `bigquery.googleapis.com/Table`, `bigqueryconnection.googleapis.com/Connection`, `bigquerydatatransfer.googleapis.com/TransferConfig`, `bigtableadmin.googleapis.com/AppProfile`, `bigtableadmin.googleapis.com/AuthorizedView`, `bigtableadmin.googleapis.com/Backup`, `bigtableadmin.googleapis.com/Cluster`, `bigtableadmin.googleapis.com/Instance`, `bigtableadmin.googleapis.com/LogicalView`, `bigtableadmin.googleapis.com/MaterializedView`, `bigtableadmin.googleapis.com/Table`, `cloudkms.googleapis.com/KeyRing`, `datacatalog.googleapis.com/EntryGroup`, `datacatalog.googleapis.com/TagTemplate`, `datacatalog.googleapis.com/Taxonomy`, `datafusion.googleapis.com/DnsPeering`, `datafusion.googleapis.com/Instance`, `dataplex.googleapis.com/Asset`, `dataplex.googleapis.com/DataScan`, `dataplex.googleapis.com/EntryGroup`, `dataplex.googleapis.com/Lake`, `dataplex.googleapis.com/Task`, `dataplex.googleapis.com/Zone`, `datastream.googleapis.com/ConnectionProfile`, `datastream.googleapis.com/PrivateConnection`, `datastream.googleapis.com/Stream`, `file.googleapis.com/Backup`, `firestore.googleapis.com/Backup`, `firestore.googleapis.com/BackupSchedule`, `firestore.googleapis.com/Database`, `metastore.googleapis.com/Service`, `netapp.googleapis.com/ActiveDirectory`, `netapp.googleapis.com/Backup`, `netapp.googleapis.com/BackupPolicy`, `netapp.googleapis.com/BackupVault`, `netapp.googleapis.com/KmsConfig`, `netapp.googleapis.com/Replication`, `netapp.googleapis.com/Snapshot`, `netapp.googleapis.com/StoragePool`, `netapp.googleapis.com/Volume`, `parametermanager.googleapis.com/Parameter`, `parametermanager.googleapis.com/ParameterVersion`, `secretmanager.googleapis.com/Secret`, `spanner.googleapis.com/Backup`, `spanner.googleapis.com/Database`, `spanner.googleapis.com/Instance`, `spanner.googleapis.com/InstancePartition`, `storagetransfer.googleapis.com/TransferJob` | +| `gcp_relations/edge.py` | Cloud DNS record sets, Certificate Manager, CA Service, Cloud Run domain mappings, Identity Platform | `certificatemanager.googleapis.com/Certificate`, `certificatemanager.googleapis.com/CertificateIssuanceConfig`, `certificatemanager.googleapis.com/CertificateMap`, `certificatemanager.googleapis.com/CertificateMapEntry`, `certificatemanager.googleapis.com/DnsAuthorization`, `certificatemanager.googleapis.com/TrustConfig`, `dns.googleapis.com/ResourceRecordSet`, `identitytoolkit.googleapis.com/Config`, `identitytoolkit.googleapis.com/DefaultSupportedIdpConfig`, `identitytoolkit.googleapis.com/InboundSamlConfig`, `identitytoolkit.googleapis.com/OauthIdpConfig`, `identitytoolkit.googleapis.com/Tenant`, `privateca.googleapis.com/CaPool`, `privateca.googleapis.com/CertificateAuthority`, `run.googleapis.com/DomainMapping` | +| `gcp_relations/governance.py` | tags and tag bindings, liens, custom constraints, Assured Workloads, authorized orgs, access levels, Infrastructure Manager, Deployment Manager | `accesscontextmanager.googleapis.com/AccessLevel`, `accesscontextmanager.googleapis.com/AuthorizedOrgsDesc`, `accesscontextmanager.googleapis.com/ServicePerimeter`, `assuredworkloads.googleapis.com/Workload`, `cloudresourcemanager.googleapis.com/Lien`, `cloudresourcemanager.googleapis.com/TagBinding`, `cloudresourcemanager.googleapis.com/TagKey`, `cloudresourcemanager.googleapis.com/TagValue`, `config.googleapis.com/Deployment`, `config.googleapis.com/Preview`, `deploymentmanager.googleapis.com/Deployment`, `orgpolicy.googleapis.com/CustomConstraint`, `orgpolicy.googleapis.com/Policy` | +| `gcp_relations/hybrid.py` | NCC hubs, route tables, groups, internal ranges, policy-based routes, regional endpoints; Interconnect; external VPN gateways; PSC accept lists; packet mirroring; sole-tenant nodes; resource policies | `compute.googleapis.com/Disk`, `compute.googleapis.com/ExternalVpnGateway`, `compute.googleapis.com/Instance`, `compute.googleapis.com/Interconnect`, `compute.googleapis.com/NodeGroup`, `compute.googleapis.com/NodeTemplate`, `compute.googleapis.com/PacketMirroring`, `compute.googleapis.com/RegionDisk`, `compute.googleapis.com/ResourcePolicy`, `compute.googleapis.com/ServiceAttachment`, `compute.googleapis.com/VpnTunnel`, `networkconnectivity.googleapis.com/Group`, `networkconnectivity.googleapis.com/Hub`, `networkconnectivity.googleapis.com/InternalRange`, `networkconnectivity.googleapis.com/PolicyBasedRoute`, `networkconnectivity.googleapis.com/RegionalEndpoint`, `networkconnectivity.googleapis.com/RouteTable` | +| `gcp_relations/identity_ext.py` | workforce / workload identity pools, OAuth clients, role permissions, deny policies, Principal Access Boundary, policy bindings, PAM entitlements, Cloud Identity groups | `cloudidentity.googleapis.com/Group`, `iam.googleapis.com/DenyPolicy`, `iam.googleapis.com/OauthClient`, `iam.googleapis.com/OauthClientCredential`, `iam.googleapis.com/PolicyBinding`, `iam.googleapis.com/PolicyV2`, `iam.googleapis.com/PrincipalAccessBoundaryPolicy`, `iam.googleapis.com/Role`, `iam.googleapis.com/WorkforcePool`, `iam.googleapis.com/WorkforcePoolProviderKey`, `iam.googleapis.com/WorkloadIdentityPool`, `iam.googleapis.com/WorkloadIdentityPoolManagedIdentity`, `iam.googleapis.com/WorkloadIdentityPoolNamespace`, `iam.googleapis.com/WorkloadIdentityPoolProviderKey`, `privilegedaccessmanager.googleapis.com/Entitlement` | +| `gcp_relations/integration.py` | Cloud Scheduler, Eventarc channels and Eventarc Advanced, Managed Kafka topics / consumer groups / Connect, Pub/Sub snapshots and schemas, Workflows definitions, Batch | `batch.googleapis.com/Job`, `cloudscheduler.googleapis.com/Job`, `eventarc.googleapis.com/Channel`, `eventarc.googleapis.com/ChannelConnection`, `eventarc.googleapis.com/Enrollment`, `eventarc.googleapis.com/GoogleApiSource`, `eventarc.googleapis.com/MessageBus`, `eventarc.googleapis.com/Pipeline`, `eventarc.googleapis.com/Trigger`, `managedkafka.googleapis.com/ConnectCluster`, `managedkafka.googleapis.com/Connector`, `managedkafka.googleapis.com/ConsumerGroup`, `managedkafka.googleapis.com/Topic`, `pubsub.googleapis.com/Schema`, `pubsub.googleapis.com/Snapshot`, `workflows.googleapis.com/Workflow` | +| `gcp_relations/ml.py` | Agent Engine, Vector Search, Discovery Engine, Dialogflow CX, Model Armor, pipelines and training jobs, Feature Store, Colab Enterprise runtimes | `aiplatform.googleapis.com/BatchPredictionJob`, `aiplatform.googleapis.com/CustomJob`, `aiplatform.googleapis.com/Dataset`, `aiplatform.googleapis.com/FeatureGroup`, `aiplatform.googleapis.com/FeatureOnlineStore`, `aiplatform.googleapis.com/Featurestore`, `aiplatform.googleapis.com/HyperparameterTuningJob`, `aiplatform.googleapis.com/Index`, `aiplatform.googleapis.com/IndexEndpoint`, `aiplatform.googleapis.com/NasJob`, `aiplatform.googleapis.com/NotebookRuntime`, `aiplatform.googleapis.com/NotebookRuntimeTemplate`, `aiplatform.googleapis.com/PipelineJob`, `aiplatform.googleapis.com/ReasoningEngine`, `aiplatform.googleapis.com/TrainingPipeline`, `aiplatform.googleapis.com/TuningJob`, `dialogflow.googleapis.com/Agent`, `dialogflow.googleapis.com/KnowledgeBase`, `discoveryengine.googleapis.com/Assistant`, `discoveryengine.googleapis.com/Collection`, `discoveryengine.googleapis.com/DataStore`, `discoveryengine.googleapis.com/Engine`, `modelarmor.googleapis.com/Template` | +| `gcp_relations/netsec.py` | Cloud NGFW, security profiles, TLS policies, authorization policies, address groups, Cloud IDS, Network Services gateways / meshes / routes, Media CDN | `compute.googleapis.com/FirewallPolicy`, `compute.googleapis.com/NetworkEdgeSecurityService`, `compute.googleapis.com/NetworkFirewallPolicy`, `compute.googleapis.com/RegionNetworkFirewallPolicy`, `ids.googleapis.com/Endpoint`, `networksecurity.googleapis.com/AddressGroup`, `networksecurity.googleapis.com/AuthorizationPolicy`, `networksecurity.googleapis.com/AuthzPolicy`, `networksecurity.googleapis.com/ClientTlsPolicy`, `networksecurity.googleapis.com/FirewallEndpoint`, `networksecurity.googleapis.com/FirewallEndpointAssociation`, `networksecurity.googleapis.com/SecurityProfile`, `networksecurity.googleapis.com/SecurityProfileGroup`, `networksecurity.googleapis.com/ServerTlsPolicy`, `networksecurity.googleapis.com/TlsInspectionPolicy`, `networkservices.googleapis.com/EdgeCacheKeyset`, `networkservices.googleapis.com/EdgeCacheOrigin`, `networkservices.googleapis.com/EdgeCacheService`, `networkservices.googleapis.com/EndpointPolicy`, `networkservices.googleapis.com/Gateway`, `networkservices.googleapis.com/GrpcRoute`, `networkservices.googleapis.com/HttpRoute`, `networkservices.googleapis.com/Mesh`, `networkservices.googleapis.com/ServiceBinding`, `networkservices.googleapis.com/ServiceLbPolicy`, `networkservices.googleapis.com/TcpRoute`, `networkservices.googleapis.com/TlsRoute` | +| `gcp_relations/ops.py` | Monitoring alert policies / channels / uptime checks, OS Config, Backup and DR | `backupdr.googleapis.com/Backup`, `backupdr.googleapis.com/BackupPlan`, `backupdr.googleapis.com/BackupPlanAssociation`, `backupdr.googleapis.com/BackupVault`, `backupdr.googleapis.com/DataSource`, `backupdr.googleapis.com/ManagementServer`, `monitoring.googleapis.com/AlertPolicy`, `monitoring.googleapis.com/NotificationChannel`, `monitoring.googleapis.com/UptimeCheckConfig`, `osconfig.googleapis.com/OSPolicyAssignment`, `osconfig.googleapis.com/PatchDeployment`, `osconfig.googleapis.com/VulnerabilityReport` | +| `gcp_relations/security_ext.py` | Security Command Center, Sensitive Data Protection, Data Access audit configs, IAM recommender, Artifact Analysis summaries | `containeranalysis.googleapis.com/VulnerabilitySummary`, `dlp.googleapis.com/DeidentifyTemplate`, `dlp.googleapis.com/DiscoveryConfig`, `dlp.googleapis.com/InspectTemplate`, `dlp.googleapis.com/JobTrigger`, `dlp.googleapis.com/StoredInfoType`, `logging.googleapis.com/AuditConfig`, `recommender.googleapis.com/IamPolicyRecommender`, `securitycenter.googleapis.com/BigQueryExport`, `securitycenter.googleapis.com/ContainerThreatDetectionSettings`, `securitycenter.googleapis.com/EventThreatDetectionSettings`, `securitycenter.googleapis.com/MuteConfig`, `securitycenter.googleapis.com/NotificationConfig`, `securitycenter.googleapis.com/ResourceValueConfig`, `securitycenter.googleapis.com/SecurityCenter`, `securitycenter.googleapis.com/SecurityHealthAnalyticsSettings`, `securitycenter.googleapis.com/Source`, `securitycenter.googleapis.com/VirtualMachineThreatDetectionSettings`, `securitycenter.googleapis.com/WebSecurityScannerSettings`, `securitycentermanagement.googleapis.com/SecurityCenterService` | + +With them, 350 asset types have dedicated extractors. `test_every_new_catalog_type_has_an_extractor` +keeps every type in the catalog's `deep` section covered. + + ### 9.5 IAM bindings and principals (`gcp_relations/iam.py`) `apply_iam_policies(assets, policies)` turns each binding into relations on the principal: @@ -987,6 +1274,160 @@ queries can carry tokens. placeholders created for a service account referenced before its project was seen are folded into the real service-account asset, and edges are re-pointed. + +### 9.6 API-backed sources (`gcp_sources/`) + +Cloud Asset Inventory does not return everything the AWS collector maps. `GCPDeepInventoryCollector._enrich()` +runs the sources in `cloudg/inventory/gcp_sources/` for the rest. Each one is a function +registered with `@source(name, family, phase=2, needs_rest=True)`; it gets a `SourceContext` +(the collector, the assets so far, the REST client, `GCPSourceOptions`) and a `SourceRun` +(call bookkeeping) and returns new assets. + +```python +from cloudg.inventory.gcp_sources.base import SourceContext, SourceRun, source + + +@source("widgets", "compute") +def widgets(ctx: SourceContext, run: SourceRun) -> list: + """Widgets of every project in scope.""" + out = [] + for project, _number in ctx.projects(): + url = f"https://widgets.googleapis.com/v1/projects/{project}/widgets" + for item in ctx.items(run, url, "widgets"): + out.append( + ctx.build( + f"//widgets.googleapis.com/{item['name']}", + "widgets.googleapis.com/Widget", + item, + project=project, + source_name="widgets", + ) + ) + return [a for a in out if a is not None] +``` + +`ctx.build()` runs the resource through the collector's own record pipeline +(`_build_asset()`): the same scrubbing, aliases, project resolution and `@_extractor` relations +as a listed resource. It returns `None` when the name (or its project number / ID variant) is +already in the inventory, so a source never duplicates what the listing found. Asset types that +only an API returns (`dns.googleapis.com/ResourceRecordSet`, `securitycenter.googleapis.com/Source`, ...) +are named like CAI types and mapped in `gcp_asset_types.yaml` (`deep`, marked `(API)`). + +Phases: 0 runs `cloud_identity` (groups are principals), then the IAM policy search runs (it +reuses the group assets instead of creating placeholders), then phases 2 and 3 (`iam_roles` and +`dlp_discovery`, which read what earlier sources built). Inside a phase, sources run concurrently +in worker threads (`gcp.source_concurrency`). + +| Source | Family | API | Maps | +|---|---|---|---| +| `cloud_identity` | identity | Cloud Identity `groups`, `memberships` | `IDENTITY_GROUP` per Google group with aliases `group:` and `gcp-principal:group:`; group `CONTAINS` each member | +| `iam_deny` | identity | IAM v2 `denypolicies` | `IAM_POLICY` that `GOVERNS / POLICY_DENIES_ACTION` its attachment point and every denied principal | +| `principal_access_boundary` | identity | IAM v3 `principalAccessBoundaryPolicies`, `policyBindings` | `IAM_POLICY` that `GOVERNS / PERMISSION_BOUNDARY_LIMITS` the allowed resources; the bound principal set `REFERENCES / PERMISSION_BOUNDARY_LIMITS` the policy | +| `pam` | identity | Privileged Access Manager `entitlements` | `PERMISSION_SET`: eligible principals `ASSUMES_ROLE / ROLE_ASSUMES_ROLE` it, it `GRANTS_ACCESS` the resource and `ROLE_HAS_POLICY` each role | +| `iam_roles` | identity | IAM `roles.get` | `IAM_ROLE` `//iam.googleapis.com/roles/` per bound predefined role, with `included_permissions` | +| `scc` | security | Security Command Center management `securityCenterServices`, SCC v1 `sources`, `notificationConfigs`, `bigQueryExports`, v1beta2 `subscription` | `SECURITY_HUB` per activated parent (tier), one asset per service (ETD / CTD / VMTD as `THREAT_DETECTOR`, SHA as `CONFIG_RECORDER`, WSS as `VULNERABILITY_SCANNER`) that `MONITORS` the parent; placeholders where SCC is not activated | +| `artifact_analysis` | security | Service Usage, Container Analysis `occurrences` | `VULNERABILITY_SCANNER` per project that `MONITORS / MONITORED_BY` each image with severity counts; placeholder when scanning is off | +| `iam_recommender` | security | Recommender `google.iam.policy.Recommender` | `ACCESS_ANALYZER` per project; `MONITORS / FINDING_AFFECTS` each principal with an excess role | +| `dlp_discovery` | security | none (reads the listing) | Placeholder per project no Sensitive Data Protection discovery config covers | +| `audit_configs` | logging | Resource Manager v3 `getIamPolicy` | `CLOUDTRAIL` per organization, folder and project with Data Access logging, `MONITORS` it and `LOGS_TO` the `_Default` bucket; placeholder where Data Access logs are off | +| `monitoring` | logging | Monitoring v3 `alertPolicies`, `notificationChannels`, `uptimeCheckConfigs`; v1 `metricsScopes` | `ALARM` / `NOTIFICATION_TOPIC`; the scoping project `MONITORS` the projects in its metrics scope | +| `dns_records` | dns | Cloud DNS `rrsets` | `DNS_RECORD` per record set, contained by its zone, `ROUTE / DNS_RESOLVED` to the IPs and hosts it answers | +| `os_config` | ops | OS Config `patchDeployments`, `osPolicyAssignments`, `vulnerabilityReports` | `SCHEDULE`, `RUNBOOK` and `VULNERABILITY_SCANNER` | +| `cloud_scheduler` | integration | Cloud Scheduler `jobs` | `SCHEDULE` that `INVOKES / SCHEDULED_BY` its target | +| `backup_dr` | backup | Backup and DR `backupVaults`, `backupPlans`, `backupPlanAssociations`, `managementServers` | `BACKUP_VAULT` / `BACKUP_PLAN` | +| `firestore_backups` | backup | Firestore `backupSchedules`, `backups` | schedule `MANAGES / BACKUP_TO` the database | +| `apigee` | serverless | Apigee `environments`, `targetservers`, `deployments` | `API_DESTINATION` per target server; proxy `ATTACHED_TO` the environments it is deployed to | +| `infra_manager` | iac | Infrastructure Manager revision `resources` | deployment `MANAGES / OWNED_BY` every resource of its latest revision | +| `deployment_manager` | iac | Deployment Manager `deployments`, `resources` | `IAC_STACK` that `MANAGES / OWNED_BY` its resources | + +Several sources also fill types that Cloud Asset Inventory can return (alert policies, OS Config, +Backup and DR, PAM, deny and boundary policies, SCC notification configs). For those the +listing wins and the source only adds what the listing missed. + +The REST client is `GCPRestClient(credentials, *, session=None, timeout=60.0, max_retries=4, sleep=time.sleep)` +in `gcp_sources/rest.py`. It wraps google-auth's `AuthorizedSession`, so it needs nothing beyond the +`gcp` extra. `get()`, `post()` (read-only RPCs such as `getIamPolicy`) and `list()` (follows +`nextPageToken`, optional `max_items`) go through the resilience governor under +`gcp / / `, and retry 429 and 5xx answers with backoff (honouring `Retry-After`). Errors are `GCPApiError` (a `CollectionError`) with `status_code`, `error_code`, +`disabled` (the API is not enabled), `not_found` and `precondition`; HTTP 401 raises +`CredentialError`. Without credentials and without an injected `rest_client`, the client is not +built and the sources that need it are `SKIPPED`. + +Every source is best effort. `SourceRun.call()` turns every API error into bookkeeping: a disabled API or a +failed precondition counts as "not in use", 404 as an empty answer, anything else as an error. +The source's coverage record `gcp_` is then `SUCCESS`, `PARTIAL` (some calls failed), +`FAILED` (nothing answered, or the source raised) or `SKIPPED` (missing prerequisite, or the API +is off everywhere it was asked). A failing source never stops the run. + +For services that are off, `ctx.placeholder(service, parent, asset_type, label)` builds the GCP +counterpart of `_disabled_asset()`: `arn = cloudg:gcp:::not-enabled`, +`enabled: false`, and the alias `gcp-security::` that the enabled asset of the +same service also carries (`gcp-security:` is a globally unique scheme for the linker). + +### 9.7 Cross-asset passes (`gcp_sources/derived.py`) + +`_finalize()` runs these before the base finalization (firewalls, project node, principal merge): + +| Pass | Result | +|---|---| +| `fold_tag_bindings` | Each `TagBinding` becomes a `REFERENCES / TAGGED_WITH` relation from the bound resource to the tag value, plus `resource_manager_tags` metadata; the binding asset is dropped | +| `link_instance_policies` | Instance resource policies by kind: instance schedules `MANAGES / SCHEDULED_BY` the VM, snapshot schedules `MANAGES / BACKUP_TO`, placement policies are a `DEPENDS_ON` | +| `link_secret_rotation` | A secret with a rotation period `INVOKES / ROTATES_SECRET` every function, service or push endpoint that consumes one of its rotation topics | +| `link_managed_labels` | `goog-dm=` labels: the Deployment Manager deployment `MANAGES / OWNED_BY` the resource; `managed-by-cnrm=true`: the project's one Config Connector cluster does | +| `fold_policy_bindings` | Principal Access Boundary bindings: the principal set `REFERENCES / PERMISSION_BOUNDARY_LIMITS` the policy | +| `link_os_config_filters` | Patch deployments and OS policy assignments `MANAGES / SCHEDULED_BY` the VMs their label, zone and name-prefix filters select | +| `authorizer_assets` | An `AUTHORIZER` per JWT provider in an API Gateway config (OpenAPI `x-google-issuer`, gRPC `authentication.providers`), contained by the config, `DEPENDS_ON` its issuer | +| `ensure_principal_placeholders` | Placeholders for `gcp-principal:` relation targets (deny rules, PAM, group members) that are not assets | + +`cap_images` keeps the newest `inventory.max_images_per_repository` Docker images per repository +before the records are built. + +### 9.8 Service families and coverage + +`gcp_tasks.py` gives every asset type a family: `type_families` in `gcp_asset_types.yaml` first, +then the API service in `families` (`compute.googleapis.com/...` is `compute`), else `other`. +`inventory.services` / `inventory.exclude_services` (`--services` / `--exclude-services`) accept +families, API services (`pubsub`), full asset types and source names, with the AWS rules: +`include` defaults to `all`, `exclude` always wins. Listed records are filtered in `_keep()`; +sources and the IAM policy search (`iam_policies`, family `identity`) by `is_selected()`. The +`hierarchy` family (organization, folders, projects) is kept unless it is excluded by name. + +Coverage per run, besides `gcp_list_assets` and `gcp_full`: + +| Record | Meaning | +|---|---| +| `gcp_assets_` | Assets of the family built from the listing; status follows the listing (`PARTIAL` when it was truncated or fell back to `SearchAllResources`). One record per selected family, also when empty | +| `gcp_` | One per selected API-backed source (see 9.6) | +| `gcp_iam_policies` | `SearchAllIamPolicies` | +| `gcp_derived_` | Only when a cross-asset pass raised (`FAILED`) | + +### 9.9 Configuration + +| Key | Default | Effect | +|---|---|---| +| `gcp.deep_sources` | `true` | Run the API-backed sources at all | +| `gcp.rest_timeout_seconds` | `60` | Per-request timeout of the REST client | +| `gcp.cloud_identity_customer_id` | none | Cloud Identity customer (`C0xxxxx`); otherwise read from the organization | +| `gcp.max_dns_records_per_zone` | `5000` | Record sets read per managed zone | +| `gcp.max_vulnerability_occurrences` | `5000` | Artifact Analysis occurrences read per project | +| `gcp.max_groups`, `gcp.max_group_members` | `2000`, `5000` | Cloud Identity groups, and members per group | +| `gcp.expand_predefined_roles`, `gcp.max_role_expansions` | `true`, `500` | Read the permissions of bound predefined roles | +| `gcp.max_scheduler_locations` | `25` | Regions asked for Cloud Scheduler jobs (where the project has resources, plus the App Engine location) | +| `gcp.source_concurrency` | `4` | Sources run at the same time | +| `inventory.services`, `inventory.exclude_services` | all, none | Families, services, types and sources to map or skip | +| `inventory.max_images_per_repository` | `20` | Newest Docker images kept per Artifact Registry repository | + +Permissions the sources need, read-only: `roles/securitycenter.adminViewer`, +`roles/cloudidentity.groups.viewer` (or a Workspace groups reader), `roles/iam.denyReviewer`, +`roles/iam.principalAccessBoundaryViewer`, `roles/privilegedaccessmanager.viewer`, +`roles/monitoring.viewer`, `roles/dns.reader`, `roles/osconfig.viewer`, +`roles/containeranalysis.occurrences.viewer`, `roles/backupdr.viewer`, +`roles/cloudscheduler.viewer`, `roles/apigee.readOnlyAdmin`, `roles/config.viewer`, +`roles/deploymentmanager.viewer`, `roles/recommender.iamViewer` and +`roles/resourcemanager.organizationViewer` / `folderViewer`. A missing one shows as that +source's `FAILED` record. + + ## 10. Hierarchies beyond AWS **Azure** (`discover_azure_hierarchy(credential, graph_client_factory=None, *, @@ -1179,6 +1620,34 @@ Put it in the module for its area (the module must be imported by `azure_graph/_ add the ARM type to `azure_arm_types.yaml`, and test it with a recorded Resource Graph row passed through `build_assets(rows, subscription_id)`. + +#### Add an Azure child resource kind + +When the type is not in Resource Graph (a proxy resource under another one), add a `ChildSpec` +to `azure_graph/children.py` and give it a task name in `families.AZURE_TASK_FAMILIES`: + +```python +ChildSpec("widgets", ("microsoft.widgets/factories",), "widgets", "2024-01-01") +``` + +The rows come back ARM-shaped, so the `@extractor("microsoft.widgets/factories/widgets")` above +applies to them unchanged. If the rows only link other assets and should not become assets +themselves, register a handler instead: + +```python +from cloudg.inventory.azure_graph.registry import relation_row + + +@relation_row("microsoft.widgets/factories/links") +def _widget_link(row, b) -> None: + b.defer(parent_resource_id(row["id"]), rel(_get(row["properties"], "targetId"), EdgeType.REFERENCES, "DEPENDS_ON")) +``` + +Test it with `tests/azure_fakes.py`: `collect_linked(tables, arm_routes)` runs the collector +against a `TypedFakeGraph` and a `FakeArm` keyed by request path, and returns the linked graph and +the coverage records. + + ### 15.4 Add a GCP extractor ```python @@ -1197,6 +1666,14 @@ def _widget(ctx: GCPContext, out: Extracted) -> None: Add the type to `gcp_asset_types.yaml` (`deep`), and test with a recorded `list_assets` record. + +Registering a second extractor for a type that already has one is fine: both run, in import +order, on the same `Extracted`, and a failing one does not stop the other. A type that only a +service API returns gets a source in `gcp_sources/` (9.6) that hands its records to +`ctx.build()`, plus an entry marked `(API)` in the catalog. Put the type's family in +`families` or `type_families` so `--services` can select it. + + ### 15.5 Add an asset type 1. Add the value to `AssetType` in `cloudg/schema/models.py`, in the right section, with a @@ -1340,6 +1817,31 @@ into `build_assets()` or inject a `graph_client_factory`; GCP tests inject a fak `AssetServiceClient` through the `client` option returning recorded `list_assets` / `search_all_iam_policies` pages. + +The API-backed sources get a `GCPRestClient(session=FakeSession(routes))` through the +`rest_client` option (`tests/gcp_fakes.py`): routes map a URL to a JSON body, an error response +or a list of pages. `tests/test_gcp_parity_extractors.py` asserts one linked edge per new +extractor; `tests/test_gcp_sources.py` covers every source, the coverage records, service +filtering and the REST client. + + + +The Azure depth tests share `tests/azure_fakes.py`: + +- `TypedFakeGraph(tables)` answers Resource Graph queries by table and applies the `type =~`, + `type in~` and `subscriptionId =~` clauses, as ARG does, so one table can hold rows for several + queries. `fail={...}` makes tables raise. +- `FakeArm(routes, forbidden=...)` answers ARM REST, Microsoft Graph and data-plane calls by path, + with 404 for unknown paths and 403 for paths ending in a `forbidden` suffix. +- `collect_linked(tables, arm_routes, dataplane=..., **options)` runs `AzureDeepInventoryCollector` + against both and returns the linked `_Graph` and the coverage records. + +`test_azure_parity_graph.py` covers the Resource Graph tables, coverage and filtering, +`test_azure_parity_rest.py` the REST children, `test_azure_parity_extractors.py` the row +extractors through `AzureAssetBuilder`, `test_azure_entra.py` Microsoft Graph, +`test_azure_kubernetes.py` in-cluster AKS and `test_azure_rest.py` the REST client. + + ### 18.4 Capturing real shapes [INVENTORY_CATALOG.md](INVENTORY_CATALOG.md) was produced by running the suite with a small diff --git a/docs/INVENTORY_REFERENCE.md b/docs/INVENTORY_REFERENCE.md index a20ad4d..43810de 100644 --- a/docs/INVENTORY_REFERENCE.md +++ b/docs/INVENTORY_REFERENCE.md @@ -184,6 +184,26 @@ Some assets stand for something that was referenced rather than collected: | Entra principal | `entra:principal/` | `IDENTITY_USER`, `IDENTITY_GROUP` or `SERVICE_PRINCIPAL` | `placeholder: true`, `principal_type`, `discovered_via: "entra-principal-reference"` | | GCP principal | `gcp-principal:` or `k8s-gke://…` | `IDENTITY_USER`, `IDENTITY_GROUP`, `SERVICE_PRINCIPAL`, `K8S_SERVICE_ACCOUNT`, ... | `discovered_via: "iam_policy"` or `"workload_reference"` | + +GCP parity placeholders: + +| Kind | `arn` | `asset_type` | Distinguishing metadata | +|---|---|---|---| +| Security service not enabled (GCP) | `cloudg:gcp:::not-enabled`, where `parent` is `projects/`, `folders/` or `organizations/` | the service's type (`SECURITY_HUB`, `THREAT_DETECTOR`, `CONFIG_RECORDER`, `VULNERABILITY_SCANNER`, `DATA_SECURITY_SCANNER`, `CLOUDTRAIL`) | `security_service` (`scc`, `event-threat-detection`, `artifact-analysis`, `dlp-discovery`, `audit-logs`, ...), `enabled: false`, `status: "not enabled"`, alias `gcp-security::` | +| GCP principal named by a deny rule, PAM entitlement or group membership | `gcp-principal:` | as above | `discovered_via: "relation_reference"` | + + + +Azure placeholders added with the parity work: + +| Kind | `arn` | `asset_type` | Distinguishing metadata | +|---|---|---|---| +| Defender plan not listed (Azure) | `cloudg:azure:defender-::not-enabled` | `THREAT_DETECTOR` | `security_service: "defender-"`, `enabled: false`, `status: "not enabled"`, alias `azure-security:defender-:`. Created for every plan of `STANDARD_PLANS` the pricings query did not return. | +| Sentinel not enabled (Azure) | `cloudg:azure:sentinel::not-enabled` | `THREAT_DETECTOR` | as above with `security_service: "sentinel"`; created when no Log Analytics workspace of the subscription is onboarded. | +| External source repository | `repo:` (lower case, no `.git`, no credentials) | `SOURCE_CONNECTION` | `external: true`, `repository_url`, `host`, `discovered_via: "repository-reference"`. The target of `READS_FROM` from web app source control, Flux, ACR tasks, Static Web Apps, Data Factory Git and dev center catalogs. | +| Entra principal from a directory role | `entra:principal/` | `SERVICE_PRINCIPAL` | `placeholder: true`, `discovered_via: "entra-principal-reference"`; a role holder Microsoft Graph did not list. | + + Example, Inspector not enabled: ```json @@ -255,10 +275,41 @@ What can appear in `arn`, in `metadata.aliases` and as a relation `target`. The | GCP project node | `//cloudresourcemanager.googleapis.com/projects/` | project `CLOUD_ACCOUNT` / container | | GCP service account ref | `serviceAccount:` | alias of every service account; how relations name them | | GCP principal | `gcp-principal:user:alice@example.com` | IAM members that are not collected resources | + + +GCP parity identifiers: + +| Format | Example | Used for | +|---|---|---| +| GCP security service | `gcp-security:scc:organizations/999` | Alias of an enabled security service and of its "not enabled" placeholder. SCC notification configs, exports and mute rules target it. | +| GCP API-only resource | `//dns.googleapis.com/projects/p/managedZones/z/rrsets/www.example.com./A` | Resources read from a service API instead of Cloud Asset Inventory, named in the same `//.googleapis.com/` form | +| GCP predefined role | `//iam.googleapis.com/roles/storage.admin` | `IAM_ROLE` assets for bound predefined roles (custom roles keep their CAI name) | +| GCP authorizer | `/authorizers/` | JWT providers of an API Gateway config | +| GCP instance by number | `gcp-instance-id:/`, `//compute.googleapis.com/projects/p/zones/z/instances/` | Aliases of a VM, for Monitoring filters and OS Config reports that name it by number | + | GKE Kubernetes SA | `k8s-gke:////` | Workload Identity Kubernetes service accounts | | Short IDs and names | `vpc-0897edd53e9ac20d7`, `orders-worker`, `sg-…` | indexed for every asset (name, last ARN segment, known `*_id` metadata keys); resolved with account/region scoping | | Endpoints | `https://sqs.us-east-1.amazonaws.com/1234…/orders`, `web-alb-1.us-east-1.elb.amazonaws.com`, `orders-uploads.s3.amazonaws.com`, `` | queue URLs, load balancer and CloudFront DNS names, bucket domains, ECR repository URIs | + +Identifiers added with the Azure parity work: + +| Format | Example | Used for | +|---|---|---| +| Entra tenant | `entra:tenant/` | the tenant `ORGANIZATION` from Microsoft Graph; target of directory-role grants and gallery sharing with a tenant | +| Entra application | `entra:application/`, alias `entra:app/` | app registrations (`APP_REGISTRATION`) | +| Entra directory role | `entra:directoryRole/` | directory role definitions (`IAM_POLICY`) | +| Entra user principal name | `entra:upn/` | alias of an Entra user | +| Client ID | `azure-client-id:` | alias of a user-assigned identity and of a service principal; what the AKS Workload ID annotation names | +| Azure role definition | `azure-role:`, plus `/subscriptions//providers/Microsoft.Authorization/roleDefinitions/` | RBAC role definitions (`IAM_POLICY`); built-in roles use `/providers/Microsoft.Authorization/roleDefinitions/` as `arn` | +| Policy definition | `azure-policy-definition:` | alias of a policy definition with a GUID name | +| Messaging entity | `.servicebus.windows.net/`, `servicebus:/`, `servicebus-queue:`, `servicebus-topic:`, `eventhub:` | Service Bus queues and topics and Event Hubs; the host form is what Stream Analytics and action groups name, the bare forms what function bindings name | +| Defender aggregates | `cloudg:azure:defender-cspm:`, `cloudg:azure:defender-alerts:` | the recommendations (`SECURITY_HUB`) and alerts (`THREAT_DETECTOR`) assets | +| Synthetic children | `` under `.../applicationGateways//httpListeners/`, `/timers/`, `/triggers/`, `/authorizers/jwt`, `/regions/` | assets built from embedded properties: Application Gateway listeners, function timers, Logic Apps recurrence triggers, APIM JWT validation, Cosmos DB replica regions | +| AKS Kubernetes object | `k8s://///` | in-cluster AKS objects, same scheme as EKS | +| OIDC issuer | the AKS `oidcIssuerProfile.issuerURL`, with and without the trailing slash | alias of the AKS cluster, so federated credentials resolve to it | + + --- ## 5. NetworkEdge @@ -670,6 +721,36 @@ Properties: `total_services`, `successful_services`, `failed_services` (`int`) a | `gcp_full`, `gcp_iam_policies` | GCP scope | Overall result, and the `searchAllIamPolicies` call (`PARTIAL` when it failed after returning some policies). | | `organization_hierarchy` | GCP discovery | Organization / folder / policy hierarchy. | + +GCP parity records (per GCP scope): + +| `service` | Meaning | +|---|---| +| `gcp_assets_` | Assets of one service family (`compute`, `network`, `data`, `security`, ...) built from the Cloud Asset Inventory listing. The status follows the listing: `PARTIAL` when it was truncated or fell back to summaries. Written for every selected family, also with `asset_count` 0. | +| `gcp_` (`gcp_scc`, `gcp_dns_records`, `gcp_cloud_identity`, `gcp_monitoring`, ...) | One API-backed source ([INVENTORY_INTERNALS.md 9.6](INVENTORY_INTERNALS.md#96-api-backed-sources-gcp_sources)). `PARTIAL` when some calls failed, `FAILED` when none answered or the source raised, `SKIPPED` when a prerequisite is missing (no credentials, no organization, no Cloud Identity customer) or the API is off everywhere it was asked. | +| `gcp_derived_` | `FAILED` when a cross-asset pass raised; absent otherwise. | +| `gcp_list_assets`, `gcp_search_resources`, `gcp_asset_mapping` | The listing, its `SearchAllResources` fallback (`PARTIAL`), and records that could not be mapped (`PARTIAL`). | + +A record that is `FAILED` or `PARTIAL` makes `gcp_full` `PARTIAL`, as before. Families and +sources left out by `inventory.services` / `exclude_services` get no record, like AWS tasks. + + + +Azure records added with the parity work. They sit in the subscription's coverage record next +to `azure_full` (and the Entra ones in a discovery record): + +| `service` | Record | Meaning | +|---|---|---| +| any Azure task name (`resources`, `policy`, `security_assessments`, `apim`, `dns_records`, `virtual_wan`, ...) | Azure subscription | One entry per task of [INVENTORY_INTERNALS.md 8.4](INVENTORY_INTERNALS.md#84-depth-beyond-the-breadth-query). `SUCCESS` with the rows or assets it produced, `PARTIAL` when some calls were refused, `FAILED` when all failed, `SKIPPED` with the reason (not selected by `inventory.services`, turned off in `azure.*`, no token for the audience, Resource Graph unavailable on the SDK path). | +| `kubernetes:` | Azure subscription | In-cluster mapping of one AKS cluster, like EKS. `SKIPPED` for clusters without Entra integration. | +| `depth` | Azure subscription | `FAILED` only if the depth stage itself raised; the breadth inventory is kept. | +| `entra` | Azure discovery | Microsoft Graph run: `PARTIAL` when a listing failed, `SKIPPED` when not selected, `FAILED` when no Graph token could be obtained. | +| `entra_tenant`, `entra_users`, `entra_groups`, `entra_group_members`, `entra_applications`, `entra_application_owners`, `entra_federated_credentials`, `entra_service_principals`, `entra_service_principal_owners`, `entra_app_role_assignments`, `entra_oauth2_grants`, `entra_directory_roles`, `entra_pim_eligibility` | Azure discovery | One per Graph listing. `FAILED` with "Graph permission missing" on a 403. | + +`azure_full` is now `PARTIAL` when any of these task records is `FAILED` or `PARTIAL`, as for +`aws_full`. + + The CLI prints the failed entries after a map run ("N collectors failed"). --- @@ -1456,6 +1537,23 @@ its output has at most one asset per resource. `AzureDeepInventoryCollector.collection_mode` is `"resource-graph"` or `"sdk"` after `collect()`. + +`AzureDeepInventoryCollector(credential, subscription_id, graph_client_factory=None, +use_resource_graph=True, *, arm_client_factory=None, dataplane_client_factory=None, +options=None, **options)` takes the `AzureDeepOptions` fields as keyword options (`services`, +`exclude_services`, `child_resources`, `max_child_parents`, `diagnostic_settings`, +`max_diagnostic_settings`, `data_plane`, `kubernetes`, `kubernetes_timeout`, +`rest_concurrency`); an unknown option raises `TypeError`. Its `coverage` attribute is a +`CollectionCoverage` with one record per task. The factories are `credential -> client` (ARM +REST: `list(path, api_version, **params)`, `get(...)`, `post(...)`) and `(credential, base_url, +scope) -> client` (data planes); tests inject fakes there. + +`discover_entra(credential, client_factory=None, *, coverage=None, max_objects=10000, +max_expansions=1000)` returns the Entra assets (`ORGANIZATION`, `IDENTITY_USER`, +`IDENTITY_GROUP`, `APP_REGISTRATION`, `SERVICE_PRINCIPAL`, `IAM_POLICY`) and records one +coverage entry per Graph listing on `coverage`. + + ### 21.3 Kubernetes | Function | Returns | @@ -1479,6 +1577,23 @@ Kubernetes object assets carry `cluster_arn`, `namespace`, `kind` and, per kind: Pod Identity associations are collected by the EKS collector itself, not through the Kubernetes API. + +AKS goes through the same mapper: + +| Function | Returns | +|---|---| +| `map_cluster_objects(reader, cluster_arn, region, account_id, provider=CloudProvider.AWS)` | as above; with `provider=CloudProvider.AZURE` the assets are Azure assets and the AKS rules below apply | +| `azure_kubernetes.cluster_endpoint(arm, cluster_id)` | `(server, ca_b64)` from the `clusters` section of the `listClusterUserCredential` kubeconfig | +| `azure_kubernetes.aks_mappable(cluster_row)` | `None`, or why the cluster is skipped (no Entra integration, stopped) | +| `azure_kubernetes.collect_aks_workloads(credential, arm, cluster_row, *, timeout=10, reader_factory=None)` | `list[CloudAsset]` for one AKS cluster, read with an Entra token for `6dae42f8-4368-4678-94ff-3960e28e3630/.default` | + +On AKS, a ServiceAccount carries `workload_identity_client_id` and `workload_identity_tenant_id` +instead of `irsa_role_arn`, and `ASSUMES_ROLE / RUNS_ON` → `azure-client-id:`, the +user-assigned identity. Service and Ingress `load_balancer_hostnames` also hold load balancer IPs, +which resolve to public IP assets, and `service.beta.kubernetes.io/azure-load-balancer-internal: +"true"` sets `internal`. EKS output is unchanged. + + ### 21.4 Catalogs | Function | Module | Returns | diff --git a/docs/PYPI.md b/docs/PYPI.md new file mode 100644 index 0000000..2995942 --- /dev/null +++ b/docs/PYPI.md @@ -0,0 +1,119 @@ +

    + cloudg logo +

    + +

    cloudg

    + +

    Map the cloud. Scan it. Merge everything.

    + +

    + PyPI version + Python versions + CI + Documentation + License: MIT +

    + +

    Read the documentation · Open the sample report

    + +cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over that inventory, and merges every finding into one deduplicated, compliance-mapped result set. It can also skip the scanning entirely and aggregate outputs you already have. + +- **Map.** `cloudg map` collects everything deployed (or default) in an account, down to the network fabric, sweeps every service for the rest, and links it all into typed relationships: S3 `INVOKES` Lambda, function `ASSUMES_ROLE` role, WAF `PROTECTS` ALB. No scanner runs and none has to be installed. +- **Scan.** `cloudg run` fans out to Prowler, ScoutSuite, Checkov and Trivy against the same accounts, then normalises, deduplicates and maps every finding against 28 compliance frameworks. +- **Merge.** `cloudg ingest` does the same for scanner output produced elsewhere, with no cloud credentials and no scanner binaries. +- **Use it.** The results leave as an interactive HTML report, JSON, GraphML, an RDF ontology, RAG chunks for LLM pipelines and Terraform files, and `cloudg mcp serve` turns all of it into an MCP server for AI assistants. + +## Install + +```bash +pip install cloudg # core: graph, ontology, normaliser, reports, cloudg ingest +pip install "cloudg[aws]" # plus the AWS SDK +pip install "cloudg[all]" # every provider SDK and the MCP SDK +``` + +| Extra | Adds | +|---|---| +| `cloudg[aws]` | boto3 and aioboto3, for collecting from AWS | +| `cloudg[azure]` | The Azure identity and management SDKs, including Resource Graph | +| `cloudg[gcp]` | google-cloud-asset and google-auth, for Cloud Asset Inventory | +| `cloudg[mcp]` | The official MCP SDK (`mcp>=1.30,<3`), for the SDK adapters and HTTP transports | +| `cloudg[all]` | Everything above | + +Python 3.11 or newer. The scanners are separate programs, not Python dependencies: install whichever of Prowler, ScoutSuite, Checkov and Trivy you want, and cloudg skips the rest with a warning. The Docker image `ghcr.io/morpheuslord/cloudg` bundles Prowler, Checkov and Trivy. The [installation guide](https://morpheuslord.github.io/cloudg/guides/installation/) has the details. + +## Quick start + +From the command line: + +```bash +# map one AWS account in every enabled region; no scanners involved +cloudg map -p aws --regions all + +# collect, scan and report in one go +cloudg run -p aws --regions us-east-1 + +# already ran the scanners? merge their output, no credentials needed +cloudg ingest --prowler ./prowler-output/ --checkov ./results_json.json --trivy ./trivy.json +``` + +Reports land in `./reports`. Open `report.html` first, or look at the [sample report](https://morpheuslord.github.io/cloudg/sample-report.html) to see what one looks like. + +From Python, the CLI is a thin layer over `CloudGEngine`: + +```python +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig()) +result = engine.run_from_reports_sync( + {"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}, + output_dir="./reports", +) +print(result.total_findings, result.severity_breakdown) +print(result.report_paths) # findings.json and report.html +``` + +```python +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["ALL"] +inventory = CloudGEngine(config).map_inventory_sync(output_dir="./reports") +print(inventory.summary["total_assets"], "assets,", inventory.summary["total_edges"], "edges") +``` + +## Documentation + +The documentation site is at [morpheuslord.github.io/cloudg](https://morpheuslord.github.io/cloudg/): + +- [Quickstart](https://morpheuslord.github.io/cloudg/guides/quickstart/) and the [guides](https://morpheuslord.github.io/cloudg/guides/introduction/): authentication, inventory mapping, AWS Organizations, scanners, compliance, reports, Docker and CI. +- [CLI reference](https://morpheuslord.github.io/cloudg/cli/): every command and flag, generated from the code. +- [Python API](https://morpheuslord.github.io/cloudg/api/): `CloudGEngine`, the inventory, graph and findings classes, and the [exceptions](https://morpheuslord.github.io/cloudg/api/exceptions/). +- [Examples](https://morpheuslord.github.io/cloudg/examples/): complete scripts with their output. +- [MCP server](https://morpheuslord.github.io/cloudg/mcp/): connecting Claude Desktop, Claude Code, Cursor or VS Code. +- [config.yaml reference](https://morpheuslord.github.io/cloudg/reference/config/providers/) and the [changelog](https://morpheuslord.github.io/cloudg/changelog/). + +The same reference material is kept as Markdown in the repository: the [feature reference](https://github.com/morpheuslord/cloudg/blob/main/docs/DOCUMENTATION.md), the [inventory reference](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_REFERENCE.md), [catalog](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_CATALOG.md) and [internals](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_INTERNALS.md), the [MCP guide](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP.md), [tool reference](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP_TOOLS.md) and [privacy reference](https://github.com/morpheuslord/cloudg/blob/main/docs/MCP_PRIVACY.md), and [rate limits and throttling](https://github.com/morpheuslord/cloudg/blob/main/docs/RESILIENCE.md). + +## Credits + +cloudg builds on other people's work, and some of it ships inside the package. + +[Prowler](https://github.com/prowler-cloud/prowler) (Apache-2.0). The 22 framework files in `cloudg/rules/frameworks/` are generated from Prowler's compliance JSON by `scripts/import_prowler_compliance.py`. That covers the control IDs, the control titles and descriptions, and the mapping from each control to Prowler check names. Every generated file says so in its header and in its `source` field. Prowler check names also appear in `cloudg/rules/check_equivalence.yaml`, and cloudg runs Prowler and parses its ASFF and OCSF output. + +Scanners. [Checkov](https://github.com/bridgecrewio/checkov) (Apache-2.0), [Trivy](https://github.com/aquasecurity/trivy) (Apache-2.0) and [ScoutSuite](https://github.com/nccgroup/ScoutSuite) (GPL-2.0) are run as separate programs, and their reports are parsed. Their check IDs (`CKV_*`, `AVD-*`) are used in the check-equivalence map. None of their code is part of cloudg. The Docker image installs Prowler, Checkov and Trivy, so their own licenses apply to those copies. + +[Cloud Custodian](https://github.com/cloud-custodian/cloud-custodian) (Apache-2.0). The policy packs in `cloudg/policies/` are written in Cloud Custodian's policy format and run with its `custodian` CLI. + +[parliament](https://github.com/duo-labs/parliament) (BSD-3-Clause) from Duo Labs does the IAM policy linting. + +Drawing engines. The estate map in report.html is drawn with the iso-kit from [Anatomy](https://github.com/wheresryan22/anatomy) (MIT, by Ryan), and its empty states use the kernel of [Hairline](https://hairline.lucasmarkes.com) 0.5.1 (MIT, by Lucas Marques). Both are copied into `cloudg/templates/vendor/` and embedded in every report, so the map works offline. The iso-kit is wrapped so it can sit inside a script element, and `cloudg/templates/vendor/NOTICE` lists exactly what changed. + +[MITRE ATT&CK®](https://attack.mitre.org/). The `mitre_attack_*.yaml` files contain ATT&CK technique IDs and descriptions. © The MITRE Corporation. This work is reproduced and distributed with the permission of The MITRE Corporation. ATT&CK is a registered trademark of The MITRE Corporation. + +Standards. The hand-written rulesets in `cloudg/rules/` reference public control numbering from the CIS Benchmarks for AWS, Azure and GCP, NIST SP 800-53, NIST CSF, PCI DSS, the AICPA SOC 2 Trust Services Criteria, the HIPAA Security Rule, GDPR, ISO/IEC 27001:2022 and the AWS Foundational Security Best Practices. The standards belong to their publishers. CIS Benchmarks is a trademark of the Center for Internet Security, and cloudg is not affiliated with or endorsed by CIS or any other publisher named here. + +The Python libraries cloudg depends on (NetworkX, rdflib, pydantic, Click, Rich and the rest) are listed in [`pyproject.toml`](https://github.com/morpheuslord/cloudg/blob/main/pyproject.toml). The full list, with the JavaScript libraries the report embeds, is on the [credits page](https://morpheuslord.github.io/cloudg/credits/). + +## License + +MIT. See [LICENSE](https://github.com/morpheuslord/cloudg/blob/main/LICENSE). diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..599e1e2 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,272 @@ +# MkDocs Material configuration for https://morpheuslord.github.io/cloudg/ +# +# Hand-written pages live in site/content/. The reference pages (CLI options, +# config.yaml, the changelog and the long docs/*.md references) are generated +# at build time by site/hooks.py and are never committed. site/AUTHORING.md +# explains how to write a page. +# +# .venv/bin/mkdocs serve -a 127.0.0.1:8807 # live preview +# .venv/bin/mkdocs build --strict # what CI runs + +site_name: cloudg +site_url: https://morpheuslord.github.io/cloudg/ +site_description: >- + cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, + runs security scanners over the same inventory and merges every finding + into one deduplicated, compliance-mapped result. +site_author: morpheuslord + +repo_url: https://github.com/morpheuslord/cloudg +repo_name: morpheuslord/cloudg +edit_uri: edit/main/site/content/ + +docs_dir: site/content +site_dir: site/dist + +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + anchors: warn + +watch: + - site/hooks.py + - site/gen + - site/overrides + - site/static + - docs + - cloudg + - CHANGELOG.md + - config.yaml + +hooks: + - site/hooks.py + +nav: + - Home: index.md + - Guides: + - Start: + - Introduction: guides/introduction.md + - Installation: guides/installation.md + - Quickstart: guides/quickstart.md + - The pipeline: guides/pipeline.md + - Mapping: + - Inventory mapping: guides/inventory-mapping.md + - Dependencies and blast radius: guides/dependencies.md + - AWS Organizations: guides/aws-organizations.md + - Scanning: + - Running scanners: guides/running-scanners.md + - Ingesting existing output: guides/ingesting.md + - Compliance frameworks: guides/compliance.md + - Analyse: + - Graph, ontology and RAG: guides/graph-ontology-rag.md + - Reports: guides/reports.md + - Sample report: sample-report.md + - Operate: + - Authentication: guides/authentication.md + - Rate limits and throttling: guides/rate-limits.md + - Docker: guides/docker.md + - CI and automation: guides/ci-automation.md + - Python recipes: guides/python-recipes.md + - Examples: + - examples/index.md + - Map and query: examples/map-and-query.md + - Ingest and report: examples/ingest-and-report.md + - Graph, ontology and RAG: examples/graph-and-exports.md + - Errors and strict mode: examples/errors.md + - CLI: + - cli/index.md + - Pipeline: + - cloudg run: cli/run.md + - cloudg collect: cli/collect.md + - cloudg scan: cli/scan.md + - cloudg report: cli/report.md + - Inventory: + - cloudg map: cli/map.md + - cloudg deps: cli/deps.md + - cloudg ingest: cli/ingest.md + - MCP: + - cloudg mcp serve: cli/mcp-serve.md + - cloudg mcp config: cli/mcp-config.md + - cloudg mcp tools: cli/mcp-tools.md + - cloudg mcp resources: cli/mcp-resources.md + - cloudg mcp prompts: cli/mcp-prompts.md + - cloudg mcp call: cli/mcp-call.md + - cloudg mcp read: cli/mcp-read.md + - Shared: + - Provider and region flags: cli/provider-flags.md + - Auth flags: cli/auth-flags.md + - Output flags: cli/output-flags.md + - Python API: + - api/index.md + - Engine: + - CloudGEngine: api/cloudgengine.md + - CloudGConfig: api/cloudgconfig.md + - Result objects: api/results.md + - Event hooks: api/event-hooks.md + - Exceptions: api/exceptions.md + - Inventory: + - InventoryMapper: api/inventorymapper.md + - InventoryResult: api/inventoryresult.md + - RelationshipLinker: api/relationshiplinker.md + - DependencyGraph: api/dependencygraph.md + - Graph: + - GraphBuilder: api/graphbuilder.md + - ReachabilityAnalyzer: api/reachabilityanalyzer.md + - CloudOntology: api/cloudontology.md + - RAGExporter: api/ragexporter.md + - Findings: + - FindingsNormaliser: api/findingsnormaliser.md + - parse_report: api/parse-report.md + - Data models: api/models.md + - Extend: + - Entry points: api/entry-points.md + - cloudg.resilience: api/resilience.md + - CloudGMCPLayer: api/mcp-layer.md + - MCP server: + - mcp/index.md + - Connect a client: mcp/connect.md + # docs/*.md entries are split into one page per section by site/hooks.py + - Server: docs/MCP.md + - Tools: docs/MCP_TOOLS.md + - Privacy and policy: docs/MCP_PRIVACY.md + - Internals: docs/MCP_INTERNALS.md + - Reference: + # one page per top-level section of config.yaml, generated by site/hooks.py + - config.yaml: config.yaml + - Outputs: + - Output files: reference/output-files.md + - Inventory structures: docs/INVENTORY_REFERENCE.md + - Catalogs: docs/INVENTORY_CATALOG.md + - Inventory internals: docs/INVENTORY_INTERNALS.md + - Resilience internals: docs/RESILIENCE.md + - About: + - Changelog: changelog.md + - Credits: credits.md + +theme: + name: material + # The release banner on every page (site/overrides/main.html; bump it with the version) + custom_dir: site/overrides + logo: assets/images/logo-mark.svg + favicon: assets/images/favicon.svg + icon: + repo: fontawesome/brands/github + palette: + # Dark by default: it is a security tool. + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: cyan + toggle: + icon: material/weather-sunny + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: blue grey + accent: cyan + toggle: + icon: material/weather-night + name: Switch to dark mode + features: + - navigation.tabs + - navigation.sections + - navigation.indexes + - navigation.top + - navigation.footer + - search.suggest + - search.highlight + - content.code.copy + - content.code.annotate + - content.code.select + - content.tabs.link + - content.action.edit + - toc.follow + +plugins: + - search + - redirects: + redirect_maps: + # folder addresses that never had a page of their own + guides/index.md: guides/introduction.md + reference/index.md: reference/config/providers.md + reference/config/index.md: reference/config/providers.md + about/index.md: changelog.md + - mkdocstrings: + default_handler: python + handlers: + python: + paths: [.] + options: + extensions: + - site/gen/griffe_rst.py:RstToMarkdown + docstring_style: google + docstring_options: + # cloudg's "Returns:" sections are prose that wraps at the same + # indent, not one "type: description" item per line + returns_multiple_items: false + docstring_section_style: table + merge_init_into_class: true + show_root_heading: true + show_root_full_path: false + show_root_members_full_path: false + show_object_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + show_signature_annotations: true + separate_signature: true + signature_crossrefs: true + show_source: true + show_bases: true + show_labels: true + members_order: source + filters: ["!^_"] + heading_level: 3 + line_length: 88 + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: [site/examples, .] + check_paths: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +extra_css: + - assets/stylesheets/extra.css + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/morpheuslord/cloudg + - icon: fontawesome/brands/python + link: https://pypi.org/project/cloudg/ + - icon: fontawesome/brands/docker + link: https://github.com/morpheuslord/cloudg/pkgs/container/cloudg + +# copyright and extra.release are set by site/hooks.py from CHANGELOG.md diff --git a/pyproject.toml b/pyproject.toml index 3a47f38..a30a39b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,10 @@ [project] name = "cloudg" -version = "0.6.0" +version = "0.7.0" description = "cloudg (cloud graphing) - multi-cloud infrastructure mapping, knowledge graphs and security intelligence for AWS, Azure and GCP" -# The PyPI project page renders the full documentation, not the repo README -readme = "docs/DOCUMENTATION.md" +# The PyPI project page: a short overview that links into the docs site. +# docs/DOCUMENTATION.md stays the full reference (the MCP layer serves it). +readme = "docs/PYPI.md" license = { text = "MIT" } authors = [{ name = "morpheuslord", email = "chiranjeevi.naidu@proton.me" }] requires-python = ">=3.11" @@ -84,6 +85,18 @@ full = [ "policy-sentry>=0.12", ] +# Builds the documentation site (mkdocs.yml): pip install --group docs +[dependency-groups] +docs = [ + "mkdocs>=1.6.1,<2", + "mkdocs-material>=9.8.0,<10", + "mkdocstrings[python]>=1.0.8,<2", + "mkdocstrings-python>=2.0.10,<3", + "mkdocs-redirects>=1.2.3,<2", + # formats the signatures mkdocstrings renders + "ruff>=0.17.2,<1", +] + [project.scripts] cloudg = "cloudg.cli:cli" cloudg-mcp = "cloudg.mcp.cli:main" diff --git a/site/AUTHORING.md b/site/AUTHORING.md index 4fdc01e..03fc8ae 100644 --- a/site/AUTHORING.md +++ b/site/AUTHORING.md @@ -1,190 +1,259 @@ # Writing pages for the cloudg docs site -The site is built by `site/build.py` from three kinds of input: +The site at https://morpheuslord.github.io/cloudg/ is built with [MkDocs](https://www.mkdocs.org/) and the [Material theme](https://squidfunk.github.io/mkdocs-material/). It has three kinds of page: -- hand-written pages in `site/content/` (guides, CLI examples, API prose) -- generated reference data (click commands, docstrings, `config.yaml`, `CHANGELOG.md`) -- the long reference documents in `docs/*.md`, which are split at their `##` headings +- hand-written pages in `site/content/` (guides, examples, CLI prose, API prose); +- reference pages generated at build time from the code and the repository files: the CLI options tables (click commands), the API signatures and parameter tables (docstrings, through mkdocstrings), the `config.yaml` reference, the changelog, and the long `docs/*.md` references split into one page per section; +- static files copied as they are: the sample report, `docs/viewer.html` and `docs/assets/`. -Build it with: +Generated pages are never committed. `site/hooks.py` creates them on every build. + +## Build and preview + +The docs tools are a dependency group in `pyproject.toml`. Install them next to cloudg in a virtual environment: ```bash -.venv/bin/python site/build.py # writes site/dist -.venv/bin/python site/build.py --serve # builds, then serves on http://127.0.0.1:8000/cloudg/ +python -m venv .venv +.venv/bin/pip install --upgrade "pip>=25.1" +.venv/bin/pip install -e . --group docs + +.venv/bin/mkdocs serve -a 127.0.0.1:8807 # live preview on http://127.0.0.1:8807/cloudg/ +.venv/bin/mkdocs build --strict # what CI runs; writes site/dist ``` +With uv: `uv pip install -e . --group docs`. `mkdocs build --strict` must finish with no warnings: a broken link, a missing anchor or a page left out of the nav fails the build in CI. Set `NO_MKDOCS_2_WARNING=1` and `DISABLE_MKDOCS_2_WARNING=true` to hide the banners some plugins print about the future of MkDocs. + +## Layout + +| Path | What it holds | +|---|---| +| `mkdocs.yml` | Site settings, theme, plugins, markdown extensions and the nav | +| `site/content/` | The docs directory: every hand-written page, plus `assets/` (logo, favicon, `extra.css`) | +| `site/overrides/main.html` | Theme override: the release banner at the top of every page | +| `site/hooks.py` | MkDocs hooks that generate the reference pages and copy the static files | +| `site/gen/` | The generators the hooks call: `cli.py`, `config_ref.py`, `changelog.py`, `split.py`, `links.py`, and `griffe_rst.py` for docstrings | +| `site/gen/config-notes.yaml` | Descriptions for `config.yaml` keys that have neither a comment in the file nor a `Field(description=...)` | +| `site/examples/` | Example scripts that pages include with snippets | +| `site/static/sample-report.html` | The sample report, built by `scripts/build_sample_report.py` | + ## Style rules - No em dashes anywhere, and no `--` or spaced en dash standing in for one. Use a comma, a colon, parentheses or two sentences. - Straight quotes only. No emoji. - Sentence-case headings. - Run every page you write through the `/human` skill's scanner before committing it. -- Every code example must work as written against the current release. Check signatures in the source; run the Python ones when you can (the sample scanner output inlined in `tests/test_ingest.py` is handy for ingest examples). +- Every code example must work as written against the current release. Check signatures in the source and run the Python ones when you can (the sample scanner output inlined in `tests/test_ingest.py` is handy for ingest examples). Prefer putting a longer example in `site/examples/` and including it, so the script on the page is a file you can run. + +## Adding a page -## Front matter +1. Create the Markdown file under `site/content/`. +2. Add it to `nav:` in `mkdocs.yml`. A page that is not in the nav fails the strict build. +3. Link to other pages with relative paths to their Markdown files, for example `[cloudg map](../cli/map.md#usage)`. MkDocs checks every link and anchor and turns them into the right URLs. Never write site-absolute links (`/cloudg/...`). -Every file starts with YAML front matter. +A page starts with optional front matter, then the body: -```yaml +```markdown --- -title: Inventory mapping -lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" -meta: # optional strip under the lede (guides only) - - [Command, "`cloudg map`"] - - [Providers, "AWS, Azure, GCP"] -source: docs/DOCUMENTATION.md # optional: what "Edit on GitHub" opens. Defaults to this file. -since: "0.5.1" # optional: shown as "Last changed in v0.5.1" +title: Inventory mapping # optional: the nav title is used otherwise +description: One sentence for search engines and link previews. --- -``` -`lede` and `meta` values are inline markdown. +`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together? +{ .lede } -## Links +**Command** `cloudg map` · **Providers** AWS, Azure, GCP +{ .page-meta } -Write site links as root-relative paths without the base: `/guides/quickstart/`, `/cli/map/#options`, `/reference/config/ratelimit/#aws.retry_budget`. The build adds the base path. The full URL map is in `site/data/nav.yaml`. +## First section +``` -## Markdown extensions +The `{ .lede }` paragraph is the larger introduction under the title. The `{ .page-meta }` line is the strip of facts under it; end it with `**Last changed** [v0.6.1](../changelog.md#0.6.1)` when a page documents behaviour that changed in a release. The body never starts with an `#` heading: the title comes from the nav or the front matter. -Standard CommonMark plus GFM tables. On top of that: +## Markdown -### Headings +The extensions are listed in `mkdocs.yml`. The ones pages use most: -`##` and `###` feed the "On this page" list. Anchors are generated from the text; force one with a trailing `{#my-anchor}`. Never skip a level. +### Headings and anchors -### Code blocks +`##` and `###` feed the table of contents. Anchors are generated from the text the way GitHub does it; set one yourself with a trailing `{ #my-anchor }`. Never skip a level. -The info string takes a language and optional attributes: +### Code blocks ````markdown -```python title="map_and_overlay.py" hl="4-6" +```python title="map_and_overlay.py" hl_lines="4-6" from cloudg import CloudGConfig, CloudGEngine ... ``` ```` -- `title` shows as the file name in the editor bar. Use a real file name for code people should save (`deploy.py`, `config.yaml`, `.github/workflows/cloudg.yml`). -- `hl` highlights lines: `hl="3"`, `hl="2,5-7"`. -- `bash` and `shell` blocks render as a terminal. Each command line gets a `$` prompt automatically; comment lines (`# ...`) are left out when the reader presses Copy. Put each command on its own line, or continue it with a trailing `\`. -- `console` is for a captured session with output. Write the prompt yourself (`$ cloudg deps ...`) and the output below it. -- Other languages (`python`, `yaml`, `json`, `toml`, `dockerfile`, `ini`, `text`) get line numbers. +- `title` shows as the file name above the block. Use a real file name for code people should save. +- `hl_lines` highlights lines: `hl_lines="3"`, `hl_lines="2 5-7"`. +- `bash` blocks hold commands only; `console` is for a captured session, with the `$` prompt written out and the output below it. +- Every block has a copy button. Annotations work too: put `# (1)!` at the end of a line and a numbered list after the block. -### Code tabs +### Code from a file -Consecutive fenced blocks that each carry a `tab` attribute become one tabbed block. Tabs with the same label are linked across the page, and the reader's choice is remembered. +`pymdownx.snippets` includes a file, so the code on the page is the script that was run. Paths are relative to `site/examples/` (then the repository root): ````markdown -```bash tab="CLI" -cloudg map -p aws --regions all +```python title="strict_scanner.py" +--8<-- "errors/strict_scanner.py" ``` +```` -```python tab="Python" title="map.py" -from cloudg import CloudGConfig, CloudGEngine -... -``` +A missing file fails the build. -```bash tab="Docker" -docker run --rm -v ~/.aws:/home/cloudg/.aws:ro -v "$PWD/reports:/app/reports" cloudg map -p aws --regions all -``` +### Tabs + +Tabs with the same label are linked across the whole site, so a reader who picks "Python" once sees Python everywhere. + +````markdown +=== "CLI" + + ```bash + cloudg map -p aws --regions all + ``` + +=== "Python" + + ```python title="map.py" + from cloudg import CloudGConfig, CloudGEngine + ... + ``` ```` -Use the labels `CLI`, `Python`, `Docker` for ways to do one thing, and `pip`, `uv`, `Docker` for installs. +Use the labels `CLI`, `Python`, `Docker` for ways to do one thing, and `pip`, `uv`, `Docker` for installs. The content of a tab is indented by four spaces. ### Diagrams -A `mermaid` block renders as a diagram. Add a caption with `caption="..."`. +A `mermaid` block renders as a diagram that follows the light and dark palette. A caption goes in a paragraph right after it: ````markdown -```mermaid caption="What cloudg run does, phase by phase" +```mermaid flowchart LR A[Collect] --> B[Graph] --> C[Scan] --> D[Normalise] --> E[Report] ``` +What cloudg run does, phase by phase. +{ .diagram-caption } ```` -Keep diagrams small enough to read at 680 px: up to about 12 nodes. Prefer `flowchart LR` or `TD`, `sequenceDiagram` for request flows and `stateDiagram-v2` for state machines. Don't set colours or `classDef` styles; the theme handles it. Quote labels that contain punctuation: `A["cloudg map --org"]`. +Keep diagrams small enough to read on a laptop: up to about 12 nodes. Prefer `flowchart LR` or `TD`, `sequenceDiagram` for request flows and `stateDiagram-v2` for state machines. Don't set colours or `classDef` styles. Quote labels that contain punctuation: `A["cloudg map --org"]`. -### Callouts +### Admonitions ```markdown -:::note -The map doesn't depend on the scanners. -::: +!!! note + The map doesn't depend on the scanners. -:::warning Admin rights -AWSControlTowerExecution has admin rights. -::: +!!! warning "AWSControlTowerExecution has admin rights" + Without `--org-role`, cloudg assumes that role in every member account. + +??? tip "Collapsed until clicked" + Use `???` for detail most readers can skip. ``` -Kinds: `note`, `tip`, `warning`, `danger`. Text after the kind replaces the default label. +Kinds: `note`, `tip`, `warning`, `danger`, `example`, `info`. -### Steps +### Steps, cells and link cards -Numbered steps, one `###` per step. Use four colons on the outer fence when a callout sits inside. +Numbered steps are `###` headings inside a `steps` block; the numbers come from the stylesheet: ```markdown -::::steps +
    + ### Install + ... -:::tip -... -::: + ### Authenticate + ... -:::: -``` -### Numbered cells +
    +``` -Two to four short items side by side, one `###` per cell. +Two to four short items side by side are cards in a grid: ```markdown -:::cells +
    +
    + ### Deep collectors -ECS, EKS, Lambda triggers... -### Breadth sweep -On AWS, a Cloud Control sweep... -::: -``` -### Link cards +131 dedicated AWS collector tasks... + +
    +
    +``` -A list of links rendered as a grid of cards. The text after the link is the description. +A list of links with a line of text each becomes a grid of cards: ```markdown -:::links -- [Inventory mapping](/guides/inventory-mapping/) What exists and how it is wired. -- [cloudg deps](/cli/deps/) Blast radius for one asset. -::: +
    + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + What exists and how it is wired. + +
    ``` ## Page kinds -### Guides: `site/content/guides/.md` +### Guides and examples -Front matter plus a body. The body starts with prose or an `##`, never with an `#` (the title comes from front matter). +Plain pages in `site/content/guides/` and `site/content/examples/`. ### CLI commands: `site/content/cli/.md` -The generator writes the usage line and the options table from the click command. Your file adds: +The front matter names the command, and a marker says where the generated "Usage" and "Options" sections go: -```yaml +```markdown --- command: map # or "mcp serve" -lede: Builds the scanner-independent inventory map. -intro: | # optional markdown shown above the usage line - ... +description: Builds the scanner-independent inventory map. --- + +One paragraph on what the command is for. + + + +## How it works +... ``` -The body follows the options table. Typical sections: `## Examples`, `## Output files`, `## Exit codes`, `## Notes`. Pages without a `command` key (the shared flag pages) are plain pages. +The tables come from the click definitions in `cloudg/cli*.py`, so they match `--help`. `cli/index.md` uses `` for the table of every command, in nav order. Typical sections after the options: `## Examples`, `## Output files`, `## Exit codes`, `## Notes`. ### Python API: `site/content/api/.md` -```yaml ---- -object: cloudg.api.CloudGEngine # import path; omit for a plain page -members: [run_pipeline, run_pipeline_sync, collect, map_inventory] # optional order/filter -lede: Runs the whole pipeline from Python, or any one phase of it. ---- +Prose and examples come first, then a `## Reference` section where mkdocstrings renders the objects from their docstrings: + +```markdown +## Reference + +::: cloudg.api.CloudGEngine + options: + members: + - run_pipeline + - map_inventory_sync ``` -The generator writes the kind tag, source link, signature and a ruled row per member (signature, docstring, parameter table). Your body follows the members. Typical sections: `## Examples`, `## Notes`. +`members` picks and orders the members; `members: false` shows the object without them. The signatures, parameter, return and raises tables, and the "source" link to GitHub come from the code. Docstrings follow the Google style; `site/gen/griffe_rst.py` turns the reST roles (`:class:`, double backticks, `::` blocks) they still use into Markdown while the site is built, without changing the source. + +### Generated reference pages + +Three kinds of nav entry are replaced by generated pages in `site/hooks.py`: + +- `docs/X.md`: the document is split into one page per `##` section under the prefix in `SPLITS`. A section over 40 KB with at least three `###` headings becomes a folder with one page per `###`. Every heading keeps the anchor GitHub gives it, so links into the single-file document keep working, and links between the documents are pointed at the page that now holds their target. +- `config.yaml`: one page per top-level section, from the comments in `config.yaml`, the `Field(description=...)` of the `CloudGConfig` models and `site/gen/config-notes.yaml`. +- `changelog.md`: `CHANGELOG.md`, with each release's anchor set to its version (`changelog/#0.6.1`). + +To change what those pages say, edit their source (the `docs/*.md` file, `config.yaml`, the models or `CHANGELOG.md`). The "Edit" button on a generated page opens its source on GitHub. + +### Sample report + +`sample-report.md` embeds `site/static/sample-report.html`, which the hooks publish at `/cloudg/sample-report.html` (and at the old address, `/cloudg/static/sample-report.html`). Rebuild the file with `python scripts/build_sample_report.py`; the page needs no change. + +## URLs + +Every page keeps the address it had on the earlier site generator: `guides/quickstart.md` is served at `/cloudg/guides/quickstart/`, `cli/index.md` at `/cloudg/cli/`. Folder addresses without a page of their own (`/cloudg/guides/`, `/cloudg/reference/`) redirect through the `redirects` plugin; add an entry under `redirect_maps` in `mkdocs.yml` when a page moves. diff --git a/site/build.py b/site/build.py deleted file mode 100644 index 6ed2ff3..0000000 --- a/site/build.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Build the cloudg documentation site into site/dist. - - python site/build.py # build - python site/build.py --serve # build, then serve on http://127.0.0.1:8000/cloudg/ - python site/build.py --strict # fail on broken internal links (used in CI) - python site/build.py --base / # build for serving at the domain root - -See site/AUTHORING.md for the page format. The generator itself lives in -site/docsgen/; this file is the command line around it. -""" - -from __future__ import annotations - -import argparse -import functools -import http.server -import os -import sys -from pathlib import Path -from urllib.parse import urlsplit - -SITE = Path(__file__).resolve().parent -REPO = SITE.parent -sys.path.insert(0, str(SITE)) -sys.path.insert(0, str(REPO)) - -from docsgen.site import Site # noqa: E402 - - -def serve(out: Path, base: str, port: int) -> None: - class Handler(http.server.SimpleHTTPRequestHandler): - def end_headers(self): - self.send_header("Cache-Control", "no-store") - super().end_headers() - - def translate_path(self, path): # noqa: D401 - parsed = urlsplit(path).path - if base != "/" and parsed.startswith(base): - parsed = "/" + parsed[len(base) :] - return str(out) + parsed - - def send_error(self, code, message=None, explain=None): - if code == 404 and (out / "404.html").exists(): - body = (out / "404.html").read_bytes() - self.send_response(404) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - return - super().send_error(code, message, explain) - - handler = functools.partial(Handler, directory=str(out)) - with http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) as httpd: - print(f"serving http://127.0.0.1:{port}{base}") - httpd.serve_forever() - - -def main() -> int: - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--out", default=str(SITE / "dist")) - ap.add_argument("--base", default=os.environ.get("DOCS_BASE", "/cloudg/")) - ap.add_argument("--strict", action="store_true", help="exit non-zero on broken internal links") - ap.add_argument("--serve", action="store_true") - ap.add_argument("--port", type=int, default=8000) - args = ap.parse_args() - - site = Site(args.base, args.strict) - site.discover() - site.render_all() - out = Path(args.out) - site.write(out) - problems = site.check_links() - for p in problems: - print(f"link: {p}", file=sys.stderr) - print(f"built {len(site.pages)} pages into {out} (base {site.base})") - if problems and args.strict: - return 1 - if args.serve: - serve(out, site.base, args.port) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/site/content/api/cloudgconfig.md b/site/content/api/cloudgconfig.md index 762177e..be990b0 100644 --- a/site/content/api/cloudgconfig.md +++ b/site/content/api/cloudgconfig.md @@ -1,10 +1,10 @@ --- -object: [cloudg.config.CloudGConfig, cloudg.config.load_config] -members: - cloudg.config.CloudGConfig: [] -lede: "`CloudGConfig` is the pydantic model behind `config.yaml`. Build it in code, load it from a file, or do both and override a few fields." +description: CloudGConfig is the pydantic model behind config.yaml. Build it in code, load it from a file, or do both and override a few fields. --- +`CloudGConfig` is the pydantic model behind `config.yaml`. Build it in code, load it from a file, or do both and override a few fields. +{ .lede } + ## Two ways to get a config `CloudGConfig()` with no arguments is a working configuration: AWS only, `us-east-1`, all five scanners enabled, ontology and RAG export on, Terraform export off. Every field has a default, so you only set what differs. @@ -19,9 +19,9 @@ config.aws.regions = ["eu-west-1", "eu-central-1"] config.scanners.enabled = ["prowler", "iam"] ``` -From a file, `load_config(path)` reads the YAML with `yaml.safe_load` and hands the dict to `CloudGConfig.model_validate()`. The file has the same shape as the model: top-level keys are the attributes below, nested keys are the fields of each section. The generated [config.yaml reference](/reference/config/providers/) documents every key with its default. +From a file, `load_config(path)` reads the YAML with `yaml.safe_load` and hands the dict to `CloudGConfig.model_validate()`. The file has the same shape as the model: top-level keys are the attributes below, nested keys are the fields of each section. The generated [config.yaml reference](../reference/config/providers.md) documents every key with its default. -```mermaid caption="What load_config does with the path it gets" +```mermaid flowchart TD P["load_config(path)"] --> N{"path is None?"} N -->|yes| D["CloudGConfig() defaults"] @@ -33,6 +33,8 @@ flowchart TD U --> C["CloudGConfig"] V -->|invalid value| X["pydantic ValidationError"] ``` +What load_config does with the path it gets +{ .diagram-caption } Two branches in that diagram return defaults instead of failing: a `None` path and a path that does not exist. A typo in the file name therefore gives you a run against `us-east-1` with every scanner enabled, and the only sign is a `WARNING cloudg.config: Config file ... not found, using defaults` line. If that matters, check `Path(path).is_file()` before calling, or use the strict loader in the examples. An empty file also gives defaults. The CLI calls the same function with the `-c` value and does not look for a `config.yaml` in the working directory on its own. @@ -77,7 +79,7 @@ Every section ignores keys it does not define, and says so. An unknown key at th | every Azure credential | `DefaultAzureCredential` | | `gcp.credentials_file` | application default credentials, which read `GOOGLE_APPLICATION_CREDENTIALS` | -Two variables change cloudg behaviour outside the config: `CLOUDG_CATALOG_DIR` overlays the inventory reference catalogs, and `CLOUDG_CACHE_DIR` moves the Cloud Control schema cache (default `~/.cache/cloudg`). [Authentication](/guides/authentication/) covers the full credential order for each provider. +Two variables change cloudg behaviour outside the config: `CLOUDG_CATALOG_DIR` overlays the inventory reference catalogs, and `CLOUDG_CACHE_DIR` moves the Cloud Control schema cache (default `~/.cache/cloudg`). [Authentication](../guides/authentication.md) covers the full credential order for each provider. ## Examples @@ -158,7 +160,6 @@ from pydantic import BaseModel from cloudg import CloudGConfig - def unknown_keys(data: object, model: type[BaseModel], path: str = "") -> list[str]: """Dotted paths of the keys in `data` that `model` does not define.""" if not isinstance(data, dict): @@ -172,7 +173,6 @@ def unknown_keys(data: object, model: type[BaseModel], path: str = "") -> list[s found += unknown_keys(value, field.annotation, f"{path}{key}.") return found - def load_strict(path: str) -> CloudGConfig: file = Path(path) if not file.is_file(): @@ -183,7 +183,6 @@ def load_strict(path: str) -> CloudGConfig: raise ValueError("unknown config keys: " + ", ".join(bad)) return CloudGConfig.model_validate(data) - if __name__ == "__main__": try: config = load_strict(sys.argv[1] if len(sys.argv) > 1 else "config.yaml") @@ -211,12 +210,36 @@ The patterns are enforced at construction: `aws.retry_mode` must be `legacy`, `s `InventoryMapper` deep-copies the config before a run, so organization discovery (which rewrites `aws.accounts`, `aws.role_name` and possibly `aws.regions`) never changes the object you passed in. +## Reference + +::: cloudg.config.CloudGConfig + options: + members: false + +::: cloudg.config.load_config + ## Related -:::links -- [config.yaml reference](/reference/config/providers/) Every key, its type and default. -- [Rate limit settings](/reference/config/ratelimit/) The `ratelimit` section in full. -- [Authentication](/guides/authentication/) Credential order for AWS, Azure and GCP. -- [CloudGEngine](/api/cloudgengine/) What runs on the config. -- [Rate limits and throttling](/guides/rate-limits/) When to change the `ratelimit` defaults. -::: +
    + +- [**config.yaml reference**](../reference/config/providers.md) + + Every key, its type and default. + +- [**Rate limit settings**](../reference/config/ratelimit.md) + + The `ratelimit` section in full. + +- [**Authentication**](../guides/authentication.md) + + Credential order for AWS, Azure and GCP. + +- [**CloudGEngine**](cloudgengine.md) + + What runs on the config. + +- [**Rate limits and throttling**](../guides/rate-limits.md) + + When to change the `ratelimit` defaults. + +
    diff --git a/site/content/api/cloudgengine.md b/site/content/api/cloudgengine.md index f9cc192..b2c7ec9 100644 --- a/site/content/api/cloudgengine.md +++ b/site/content/api/cloudgengine.md @@ -1,14 +1,15 @@ --- -object: cloudg.api.CloudGEngine -members: [run_pipeline, run_pipeline_sync, collect, scan, analyze, ingest_reports, normalise_findings, run_from_reports, run_from_reports_sync, map_inventory, map_inventory_sync] -lede: "The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: call the whole pipeline in one line, or drive each phase yourself." +description: 'The engine behind cloudg run, cloudg ingest and cloudg map: call the whole pipeline in one line, or drive each phase yourself.' --- +The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: call the whole pipeline in one line, or drive each phase yourself. +{ .lede } + ## Which method to call -`CloudGEngine` takes one argument, a [`CloudGConfig`](/api/cloudgconfig/), and keeps it on `engine.config`. Everything else is a method call. The methods fall into three groups, and they share very little state: the engine holds the config and the six [event hooks](/api/event-hooks/), nothing more. You can call `collect()` twice, or call `analyze()` on assets you built by hand, and nothing on the engine gets in the way. +`CloudGEngine` takes one argument, a [`CloudGConfig`](cloudgconfig.md), and keeps it on `engine.config`. Everything else is a method call. The methods fall into three groups, and they share very little state: the engine holds the config and the six [event hooks](event-hooks.md), nothing more. You can call `collect()` twice, or call `analyze()` on assets you built by hand, and nothing on the engine gets in the way. -```mermaid caption="What each entry point runs" +```mermaid flowchart LR RP["run_pipeline()"] --> C["collect()"] RP --> S["scan()"] @@ -21,6 +22,8 @@ flowchart LR RFR --> R MI["map_inventory()"] --> IM["InventoryMapper"] ``` +What each entry point runs +{ .diagram-caption } | You have | Call | Cloud access | Scanner binaries | |---|---|---|---| @@ -40,10 +43,10 @@ The `*_sync` variants wrap the async method in `asyncio.run()`. Use them from pl 1. `collect()` runs the collectors for every provider in `config.providers`, across `aws.regions`, `aws.accounts`, the Azure subscriptions and the GCP projects you configured. 2. `scan()` builds a graph of the collected assets and runs the reachability analysis on it, then starts every scanner in `scanners.enabled` in a thread pool with one worker per job: Prowler and ScoutSuite once per provider, with `aws.profile` and the rest of the configured AWS credentials and regions; Checkov once per IaC directory; Trivy once over `scanners.trivy_images`, or over the IaC directories when there are no images; scanner plugins by name. The IAM policy linter runs when `iam` is in the list and there are assets. Unknown names are logged and skipped. 3. `analyze()` rebuilds the graph and records node and edge counts, reachability findings and lateral movement paths. It then writes the ontology (`ontology.enabled`), the RAG chunks (`rag.enabled`) and the Terraform recreation (`terraform.enabled`). -4. The scanner findings and the reachability findings go through [`FindingsNormaliser`](/api/findingsnormaliser/): deduplication, cross-scanner merging, CVSS rescoring and compliance mapping. The reachability analysis ran in both step 2 and step 3. Both runs produce the same check on the same resource, so the deduplication pass folds the copies into one. +4. The scanner findings and the reachability findings go through [`FindingsNormaliser`](findingsnormaliser.md): deduplication, cross-scanner merging, CVSS rescoring and compliance mapping. The reachability analysis ran in both step 2 and step 3. Both runs produce the same check on the same resource, so the deduplication pass folds the copies into one. 5. The reports in `report.formats` are written: `findings.json` and `report.html` with the D3 graph embedded, and `topology.svg`. -Everything ends up in one [`PipelineResult`](/api/results/). +Everything ends up in one [`PipelineResult`](results.md). ### IaC targets for Checkov @@ -53,7 +56,7 @@ Everything ends up in one [`PipelineResult`](/api/results/). ### Full pipeline -This needs read access to the account (any method from [Authentication](/guides/authentication/) works) and the scanner binaries you enable. Missing binaries are skipped. +This needs read access to the account (any method from [Authentication](../guides/authentication.md) works) and the scanner binaries you enable. Missing binaries are skipped. ```python title="nightly_pipeline.py" import asyncio @@ -61,7 +64,6 @@ import json from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.profile = "security-audit" @@ -82,7 +84,6 @@ async def main() -> None: if cov.failed_services: print("incomplete:", cov.to_summary()) - asyncio.run(main()) ``` @@ -91,24 +92,7 @@ asyncio.run(main()) No credentials and no scanners: point the engine at reports produced elsewhere. Keys are tool names (`prowler`, `scoutsuite`, `checkov`, `trivy`); values are lists of files or directories. ```python title="ingest_only.py" -from cloudg import CloudGConfig, CloudGEngine - -engine = CloudGEngine(CloudGConfig()) -engine.on_phase_start = lambda phase: print("phase:", phase) - -result = engine.run_from_reports_sync( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy.json"], - }, - output_dir="./reports", -) - -print(result.total_findings, result.severity_breakdown) -print(result.report_paths) -for f in result.findings: - print(f.severity.value, f.risk_score, f.source_tool, f.title) +--8<-- "ingest/ingest_only.py" ``` With one failing Prowler check, one Checkov failure and one Trivy CVE, it prints: @@ -120,7 +104,7 @@ phase: normalisation phase: reporting 3 {'CRITICAL': 1, 'HIGH': 2} {'json': PosixPath('reports/findings.json'), 'html': PosixPath('reports/report.html')} -CRITICAL 9.5 trivy [Trivy] CVE-2024-0001: openssl (alpine) +CRITICAL 9.7 trivy [Trivy] CVE-2024-0001: openssl (alpine) HIGH 7.5 prowler S3 bucket default encryption HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled ``` @@ -129,25 +113,13 @@ The result has no assets, so `graph_nodes`, `ontology_triples` and `attack_paths ### Map only -`map_inventory()` runs the deep inventory collectors and the [relationship linker](/api/relationshiplinker/). No scanner is involved. Passing `output_dir` writes the map files; passing `findings` as well writes `asset-map.json` and `compliance-map.json`. +`map_inventory()` runs the deep inventory collectors and the [relationship linker](relationshiplinker.md). No scanner is involved. Passing `output_dir` writes the map files; passing `findings` as well writes `asset-map.json` and `compliance-map.json`. ```python title="map_only.py" -from cloudg import CloudGConfig, CloudGEngine - -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["ALL"] -config.inventory.exclude_services = ["kubernetes"] - -engine = CloudGEngine(config) -inventory = engine.map_inventory_sync(output_dir="./inventory", tagging_sweep=False) - -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("internet exposed:", summary["internet_exposed"]) -print("top services:", list(summary["assets_by_service"].items())[:5]) +--8<-- "map/map_only.py" ``` -The return value is an [`InventoryResult`](/api/inventoryresult/). Unlike `collect()`, this method re-raises when the mapping fails, after calling `on_error("inventory_mapping", exc)`. +The return value is an [`InventoryResult`](inventoryresult.md). Unlike `collect()`, this method re-raises when the mapping fails, after calling `on_error("inventory_mapping", exc)`. ### Phases by hand @@ -188,7 +160,6 @@ edges = [ NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), ] - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) engine.on_phase_start = lambda phase: print("phase:", phase) @@ -206,7 +177,6 @@ async def main() -> None: print("RAG chunks:", analysis.rag_chunks_path) print(scan_result.summary["severity_breakdown"]) - asyncio.run(main()) ``` @@ -238,7 +208,7 @@ engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") engine.run_from_reports_sync({"trivy": ["./trivy.json"]}, output_dir="./reports") ``` -[Event hooks](/api/event-hooks/) lists every hook, when it fires and what it receives, with longer examples. +[Event hooks](event-hooks.md) lists every hook, when it fires and what it receives, with longer examples. ## Notes @@ -248,19 +218,55 @@ engine.run_from_reports_sync({"trivy": ["./trivy.json"]}, output_dir="./reports" `on_finding` fires once per finding as each scanner finishes in `scan()` (and when `ingest_reports()` returns); `on_scan_complete` fires once at the end. Both run before normalisation, so the findings have not been deduplicated or rescored yet. They receive deep copies, which normalisation does not change later. For the final list, read `result.findings` after the run. -`map_inventory(findings=...)` stores the overlay on the returned result as `result.asset_map` and `result.compliance_map`, with or without `output_dir`. With `output_dir` set, it also writes `asset-map.json` and `compliance-map.json` there. To overlay findings on a map you already hold, call [`InventoryMapper.export_merged()`](/api/inventorymapper/) yourself. +`map_inventory(findings=...)` stores the overlay on the returned result as `result.asset_map` and `result.compliance_map`, with or without `output_dir`. With `output_dir` set, it also writes `asset-map.json` and `compliance-map.json` there. To overlay findings on a map you already hold, call [`InventoryMapper.export_merged()`](inventorymapper.md) yourself. The Terraform recreation goes to `/terraform`, following the `output_dir` you pass to `run_pipeline()`, `analyze()` or `scan()`. A `terraform.output_dir` set in the config replaces it, except the old default `./reports/terraform`, which counts as unset. `scanners.timeout_seconds` (default 3600) is the subprocess timeout of each scanner job: each Prowler or ScoutSuite run, each Checkov directory, each Trivy image or directory. A job that runs over is killed and reported through `on_error` under its job name, and the findings it produced before that are kept. A scanner that fails to launch is reported the same way. +## Reference + +::: cloudg.api.CloudGEngine + options: + members: + - run_pipeline + - run_pipeline_sync + - collect + - scan + - analyze + - ingest_reports + - normalise_findings + - run_from_reports + - run_from_reports_sync + - map_inventory + - map_inventory_sync + ## Related -:::links -- [Event hooks](/api/event-hooks/) The six callbacks and when each one fires. -- [Result objects](/api/results/) PipelineResult, CollectionResult and AnalysisResult. -- [CloudGConfig](/api/cloudgconfig/) Building the config the engine runs on. -- [The pipeline](/guides/pipeline/) The same phases from the CLI side. -- [Python recipes](/guides/python-recipes/) Longer integration examples. -- [cloudg run](/cli/run/) The command that wraps `run_pipeline()`. -::: +
    + +- [**Event hooks**](event-hooks.md) + + The six callbacks and when each one fires. + +- [**Result objects**](results.md) + + PipelineResult, CollectionResult and AnalysisResult. + +- [**CloudGConfig**](cloudgconfig.md) + + Building the config the engine runs on. + +- [**The pipeline**](../guides/pipeline.md) + + The same phases from the CLI side. + +- [**Python recipes**](../guides/python-recipes.md) + + Longer integration examples. + +- [**cloudg run**](../cli/run.md) + + The command that wraps `run_pipeline()`. + +
    diff --git a/site/content/api/cloudontology.md b/site/content/api/cloudontology.md index 27388f9..a1d5eb1 100644 --- a/site/content/api/cloudontology.md +++ b/site/content/api/cloudontology.md @@ -1,16 +1,19 @@ --- title: CloudOntology -object: cloudg.graph.ontology.CloudOntology -members: [build, graph, triple_count, query, query_internet_exposed, query_by_relation_group, query_asset_neighbourhood, query_compliance_gaps, save, to_turtle, to_jsonld, to_rdfxml, to_ntriples, stats] -lede: Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. -since: "0.6.0" +description: Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. --- -The ontology is the semantic view of an inventory. Where [GraphBuilder](/api/graphbuilder/) keeps the collector's edge types (`SECURITY_GROUP_RULE`, `ATTACHED_TO`), `CloudOntology` reads each edge together with its endpoint types, ports and CIDR and turns it into relations that say what the edge means: `INTERNET_REACHABLE`, `ONLY_SSH`, `PROTECTED_BY_SG`, `VPC_CONTAINS_SUBNET`. `cloudg run` builds it after the scanners finish, so findings are part of the graph, and writes `ontology.ttl` and `ontology.jsonld` by default. +Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + +The ontology is the semantic view of an inventory. Where [GraphBuilder](graphbuilder.md) keeps the collector's edge types (`SECURITY_GROUP_RULE`, `ATTACHED_TO`), `CloudOntology` reads each edge together with its endpoint types, ports and CIDR and turns it into relations that say what the edge means: `INTERNET_REACHABLE`, `ONLY_SSH`, `PROTECTED_BY_SG`, `VPC_CONTAINS_SUBNET`. `cloudg run` builds it after the scanners finish, so findings are part of the graph, and writes `ontology.ttl` and `ontology.jsonld` by default. ## What build() writes -```mermaid caption="Where the triples come from" +```mermaid flowchart LR S["Schema: classes and properties"] --> G["rdflib Graph"] A["Assets"] --> I["Individuals with data properties"] --> G @@ -19,6 +22,8 @@ flowchart LR A --> AR["Asset-level relations"] --> G F["Findings"] --> X["AssetIndex resolves the asset"] --> G ``` +Where the triples come from +{ .diagram-caption } Three namespaces are bound on the graph, so they appear as prefixes in every serialisation and in your SPARQL: @@ -87,103 +92,7 @@ Names are matched without regard to case. `save()` writes Turtle, with a warning Four assets, two edges and one scanner finding that names the database by ARN. The query lists every relation in the graph with readable names on both ends, skipping the data properties. ```python title="ontology_demo.py" -"""Build the RDF ontology for a small estate, save it and query it with SPARQL.""" - -from pathlib import Path - -from rdflib import Graph - -from cloudg.graph.ontology import CloudOntology, RelationGroup -from cloudg.schema.models import ( - AssetType, - CloudAsset, - CloudProvider, - EdgeType, - Finding, - NetworkEdge, - Severity, -) - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY_ARN = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" - - -def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - **extra, - ) - - -assets = [ - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", - tags={"owner": "payments"}), - asset("orders-key", AssetType.KMS_KEY, KEY_ARN), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", - metadata={"storage_encrypted": True, "kms_key_id": KEY_ARN}), -] - -edges = [ - NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range="22", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), -] - -# A scanner finding that names the database by ARN, not by asset id -findings = [ - Finding( - resource_id=f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", - severity=Severity.HIGH, - title="RDS instance has no deletion protection", - description="Deletion protection is disabled.", - source_tool="prowler", - compliance_frameworks=["CIS"], - ) -] - -onto = CloudOntology() -onto.build(assets, edges, findings) -stats = onto.stats() -print(stats["total_triples"], "triples,", stats["classes_used"], "classes,", - stats["individuals"], "individuals") -print("groups:", dict(sorted(stats["relation_group_counts"].items()))) - -# The graph binds the cm:, cmr: and cmp: prefixes, so queries can use them -rows = onto.query(""" - SELECT ?subject ?relation ?object WHERE { - ?s ?p ?o . - FILTER(STRSTARTS(STR(?p), STR(cmp:))) - FILTER(!STRSTARTS(STR(?p), STR(cmp:has))) - FILTER(?p != cmp:isInternetExposed) - ?s cmp:hasName ?subject . - OPTIONAL { ?o cmp:hasName ?oname } - BIND(STRAFTER(STR(?p), STR(cmp:)) AS ?relation) - BIND(COALESCE(?oname, STR(?o)) AS ?object) - } ORDER BY ?subject ?relation -""") -for row in rows: - print(f" {row['subject']} {row['relation']} {row['object']}") - -print("internet reachable:", [r["name"] for r in onto.query_internet_exposed()]) -print("security group:", len(onto.query_by_relation_group(RelationGroup.SECURITY)), "triples") -print("compliance gaps:", [(r["severity"], r["resourceName"]) for r in onto.query_compliance_gaps()]) - -out = Path("ontology-out") -for fmt, name in [("turtle", "ontology.ttl"), ("json-ld", "ontology.jsonld"), - ("xml", "ontology.rdf"), ("nt", "ontology.nt")]: - onto.save(out / name, fmt=fmt) - -# Any RDF tool can read the files back; here rdflib itself -reloaded = Graph().parse(out / "ontology.ttl", format="turtle") -print("reloaded", len(reloaded), "triples from", sorted(p.name for p in out.iterdir())) +--8<-- "graph/ontology_demo.py" ``` ```console @@ -209,7 +118,7 @@ Things to notice: - 636 of the 689 triples are schema. The `stats()` counts cover classes and individuals actually used. - `ENCRYPTED_BY_KMS` points at `orders-key` because the database's `kms_key_id` matched the key asset's ARN. With no collected key, the object would be the raw key reference. - The finding attached to `orders-db` although its `resource_id` was an ARN. -- `query_internet_exposed()` lists the objects of `INTERNET_REACHABLE` triples, which here is the security group. It reads the rule edges, not the reachability walk; for "which workloads can the internet reach" use [ReachabilityAnalyzer](/api/reachabilityanalyzer/). +- `query_internet_exposed()` lists the objects of `INTERNET_REACHABLE` triples, which here is the security group. It reads the rule edges, not the reachability walk; for "which workloads can the internet reach" use [ReachabilityAnalyzer](reachabilityanalyzer.md). - The `OWNED_BY` object is a tag node (`tag_payments`) with no `cmp:hasName`, so the query printed its IRI. Saving N-Triples prints a harmless `UserWarning: NTSerializer always uses UTF-8 encoding` from rdflib. @@ -255,12 +164,48 @@ The same query works against `ontology.jsonld` with `Graph().parse("ontology.jso - Each `build()` call adds to the same graph. Create a new `CloudOntology` per inventory. - The JSON-LD output uses rdflib's serialiser without a framing context, so it is a flat list of nodes. +## Reference + +::: cloudg.graph.ontology.CloudOntology + options: + members: + - build + - graph + - triple_count + - query + - query_internet_exposed + - query_by_relation_group + - query_asset_neighbourhood + - query_compliance_gaps + - save + - to_turtle + - to_jsonld + - to_rdfxml + - to_ntriples + - stats + ## Related -:::links -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) When to use which view of the inventory. -- [RAGExporter](/api/ragexporter/) Relation-group chunks built from the same inference rules. -- [GraphBuilder](/api/graphbuilder/) The NetworkX view of the same edges. -- [cloudg run](/cli/run/) The `--ontology/--no-ontology` switch and the files it writes. -- [Output files](/reference/output-files/) Every file the pipeline writes, ontology included. -::: +
    + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + When to use which view of the inventory. + +- [**RAGExporter**](ragexporter.md) + + Relation-group chunks built from the same inference rules. + +- [**GraphBuilder**](graphbuilder.md) + + The NetworkX view of the same edges. + +- [**cloudg run**](../cli/run.md) + + The `--ontology/--no-ontology` switch and the files it writes. + +- [**Output files**](../reference/output-files.md) + + Every file the pipeline writes, ontology included. + +
    diff --git a/site/content/api/dependencygraph.md b/site/content/api/dependencygraph.md index dc88fe1..e9afaa2 100644 --- a/site/content/api/dependencygraph.md +++ b/site/content/api/dependencygraph.md @@ -1,11 +1,10 @@ --- -object: [cloudg.inventory.dependencies.DependencyGraph, cloudg.inventory.dependencies.AssetIndex] -members: - cloudg.inventory.dependencies.DependencyGraph: [find, depends_on, dependents, direct_counts, tree, shared_dependencies, blast_radius] - cloudg.inventory.dependencies.AssetIndex: [add, find, resolve, resolve_finding] -lede: "What does this asset need, and what breaks if it goes away? `DependencyGraph` answers both over any set of assets and edges; `AssetIndex` is the matcher that finds the asset a reference or a finding is about." +description: What does this asset need, and what breaks if it goes away? DependencyGraph answers both over any set of assets and edges; AssetIndex is the matcher that finds the asset a reference or a finding is about. --- +What does this asset need, and what breaks if it goes away? `DependencyGraph` answers both over any set of assets and edges; `AssetIndex` is the matcher that finds the asset a reference or a finding is about. +{ .lede } + ## From edges to dependencies Edges in an inventory map point in their natural direction: a function `ASSUMES_ROLE` a role, a queue `INVOKES` a function, a WAF `PROTECTS` an API. That direction is not always the direction of dependency. The function needs its role, so the arrow and the dependency agree. The function also needs the queue that triggers it, but the `INVOKES` arrow points from the queue to the function. @@ -18,7 +17,7 @@ Edges in an inventory map point in their natural direction: a function `ASSUMES_ | reverse: the target depends on the source | `CONTAINS`, `INVOKES`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS` | a function depends on the queue that triggers it, an API on the WAF in front of it, a resource on the stack that manages it | | ignored | `SECURITY_GROUP_RULE`, `NACL_RULE`, `INTERNET_EXPOSED` | network rules are not dependencies | -```mermaid caption="Natural edges on the left, dependency arrows on the right" +```mermaid flowchart LR subgraph natural ["inventory edges"] Q1["queue"] -->|INVOKES| F1["function"] @@ -29,12 +28,14 @@ flowchart LR F2 -->|needs| R2["role"] end ``` +Natural edges on the left, dependency arrows on the right +{ .diagram-caption } On that view, `depends_on(x)` walks upstream (everything `x` needs) and `dependents(x)` walks downstream (everything that needs `x`, its blast radius). Both are breadth-first, visit each asset once, stop at `max_depth` (default 10) and return `DependencyLink` records with `asset_id`, `via_edge`, `relationship`, `depth` and `parent_id`, the asset one hop closer to the start. Edges whose source or target is not in `assets` are skipped. With `include_hierarchy=False`, `CONTAINS` edges into `ORGANIZATION`, `ORG_UNIT` and `CLOUD_ACCOUNT` nodes are skipped too, because otherwise every account would top every blast radius list. `InventoryResult.dependency_graph()` builds the graph with this default. -This is an availability and change-impact view: what stops working when something is deleted, disabled or misconfigured. Compromise spreads differently. A stolen function credential exposes the role it assumes, which is the forward direction of `ASSUMES_ROLE` in the map itself, not a dependent of the role. For attack paths use the graph analysis in [`CloudGEngine.analyze()`](/api/cloudgengine/) or walk the identity edges directly. +This is an availability and change-impact view: what stops working when something is deleted, disabled or misconfigured. Compromise spreads differently. A stolen function credential exposes the role it assumes, which is the forward direction of `ASSUMES_ROLE` in the map itself, not a dependent of the role. For attack paths use the graph analysis in [`CloudGEngine.analyze()`](cloudgengine.md) or walk the identity edges directly. ### The reports @@ -66,58 +67,7 @@ Ambiguous names and tails never match at their own tier. Tiers 5 and 6 are also Runs offline on six linked assets. ```python title="blast_radius.py" -from cloudg.inventory import RelationshipLinker -from cloudg.inventory.dependencies import DependencyGraph -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" -ROLE = f"arn:aws:iam::{ACCOUNT}:role/orders-worker-role" -QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" -TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" -FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" -API = f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3d4e5" - - -def aws(arn, name, asset_type, *relations, region=REGION, exposed=False): - return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, - region=region, account_id=ACCOUNT, is_internet_exposed=exposed, - metadata={"relations": list(relations)}) - - -def rel(target, edge, relationship=None, reverse=False): - return {"target": target, "edge": edge, "relationship": relationship, "reverse": reverse} - - -assets = [ - aws(KEY, "orders-key", AssetType.KMS_KEY), - aws(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global"), - aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), - aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), - aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, - rel(ROLE, "ASSUMES_ROLE", "RUNS_ON"), - rel(TABLE, "REFERENCES", "WRITES_TO"), - rel(QUEUE, "INVOKES", "TRIGGERED_BY", reverse=True)), - aws(API, "orders-api", AssetType.API_GATEWAY, rel(FUNCTION, "INVOKES"), exposed=True), -] -edges = RelationshipLinker(assets).link() -graph = DependencyGraph(assets, edges) - -key = graph.find("orders-key") # ID, ARN, unique name or unique ARN tail -print("If", key.name, "is disabled:") -for link in graph.dependents(key.id): - asset = graph.assets[link.asset_id] - parent = graph.assets[link.parent_id].name - print(f" depth {link.depth}: {asset.name} ({asset.asset_type.value}) needs {parent} via {link.via_edge}") - -worker = graph.find(FUNCTION) -print("orders-worker needs:", [graph.assets[l.asset_id].name for l in graph.depends_on(worker.id)]) -print("direct (needs, needed by):", graph.direct_counts(worker.id)) - -for row in graph.blast_radius(top=3): - print(f"{row['name']:20} {row['transitive_dependents']} dependents, " - f"{row['internet_exposed_dependents']} internet-exposed") +--8<-- "map/blast_radius.py" ``` ```console @@ -141,12 +91,10 @@ Two results are worth a second look. The function appears at depth 2 under the k from cloudg.inventory.dependencies import AssetIndex from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, Finding, Severity - def asset(arn, name, asset_type): return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, region="eu-west-1", account_id="123456789012") - assets = [ asset("arn:aws:rds:eu-west-1:123456789012:db:orders", "orders", AssetType.RDS_INSTANCE), asset("arn:aws:dynamodb:eu-west-1:123456789012:table/orders", "orders", AssetType.DYNAMODB_TABLE), @@ -156,12 +104,10 @@ assets = [ index = AssetIndex(assets) names = {a.id: f"{a.asset_type.value}:{a.name}" for a in assets} - def finding(resource_id, resource_arn=None): return Finding(resource_id=resource_id, resource_arn=resource_arn, severity=Severity.MEDIUM, title="example", description="example", source_tool="custom") - cases = [ finding("x", "arn:aws:rds:eu-west-1:123456789012:db:orders"), # exact ARN finding("orders"), # name shared by two assets @@ -194,11 +140,45 @@ The second line shows the limit of tier 3. `orders` is the name of two assets, s The module also has two functions that work on plain asset and edge lists: `cross_account_edges(assets, edges)` (every edge between two different accounts, hierarchy containment excluded) and `security_coverage(assets, edges)` (security services by account and region, the gaps, workloads without vulnerability scanning, internet-facing entry points without a WAF). `InventoryResult.analysis()` calls both. +## Reference + +::: cloudg.inventory.dependencies.DependencyGraph + options: + members: + - find + - depends_on + - dependents + - direct_counts + - tree + - shared_dependencies + - blast_radius + +::: cloudg.inventory.dependencies.AssetIndex + options: + members: + - add + - find + - resolve + - resolve_finding + ## Related -:::links -- [Dependencies and blast radius](/guides/dependencies/) The guide, with `cloudg deps`. -- [cloudg deps](/cli/deps/) The CLI front end to `tree()`. -- [InventoryResult](/api/inventoryresult/) `dependency_graph()` and `analysis()`. -- [RelationshipLinker](/api/relationshiplinker/) Where the edges come from. -::: +
    + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + The guide, with `cloudg deps`. + +- [**cloudg deps**](../cli/deps.md) + + The CLI front end to `tree()`. + +- [**InventoryResult**](inventoryresult.md) + + `dependency_graph()` and `analysis()`. + +- [**RelationshipLinker**](relationshiplinker.md) + + Where the edges come from. + +
    diff --git a/site/content/api/entry-points.md b/site/content/api/entry-points.md index e17f42b..fa0584e 100644 --- a/site/content/api/entry-points.md +++ b/site/content/api/entry-points.md @@ -1,10 +1,11 @@ --- title: Entry points -object: cloudg.registry.PluginRegistry -members: [get_collector, get_scanner, list_collectors, list_scanners] -lede: Ship a collector or a scanner as its own package, and `PluginRegistry` finds it through Python entry points. +description: Ship a collector or a scanner as its own package, and PluginRegistry finds it through Python entry points. --- +Ship a collector or a scanner as its own package, and `PluginRegistry` finds it through Python entry points. +{ .lede } + cloudg looks for plugins in two entry point groups, `cloudg.collectors` and `cloudg.scanners`. A package that declares an entry point in one of them becomes visible to `PluginRegistry` as soon as it is installed in the same environment, with no change to cloudg. cloudg registers its own built-ins the same way in its `pyproject.toml`: ```toml title="pyproject.toml" @@ -27,7 +28,7 @@ Scanner plugins run wherever the built-in scanners run. Name one in `scanners.en Discovery is lazy and happens once per `PluginRegistry` instance, on the first `get_*` or `list_*` call. -```mermaid caption="What the first get or list call does" +```mermaid flowchart TD A["First get_* or list_* call"] --> B["Read entry points in cloudg.collectors"] B --> N{"Built-in name?"} @@ -41,6 +42,8 @@ flowchart TD F --> H["Import the built-ins from the table in registry.py"] H --> I["Cached for this registry"] ``` +What the first get or list call does +{ .diagram-caption } The built-ins always come from the table in `cloudg/registry.py`, which maps the same eight names as the entry points above to the same classes, so they load the same way from an installed package and from a source checkout that was never installed. A built-in whose import fails (a missing optional SDK, say) is left out quietly, at debug log level. @@ -121,7 +124,6 @@ from cloudg.collectors.base import BaseCollector from cloudg.inventory.linker import RelationshipLinker from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, NetworkEdge - class CMDBCollector(BaseCollector): """Turns a CMDB JSON export into CloudAssets and linked edges.""" @@ -174,7 +176,6 @@ from cloudg.schema.models import CloudAsset, Finding, Severity CHECK_ID = "TAGPOLICY_001" - class TagPolicyScanner: """Follows the built-in scanners' shape: is_available, run, parse_report.""" @@ -213,13 +214,17 @@ class TagPolicyScanner: Install it into the environment cloudg runs in, editable while you work on it: -```bash tab="pip" -pip install -e ./cloudg-cmdb -``` +=== "pip" -```bash tab="uv" -uv pip install -e ./cloudg-cmdb -``` + ```bash + pip install -e ./cloudg-cmdb + ``` + +=== "uv" + + ```bash + uv pip install -e ./cloudg-cmdb + ``` ### Use the plugins @@ -296,15 +301,41 @@ To merge your scanner's findings with an equivalent check from Prowler, Checkov - A plugin that fails to import is logged as a warning (`Failed to load scanner plugin ...`) and skipped; the others still load. It goes to the `cloudg.registry` logger, so configure `logging` in your script to see it. - `get_collector()` and `get_scanner()` raise `KeyError` for an unknown name, and the message lists the names that are registered. -- The relation format the linker reads (`target`, `edge`, `relationship`, `reverse`, `properties`) is specified in section 7 of the inventory reference, and the [RelationshipLinker](/api/relationshiplinker/) page shows the linker itself. Unknown `edge` values are linked as `REFERENCES`. -- Assets from a plugin collector work with everything downstream: [GraphBuilder](/api/graphbuilder/), the reachability analysis, the ontology, and `InventoryResult(assets=..., edges=...).export(dir)` for an `inventory-map.json` that `cloudg deps` can read. +- The relation format the linker reads (`target`, `edge`, `relationship`, `reverse`, `properties`) is specified in section 7 of the inventory reference, and the [RelationshipLinker](relationshiplinker.md) page shows the linker itself. Unknown `edge` values are linked as `REFERENCES`. +- Assets from a plugin collector work with everything downstream: [GraphBuilder](graphbuilder.md), the reachability analysis, the ontology, and `InventoryResult(assets=..., edges=...).export(dir)` for an `inventory-map.json` that `cloudg deps` can read. + +## Reference + +::: cloudg.registry.PluginRegistry + options: + members: + - get_collector + - get_scanner + - list_collectors + - list_scanners ## Related -:::links -- [FindingsNormaliser](/api/findingsnormaliser/) Dedupe, scoring and compliance mapping for any findings list. -- [Data models](/api/models/) `CloudAsset`, `NetworkEdge` and `Finding`, field by field. -- [RelationshipLinker](/api/relationshiplinker/) How declared relations become edges. -- [Python recipes](/guides/python-recipes/) More ways to combine the pipeline phases from code. -- [cloudg report](/cli/report/) Render HTML and SVG from a `findings.json`. -::: +
    + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Dedupe, scoring and compliance mapping for any findings list. + +- [**Data models**](models.md) + + `CloudAsset`, `NetworkEdge` and `Finding`, field by field. + +- [**RelationshipLinker**](relationshiplinker.md) + + How declared relations become edges. + +- [**Python recipes**](../guides/python-recipes.md) + + More ways to combine the pipeline phases from code. + +- [**cloudg report**](../cli/report.md) + + Render HTML and SVG from a `findings.json`. + +
    diff --git a/site/content/api/event-hooks.md b/site/content/api/event-hooks.md index 9ecfa07..4762c3a 100644 --- a/site/content/api/event-hooks.md +++ b/site/content/api/event-hooks.md @@ -1,8 +1,11 @@ --- title: Event hooks -lede: "Six optional callbacks on `CloudGEngine` report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it." +description: 'Six optional callbacks on CloudGEngine report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it.' --- +Six optional callbacks on `CloudGEngine` report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it. +{ .lede } + ## The six hooks Every hook starts as `None` in `CloudGEngine.__init__`. Assign a callable before you call a method, and reassign or reset it to `None` between runs as you like. The callback types are defined in `cloudg.api` as `typing.Callable` aliases, so you can annotate your own handlers with them. @@ -30,7 +33,7 @@ on_error: OnError = alert The diagram follows `run_pipeline()`, the one method that can call all six. -```mermaid caption="Hook calls during run_pipeline()" +```mermaid sequenceDiagram participant App as Your code participant E as CloudGEngine @@ -49,6 +52,8 @@ sequenceDiagram E->>App: on_phase_start("reporting") E-->>App: PipelineResult ``` +Hook calls during run_pipeline() +{ .diagram-caption } Other entry points use a subset: @@ -79,7 +84,7 @@ Other entry points use a subset: | `prowler-`, `scoutsuite-` | one scanner run per provider, such as `prowler-aws`, including a run killed at `scanners.timeout_seconds` | | `checkov-` | one Checkov run per IaC directory | | `trivy`, `trivy-fs`, `iam` | the Trivy image scan, the Trivy filesystem scan, the IAM policy linter | -| a plugin's name | a [scanner plugin](/api/entry-points/) run | +| a plugin's name | a [scanner plugin](entry-points.md) run | | `graph_build`, `ontology`, `rag_export` | the steps of `analyze()` | | `normalisation`, `reporting` | the last two pipeline phases | @@ -162,14 +167,12 @@ SPOOL = "./cloudg-findings.jsonl" # tailed by the SIEM agent (Fluent Bit, Splu STOP = object() outbox: queue.Queue = queue.Queue() - def writer() -> None: with open(SPOOL, "a", encoding="utf-8") as spool: while (item := outbox.get()) is not STOP: spool.write(json.dumps(item) + "\n") spool.flush() - thread = threading.Thread(target=writer, daemon=True) thread.start() @@ -189,74 +192,44 @@ thread.join(timeout=30) print(f"forwarded {result.total_findings} findings to {SPOOL}") ``` -Each line is one finding as `Finding.model_dump(mode="json")` produces it (see [Data models](/api/models/) for the fields), plus `"kind": "finding"`. +Each line is one finding as `Finding.model_dump(mode="json")` produces it (see [Data models](models.md) for the fields), plus `"kind": "finding"`. ### Fail fast Raising inside `on_error` does nothing, because the engine swallows it. Record the failure instead, and check between phases you drive yourself. `run_pipeline()` has no such checkpoints, so this uses the individual phase methods. ```python title="fail_fast.py" -import asyncio -import sys - -from cloudg import CloudGConfig, CloudGEngine - - -class PhaseFailed(RuntimeError): - pass +--8<-- "errors/fail_fast.py" +``` +A failed provider never reaches `on_error`; it is a `FAILED` entry in `collection.coverage`. That is why the script checks coverage as well as the failure list after collecting. On a failure the script prints one line in the form `cloudg: failed during : ` to stderr and exits with status 2, so a CI job or a scheduler sees the run as failed. -async def main() -> int: - engine = CloudGEngine(CloudGConfig(providers=["aws"])) - failures: list[tuple[str, Exception]] = [] - engine.on_error = lambda phase, exc: failures.append((phase, exc)) +## Notes - def checkpoint(step: str) -> None: - if failures: - phase, exc = failures[0] - raise PhaseFailed(f"{phase} failed during {step}: {exc}") +Hooks are per engine instance. Two engines in one process do not share them, and there is no global registry. - collection = await engine.collect() - checkpoint("collection") - broken = [c.to_summary() for c in collection.coverage if c.failed_services] - if broken or not collection.assets: - raise PhaseFailed(f"collection incomplete: {broken or 'no assets'}") +`on_collection_complete` does not fire when `collect()` fails as a whole; you get `on_error("collection", exc)` and an empty `CollectionResult` instead. `on_analysis_complete` fires even when every analysis step failed, with whatever was filled in. - findings = await engine.scan(collection.assets, collection.edges, output_dir="./reports") - checkpoint("scanning") +Nothing fires for findings produced by `analyze()`. Its reachability findings reach you through `AnalysisResult.reachability_findings` and, after `run_pipeline()`, through `PipelineResult.findings`. - analysis = await engine.analyze(collection.assets, collection.edges, findings, "./reports") - checkpoint("analysis") +## Related - scan_result = engine.normalise_findings( - findings + analysis.reachability_findings, assets=collection.assets - ) - print(scan_result.summary) - return 0 +
    +- [**CloudGEngine**](cloudgengine.md) -try: - sys.exit(asyncio.run(main())) -except PhaseFailed as exc: - print(f"cloudg: {exc}", file=sys.stderr) - sys.exit(2) -``` + The methods that call the hooks. -A failed provider never reaches `on_error`; it is a `FAILED` entry in `collection.coverage`. That is why the script checks coverage as well as the failure list after collecting. On a failure the script prints one line in the form `cloudg: failed during : ` to stderr and exits with status 2, so a CI job or a scheduler sees the run as failed. +- [**Result objects**](results.md) -## Notes + What `on_collection_complete` and `on_analysis_complete` receive. -Hooks are per engine instance. Two engines in one process do not share them, and there is no global registry. +- [**Data models**](models.md) -`on_collection_complete` does not fire when `collect()` fails as a whole; you get `on_error("collection", exc)` and an empty `CollectionResult` instead. `on_analysis_complete` fires even when every analysis step failed, with whatever was filled in. + The `Finding` that `on_finding` receives. -Nothing fires for findings produced by `analyze()`. Its reachability findings reach you through `AnalysisResult.reachability_findings` and, after `run_pipeline()`, through `PipelineResult.findings`. +- [**CI and automation**](../guides/ci-automation.md) -## Related + Running cloudg unattended. -:::links -- [CloudGEngine](/api/cloudgengine/) The methods that call the hooks. -- [Result objects](/api/results/) What `on_collection_complete` and `on_analysis_complete` receive. -- [Data models](/api/models/) The `Finding` that `on_finding` receives. -- [CI and automation](/guides/ci-automation/) Running cloudg unattended. -::: +
    diff --git a/site/content/api/exceptions.md b/site/content/api/exceptions.md new file mode 100644 index 0000000..18f8b8e --- /dev/null +++ b/site/content/api/exceptions.md @@ -0,0 +1,205 @@ +--- +title: Exceptions +description: Every exception cloudg raises on purpose, where each one comes from, and how to get exceptions from the scanner wrappers instead of recorded failures. +--- + +Every exception cloudg raises on purpose lives in `cloudg.exceptions` and derives from `CloudGError`, so one `except CloudGError` catches anything cloudg itself raised. Errors from the cloud SDKs and the operating system pass through untouched. +{ .lede } + +**Module** `cloudg.exceptions` · **Also exported from** `cloudg` · **Last changed** [v0.6.1](../changelog.md#0.6.1) +{ .page-meta } + +## The class tree + +The narrower classes keep the built-in base they always had, so code written against an older release still works: `ProviderSDKMissingError` is still an `ImportError`, `CredentialError` is still a `RuntimeError`, and `ConfigurationError` is still a `ValueError`. + +```mermaid +flowchart LR + E["CloudGError"] --> CFG["ConfigurationError
    (ValueError)"] + E --> SDK["ProviderSDKMissingError
    (ImportError)"] + E --> COL["CollectionError
    (RuntimeError)"] + COL --> CRED["CredentialError"] + COL --> ORG["OrganizationScopeError
    (ValueError)"] + E --> RES["ResilienceError"] + E --> LOR["LiveOperationRejected"] + E --> SCN["ScannerError"] + SCN --> NI["ScannerNotInstalledError"] + SCN --> TO["ScannerTimeoutError"] + SCN --> EX["ScannerExecutionError"] + SCN --> OUT["ScannerOutputError"] + OUT --> RPE["ReportParseError"] + SCN --> SF["ScannerFailed
    (RuntimeError)"] +``` +The built-in base each class keeps is in brackets. +{ .diagram-caption } + +`ResilienceError` and `LiveOperationRejected` live in `cloudg.resilience`, next to their subclasses (`CircuitOpenError`, `DeadlineExceededError`, `RetryBudgetExhaustedError`, `CooldownActive`, `LiveOperationBusy`, `CallerQuotaExceeded`, `LiveOperationTimeout`). [cloudg.resilience](resilience.md) covers them. Everything else imports from `cloudg.exceptions` or straight from `cloudg`: + +```python +from cloudg import CloudGError, ScannerError +from cloudg.exceptions import OrganizationScopeError, ScannerTimeoutError +from cloudg.resilience import CircuitOpenError, LiveOperationRejected +``` + +## When each one is raised + +| Exception | Raised when | Useful attributes | +|---|---|---| +| `ConfigurationError` | A configuration value is missing, malformed or contradicts another. `cloudg.ingest.parse_report()` raises it for a tool name it does not support. | | +| `ProviderSDKMissingError` | A provider SDK is not installed: boto3 for AWS, azure-identity for Azure, google-auth or google-cloud-asset for GCP. The message names the pip extra. | `provider`, `extra`, `missing` | +| `CollectionError` | Collecting from a provider failed as a whole, for example when GCP rejects both ListAssets and SearchAllResources for a scope. | | +| `CredentialError` | Role assumption (`AssumeRole`, `AssumeRoleWithWebIdentity`) failed, no credentials resolved for `aws.role_arn`, or an account was asked for without a role to reach it. | `provider`, `detail` | +| `OrganizationScopeError` | An `--ou` (`aws.organization.include_ous`) filter matches nothing in the organization. cloudg stops instead of quietly mapping only the caller's account. | | +| `ScannerNotInstalledError` | A scanner wrapper in strict mode finds no executable on `PATH`. | `executable`, `install_hint` | +| `ScannerTimeoutError` | A scanner ran longer than `timeout_seconds` and was killed. | `timeout_seconds`, `target` | +| `ScannerExecutionError` | A scanner process could not be started, or crashed. | `returncode`, `output`, `target` | +| `ScannerOutputError` | A scanner ran, but its output could not be read or parsed. | `path` | +| `ReportParseError` | A report handed to a wrapper's `parse_report(path, strict=True)` is unreadable, or a directory holds no report. | `path` | +| `ScannerFailed` | A scanner did not finish, as reported by `CloudGEngine.scan()` through `on_error`, or by `raise_for_errors()` when a wrapper recorded several failures. | `findings`, `errors`, `failures` | + +Every `ScannerError` also carries `scanner` (the wrapper's short name, such as `"checkov"`), `detail` (the reason in a few words) and `target` (the image, directory or file being scanned, when there is one). `str(exc)` is `": "`. + +## Recorded failures and strict mode + +The scanner wrappers record failures instead of raising them, so one broken scanner never stops a pipeline. Each wrapper keeps two lists: + +- `errors`: why the last run did not finish (timeouts, launch failures), as text; +- `failures`: every `ScannerError` the wrapper met, including output it could not parse. Unparseable output does not stop a run, so it is in `failures` but not in `errors`. + +There are two ways to turn those records into exceptions: + +```mermaid +flowchart TD + R["scanner.run()"] --> Q{"strict=True?"} + Q -->|yes| RAISE["first failure raised at once"] + Q -->|no| REC["failure appended to .failures"] + REC --> D["run finishes, partial findings returned"] + D --> RFE["scanner.raise_for_errors()"] + RFE --> ONE["one failure: raised as it is"] + RFE --> MANY["several: ScannerFailed with .failures"] + RFE --> NONE["none: returns quietly"] +``` + +A missing executable is a warning by default and is not recorded as a failure, because "Trivy is not installed" is a setup choice rather than a broken run. Pass `strict=True` to get a `ScannerNotInstalledError` instead. + +## Examples + +Each tab is a complete script. They run without cloud credentials: with no scanner installed, the strict scanner prints the install hint, and the report parser needs only a file. + +=== "strict=True" + + The first failure is raised as soon as it happens, so the `except` clauses read like an ordinary call. + + ```python title="strict_scanner.py" + --8<-- "errors/strict_scanner.py" + ``` + + Without Checkov on `PATH` it prints: + + ```console + $ python strict_scanner.py + checkov is missing. pip install checkov + ``` + +=== "raise_for_errors()" + + Run everything first and keep the partial results, then decide what counts as a failure. + + ```python title="raise_for_errors.py" + --8<-- "errors/raise_for_errors.py" + ``` + + With a `trivy` on `PATH` that prints something other than JSON, both images fail to parse, so the two failures come back together as one `ScannerFailed`: + + ```console + $ python raise_for_errors.py + 0 findings, 2 failures recorded + - ScannerOutputError output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + - ScannerOutputError output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +=== "Around CloudGEngine" + + `CloudGEngine` reports failures instead of raising them: `collect()` records them per service in the coverage records, and scanner failures reach the `on_error` hook as exception objects. A missing SDK or a refused role is easiest to catch before the run, by building the session yourself. + + ```python title="engine_errors.py" + --8<-- "errors/engine_errors.py" + ``` + + Without boto3 it stops at the first check: + + ```console + $ python engine_errors.py + boto3 is not installed: pip install 'cloudg[aws]' + ``` + + `build_aws_session()` only contacts AWS when it has a role to assume, so a wrong key surfaces later, as failed services in `result.coverage` and lines in `result.errors`. + +=== "parse_report(path, strict=True)" + + Each scanner wrapper has a `parse_report` class method for output that already exists. With `strict=True`, a file that is not a report raises `ReportParseError` instead of being skipped with a warning. + + ```python title="strict_parse_report.py" + --8<-- "errors/strict_parse_report.py" + ``` + + ```console + $ echo "not json" > results_json.json + $ python strict_parse_report.py results_json.json + checkov report results_json.json is unreadable: output is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + + `cloudg.ingest.parse_report(tool, path)`, the function behind `cloudg ingest`, has no `strict` flag: it logs and skips unreadable files, and raises `ConfigurationError` (a `ValueError`) for an unknown tool and `FileNotFoundError` for a missing path. Call the wrapper's own `parse_report` when a bad file should stop the job. + +## Catching everything cloudg raises + +```python title="catch_all.py" +from cloudg import CloudGError +from cloudg.ingest import parse_report + +try: + findings = parse_report("trivvy", "./trivy.json") +except CloudGError as exc: + print("cloudg refused:", exc) +except FileNotFoundError as exc: + print("no such report:", exc) +``` + +```console +$ python catch_all.py +cloudg refused: Unsupported tool 'trivvy'. Supported: prowler, scoutsuite, checkov, trivy +``` + +The misspelt tool name raises `ConfigurationError`, which the first clause catches. A missing file raises the built-in `FileNotFoundError`, which is not a `CloudGError`: `except CloudGError` never hides an operating system error, an SDK exception or a bug in your own code. + +## Reference + +::: cloudg.exceptions + options: + show_root_heading: false + show_source: true + members_order: source + +## Related + +
    + +- [**cloudg.resilience**](resilience.md) + + The rate limiter, circuit breakers and the errors they raise. + +- [**parse_report**](parse-report.md) + + Turning existing scanner output into findings. + +- [**Event hooks**](event-hooks.md) + + `on_error` and the other callbacks on `CloudGEngine`. + +- [**Running scanners**](../guides/running-scanners.md) + + What each scanner needs and how failures show up on the CLI. + +
    diff --git a/site/content/api/findingsnormaliser.md b/site/content/api/findingsnormaliser.md index b145a17..56b6e22 100644 --- a/site/content/api/findingsnormaliser.md +++ b/site/content/api/findingsnormaliser.md @@ -1,16 +1,17 @@ --- -object: cloudg.normaliser.FindingsNormaliser -members: [normalise] -lede: "Takes findings from any number of scanners and returns one `ScanResult`: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped." +description: 'Takes findings from any number of scanners and returns one ScanResult: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped.' --- +Takes findings from any number of scanners and returns one `ScanResult`: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped. +{ .lede } + ## What normalise does `FindingsNormaliser(rules_dir=None, load_external=True)` loads its rules once, at construction: every `*.yaml` file under `rules_dir` (searched recursively) as a compliance ruleset, and `check_equivalence.yaml` from the top of `rules_dir` as the cross-scanner equivalence map. With `rules_dir=None` it uses the directory shipped inside the package, `cloudg/rules/`, which holds CIS, NIST 800-53, PCI DSS, SOC 2, ISO 27001, HIPAA and GDPR rulesets plus the per-provider framework mappings under `frameworks/`. `load_external=False` (the `rulesets.load_external` key) skips the rulesets and keeps only the equivalence map. `normalise(*finding_lists, assets=None, passed_checks=None)` accepts any number of lists, one per source, and runs them through one pass: -```mermaid caption="FindingsNormaliser.normalise()" +```mermaid flowchart TD A["all lists concatenated"] --> B["pass 1: dedupe within a scanner"] B --> C["pass 2: merge across scanners"] @@ -20,6 +21,8 @@ flowchart TD F --> G["one ComplianceResult per framework control"] G --> H["ScanResult"] ``` +FindingsNormaliser.normalise() +{ .diagram-caption } ### Deduplication @@ -31,7 +34,7 @@ A merge keeps the finding with the higher severity, joins the tool names into `s ### Scoring -A `cvss_score` of 9.0 or more sets the severity to `CRITICAL`. A score from 7.0 to 8.9 raises anything below `HIGH` to `HIGH`. Severity never goes down. `risk_score` on the finding is computed from severity and CVSS whenever you read it; see [Data models](/api/models/). +A `cvss_score` of 9.0 or more sets the severity to `CRITICAL`. A score from 7.0 to 8.9 raises anything below `HIGH` to `HIGH`. Severity never goes down. `risk_score` on the finding is computed from severity and CVSS whenever you read it; see [Data models](models.md). ### Compliance mapping @@ -53,37 +56,7 @@ These results have status `FAIL`. Passing checks add `PASS` results: a ruleset c Five findings about one bucket and one image, constructed by hand so each rule shows. Runs offline. ```python title="normalise_demo.py" -from cloudg.normaliser import FindingsNormaliser -from cloudg.schema.models import Finding, Severity - -BUCKET = "arn:aws:s3:::orders-archive" - - -def finding(tool, check, title, severity, **extra): - return Finding(resource_id=BUCKET, resource_arn=BUCKET, source_tool=tool, - source_finding_id=check, title=title, description=title, - severity=severity, **extra) - - -prowler = [ - finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", - "S3 bucket default encryption", Severity.MEDIUM), - # the same check reported twice (two output formats, say): a within-scanner duplicate - finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", - "S3 bucket default encryption", Severity.MEDIUM), -] -trivy = [ - # equivalent check (AVD-AWS-0088), same resource, same normalised title - finding("trivy", "AVD-AWS-0088", "[Trivy] S3 bucket default encryption", Severity.HIGH), - # a CVE with a CVSS score of 9.8 reported as MEDIUM - finding("trivy", "CVE-2024-0001", "openssl: buffer overflow", Severity.MEDIUM, cvss_score=9.8), -] -untagged = [finding("custom", None, "CloudTrail logging disabled in eu-west-1", Severity.LOW)] - -result = FindingsNormaliser().normalise(prowler, trivy, untagged) -for f in result.findings: - print(f"{f.severity.value:8} {f.risk_score:4} {f.source_tool:16} {f.title} {f.compliance_frameworks[:4]}") -print(len(result.compliance), "compliance results") +--8<-- "ingest/normalise_demo.py" ``` ```console @@ -171,11 +144,31 @@ Merging across scanners needs the same resource identifier on both sides. Checko The normaliser keeps the last result on the instance, but nothing reads it back; treat `normalise()` as a function and use its return value. One instance can normalise any number of batches. +## Reference + +::: cloudg.normaliser.FindingsNormaliser + options: + members: + - normalise + ## Related -:::links -- [Compliance frameworks](/guides/compliance/) Which frameworks cloudg maps and how to read the results. -- [parse_report](/api/parse-report/) Getting findings out of scanner output. -- [Data models](/api/models/) Finding, ComplianceResult and ScanResult. -- [Ingesting existing output](/guides/ingesting/) The CLI path through the same normaliser. -::: +
    + +- [**Compliance frameworks**](../guides/compliance.md) + + Which frameworks cloudg maps and how to read the results. + +- [**parse_report**](parse-report.md) + + Getting findings out of scanner output. + +- [**Data models**](models.md) + + Finding, ComplianceResult and ScanResult. + +- [**Ingesting existing output**](../guides/ingesting.md) + + The CLI path through the same normaliser. + +
    diff --git a/site/content/api/graphbuilder.md b/site/content/api/graphbuilder.md index be6e973..e031ba4 100644 --- a/site/content/api/graphbuilder.md +++ b/site/content/api/graphbuilder.md @@ -1,11 +1,14 @@ --- title: GraphBuilder -object: cloudg.graph.builder.GraphBuilder -members: [build, graph, find_attack_paths, find_lateral_movement_paths, compute_centrality, subgraph, to_d3_json, to_cytoscape_json, to_json_str, save_graphml, load_graphml] -lede: Turns a list of `CloudAsset` and `NetworkEdge` objects into a NetworkX `DiGraph`, then searches, scores and exports it. -since: "0.6.0" +description: Turns a list of CloudAsset and NetworkEdge objects into a NetworkX DiGraph, then searches, scores and exports it. --- +Turns a list of `CloudAsset` and `NetworkEdge` objects into a NetworkX `DiGraph`, then searches, scores and exports it. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `GraphBuilder` is the first thing `cloudg run` does after collection, and `InventoryResult.export()` uses it to write `inventory-map.graphml` and `inventory-graph.json`. It needs no credentials and no network: hand it assets and edges from a collector, a saved `inventory-map.json` or your own code, and it gives you a plain `networkx.DiGraph` you can keep working on with the whole NetworkX library. ## What build() does @@ -14,7 +17,7 @@ Each asset becomes a node keyed by `asset.id`. The node carries `name`, `asset_t Each edge then joins `source_id` to `target_id`. Collectors write edges whose endpoints are not assets at all: a CIDR such as `0.0.0.0/0`, an Azure service tag or an unresolved reference. For those, `build()` creates a placeholder node with `asset_type` and `provider` set to `EXTERNAL` and `is_external=True`. -```mermaid caption="How build() places one NetworkEdge" +```mermaid flowchart TD A["NetworkEdge"] --> B{"Endpoints are nodes?"} B -->|no| C["Add EXTERNAL placeholder"] --> D @@ -24,6 +27,8 @@ flowchart TD F -->|yes| G["Merge ports, protocols, descriptions"] F -->|no| H["Replace attributes"] ``` +How build() places one NetworkEdge +{ .diagram-caption } A `DiGraph` holds one edge per ordered pair. Collectors often emit several rules for one pair (two ingress rules from `0.0.0.0/0` to the same security group, for 22 and 443), so since 0.6.0 `SECURITY_GROUP_RULE`, `NACL_RULE` and `INTERNET_EXPOSED` edges on the same pair are merged when their `cidr` and `direction` match: @@ -41,114 +46,10 @@ An empty port list on a TCP, UDP or all-protocol filter rule (GCP writes these) ### Build, search and export a small estate -The script builds eight assets and eleven edges by hand, so it runs offline. It also groups the nodes into communities with python-louvain, which cloudg installs; the [RAG exporter](/api/ragexporter/) runs the same `best_partition` call for its community chunks. +The script builds eight assets and eleven edges by hand, so it runs offline. It also groups the nodes into communities with python-louvain, which cloudg installs; the [RAG exporter](ragexporter.md) runs the same `best_partition` call for its community chunks. ```python title="build_graph.py" -"""Build a graph from a hand-made estate and look at it from a few angles.""" - -import json -from pathlib import Path - -import community as community_louvain # python-louvain, a cloudg dependency - -from cloudg.graph.builder import GraphBuilder -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge - -ACCOUNT = "123456789012" - - -def asset(asset_id: str, name: str, asset_type: AssetType, arn_suffix: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=name, - asset_type=asset_type, - provider=CloudProvider.AWS, - region="eu-west-1", - account_id=ACCOUNT, - arn=f"arn:aws:{arn_suffix}", - **extra, - ) - - -assets = [ - asset("sg-alb", "sg-alb", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-alb"), - asset("sg-web", "sg-web", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), - asset("alb", "web-alb", AssetType.LOAD_BALANCER, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:loadbalancer/app/web-alb/1"), - asset("tg", "web-tg", AssetType.TARGET_GROUP, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:targetgroup/web-tg/1"), - asset("web-1", "web-1", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web1", is_internet_exposed=True), - asset("web-2", "web-2", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web2"), - asset("app-role", "app-role", AssetType.IAM_ROLE, f"iam::{ACCOUNT}:role/app-role"), - asset("orders-db", "orders-db", AssetType.RDS_INSTANCE, f"rds:eu-west-1:{ACCOUNT}:db:orders-db"), -] - - -def rule(source: str, target: str, port: str, cidr: str = "", description: str = "") -> NetworkEdge: - return NetworkEdge( - source_id=source, - target_id=target, - edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range=port, - protocol="TCP", - cidr=cidr or None, - direction="ingress", - description=description or None, - ) - - -edges = [ - # Two rules on the same pair: GraphBuilder merges them into one edge - rule("0.0.0.0/0", "sg-alb", "443", "0.0.0.0/0", "HTTPS from anywhere"), - rule("0.0.0.0/0", "sg-alb", "80", "0.0.0.0/0", "HTTP redirect"), - rule("sg-alb", "sg-web", "8080", description="ALB to web tier"), - NetworkEdge(source_id="alb", target_id="sg-alb", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - NetworkEdge(source_id="alb", target_id="tg", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="tg", target_id="web-1", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="tg", target_id="web-2", edge_type=EdgeType.LOAD_BALANCER_TARGET), - NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.IAM_TRUST), - NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), -] - -builder = GraphBuilder() -graph = builder.build(assets, edges) -print(f"{graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges from {len(edges)} NetworkEdges") - -merged = graph.edges["0.0.0.0/0", "sg-alb"] -print("merged rule:", merged["port_range"], "|", merged["description"]) -print("placeholder:", graph.nodes["0.0.0.0/0"]["asset_type"], graph.nodes["0.0.0.0/0"]["is_external"]) - - -def names(path: list[str]) -> str: - return " -> ".join(graph.nodes[n]["name"] for n in path) - - -# Paths follow edges in their stored direction -for path in builder.find_attack_paths("0.0.0.0/0", "sg-web"): - print("internet path:", names(path)) -for path in builder.find_attack_paths("web-1", "orders-db"): - print("identity path:", names(path)) -for path in builder.find_lateral_movement_paths(): - print("lateral path:", names(path)) - -centrality = builder.compute_centrality() -top = sorted(centrality.items(), key=lambda kv: kv[1]["betweenness"], reverse=True)[:3] -for node_id, scores in top: - print(f"betweenness {scores['betweenness']:.3f} {graph.nodes[node_id]['name']}") - -# Louvain works on undirected graphs; the RAG exporter does the same -partition = community_louvain.best_partition(graph.to_undirected(), random_state=42) -groups: dict[int, list[str]] = {} -for node_id, community_id in partition.items(): - groups.setdefault(community_id, []).append(graph.nodes[node_id]["name"]) -for community_id, members in sorted(groups.items()): - print(f"community {community_id}: {sorted(members)}") - -out = Path("graph-out") -builder.save_graphml(out / "estate.graphml") -(out / "estate-d3.json").write_text(builder.to_json_str()) -(out / "estate-cytoscape.json").write_text(json.dumps(builder.to_cytoscape_json(), indent=2)) -print("wrote", sorted(p.name for p in out.iterdir())) +--8<-- "graph/build_graph.py" ``` Output with cloudg 0.6.0 and python-louvain 0.16: @@ -171,7 +72,7 @@ community 3: ['web-alb', 'web-tg'] wrote ['estate-cytoscape.json', 'estate-d3.json', 'estate.graphml'] ``` -Eight assets plus the `0.0.0.0/0` placeholder make nine nodes, and the two rules into `sg-alb` became one edge. There is no internet path to `web-1` here even though the load balancer targets it: `alb -> sg-alb` is an `ATTACHED_TO` edge pointing from the resource to its group, and `find_attack_paths` walks edges only in the direction they are stored. The [reachability analysis](/api/reachabilityanalyzer/) knows which way traffic actually moves; walk `network_flow_graph(graph)` instead when you want flow-aware paths. +Eight assets plus the `0.0.0.0/0` placeholder make nine nodes, and the two rules into `sg-alb` became one edge. There is no internet path to `web-1` here even though the load balancer targets it: `alb -> sg-alb` is an `ATTACHED_TO` edge pointing from the resource to its group, and `find_attack_paths` walks edges only in the direction they are stored. The [reachability analysis](reachabilityanalyzer.md) knows which way traffic actually moves; walk `network_flow_graph(graph)` instead when you want flow-aware paths. Leave out `random_state` and Louvain may number and split the communities differently from one run to the next. @@ -205,14 +106,47 @@ Edges whose other end lies outside the filter bring that end back as an `EXTERNA - `compute_centrality()` returns normalised degree, in-degree, out-degree and betweenness per node. Betweenness is exact (`nx.betweenness_centrality`), so it is slow on graphs of tens of thousands of nodes. - `subgraph(node_ids)` returns an independent copy, so changes to it do not touch `builder.graph`. - `save_graphml()` creates missing parent directories and writes with `nx.write_graphml_xml` through `graphml_safe()`, a copy of the graph without `None` attributes and with lists and dicts stored as JSON strings. `builder.graph` itself is not changed. `load_graphml()` reads back through `nx.read_graphml`, which returns a `DiGraph` with string attributes; booleans such as `is_internet_exposed` come back as Python booleans because GraphML stores their type. -- The D3 export (`to_d3_json`, `to_json_str`) has `nodes` and `links`. Cytoscape's has `elements.nodes` and `elements.edges` with a smaller attribute set (no ARN or CIDR). The field reference for the D3 form is [inventory-graph.json](/reference/inventory/inventory-graph-json/). +- The D3 export (`to_d3_json`, `to_json_str`) has `nodes` and `links`. Cytoscape's has `elements.nodes` and `elements.edges` with a smaller attribute set (no ARN or CIDR). The field reference for the D3 form is [inventory-graph.json](../reference/inventory/inventory-graph-json.md). + +## Reference + +::: cloudg.graph.builder.GraphBuilder + options: + members: + - build + - graph + - find_attack_paths + - find_lateral_movement_paths + - compute_centrality + - subgraph + - to_d3_json + - to_cytoscape_json + - to_json_str + - save_graphml + - load_graphml ## Related -:::links -- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) Internet exposure, flow-aware walks and blast radius over this graph. -- [CloudOntology](/api/cloudontology/) The same assets and edges as RDF triples. -- [RAGExporter](/api/ragexporter/) Chunks built from the graph's communities. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) How the three analyses fit together. -- [inventory-map.graphml](/reference/inventory/inventory-map-graphml/) Node and edge attributes, field by field. -::: +
    + +- [**ReachabilityAnalyzer**](reachabilityanalyzer.md) + + Internet exposure, flow-aware walks and blast radius over this graph. + +- [**CloudOntology**](cloudontology.md) + + The same assets and edges as RDF triples. + +- [**RAGExporter**](ragexporter.md) + + Chunks built from the graph's communities. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + How the three analyses fit together. + +- [**inventory-map.graphml**](../reference/inventory/inventory-map-graphml.md) + + Node and edge attributes, field by field. + +
    diff --git a/site/content/api/index.md b/site/content/api/index.md new file mode 100644 index 0000000..776cef5 --- /dev/null +++ b/site/content/api/index.md @@ -0,0 +1,202 @@ +--- +title: Python API +description: cloudg is a library first. The CLI is a thin layer over CloudGEngine, so anything a command does you can do from Python, one phase at a time if you like. +--- + +cloudg is a library first. The CLI is a thin layer over `CloudGEngine`, so anything a command does you can do from Python, one phase at a time if you like. +{ .lede } + +**Package** `cloudg` · **Python** 3.11 or newer · **Install** `pip install cloudg` +{ .page-meta } + +## Start here + +Most programs need three names, all importable from the package root: `CloudGConfig` to say what to collect, `CloudGEngine` to run it, and the result object that comes back. + +=== "Ingest existing reports" + + No credentials and no scanner binaries: point the engine at scanner output produced elsewhere. + + ```python title="ingest_only.py" + from cloudg import CloudGConfig, CloudGEngine + + engine = CloudGEngine(CloudGConfig()) + result = engine.run_from_reports_sync( + {"trivy": ["./trivy.json"], "checkov": ["./results_json.json"]}, + output_dir="./reports", + ) + + print(result.total_findings, result.severity_breakdown) + for f in result.findings: + print(f.severity.value, f.source_tool, f.title) + ``` + +=== "Map an account" + + Read-only credentials and the `aws` extra. No scanner runs. + + ```python title="map_account.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] + + inventory = CloudGEngine(config).map_inventory_sync(output_dir="./reports") + print(inventory.summary["total_assets"], "assets") + print(inventory.summary["internet_exposed"], "internet-exposed") + ``` + +=== "Full pipeline" + + What `cloudg run` does: collect, scan, analyse and report. + + ```python title="pipeline.py" + import asyncio + + from cloudg import CloudGConfig, CloudGEngine + + + async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.scanners.enabled = ["prowler"] + result = await CloudGEngine(config).run_pipeline(output_dir="./reports") + print(result.to_summary()) + + + asyncio.run(main()) + ``` + +## How the pieces fit + +```mermaid +flowchart LR + CFG["CloudGConfig"] --> ENG["CloudGEngine"] + ENG -->|collect| ASSETS["CloudAsset, NetworkEdge"] + ENG -->|scan or ingest| FIND["Finding"] + ASSETS --> GB["GraphBuilder"] + GB --> RA["ReachabilityAnalyzer"] + ASSETS --> ONT["CloudOntology, RAGExporter"] + FIND --> NORM["FindingsNormaliser"] + RA --> NORM + NORM --> RES["PipelineResult"] + ENG -->|map_inventory| IM["InventoryMapper"] + IM --> IR["InventoryResult"] + IR --> DG["DependencyGraph"] +``` + +Each box is a page below. The engine wires them together, but every one of them also works on its own: build a graph from assets you loaded yourself, normalise findings from a CMDB export, or link assets that came from your own collector. + +## Reference + +### Engine + +
    + +- [**CloudGEngine**](cloudgengine.md) + + The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: the whole pipeline in one call, or each phase yourself. + +- [**CloudGConfig**](cloudgconfig.md) + + The pydantic model behind `config.yaml`. Build it in code, load it from a file, or both. + +- [**Result objects**](results.md) + + `PipelineResult`, `CollectionResult` and `AnalysisResult`: what the engine hands back. + +- [**Event hooks**](event-hooks.md) + + Six optional callbacks that report phases, findings, results and failures during a run. + +- [**Exceptions**](exceptions.md) + + The `CloudGError` hierarchy, strict mode for the scanner wrappers and `raise_for_errors()`. + +
    + +### Inventory + +
    + +- [**InventoryMapper**](inventorymapper.md) + + Maps what is deployed and how it connects, then overlays scanner findings on the map. + +- [**InventoryResult**](inventoryresult.md) + + The map as one object: assets, edges, summary, dependency analysis, save and load. + +- [**RelationshipLinker**](relationshiplinker.md) + + Turns a flat list of assets from any source into a connected map. + +- [**DependencyGraph**](dependencygraph.md) + + What an asset needs, and what breaks if it goes away. + +
    + +### Graph + +
    + +- [**GraphBuilder**](graphbuilder.md) + + Assets and edges as a NetworkX `DiGraph`: attack paths, centrality, exports. + +- [**ReachabilityAnalyzer**](reachabilityanalyzer.md) + + What the internet can reach, along the edges traffic can actually take. + +- [**CloudOntology**](cloudontology.md) + + An rdflib graph with 64 typed relations, exported as RDF and queried with SPARQL. + +- [**RAGExporter**](ragexporter.md) + + Text chunks with metadata, ready to embed for retrieval-augmented generation. + +
    + +### Findings + +
    + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Findings from any number of scanners, merged, rescored, sorted and mapped to controls. + +- [**parse_report**](parse-report.md) + + Native Prowler, ScoutSuite, Checkov and Trivy output as `Finding` objects. + +- [**Data models**](models.md) + + The pydantic models every part of cloudg exchanges. + +
    + +### Extend + +
    + +- [**Entry points**](entry-points.md) + + Ship a collector or a scanner as its own package; `PluginRegistry` finds it. + +- [**cloudg.resilience**](resilience.md) + + Your own cloud API calls through cloudg's rate limits, circuit breakers and retry budget. + +- [**CloudGMCPLayer**](mcp-layer.md) + + The MCP catalog as a Python object: call it in-process, mount it, or export it. + +
    + +## Conventions + +- Signatures, parameter tables and the "source" links on these pages are generated from the code at build time, so they always match the release the site was built from. The prose and the examples above each reference are written by hand. +- Methods that end in `_sync` wrap the async method of the same name in `asyncio.run()`. Use them from scripts; inside a running event loop, `await` the async method. +- Everything cloudg raises on purpose derives from `CloudGError`. See [Exceptions](exceptions.md). diff --git a/site/content/api/inventorymapper.md b/site/content/api/inventorymapper.md index 40d2ee3..692362d 100644 --- a/site/content/api/inventorymapper.md +++ b/site/content/api/inventorymapper.md @@ -1,14 +1,15 @@ --- -object: cloudg.inventory.InventoryMapper -members: [map_inventory, map_inventory_sync, build_asset_map, build_compliance_map, export_merged] -lede: "Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up." +description: Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up. --- +Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up. +{ .lede } + ## What a mapping run does `InventoryMapper(config, tagging_sweep=None)` stores the config and reads the `inventory` section from it. `tagging_sweep` overrides `config.inventory.tagging_sweep` for this mapper only. Nothing touches the cloud until you call `map_inventory()`. -```mermaid caption="InventoryMapper.map_inventory(), step by step" +```mermaid flowchart TD A["deep copy of the config"] --> B{"aws.organization.enabled?"} B -->|yes| C["discover the org, pick accounts and member role"] @@ -20,6 +21,8 @@ flowchart TD G --> H["account hierarchy nodes"] H --> I["InventoryResult"] ``` +InventoryMapper.map_inventory(), step by step +{ .diagram-caption } In more detail: @@ -28,12 +31,12 @@ In more detail: 3. With Azure in `providers` and `azure.map_management_groups` on, the management group and Azure Policy hierarchy is read. With GCP, `gcp.organization_id` set and `gcp.map_hierarchy` on, the organization, folders, org policies and VPC Service Controls perimeters are read. 4. The deep collectors run per account and region (AWS), per subscription (Azure) and per project or organization (GCP), with the `inventory` options applied: service families, exclusions, Kubernetes, the tagging and Cloud Control sweeps, IAM resource edges and the rest. 5. Assets that share an ARN (the same resource seen from two regions or from a collector and a sweep) collapse into one. The detailed copy wins over a sweep or placeholder copy, and their declared relations are merged. -6. The [`RelationshipLinker`](/api/relationshiplinker/) resolves declared relations and applies its provider rules, with `inventory.link_references` deciding whether the generic reference scan runs. Placeholder `CLOUD_ACCOUNT` nodes are added for accounts that were referenced but not mapped. +6. The [`RelationshipLinker`](relationshiplinker.md) resolves declared relations and applies its provider rules, with `inventory.link_references` deciding whether the generic reference scan runs. Placeholder `CLOUD_ACCOUNT` nodes are added for accounts that were referenced but not mapped. 7. With `inventory.account_hierarchy` on (the default), account nodes are added and linked to the top-level resources they contain. Cloud API throttling does not fail a map. Calls are rate limited and retried according to `config.ratelimit`; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` in `coverage`, and the run's throttling telemetry lands in `result.throttling`, with each message also logged as a warning. If the code that calls `map_inventory()` already runs inside a `cloudg.resilience.stats_scope()`, the mapper reports into that scope. -`map_inventory()` returns an [`InventoryResult`](/api/inventoryresult/). Calling the engine's `map_inventory()` instead gets you the same result plus an `on_phase_start("inventory_mapping")` hook and optional export. +`map_inventory()` returns an [`InventoryResult`](inventoryresult.md). Calling the engine's `map_inventory()` instead gets you the same result plus an `on_phase_start("inventory_mapping")` hook and optional export. ## Merging findings into a map @@ -41,7 +44,7 @@ The last three methods never call a cloud API or a scanner. They take a finished `build_asset_map(result, findings)` returns one entry per asset in the map: `id`, `arn`, `name`, `type`, `provider`, `region`, `account_id`, `internet_exposed`, `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the most findings first, plus `total_assets` and `assets_with_findings` at the top level. Findings are matched to assets through `AssetIndex.resolve_finding()`, the matcher the ontology and RAG exports use: the asset's internal id, then an ARN taken from `resource_id` or `resource_arn`, then a unique name, then a unique ARN tail. A name that several assets share matches none of them, so a finding for a table named `orders` does not land on a queue also named `orders`. -`build_compliance_map(result, findings)` groups by compliance framework. Each framework gets `findings`, `severity_breakdown`, `affected_assets` (the ARN of the asset the finding matched in the same way, or the finding's own `resource_arn` when no asset matched) and `affected_asset_count`. It reads `finding.compliance_frameworks` as it is, so run the findings through [`FindingsNormaliser`](/api/findingsnormaliser/) first if you want the framework mapping cloudg adds on top of what the scanner reported. +`build_compliance_map(result, findings)` groups by compliance framework. Each framework gets `findings`, `severity_breakdown`, `affected_assets` (the ARN of the asset the finding matched in the same way, or the finding's own `resource_arn` when no asset matched) and `affected_asset_count`. It reads `finding.compliance_frameworks` as it is, so run the findings through [`FindingsNormaliser`](findingsnormaliser.md) first if you want the framework mapping cloudg adds on top of what the scanner reported. `export_merged(result, findings, output_dir)` writes both: `asset-map.json` and `compliance-map.json`, and returns their paths under the keys `asset_map` and `compliance_map`. @@ -81,34 +84,14 @@ for cov in inventory.coverage: inventory.export("./inventory") ``` -[AWS Organizations](/guides/aws-organizations/) covers the role, the account selection and Control Tower in detail. +[AWS Organizations](../guides/aws-organizations.md) covers the role, the account selection and Control Tower in detail. ### Overlay findings that arrive later Mapping and scanning often run on different schedules or different hosts. This script runs offline: it loads a map exported earlier, ingests Prowler and Trivy output, normalises it, and writes the merged maps next to the inventory. ```python title="overlay_findings.py" -from cloudg import CloudGConfig, InventoryMapper, InventoryResult -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser - -# A map written earlier by `cloudg map -o ./inventory` or InventoryResult.export() -inventory = InventoryResult.load("./inventory") - -# Scanner output that arrived later, normalised so compliance mapping is filled in -raw = ingest_reports({"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}) -findings = FindingsNormaliser().normalise(raw).findings - -mapper = InventoryMapper(CloudGConfig(providers=["aws"])) -paths = mapper.export_merged(inventory, findings, "./inventory") -print({name: str(path) for name, path in paths.items()}) - -compliance = mapper.build_compliance_map(inventory, findings) -for framework, entry in list(compliance["frameworks"].items())[:3]: - print(framework, entry["findings"], entry["affected_assets"]) - -asset_map = mapper.build_asset_map(inventory, findings) -print(asset_map["total_assets"], "assets,", asset_map["assets_with_findings"], "with findings") +--8<-- "map/overlay_findings.py" ``` ```console @@ -132,12 +115,39 @@ The mapper needs a config only to exist; the merge methods ignore it. `Inventory `InventoryResult.coverage` is saved in `inventory-map.json` under `coverage`, and `InventoryResult.load()` reads it back. The CLI also prints the failed collectors after a `cloudg map` run. +## Reference + +::: cloudg.inventory.InventoryMapper + options: + members: + - map_inventory + - map_inventory_sync + - build_asset_map + - build_compliance_map + - export_merged + ## Related -:::links -- [InventoryResult](/api/inventoryresult/) What `map_inventory()` returns. -- [RelationshipLinker](/api/relationshiplinker/) The linking step on its own. -- [DependencyGraph](/api/dependencygraph/) Blast radius and dependency walks over the map. -- [Inventory mapping](/guides/inventory-mapping/) The guide, with the CLI side. -- [cloudg map](/cli/map/) The command that wraps this class. -::: +
    + +- [**InventoryResult**](inventoryresult.md) + + What `map_inventory()` returns. + +- [**RelationshipLinker**](relationshiplinker.md) + + The linking step on its own. + +- [**DependencyGraph**](dependencygraph.md) + + Blast radius and dependency walks over the map. + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + The guide, with the CLI side. + +- [**cloudg map**](../cli/map.md) + + The command that wraps this class. + +
    diff --git a/site/content/api/inventoryresult.md b/site/content/api/inventoryresult.md index 698b427..672fe24 100644 --- a/site/content/api/inventoryresult.md +++ b/site/content/api/inventoryresult.md @@ -1,14 +1,15 @@ --- -object: cloudg.inventory.InventoryResult -members: [summary, dependency_graph, analysis, export, load] -lede: "The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from." +description: 'The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from.' --- +The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from. +{ .lede } + ## Where it comes from `InventoryResult` is a dataclass defined in `cloudg/inventory/mapper_result.py` and importable from `cloudg`, `cloudg.inventory` and `cloudg.inventory.mapper`. You get one from `InventoryMapper.map_inventory()`, from `CloudGEngine.map_inventory()`, or from `InventoryResult.load()` on a map saved earlier. You can also build one yourself from any list of `CloudAsset` and `NetworkEdge` objects; every field has a default, and the methods work the same on a hand-built result. -```mermaid caption="An inventory map's round trip through the file system" +```mermaid flowchart LR M["map_inventory()"] --> R["InventoryResult"] Y["your own assets and edges"] --> R @@ -17,12 +18,14 @@ flowchart LR R -->|"dependency_graph()"| G["DependencyGraph"] R -->|"analysis()"| A["dependency and coverage report"] ``` +An inventory map's round trip through the file system +{ .diagram-caption } ## The parts that need explaining -`summary` is a property, not a stored field, and it is recomputed from `assets` and `edges` on every access. On a map with tens of thousands of assets that is a noticeable amount of work, so read it once into a variable. Its keys are counts and breakdowns: `total_assets`, `total_edges`, `providers`, `assets_by_type`, `assets_by_service`, `assets_by_region`, `assets_by_account`, `edges_by_type`, `edges_by_relationship`, `unlinked_assets`, `internet_exposed`, `accounts`, `cross_account_edges`, `external_accounts`, `security_service_gaps`, `unresolved_references`, and `organization` and `throttling` when those exist. The [summary reference](/reference/inventory/summary/) says how each one is computed. `assets_by_service` is derived from the identifier: the service field of an ARN, the provider namespace of an Azure ID, the API host of a GCP name. +`summary` is a property, not a stored field, and it is recomputed from `assets` and `edges` on every access. On a map with tens of thousands of assets that is a noticeable amount of work, so read it once into a variable. Its keys are counts and breakdowns: `total_assets`, `total_edges`, `providers`, `assets_by_type`, `assets_by_service`, `assets_by_region`, `assets_by_account`, `edges_by_type`, `edges_by_relationship`, `unlinked_assets`, `internet_exposed`, `accounts`, `cross_account_edges`, `external_accounts`, `security_service_gaps`, `unresolved_references`, and `organization` and `throttling` when those exist. The [summary reference](../reference/inventory/summary.md) says how each one is computed. `assets_by_service` is derived from the identifier: the service field of an ARN, the provider namespace of an Azure ID, the API host of a GCP name. -`dependency_graph(include_hierarchy=False)` builds a new [`DependencyGraph`](/api/dependencygraph/) on each call. With the default, `CONTAINS` edges into organization, OU and account nodes are left out, so "everything in this account depends on the account" does not dominate every blast radius. +`dependency_graph(include_hierarchy=False)` builds a new [`DependencyGraph`](dependencygraph.md) on each call. With the default, `CONTAINS` edges into organization, OU and account nodes are left out, so "everything in this account depends on the account" does not dominate every blast radius. `analysis(top=25)` runs the dependency graph and the coverage checks and returns a dict with five keys: `shared_dependencies` (assets many others directly depend on), `largest_blast_radius`, `cross_account_edges` (one entry per edge between two accounts, with `external` true when one side is an unmapped account), `security_coverage` (which security services run in which account and region, the gaps, workloads no vulnerability scanner covers, internet-facing entry points without a WAF) and `unresolved_references`. This is what `export()` saves as `inventory-dependencies.json`. @@ -43,59 +46,10 @@ The missing `raw_data` matters if you link a loaded map again: a few linker rule ### Build, export, reload, analyse -Runs offline. Four assets of an orders service are linked by the [`RelationshipLinker`](/api/relationshiplinker/), one of them points at a role in an account that was never mapped, and the result goes through a full save and load. +Runs offline. Four assets of an orders service are linked by the [`RelationshipLinker`](relationshiplinker.md), one of them points at a role in an account that was never mapped, and the result goes through a full save and load. ```python title="inventory_roundtrip.py" -from cloudg import InventoryResult -from cloudg.inventory import RelationshipLinker -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - -ACCOUNT = "123456789012" -REGION = "eu-west-1" -KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" -QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" -TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" -FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" - - -def aws(arn, name, asset_type, **metadata): - return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, - region=REGION, account_id=ACCOUNT, metadata=metadata) - - -assets = [ - aws(KEY, "orders-key", AssetType.KMS_KEY), - aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, kms_key_id=KEY), - aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, kms_key_id=KEY), - aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, kms_key_id=KEY, relations=[ - {"target": QUEUE, "edge": "INVOKES", "relationship": "TRIGGERED_BY", "reverse": True}, - {"target": TABLE, "edge": "REFERENCES", "relationship": "WRITES_TO"}, - {"target": "arn:aws:iam::999999999999:role/partner-reader", "edge": "ASSUMES_ROLE"}, - ]), -] - -linker = RelationshipLinker(assets) -edges = linker.link() -inventory = InventoryResult( - assets=assets + linker.external_assets, - edges=edges, - providers=["aws"], - regions={"aws": [REGION]}, - unresolved_references=linker.unresolved, -) - -paths = inventory.export("./inventory") -print(sorted(paths)) - -again = InventoryResult.load("./inventory") # the directory or inventory-map.json -s = again.summary -print(s["total_assets"], s["total_edges"], s["edges_by_type"]) -print("external accounts:", s["external_accounts"], "cross-account edges:", s["cross_account_edges"]) - -analysis = again.analysis(top=5) -print(list(analysis)) -print([(d["name"], d["direct_dependents"]) for d in analysis["shared_dependencies"]]) -print(analysis["cross_account_edges"]) +--8<-- "map/inventory_roundtrip.py" ``` ```console @@ -115,19 +69,7 @@ Five assets, not four: the role in account `999999999999` was not mapped, so the Any map exported by `cloudg map -o ./inventory` loads the same way. No credentials are needed. ```python title="whats_exposed.py" -from cloudg import InventoryResult - -inventory = InventoryResult.load("./inventory") -summary = inventory.summary - -print(f"{summary['internet_exposed']} of {summary['total_assets']} assets face the internet") -for asset in inventory.assets: - if asset.is_internet_exposed: - print(f" {asset.asset_type.value:16} {asset.account_id or '-':14} {asset.display_id}") - -print("unresolved references:", summary["unresolved_references"]) -for ref in inventory.unresolved_references[:10]: - print(f" {ref['source_name']} -> {ref['target']} ({ref['edge_type']})") +--8<-- "map/whats_exposed.py" ``` ## Notes @@ -140,11 +82,35 @@ for ref in inventory.unresolved_references[:10]: Building an `InventoryResult` yourself does not deduplicate or link anything. Pass assets that are already unique by ARN and edges you produced (with the linker or otherwise), or the summary counts will include the duplicates. +## Reference + +::: cloudg.inventory.InventoryResult + options: + members: + - summary + - dependency_graph + - analysis + - export + - load + ## Related -:::links -- [InventoryMapper](/api/inventorymapper/) The class that produces this result. -- [DependencyGraph](/api/dependencygraph/) What `dependency_graph()` returns. -- [InventoryResult reference](/reference/inventory/inventoryresult/) Every field and file, specified. -- [Output files](/reference/output-files/) All files cloudg writes, side by side. -::: +
    + +- [**InventoryMapper**](inventorymapper.md) + + The class that produces this result. + +- [**DependencyGraph**](dependencygraph.md) + + What `dependency_graph()` returns. + +- [**InventoryResult reference**](../reference/inventory/inventoryresult.md) + + Every field and file, specified. + +- [**Output files**](../reference/output-files.md) + + All files cloudg writes, side by side. + +
    diff --git a/site/content/api/mcp-layer.md b/site/content/api/mcp-layer.md index 50c015a..dcea76a 100644 --- a/site/content/api/mcp-layer.md +++ b/site/content/api/mcp-layer.md @@ -1,17 +1,14 @@ --- title: CloudGMCPLayer -object: - - cloudg.mcp.CloudGMCPLayer - - cloudg.mcp.register_into - - cloudg.mcp.build_server - - cloudg.mcp.export_definitions -members: - cloudg.mcp.CloudGMCPLayer: [call_tool, read_resource, get_prompt, complete, list_tools, list_resources, list_resource_templates, list_prompts, tools_wire, get_tool, register_into, serve, use, set_prefix, on_change, notify_change, context] -lede: The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. -source: docs/MCP.md -since: "0.6.0" +description: The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. --- +The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `cloudg mcp serve` is a thin wrapper around this class. Everything the server does, from the 74 tools to the policy that hides, redacts and pseudonymises, lives in `CloudGMCPLayer`, so you can embed the same catalog wherever your agent or server already runs. Importing `cloudg.mcp` pulls in nothing beyond cloudg's own dependencies; the official `mcp` SDK (the `cloudg[mcp]` extra) or `fastmcp` is imported only when you mount into one of them. There are three ways in, and all of them end at the same four methods: @@ -22,7 +19,7 @@ There are three ways in, and all of them end at the same four methods: | An MCP server built on the SDK or fastmcp | `layer.register_into(server)` | cloudg's tools, resources and prompts next to the host's own | | A gateway or framework that speaks neither | `export_definitions(layer)` | MCP wire dicts plus async handlers, for one principal | -```mermaid caption="Every route goes through the same layer methods" +```mermaid flowchart LR A["Your code"] --> L["layer.call_tool / read_resource / get_prompt / complete"] B["SDK or fastmcp server via register_into"] --> L @@ -35,6 +32,8 @@ flowchart LR P --> H["Handler"] H --> O["Output transforms"] ``` +Every route goes through the same layer methods +{ .diagram-caption } A tool the policy hides from the caller is indistinguishable from one that does not exist. Every other failure of a tool call (bad arguments, access denied, a handler exception, a timeout) comes back as a `ToolResult` with `is_error=True` that a model can read and recover from, rather than as an exception. Resources and prompts raise `MCPLayerError` subclasses instead. @@ -78,12 +77,10 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, ACCOUNT = "123456789012" - def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: return CloudAsset(id=asset_id, name=asset_id, asset_type=asset_type, provider=CloudProvider.AWS, region="eu-west-1", account_id=ACCOUNT, arn=arn, **extra) - inventory = InventoryResult( assets=[ asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), @@ -99,7 +96,6 @@ inventory = InventoryResult( providers=["aws"], ) - async def main() -> None: workspace = Workspace(allowed_roots=["."]) workspace.add_inventory(inventory, "demo") @@ -132,7 +128,6 @@ async def main() -> None: with open("mcp-audit.jsonl") as fh: print("audit:", fh.readline().strip()) - asyncio.run(main()) ``` @@ -172,13 +167,11 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider server = MCPServer("platform-tools") - @server.tool() def ticket_count(team: str) -> int: """How many open tickets a team has (the host's own tool).""" return 7 - inventory = InventoryResult( assets=[ CloudAsset(id=f"vm-{i}", name=f"vm-{i}", asset_type=AssetType.EC2, @@ -193,7 +186,6 @@ workspace.add_inventory(inventory, "demo") layer = CloudGMCPLayer(policy="standard", workspace=workspace) binding = layer.register_into(server, prefix="cloudg_") # mount after the host's own tools - async def main() -> None: async with Client(server) as client: names = [t.name for t in (await client.list_tools()).tools] @@ -210,7 +202,6 @@ async def main() -> None: "arguments": {"group_by": "type"}}) print("dispatch:", reply["structuredContent"]["groups"], reply["isError"]) - asyncio.run(main()) ``` @@ -240,7 +231,7 @@ layer = CloudGMCPLayer(policy="strict", workspace=workspace, prefix="cloudg_") layer.serve("stdio") # blocks until the client disconnects ``` -For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MCP_TOKEN:analyst:ci"])` takes the same options as the CLI. The token spec format and the other transport settings are on the [cloudg mcp serve](/cli/mcp-serve/) page. +For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MCP_TOKEN:analyst:ci"])` takes the same options as the CLI. The token spec format and the other transport settings are on the [cloudg mcp serve](../cli/mcp-serve.md) page. ## Notes @@ -252,12 +243,57 @@ For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MC - Sync tool handlers run in a worker thread, so a slow graph build does not block the event loop. `notify_change()` is safe to call from those threads. - A policy with `vault.path` set persists its pseudonym vault: `serve()` saves it when serving ends, and any process saves a changed vault at interpreter exit. Call `layer.policy.save_vault()` to save at a moment you choose, for example before a long-running worker is killed. +## Reference + +::: cloudg.mcp.CloudGMCPLayer + options: + members: + - call_tool + - read_resource + - get_prompt + - complete + - list_tools + - list_resources + - list_resource_templates + - list_prompts + - tools_wire + - get_tool + - register_into + - serve + - use + - set_prefix + - on_change + - notify_change + - context + +::: cloudg.mcp.register_into + +::: cloudg.mcp.build_server + +::: cloudg.mcp.export_definitions + ## Related -:::links -- [MCP server overview](/mcp/) What the catalog covers and how clients connect. -- [In-process use](/mcp/server/in-process-use/) Every layer method and the `ToolContext` handlers receive. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each of the seven policies allows and transforms. -- [Tool index](/mcp/tools/tool-index/) All 74 tools by category. -- [cloudg mcp serve](/cli/mcp-serve/) The same layer behind stdio or HTTP. -::: +
    + +- [**MCP server overview**](../mcp/index.md) + + What the catalog covers and how clients connect. + +- [**In-process use**](../mcp/server/in-process-use.md) + + Every layer method and the `ToolContext` handlers receive. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each of the seven policies allows and transforms. + +- [**Tool index**](../mcp/tools/tool-index.md) + + All 74 tools by category. + +- [**cloudg mcp serve**](../cli/mcp-serve.md) + + The same layer behind stdio or HTTP. + +
    diff --git a/site/content/api/models.md b/site/content/api/models.md index b0fc811..790c530 100644 --- a/site/content/api/models.md +++ b/site/content/api/models.md @@ -1,22 +1,15 @@ --- -object: [cloudg.schema.models.Finding, cloudg.schema.models.CloudAsset, cloudg.schema.models.NetworkEdge, cloudg.schema.models.ComplianceResult, cloudg.schema.models.ScanResult, cloudg.schema.models.Severity, cloudg.schema.models.CloudProvider, cloudg.schema.models.ComplianceStatus] -members: - cloudg.schema.models.Finding: [risk_score] - cloudg.schema.models.CloudAsset: [display_id] - cloudg.schema.models.NetworkEdge: [] - cloudg.schema.models.ComplianceResult: [] - cloudg.schema.models.ScanResult: [summary] - cloudg.schema.models.Severity: [] - cloudg.schema.models.CloudProvider: [] - cloudg.schema.models.ComplianceStatus: [] -lede: "The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the `ScanResult` that holds them together." +description: 'The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the ScanResult that holds them together.' --- +The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the `ScanResult` that holds them together. +{ .lede } + ## How the models fit together Every collector produces `CloudAsset` and `NetworkEdge` objects, every scanner parser produces `Finding` objects, and the normaliser bundles findings and their `ComplianceResult` entries into a `ScanResult`. The links between them are plain string ids, not object references: -```mermaid caption="The id fields that connect the models" +```mermaid flowchart LR SR["ScanResult"] -->|assets| CA["CloudAsset"] SR -->|edges| NE["NetworkEdge"] @@ -26,8 +19,10 @@ flowchart LR F -->|"resource_id, resource_arn"| CA CR -->|finding_ids| F ``` +The id fields that connect the models +{ .diagram-caption } -Those references are not checked. An edge may point at an id no asset has (cloudg's own security group rules use the CIDR `0.0.0.0/0` as `source_id`, and the graph builder turns it into a node of its own), and a finding's `resource_id` is often an ARN or a Terraform address instead of an asset id. [`AssetIndex`](/api/dependencygraph/) is the matcher cloudg uses to resolve such references. +Those references are not checked. An edge may point at an id no asset has (cloudg's own security group rules use the CIDR `0.0.0.0/0` as `source_id`, and the graph builder turns it into a node of its own), and a finding's `resource_id` is often an ARN or a Terraform address instead of an asset id. [`AssetIndex`](dependencygraph.md) is the matcher cloudg uses to resolve such references. All five models are configured with `extra="allow"`: a field the model does not define is kept on the instance and written back out by `model_dump()`. That makes it easy to carry your own data (`CloudAsset(..., owner="team-orders")`) through cloudg's pipeline, and it also means a typo in a keyword argument is not an error. @@ -37,9 +32,9 @@ All five models are configured with `extra="allow"`: a field the model does not All of them are `str` enums, so `Severity.HIGH == "HIGH"` is true, and pydantic accepts the plain string wherever the enum is expected. Values outside the enum raise a `ValidationError`. -`AssetType` has 156 values, from `EC2` and `S3_BUCKET` to `K8S_WORKLOAD`, `LANDING_ZONE` and `AI_GUARDRAIL`, plus `OTHER` for anything without a better fit. The [asset types by category](/reference/catalog/asset-types-by-category/) page lists them all, grouped, with the metadata each type carries. +`AssetType` has 156 values, from `EC2` and `S3_BUCKET` to `K8S_WORKLOAD`, `LANDING_ZONE` and `AI_GUARDRAIL`, plus `OTHER` for anything without a better fit. The [asset types by category](../reference/catalog/asset-types-by-category.md) page lists them all, grouped, with the metadata each type carries. -`EdgeType` has 20 values. Eleven describe network and identity structure (`SECURITY_GROUP_RULE`, `NACL_RULE`, `ROUTE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `CONTAINS`, `PEERING`, `LOAD_BALANCER_TARGET`, `INTERNET_EXPOSED`, `ATTACHED_TO`, `REFERENCES`) and nine are the typed inventory relationships (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`), always read "source verb target". The finer `relationship` label on an edge comes from a separate vocabulary, documented under [relationship vocabulary](/reference/catalog/relationship-vocabulary/). +`EdgeType` has 20 values. Eleven describe network and identity structure (`SECURITY_GROUP_RULE`, `NACL_RULE`, `ROUTE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `CONTAINS`, `PEERING`, `LOAD_BALANCER_TARGET`, `INTERNET_EXPOSED`, `ATTACHED_TO`, `REFERENCES`) and nine are the typed inventory relationships (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`), always read "source verb target". The finer `relationship` label on an edge comes from a separate vocabulary, documented under [relationship vocabulary](../reference/catalog/relationship-vocabulary.md). ## Serialised forms @@ -196,7 +191,7 @@ for f in sorted(urgent, key=lambda f: f.risk_score, reverse=True): print(f"{f.risk_score:4} {f.severity.value:8} {f.resource_arn or f.resource_id} {f.title}") ``` -[parse_report](/api/parse-report/) shows how to write such a file from scanner output. +[parse_report](parse-report.md) shows how to write such a file from scanner output. ## Notes @@ -212,12 +207,65 @@ Assignment is not validated. `finding.severity = "SEVERE"` succeeds, and the pla `NetworkEdge.direction` defaults to `"ingress"` and `ports` to an empty list; most collectors fill `port_range` (`"80-443"`, `"22"`) instead of `ports`. Parallel security group rules between the same two nodes are separate edges in the data and are merged into one edge only inside the graph builder. +## Reference + +::: cloudg.schema.models.Finding + options: + members: + - risk_score + +::: cloudg.schema.models.CloudAsset + options: + members: + - display_id + +::: cloudg.schema.models.NetworkEdge + options: + members: false + +::: cloudg.schema.models.ComplianceResult + options: + members: false + +::: cloudg.schema.models.ScanResult + options: + members: + - summary + +::: cloudg.schema.models.Severity + options: + members: false + +::: cloudg.schema.models.CloudProvider + options: + members: false + +::: cloudg.schema.models.ComplianceStatus + options: + members: false + ## Related -:::links -- [Asset types by category](/reference/catalog/asset-types-by-category/) All 156 `AssetType` values, grouped. -- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The `relationship` labels on edges. -- [Result objects](/api/results/) The dataclasses that carry these models out of the engine. -- [FindingsNormaliser](/api/findingsnormaliser/) Where findings get merged, rescored and mapped. -- [Output files](/reference/output-files/) The JSON files these models are written to. -::: +
    + +- [**Asset types by category**](../reference/catalog/asset-types-by-category.md) + + All 156 `AssetType` values, grouped. + +- [**Relationship vocabulary**](../reference/catalog/relationship-vocabulary.md) + + The `relationship` labels on edges. + +- [**Result objects**](results.md) + + The dataclasses that carry these models out of the engine. + +- [**FindingsNormaliser**](findingsnormaliser.md) + + Where findings get merged, rescored and mapped. + +- [**Output files**](../reference/output-files.md) + + The JSON files these models are written to. + +
    diff --git a/site/content/api/parse-report.md b/site/content/api/parse-report.md index 251f770..37df354 100644 --- a/site/content/api/parse-report.md +++ b/site/content/api/parse-report.md @@ -1,8 +1,10 @@ --- -object: [cloudg.ingest.parse_report, cloudg.ingest.ingest_reports] -lede: "Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg `Finding` objects. No engine, no credentials, no scanner binaries." +description: Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg Finding objects. No engine, no credentials, no scanner binaries. --- +Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg `Finding` objects. No engine, no credentials, no scanner binaries. +{ .lede } + ## Two functions, one dispatcher `cloudg.ingest` is a small module with two public functions and one constant. `SUPPORTED_TOOLS` is the tuple `("prowler", "scoutsuite", "checkov", "trivy")`. @@ -11,7 +13,7 @@ lede: "Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into clo `ingest_reports(reports)` takes a dict of tool name to a list of paths and calls `parse_report` for each pair, collecting everything into one list. It is what `CloudGEngine.ingest_reports()` and `cloudg ingest` use underneath. -```mermaid caption="How ingest_reports handles each path" +```mermaid flowchart TD R["reports dict"] --> L["for each tool, for each path"] L --> P["parse_report(tool, path)"] @@ -23,6 +25,8 @@ flowchart TD N --> K F --> C["combined list"] ``` +How ingest_reports handles each path +{ .diagram-caption } `parse_report` raises both errors. `ingest_reports` catches exactly those two, logs them on the `cloudg.ingest` logger at ERROR level, and carries on with the next path. A file that exists but does not parse (truncated JSON, the wrong tool's output) is handled one level down: the scanner wrapper logs a warning and returns what it could read, often an empty list. Neither function raises for bad content. @@ -35,9 +39,9 @@ flowchart TD | `checkov` | the JSON report | `results_json.json` anywhere below it, else every `*.json` | `checkov -d ./iac --output json > results_json.json` | | `trivy` | an `image` or `fs` JSON report | every `*.json` below it | `trivy image --format json myrepo/app:latest > trivy-image.json` | -A few rules per tool decide what becomes a finding. Prowler records are read as OCSF or ASFF, record by record. Passing checks (ASFF `Compliance.Status` of `PASSED`, OCSF `status_code` of `PASS`) give no finding, and the check name inside the finding id (`prowler--...` or `prowler---...`) becomes the deduplication key. ScoutSuite yields one finding per flagged item, with `danger`, `warning` and `caution` mapped to CRITICAL, HIGH and MEDIUM. Checkov findings come from `results.failed_checks`, whether the report is one object or a list of them (several frameworks). Trivy reports are split by their `ArtifactType`: `container_image` reports give vulnerability and secret findings, anything else also gives misconfigurations. [Input requirements](/guides/ingesting/#input-requirements) lists every field read. +A few rules per tool decide what becomes a finding. Prowler records are read as OCSF or ASFF, record by record. Passing checks (ASFF `Compliance.Status` of `PASSED`, OCSF `status_code` of `PASS`) give no finding, and the check name inside the finding id (`prowler--...` or `prowler---...`) becomes the deduplication key. ScoutSuite yields one finding per flagged item, with `danger`, `warning` and `caution` mapped to CRITICAL, HIGH and MEDIUM. Checkov findings come from `results.failed_checks`, whether the report is one object or a list of them (several frameworks). Trivy reports are split by their `ArtifactType`: `container_image` reports give vulnerability and secret findings, anything else also gives misconfigurations. [Input requirements](../guides/ingesting.md#input-requirements) lists every field read. -The findings come back raw. Their severities are what the scanner said, duplicates are still there, and only scanner-native compliance tags are set. Pass them through [`FindingsNormaliser`](/api/findingsnormaliser/) (or `CloudGEngine.normalise_findings()`) before you count or report on them. +The findings come back raw. Their severities are what the scanner said, duplicates are still there, and only scanner-native compliance tags are set. Pass them through [`FindingsNormaliser`](findingsnormaliser.md) (or `CloudGEngine.normalise_findings()`) before you count or report on them. For Prowler, the list returned is a `FindingList`: a plain `list` subclass with a `passed_checks` set naming the checks that passed. `ingest_reports` returns a `FindingList` too, with the passed checks of every Prowler report in it. Hand the list to `FindingsNormaliser.normalise()` as it is and the compliance controls those checks map to get PASS results. `+=` keeps the left list's attribute (as in the example below), but `a + b` and `list(a)` build a plain list without it; pass `passed_checks=` to `normalise()` yourself in that case. @@ -48,29 +52,7 @@ For Prowler, the list returned is a `FindingList`: a plain `list` subclass with Runs offline against one Prowler directory, one Checkov file and one Trivy file. Logging is on so the skipped paths are visible. ```python title="parse_first.py" -import logging - -from cloudg.ingest import SUPPORTED_TOOLS, ingest_reports, parse_report -from cloudg.normaliser import FindingsNormaliser - -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - -print(SUPPORTED_TOOLS) -findings = parse_report("prowler", "./prowler-output/") -findings += parse_report("checkov", "./results_json.json") -print(len(findings), "findings from two reports") - -# A typo'd tool name or a missing path is skipped by ingest_reports, not raised -more = ingest_reports({"trivy": ["./trivy.json", "./missing.json"], "trivvy": ["./trivy.json"]}) -print(len(more), "findings from the batch") - -try: - parse_report("grype", "./trivy.json") -except ValueError as exc: - print("ValueError:", exc) - -scan_result = FindingsNormaliser().normalise(findings, more) -print(scan_result.summary["severity_breakdown"], len(scan_result.compliance), "control results") +--8<-- "ingest/parse_first.py" ``` ```console @@ -97,22 +79,7 @@ The log lines appear first because logging writes to stderr and `print` to stdou `cloudg map --findings` reads a JSON list of findings. This writes one from scanner output collected in CI, so a later map run on another host can overlay it. ```python title="export_findings.py" -import json -from pathlib import Path - -from cloudg.ingest import ingest_reports - -findings = ingest_reports( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy.json"], - } -) -out = Path("./reports/raw-findings.json") -out.parent.mkdir(parents=True, exist_ok=True) -out.write_text(json.dumps([f.model_dump(mode="json") for f in findings], indent=2)) -print(f"wrote {len(findings)} findings to {out}") +--8<-- "ingest/export_findings.py" ``` ```bash @@ -136,13 +103,32 @@ Directories are searched recursively, and for Prowler and Trivy every `*.json` f `parse_report` passes the path to the scanner wrappers as a string. A `pathlib.Path` is fine as input to both functions. -The `resource_id` of a finding is whatever the scanner identified: an ARN for Prowler, a Terraform address such as `aws_s3_bucket.data` for Checkov, an image name for Trivy image scans. Matching those to collected assets is the job of [`AssetIndex`](/api/dependencygraph/) and of the asset map in [`InventoryMapper`](/api/inventorymapper/). +The `resource_id` of a finding is whatever the scanner identified: an ARN for Prowler, a Terraform address such as `aws_s3_bucket.data` for Checkov, an image name for Trivy image scans. Matching those to collected assets is the job of [`AssetIndex`](dependencygraph.md) and of the asset map in [`InventoryMapper`](inventorymapper.md). + +## Reference + +::: cloudg.ingest.parse_report + +::: cloudg.ingest.ingest_reports ## Related -:::links -- [Ingesting existing output](/guides/ingesting/) The guide, with the CLI and the input formats in detail. -- [cloudg ingest](/cli/ingest/) The command built on these functions. -- [FindingsNormaliser](/api/findingsnormaliser/) The step that should follow parsing. -- [CloudGEngine](/api/cloudgengine/) `ingest_reports()` and `run_from_reports()` with hooks. -::: +
    + +- [**Ingesting existing output**](../guides/ingesting.md) + + The guide, with the CLI and the input formats in detail. + +- [**cloudg ingest**](../cli/ingest.md) + + The command built on these functions. + +- [**FindingsNormaliser**](findingsnormaliser.md) + + The step that should follow parsing. + +- [**CloudGEngine**](cloudgengine.md) + + `ingest_reports()` and `run_from_reports()` with hooks. + +
    diff --git a/site/content/api/ragexporter.md b/site/content/api/ragexporter.md index d49156b..ffaa7a7 100644 --- a/site/content/api/ragexporter.md +++ b/site/content/api/ragexporter.md @@ -1,11 +1,14 @@ --- title: RAGExporter -object: cloudg.graph.rag_export.RAGExporter -members: [export_all, export_entity_chunks, export_community_chunks, export_relation_chunks] -lede: Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. -since: "0.6.0" +description: Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. --- +Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + A language model answering questions about your cloud needs the right slice of the inventory in its context window, not the whole `inventory-map.json`. `RAGExporter` writes three kinds of chunk, each aimed at a different kind of question, as one JSON object per line. Embed the `content` field, keep `metadata` for filtering, and you have a retrieval corpus. Nothing in the exporter calls a model or an embedding API. ## Chunk kinds @@ -26,7 +29,7 @@ Every chunk has the same five keys: `chunk_id`, `chunk_type`, `content` (the tex `severity_max` is the worst severity among the matched findings, or `"NONE"`. `compliance_frameworks` lists the frameworks of every matched finding, including findings cut from the chunk's text. A community's `risk_score` is `0.3` per member plus `0.5` per finding plus `2.0` per internet-exposed member, capped at 10. -The relations are the ones the [ontology](/api/cloudontology/) uses: `infer_relations()` on each edge (so a security group rule from `0.0.0.0/0` on port 443 becomes `INGRESS_ALLOWED`, `INTERNET_REACHABLE` and `ONLY_HTTPS`), plus the asset-level relations from metadata (`ENCRYPTED_BY_KMS`, VPC containment from `vpc_id`, tag ownership, key rotation, EC2 security group membership). Relation-group chunks carry both; entity chunks carry the edge relations only. +The relations are the ones the [ontology](cloudontology.md) uses: `infer_relations()` on each edge (so a security group rule from `0.0.0.0/0` on port 443 becomes `INGRESS_ALLOWED`, `INTERNET_REACHABLE` and `ONLY_HTTPS`), plus the asset-level relations from metadata (`ENCRYPTED_BY_KMS`, VPC containment from `vpc_id`, tag ownership, key rotation, EC2 security group membership). Relation-group chunks carry both; entity chunks carry the edge relations only. Findings attach to assets through `AssetIndex`, so a finding whose `resource_id` is an ARN or a unique name still counts toward the right entity and community. @@ -39,7 +42,7 @@ Findings attach to assets through `AssetIndex`, so a finding whose `resource_id` | `rag_chunks.jsonl` | one chunk per line: entity chunks first, then community, then relation-group | | `rag_metadata_index.json` | `total_chunks`, `entity_chunks`, `community_chunks`, `relation_group_chunks`, `chunk_ids` (in file order) and `chunk_types` | -```mermaid caption="What export_all() does" +```mermaid flowchart LR A["assets, edges"] --> E["export_entity_chunks"] G["graph"] --> L["Louvain communities"] --> C["export_community_chunks"] @@ -51,6 +54,8 @@ flowchart LR R --> J J --> I["rag_metadata_index.json"] ``` +What export_all() does +{ .diagram-caption } `cloudg run` writes these files twice. Phase 2c exports right after the graph is built, when the only findings are the reachability ones. After the scanners finish, a silent second pass rewrites both files with every finding. If that second pass fails, the files from phase 2c stay, with only reachability findings in them. `--no-rag-export` or `rag.enabled: false` turns the export off. @@ -61,120 +66,7 @@ flowchart LR The script needs nothing beyond cloudg. It exports the chunks, then turns every chunk into an `{id, text, metadata}` record with plain-text content and scalar metadata, which is the shape most vector stores accept. ```python title="rag_demo.py" -"""Export RAG chunks for a small estate and reshape them for a vector store.""" - -import json -from pathlib import Path - -from cloudg.graph.builder import GraphBuilder -from cloudg.graph.rag_export import RAGExporter -from cloudg.schema.models import ( - AssetType, - CloudAsset, - CloudProvider, - EdgeType, - Finding, - NetworkEdge, - Severity, -) - -ACCOUNT = "123456789012" -REGION = "eu-west-1" - - -def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - **extra, - ) - - -assets = [ - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", - is_internet_exposed=True), - asset("web-2", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web2"), - asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), -] -edges = [ - NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range="443", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), - NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), - NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), - NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.ASSUMES_ROLE), - NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), -] -findings = [ - Finding( - resource_id="web-1", - severity=Severity.HIGH, - title="Instance metadata service v1 enabled", - description="IMDSv2 is not enforced.", - source_tool="prowler", - compliance_frameworks=["CIS"], - ) -] - -graph = GraphBuilder().build(assets, edges) -paths = RAGExporter().export_all(assets, edges, graph, findings, output_dir="rag-out") -print({kind: path.name for kind, path in paths.items()}) - -index = json.loads(paths["index"].read_text()) -print({k: v for k, v in index.items() if k.endswith("chunks")}) - -chunks = [json.loads(line) for line in paths["chunks"].read_text().splitlines()] - -# The content uses a few symbols; plain words embed and display more reliably -MARKERS = { - "\N{WARNING SIGN}": "WARNING:", - "\N{RIGHTWARDS ARROW}": "->", - "\N{LEFTWARDS ARROW}": "<-", - "\N{BULLET}": "-", -} - - -def clean(text: str) -> str: - for symbol, word in MARKERS.items(): - text = text.replace(symbol, word) - return text - - -def scalar(value): - """Most vector stores accept only str, int, float and bool metadata.""" - if isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, list): - return ",".join(map(str, value)) - return json.dumps(value, sort_keys=True) - - -records = [ - { - "id": c["chunk_id"], - "text": clean(c["content"]), - "metadata": {"chunk_type": c["chunk_type"], - **{k: scalar(v) for k, v in c["metadata"].items()}}, - } - for c in chunks -] -out = Path("rag-out") / "vector-records.jsonl" -out.write_text("".join(json.dumps(r) + "\n" for r in records)) -print(len(records), "records ->", out) - -web_1 = next(r for r in records if r["id"] == "entity::web-1") -print(web_1["text"]) -print(web_1["metadata"]) - -# Pre-filter on metadata before any similarity search -urgent = [r["id"] for r in records if r["metadata"].get("severity_max") in ("CRITICAL", "HIGH")] -print("high or critical:", urgent) +--8<-- "graph/rag_demo.py" ``` ```console @@ -239,12 +131,38 @@ for chunk in RAGExporter().export_relation_chunks(result.edges, assets_by_id): - `content` uses four non-ASCII symbols: a warning sign (U+26A0) before exposure lines, arrows (U+2192, U+2190) before outgoing and incoming relations, and a bullet (U+2022) before community members. `rag_chunks.jsonl` stores them as `\u` escapes, and they come back as the symbols when parsed. The example swaps them for words before embedding. - `chunk_types` in the index comes from a set, so its order varies between runs. +## Reference + +::: cloudg.graph.rag_export.RAGExporter + options: + members: + - export_all + - export_entity_chunks + - export_community_chunks + - export_relation_chunks + ## Related -:::links -- [CloudOntology](/api/cloudontology/) The same relations as RDF. -- [GraphBuilder](/api/graphbuilder/) Builds the graph the community chunks are cut from. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) The analysis phase end to end. -- [cloudg run](/cli/run/) The `--rag-export` switch. -- [Output files](/reference/output-files/) Where `rag_chunks.jsonl` sits among the other outputs. -::: +
    + +- [**CloudOntology**](cloudontology.md) + + The same relations as RDF. + +- [**GraphBuilder**](graphbuilder.md) + + Builds the graph the community chunks are cut from. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + The analysis phase end to end. + +- [**cloudg run**](../cli/run.md) + + The `--rag-export` switch. + +- [**Output files**](../reference/output-files.md) + + Where `rag_chunks.jsonl` sits among the other outputs. + +
    diff --git a/site/content/api/reachabilityanalyzer.md b/site/content/api/reachabilityanalyzer.md index 2022c3d..c0970a2 100644 --- a/site/content/api/reachabilityanalyzer.md +++ b/site/content/api/reachabilityanalyzer.md @@ -1,15 +1,14 @@ --- title: ReachabilityAnalyzer -object: - - cloudg.graph.reachability.ReachabilityAnalyzer - - cloudg.graph.reachability.finding_id - - cloudg.graph.reachability.network_flow_graph -members: - cloudg.graph.reachability.ReachabilityAnalyzer: [internet_entry_points, find_internet_exposed, flow_hops, flow_successors, generate_findings, compute_blast_radius] -lede: Walks a `GraphBuilder` graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. -since: "0.6.0" +description: Walks a GraphBuilder graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. --- +Walks a `GraphBuilder` graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + `cloudg run` calls `ReachabilityAnalyzer(graph).generate_findings()` right after it builds the graph, and the results show up in `findings.json` with `source_tool` set to `cloudg-reachability`. The class answers two different questions, and they follow different edges: - What can the internet reach? `find_internet_exposed()` walks network-flow edges only, with the rules below. @@ -21,7 +20,7 @@ The analyzer works on the graph you pass in and changes it: `find_internet_expos Release 0.6.0 changed the walk. Before it, the analysis followed every edge type out of an exposed node, so a database whose only link to the internet was an IAM role granted access to it came out as a CRITICAL "Internet-exposed RDS_INSTANCE". Now an edge is followed only when traffic can travel along it, and in the direction it travels. -```mermaid caption="Edges the exposure walk follows, in the direction traffic moves" +```mermaid flowchart LR I["0.0.0.0/0, ::/0, Internet tag"] -->|"SG or NACL rule, ingress"| G["Security group, NSG, NACL"] G -->|"ATTACHED_TO, walked backwards"| R["Resource in the group"] @@ -32,6 +31,8 @@ flowchart LR N["VPC, VNet, subnet"] -->|"CONTAINS"| M["Resource placed in it"] S["Route or peering source"] -->|"ROUTE, PEERING"| D["Destination"] ``` +Edges the exposure walk follows, in the direction traffic moves +{ .diagram-caption } | Edge type | Followed | Direction | |---|---|---| @@ -80,107 +81,7 @@ The key is the node's ARN, or the node id when there is no ARN (CIDR placeholder Ten assets, fourteen edges, no credentials. The estate has a public load balancer in front of `web-1`, an SSH rule left open on the web tier's group, a database reachable only from a private subnet and through an IAM grant, and a Lambda function behind a public API Gateway. ```python title="exposure.py" -"""Internet exposure, flow hops, finding ids and blast radius on a small estate.""" - -from cloudg.graph.builder import GraphBuilder -from cloudg.graph.reachability import ( - RULE_UNEXPECTED_EXPOSURE, - ReachabilityAnalyzer, - asset_key, - finding_id, - network_flow_graph, -) -from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge - -ACCOUNT = "123456789012" -REGION = "eu-west-1" - - -def asset(asset_id: str, asset_type: AssetType, arn: str) -> CloudAsset: - return CloudAsset( - id=asset_id, - name=asset_id, - asset_type=asset_type, - provider=CloudProvider.AWS, - region=REGION, - account_id=ACCOUNT, - arn=arn, - ) - - -assets = [ - asset("sg-alb", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-alb"), - asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), - asset("sg-db", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-db"), - asset("web-alb", AssetType.LOAD_BALANCER, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:loadbalancer/app/web-alb/1"), - asset("web-tg", AssetType.TARGET_GROUP, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:targetgroup/web-tg/1"), - asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1"), - asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), - asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), - asset("orders-api", AssetType.API_GATEWAY, f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3"), - asset("orders-fn", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-fn"), -] - - -def ingress(source: str, target: str, ports: str, cidr: str | None = None) -> NetworkEdge: - return NetworkEdge( - source_id=source, - target_id=target, - edge_type=EdgeType.SECURITY_GROUP_RULE, - port_range=ports, - protocol="TCP", - cidr=cidr, - direction="ingress", - ) - - -def edge(source: str, target: str, edge_type: EdgeType, **extra) -> NetworkEdge: - return NetworkEdge(source_id=source, target_id=target, edge_type=edge_type, **extra) - - -edges = [ - ingress("0.0.0.0/0", "sg-alb", "443", cidr="0.0.0.0/0"), - ingress("0.0.0.0/0", "sg-web", "22", cidr="0.0.0.0/0"), # SSH left open - ingress("sg-alb", "sg-web", "8080"), - ingress("10.0.1.0/24", "sg-db", "5432", cidr="10.0.1.0/24"), # private source - # The default egress rule points from the group to the internet - edge("sg-web", "0.0.0.0/0", EdgeType.SECURITY_GROUP_RULE, port_range="0-65535", - protocol="ALL", cidr="0.0.0.0/0", direction="egress"), - edge("web-alb", "sg-alb", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("web-1", "sg-web", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("orders-db", "sg-db", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), - edge("web-alb", "web-tg", EdgeType.LOAD_BALANCER_TARGET), - edge("web-tg", "web-1", EdgeType.LOAD_BALANCER_TARGET), - edge("web-1", "app-role", EdgeType.ASSUMES_ROLE), - edge("app-role", "orders-db", EdgeType.GRANTS_ACCESS), - edge("0.0.0.0/0", "orders-api", EdgeType.INTERNET_EXPOSED, cidr="0.0.0.0/0"), - edge("orders-api", "orders-fn", EdgeType.INVOKES), -] - -graph = GraphBuilder().build(assets, edges) -analyzer = ReachabilityAnalyzer(graph) - -print("entry points:", sorted(analyzer.internet_entry_points())) -print("exposed:", sorted(analyzer.find_internet_exposed())) - -print("hops out of sg-web:") -for nxt, data, walked_back in analyzer.flow_hops("sg-web"): - print(f" -> {nxt} via {data['edge_type']}{' (reversed)' if walked_back else ''}") - -for finding in analyzer.generate_findings(): - print(f"{finding.severity.value:8} {finding.title} id={finding.id[:8]}") - -# The id can be recomputed from the rule name and the asset's ARN -expected = finding_id(RULE_UNEXPECTED_EXPOSURE, asset_key(graph, "web-1")) -print("web-1 exposure id:", expected[:8]) - -flow = network_flow_graph(graph) -print("flow edges:", flow.number_of_edges(), "of", graph.number_of_edges()) -print("flow reversed:", sorted((u, v) for u, v, d in flow.edges(data=True) if d["reversed"])) - -radius = analyzer.compute_blast_radius("web-1") -print("blast radius of web-1:", sorted(radius["reachable_nodes"]), - "depth", radius["depth"], "risk", radius["risk_score"]) +--8<-- "graph/exposure.py" ``` ```console @@ -244,12 +145,44 @@ The first path is the over-approximation at work: `sg-alb` admits 443 from the i - The open-port title keeps the historical `0.0.0.0/0` wording for both `0.0.0.0/0` and `::/0`, and names the Azure tag as written (`Internet`, `Any`, `*`) otherwise. - Stored findings from 0.5.x carry random ids. They will not match the 0.6.0 ids; re-baseline suppressions after upgrading. +## Reference + +::: cloudg.graph.reachability.ReachabilityAnalyzer + options: + members: + - internet_entry_points + - find_internet_exposed + - flow_hops + - flow_successors + - generate_findings + - compute_blast_radius + +::: cloudg.graph.reachability.finding_id + +::: cloudg.graph.reachability.network_flow_graph + ## Related -:::links -- [GraphBuilder](/api/graphbuilder/) Builds the graph this class walks. -- [Dependencies and blast radius](/guides/dependencies/) The inventory-level blast radius behind `cloudg deps`. -- [DependencyGraph](/api/dependencygraph/) Dependency walks over a saved map. -- [Data models](/api/models/) The `Finding` model these results use. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) Where reachability sits in the analysis phase. -::: +
    + +- [**GraphBuilder**](graphbuilder.md) + + Builds the graph this class walks. + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + The inventory-level blast radius behind `cloudg deps`. + +- [**DependencyGraph**](dependencygraph.md) + + Dependency walks over a saved map. + +- [**Data models**](models.md) + + The `Finding` model these results use. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + Where reachability sits in the analysis phase. + +
    diff --git a/site/content/api/relationshiplinker.md b/site/content/api/relationshiplinker.md index 91fd651..03decd6 100644 --- a/site/content/api/relationshiplinker.md +++ b/site/content/api/relationshiplinker.md @@ -1,16 +1,17 @@ --- -object: cloudg.inventory.linker.RelationshipLinker -members: [link, seed_existing, resolve] -lede: "Turns a flat list of `CloudAsset` objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg's collectors, your own plugin, a CMDB export, a previous run." +description: 'Turns a flat list of CloudAsset objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg''s collectors, your own plugin, a CMDB export, a previous run.' --- +Turns a flat list of `CloudAsset` objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg's collectors, your own plugin, a CMDB export, a previous run. +{ .lede } + ## How linking works `RelationshipLinker(assets, materialize_external=True)` indexes every asset by the identifiers other resources use to point at it: ARN or resource ID, name, the short ID at the end of the ARN (`i-0b1c...`, `sg-0a1b...`), metadata keys such as `queue_url`, `repository_uri` or `zone_id`, every entry in `metadata["aliases"]`, and a few derived forms (an S3 bucket's `.s3.amazonaws.com`, a load balancer's DNS name). Building the index makes no API calls and keeps no reference to a cloud session. `link()` then makes three passes and returns the new edges: -```mermaid caption="The three passes of RelationshipLinker.link()" +```mermaid flowchart TD I["identifier index over all assets"] --> P1["pass 1: declared relations in metadata.relations"] P1 --> P2["pass 2: provider rules on metadata keys"] @@ -22,6 +23,8 @@ flowchart TD P1 -.->|"target in an unmapped account"| X["external CLOUD_ACCOUNT placeholder"] P1 -.->|"target not found"| U["linker.unresolved"] ``` +The three passes of RelationshipLinker.link() +{ .diagram-caption } Pass 1 reads `metadata["relations"]`, the list each collector fills with what an asset talks to. Pass 2 applies provider-aware rules to well-known metadata keys: `security_groups` and `nsg_id` (attachment), `subnet_id` and `vpc_config` (containment), `role_arn` (execution roles), `kms_key_id` (encryption), `origins` and `web_acl_id` (CloudFront and WAF), route table routes, ENI and volume attachments, VPC peering, Azure NIC wiring and the GCP `parent_full_resource_name`. Pass 3 walks every other string in the metadata, up to six levels deep, and links anything that looks like an ARN, an Azure resource ID, a GCP full resource name or an AWS short ID and resolves to another asset. Pairs already joined in either direction are skipped, so the generic pass only adds links nothing else found. @@ -34,8 +37,8 @@ Each entry in `metadata["relations"]` is a dict: | Key | Required | Meaning | |---|---|---| | `target` | yes | Any identifier of the other asset: ARN, resource ID, name, short ID, alias. | -| `edge` | no | An [`EdgeType`](/api/models/) value such as `INVOKES` or `ASSUMES_ROLE`. Missing or unknown values become `REFERENCES`. | -| `relationship` | no | A finer label, copied to the edge's `relationship` (`TRIGGERED_BY`, `ENCRYPTED_BY_KMS`, `RUNS_ON`). The [relationship vocabulary](/reference/catalog/relationship-vocabulary/) lists the names cloudg's collectors use. | +| `edge` | no | An [`EdgeType`](models.md) value such as `INVOKES` or `ASSUMES_ROLE`. Missing or unknown values become `REFERENCES`. | +| `relationship` | no | A finer label, copied to the edge's `relationship` (`TRIGGERED_BY`, `ENCRYPTED_BY_KMS`, `RUNS_ON`). The [relationship vocabulary](../reference/catalog/relationship-vocabulary.md) lists the names cloudg's collectors use. | | `reverse` | no | When true, the edge runs from the target to the declaring asset. Use it when the declaring side is the passive one: "my queue triggers me", "this account is trusted by me". | | `description` | no | Copied to the edge's `description`. | | `properties` | no | Copied to the edge's `properties`. | @@ -71,13 +74,11 @@ QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" - def cmdb_row(arn, name, asset_type, region=REGION, **metadata): """One record from a CMDB export, turned into a CloudAsset.""" return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, region=region, account_id=ACCOUNT, metadata=metadata) - assets = [ cmdb_row(KEY, "orders-key", AssetType.KMS_KEY), cmdb_row(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global", relations=[ @@ -133,7 +134,7 @@ Reading the output line by line: - `role_arn` on the function produced `ASSUMES_ROLE` / `RUNS_ON` through a pass 2 rule, with no declared relation. - The log group is not in the export, so that relation is in `unresolved`. -To feed the result into the rest of cloudg, wrap it in an [`InventoryResult`](/api/inventoryresult/) and call `export()`, or hand `assets` and `edges` to [`CloudGEngine.analyze()`](/api/cloudgengine/). +To feed the result into the rest of cloudg, wrap it in an [`InventoryResult`](inventoryresult.md) and call `export()`, or hand `assets` and `edges` to [`CloudGEngine.analyze()`](cloudgengine.md). ### Link only what was declared @@ -193,12 +194,37 @@ The linker never modifies the assets you pass in. It only reads `metadata`, `arn Errors inside a pass are caught per asset and logged at DEBUG on `cloudg.inventory.linker`, so one malformed asset cannot stop the run. If edges you expect are missing, turn on DEBUG for that logger. +## Reference + +::: cloudg.inventory.linker.RelationshipLinker + options: + members: + - link + - seed_existing + - resolve + ## Related -:::links -- [InventoryMapper](/api/inventorymapper/) Runs the linker as part of every map. -- [DependencyGraph](/api/dependencygraph/) Dependency questions over the linked edges. -- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The relationship names and what they mean. -- [Asset types by category](/reference/catalog/asset-types-by-category/) Which types exist and what they carry. -- [Inventory mapping](/guides/inventory-mapping/) The linker in the context of a full map. -::: +
    + +- [**InventoryMapper**](inventorymapper.md) + + Runs the linker as part of every map. + +- [**DependencyGraph**](dependencygraph.md) + + Dependency questions over the linked edges. + +- [**Relationship vocabulary**](../reference/catalog/relationship-vocabulary.md) + + The relationship names and what they mean. + +- [**Asset types by category**](../reference/catalog/asset-types-by-category.md) + + Which types exist and what they carry. + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + The linker in the context of a full map. + +
    diff --git a/site/content/api/resilience.md b/site/content/api/resilience.md index f04deec..3cac7d9 100644 --- a/site/content/api/resilience.md +++ b/site/content/api/resilience.md @@ -1,29 +1,19 @@ --- title: cloudg.resilience -object: - - cloudg.resilience.call_with_resilience - - cloudg.resilience.call_with_resilience_sync - - cloudg.resilience.resilient - - cloudg.resilience.Scope - - cloudg.resilience.RetryPolicy - - cloudg.resilience.stats_scope - - cloudg.resilience.configure - - cloudg.resilience.get_governor - - cloudg.resilience.Governor - - cloudg.resilience.classify -members: - cloudg.resilience.Scope: [service_scope, account_scope] - cloudg.resilience.Governor: [summary, budget, try_retry, breaker_scope, provider_enabled, reset] -lede: Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. -source: docs/RESILIENCE.md -since: "0.6.0" +description: Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. --- +Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every cloud call cloudg makes goes through one process-wide governor: adaptive token buckets per provider, account, region and service, a concurrency cap per account, a circuit breaker per bucket and a retry budget per provider. If your own code calls the same APIs in the same process (extra enumeration next to an `InventoryMapper` run, an MCP tool of your own, a CI job that maps and then queries), wrap those calls with `call_with_resilience` and they queue behind cloudg's instead of competing with them. A throttle your call hits slows cloudg's calls to that scope too, and the other way round. ## What happens to one call -```mermaid caption="call_with_resilience, step by step" +```mermaid flowchart TD A["call_with_resilience(fn, scope)"] --> B{"Breaker open?"} B -->|yes| X["CircuitOpenError, nothing sent"] @@ -38,6 +28,8 @@ flowchart TD F -->|yes| G["Back off, never under Retry-After"] --> B F -->|no| Z["Re-raise, breaker counts a failure"] ``` +call_with_resilience, step by step +{ .diagram-caption } A throttling error also halves the scope's rate (at most once per second) and pauses the scope for the server's Retry-After, capped at 300 seconds. After five seconds without a throttle each success gives back 5% of the configured rate. The backoff before a retry is decorrelated jitter, `min(cap, uniform(0.5, 3 x previous))`. Retries stop at the first of `max_retries` (8 by default), the per-call deadline (900 seconds) or an empty retry budget (500 per provider; the first two retries of each call are free, and each success refunds 0.1). When retrying stops, your original exception is re-raised, not a wrapper. @@ -92,7 +84,6 @@ configure(RateLimitConfig.model_validate( attempts = {"describe": 0} - def describe_vpcs() -> dict: """Stands in for ec2.describe_vpcs(): throttled twice, then answers.""" attempts["describe"] += 1 @@ -101,14 +92,12 @@ def describe_vpcs() -> dict: "ResponseMetadata": {"HTTPStatusCode": 503}}, "DescribeVpcs") return {"Vpcs": [{"VpcId": "vpc-0a1b2c3d"}]} - @resilient(lambda name, **_: Scope("aws", ACCOUNT, "eu-west-1", "secretsmanager", "DescribeSecret"), policy=RetryPolicy(max_retries=2, deadline=30.0)) async def describe_secret(name: str) -> str: """Stands in for an async SDK call; the decorator paces every call.""" return f"arn:aws:secretsmanager:eu-west-1:{ACCOUNT}:secret:{name}" - async def main() -> None: ec2 = Scope("aws", ACCOUNT, "eu-west-1", "ec2", "DescribeVpcs") loop = asyncio.get_running_loop() @@ -125,7 +114,6 @@ async def main() -> None: print("totals:", summary["totals"]) print("slowed:", {k: v["rate_rps"] for k, v in summary["slowed_buckets"].items()}) - asyncio.run(main()) ``` @@ -157,7 +145,6 @@ import boto3 from cloudg.resilience import Scope, call_with_resilience, stats_scope - async def main() -> None: ec2 = boto3.client("ec2", region_name="eu-west-1") scope = Scope("aws", "123456789012", "eu-west-1", "ec2", "DescribeVpcs") @@ -165,7 +152,6 @@ async def main() -> None: vpcs = await call_with_resilience(ec2.describe_vpcs, scope=scope, MaxResults=50) print(len(vpcs["Vpcs"]), stats.messages()) - asyncio.run(main()) ``` @@ -181,12 +167,62 @@ boto3 also retries inside the call, so a throttled request can be retried both b - `reset_governor()` replaces the governor with a fresh one and forgets every learned rate. It exists for tests. `get_governor().reset()` clears the same learned state on the current governor and keeps its configuration. - `cloudg.retry.with_retry` is the older decorator. It keeps its signature and also retries provider throttling recognised from codes and statuses, but it does not use the shared buckets or breakers. +## Reference + +::: cloudg.resilience.call_with_resilience + +::: cloudg.resilience.call_with_resilience_sync + +::: cloudg.resilience.resilient + +::: cloudg.resilience.Scope + options: + members: + - service_scope + - account_scope + +::: cloudg.resilience.RetryPolicy + +::: cloudg.resilience.stats_scope + +::: cloudg.resilience.configure + +::: cloudg.resilience.get_governor + +::: cloudg.resilience.Governor + options: + members: + - summary + - budget + - try_retry + - breaker_scope + - provider_enabled + - reset + +::: cloudg.resilience.classify + ## Related -:::links -- [Rate limits and throttling](/guides/rate-limits/) What cloudg does under throttling, and how to tune it. -- [ratelimit settings](/reference/config/ratelimit/) Every `ratelimit` key in `config.yaml`. -- [How one call is paced](/reference/resilience/how-one-call-is-paced/) Buckets, AIMD, breakers and the retry budget in depth. -- [Python API](/reference/resilience/python-api/) `LiveOperationGuard`, the bulkhead and other lower-level pieces. -- [CloudGMCPLayer](/api/mcp-layer/) Live MCP tools run under the same governor. -::: +
    + +- [**Rate limits and throttling**](../guides/rate-limits.md) + + What cloudg does under throttling, and how to tune it. + +- [**ratelimit settings**](../reference/config/ratelimit.md) + + Every `ratelimit` key in `config.yaml`. + +- [**How one call is paced**](../reference/resilience/how-one-call-is-paced.md) + + Buckets, AIMD, breakers and the retry budget in depth. + +- [**Python API**](../reference/resilience/python-api.md) + + `LiveOperationGuard`, the bulkhead and other lower-level pieces. + +- [**CloudGMCPLayer**](mcp-layer.md) + + Live MCP tools run under the same governor. + +
    diff --git a/site/content/api/results.md b/site/content/api/results.md index 48033b3..a3ca933 100644 --- a/site/content/api/results.md +++ b/site/content/api/results.md @@ -1,12 +1,10 @@ --- -object: [cloudg.api.PipelineResult, cloudg.api.CollectionResult, cloudg.api.AnalysisResult] -members: - cloudg.api.PipelineResult: [total_assets, total_findings, severity_breakdown, to_summary] - cloudg.api.CollectionResult: [] - cloudg.api.AnalysisResult: [] -lede: "Three plain dataclasses carry what the engine produces. `PipelineResult` is the one most callers keep; the other two come back from single phases." +description: Three plain dataclasses carry what the engine produces. PipelineResult is the one most callers keep; the other two come back from single phases. --- +Three plain dataclasses carry what the engine produces. `PipelineResult` is the one most callers keep; the other two come back from single phases. +{ .lede } + ## Which method returns which | Returned by | Type | Holds | @@ -21,7 +19,7 @@ All three are `@dataclass` classes with defaults for every field, so you can als `run_pipeline()` gets a `CollectionResult` from `collect()` and an `AnalysisResult` from `analyze()`, normalises the findings, writes the reports, and copies a subset of fields into one `PipelineResult`. -```mermaid caption="Where each PipelineResult field comes from" +```mermaid flowchart LR C["CollectionResult"] -->|"assets, edges, coverage, providers_scanned, regions_scanned"| P["PipelineResult"] A["AnalysisResult"] -->|"graph_nodes, graph_edges, ontology_triples, rag_chunks_path, terraform_paths, attack_paths"| P @@ -29,6 +27,8 @@ flowchart LR R["report writers"] -->|"report_paths"| P E["on_error calls, failed collection targets"] -->|errors| P ``` +Where each PipelineResult field comes from +{ .diagram-caption } Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_count` and `reachability_findings` have no counterpart on `PipelineResult` (the reachability findings are merged into `findings` instead), and `duration_ms` on the pipeline result is the whole run, not the collection time. If you need those, drive the phases yourself with `collect()` and `analyze()`. @@ -38,7 +38,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou `findings` holds the normalised list when normalisation succeeded: deduplicated, rescored, sorted from CRITICAL to INFO. If normalisation failed, it holds the raw findings instead, `scan_result` is `None`, and `errors` says why. -`scan_result` is the [`ScanResult`](/api/models/) the normaliser produced. It is where the compliance mapping lives (`scan_result.compliance`, one `ComplianceResult` per framework control), and its `summary` adds `total_edges` and `compliance_frameworks` to the counts. In `run_pipeline()` its `edges` are set to the collected edges. +`scan_result` is the [`ScanResult`](models.md) the normaliser produced. It is where the compliance mapping lives (`scan_result.compliance`, one `ComplianceResult` per framework control), and its `summary` adds `total_edges` and `compliance_frameworks` to the counts. In `run_pipeline()` its `edges` are set to the collected edges. `severity_breakdown` counts `finding.severity.value` over `findings`, recomputed on every access. Only severities that occur appear as keys, so read it with `.get("CRITICAL", 0)`. `total_assets` and `total_findings` are `len()` of the lists. @@ -50,7 +50,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou `attack_paths` is a `list[list[str]]`. Each inner list is a path of graph node ids, starting at an internet-exposed or external node and ending at the target of an `IAM_TRUST` edge, at most eight hops long. Node ids are asset ids, so map them back with `{a.id: a for a in result.assets}`. The search stops after about a hundred paths. -`errors` holds one string of the form `": "` per failure of the run: everything reported through the `on_error` hook (scanners, scanner credentials, graph, ontology, RAG, Terraform, normalisation, reporting; see [Event hooks](/api/event-hooks/)), plus, for `run_pipeline()`, one `collection: : ` line per target that could not be collected. An empty list means nothing failed. +`errors` holds one string of the form `": "` per failure of the run: everything reported through the `on_error` hook (scanners, scanner credentials, graph, ontology, RAG, Terraform, normalisation, reporting; see [Event hooks](event-hooks.md)), plus, for `run_pipeline()`, one `collection: : ` line per target that could not be collected. An empty list means nothing failed. `AnalysisResult.ontology_path` is the last file the ontology was saved to. With the default `export_formats: [turtle, json-ld]` that is `ontology.jsonld`; `ontology.ttl` is written next to it. `rag_chunk_count` is the number of chunks written to `rag_chunks_path`, one per line. @@ -61,37 +61,7 @@ Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_cou Runs offline against existing Checkov and Trivy output. ```python title="save_result.py" -import json -import sys -from pathlib import Path - -from cloudg import CloudGConfig, CloudGEngine - -result = CloudGEngine(CloudGConfig()).run_from_reports_sync( - {"checkov": ["./results_json.json"], "trivy": ["./trivy.json"]}, - output_dir="./reports", -) - -# Headline numbers: plain JSON types only, safe to json.dumps -summary = result.to_summary() -print(json.dumps(summary, indent=2)) - -# Everything else: dump the pydantic models and stringify the paths yourself -payload = { - "summary": summary, - "findings": [f.model_dump(mode="json") for f in result.findings], - "compliance": [c.model_dump(mode="json") for c in result.scan_result.compliance] - if result.scan_result - else [], - "report_paths": {kind: str(path) for kind, path in result.report_paths.items()}, -} -Path("./reports/run-summary.json").write_text(json.dumps(payload, indent=2)) - -breakdown = result.severity_breakdown # only severities that occur -blocking = breakdown.get("CRITICAL", 0) + breakdown.get("HIGH", 0) -if blocking: - print(f"{blocking} critical/high findings, failing the build", file=sys.stderr) - sys.exit(1) +--8<-- "ingest/save_result.py" ``` ```console @@ -129,12 +99,10 @@ from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, ACCOUNT = "123456789012" - def aws(name, asset_type, arn, exposed=False): return CloudAsset(name=name, asset_type=asset_type, arn=arn, provider=CloudProvider.AWS, region="eu-west-1", account_id=ACCOUNT, is_internet_exposed=exposed) - api = aws("orders-api", AssetType.API_GATEWAY, "arn:aws:apigateway:eu-west-1::/restapis/a1b2c3d4e5", exposed=True) fn = aws("orders-worker", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:eu-west-1:{ACCOUNT}:function:orders-worker") role = aws("orders-worker", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/orders-worker") @@ -148,7 +116,6 @@ edges = [ NetworkEdge(source_id=role.id, target_id=admin.id, edge_type=EdgeType.IAM_TRUST), ] - async def main() -> None: config = CloudGConfig() config.ontology.enabled = False @@ -160,7 +127,6 @@ async def main() -> None: for path in analysis.attack_paths: print(" -> ".join(names.get(node, node) for node in path)) - asyncio.run(main()) ``` @@ -193,11 +159,42 @@ The dataclasses are mutable and hold the same objects the engine worked on. Chan `total_assets` counts what was collected, not what was scanned: a scanner like Checkov reports findings against IaC resources that may not be in `assets` at all. +## Reference + +::: cloudg.api.PipelineResult + options: + members: + - total_assets + - total_findings + - severity_breakdown + - to_summary + +::: cloudg.api.CollectionResult + options: + members: false + +::: cloudg.api.AnalysisResult + options: + members: false + ## Related -:::links -- [CloudGEngine](/api/cloudgengine/) The methods that return these objects. -- [Data models](/api/models/) Finding, CloudAsset, ScanResult and the enums. -- [Event hooks](/api/event-hooks/) The `on_error` calls that `errors` records. -- [Output files](/reference/output-files/) What the report paths point at. -::: +
    + +- [**CloudGEngine**](cloudgengine.md) + + The methods that return these objects. + +- [**Data models**](models.md) + + Finding, CloudAsset, ScanResult and the enums. + +- [**Event hooks**](event-hooks.md) + + The `on_error` calls that `errors` records. + +- [**Output files**](../reference/output-files.md) + + What the report paths point at. + +
    diff --git a/site/static/img/favicon.svg b/site/content/assets/images/favicon.svg similarity index 100% rename from site/static/img/favicon.svg rename to site/content/assets/images/favicon.svg diff --git a/site/content/assets/images/logo-mark.svg b/site/content/assets/images/logo-mark.svg new file mode 100644 index 0000000..23afee1 --- /dev/null +++ b/site/content/assets/images/logo-mark.svg @@ -0,0 +1 @@ + diff --git a/site/content/assets/stylesheets/extra.css b/site/content/assets/stylesheets/extra.css new file mode 100644 index 0000000..4fe6f18 --- /dev/null +++ b/site/content/assets/stylesheets/extra.css @@ -0,0 +1,264 @@ +/* cloudg docs: additions to the Material theme. + Colours come from Material's own variables, so both palettes work. */ + +:root { + --cg-brand: #ec3013; + --cg-radius: 0.4rem; +} + +/* ---- page lede and the metadata strip under it ---------------------------- */ + +.md-typeset .lede { + font-size: 0.92rem; + line-height: 1.6; + color: var(--md-default-fg-color--light); + margin-top: -0.4rem; +} + +.md-typeset .page-meta, +.md-typeset .release-date { + font-size: 0.72rem; + color: var(--md-default-fg-color--light); + border-left: 0.15rem solid var(--md-accent-fg-color); + padding-left: 0.6rem; +} + +.md-typeset .table-note { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .diagram-caption { + margin-top: -0.6rem; + text-align: center; + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .mermaid { + text-align: center; +} + +/* ---- numbered steps --------------------------------------------------------- */ + +.md-typeset .steps { + counter-reset: step; + margin-left: 1rem; + padding-left: 1.6rem; + border-left: 0.1rem solid var(--md-default-fg-color--lightest); +} + +.md-typeset .steps > h3 { + counter-increment: step; + position: relative; +} + +.md-typeset .steps > h3::before { + content: counter(step); + position: absolute; + left: -2.75rem; + top: -0.1rem; + width: 1.7rem; + height: 1.7rem; + line-height: 1.7rem; + border-radius: 50%; + text-align: center; + font-size: 0.8rem; + font-weight: 700; + color: var(--md-accent-bg-color); + background: var(--md-accent-fg-color); +} + +/* ---- cards ------------------------------------------------------------------ */ + +.md-typeset .grid.cells > .card h3 { + margin-top: 0; + font-size: 0.85rem; +} + +.md-typeset .grid.cells > .card p:last-child { + margin-bottom: 0; +} + +.md-typeset .link-cards > ul > li > p:first-child a { + display: block; +} + +.md-typeset .link-cards > ul > li > p:first-child a::after { + content: " \2192"; + color: var(--md-accent-fg-color); +} + +.md-typeset .link-cards > ul > li > p:not(:first-child) { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +/* ---- API reference ------------------------------------------------------------ */ + +.md-typeset .doc-heading .doc-source-link { + float: right; + margin-left: 0.6rem; + padding: 0 0.45rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + font-family: var(--md-text-font-family); + font-size: 0.6rem; + font-weight: 400; + line-height: 1.5; + color: var(--md-default-fg-color--light); +} + +.md-typeset .doc-heading .doc-source-link:hover { + color: var(--md-accent-fg-color); + border-color: var(--md-accent-fg-color); +} + +.md-typeset .doc-object > .doc-contents { + padding-left: 0.2rem; +} + +.md-typeset h2.doc-heading, +.md-typeset h3.doc-heading { + border-top: 0.05rem solid var(--md-default-fg-color--lightest); + padding-top: 0.8rem; +} + +.md-typeset h4.doc-heading { + margin-top: 1.6rem; +} + +/* ---- config reference ------------------------------------------------------------ */ + +.md-typeset a.config-key { + white-space: nowrap; +} + +.md-typeset a.config-key:target code { + outline: 0.1rem solid var(--md-accent-fg-color); +} + +/* ---- home page ------------------------------------------------------------------- */ + +.md-typeset .cg-hero { + display: grid; + grid-template-columns: minmax(0, 1.25fr) minmax(0, 1fr); + gap: 2rem; + align-items: center; + margin: 0.5rem 0 2rem; +} + +.md-typeset .cg-hero h1 { + margin: 0.4rem 0 0.8rem; + font-size: 2.1rem; + font-weight: 800; + line-height: 1.15; + letter-spacing: -0.02em; + color: var(--md-default-fg-color); +} + +.md-typeset .cg-kicker { + font-family: var(--md-code-font-family); + font-size: 0.65rem; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--md-accent-fg-color); +} + +.md-typeset .cg-hero .lede { + margin-top: 0; +} + +.md-typeset .cg-hero .md-button { + margin: 0.3rem 0.4rem 0.3rem 0; +} + +.md-typeset .cg-hero-art { + text-align: center; +} + +.md-typeset .cg-hero-art img { + width: 100%; + max-width: 22rem; +} + +.md-typeset .cg-stats { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.8rem; + margin: 0 0 2rem; +} + +.md-typeset .cg-stat { + display: block; + padding: 0.8rem 1rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + color: inherit; +} + +.md-typeset a.cg-stat:hover { + border-color: var(--md-accent-fg-color); +} + +.md-typeset .cg-stat strong { + display: block; + font-size: 1.4rem; + font-weight: 800; + line-height: 1.2; + color: var(--md-default-fg-color); +} + +.md-typeset .cg-stat span { + font-size: 0.7rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset .cards .cg-num { + font-family: var(--md-code-font-family); + font-size: 0.6rem; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--md-accent-fg-color); +} + +.md-typeset .sample-report-frame { + width: 100%; + height: calc(100vh - 9rem); + min-height: 32rem; + border: 0.05rem solid var(--md-default-fg-color--lightest); + border-radius: var(--cg-radius); + background: #fff; +} + +@media screen and (max-width: 59.9375em) { + .md-typeset .cg-hero { + grid-template-columns: minmax(0, 1fr); + } + + .md-typeset .cg-hero-art { + display: none; + } + + .md-typeset .cg-stats { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .md-typeset .cg-hero h1 { + font-size: 1.6rem; + } +} + +/* ---- the example map on the home page ------------------------------------------- */ + +.cg-hero-art .hero-graph { width: 100%; height: auto; } +.hg-cloud { fill: var(--md-default-bg-color); stroke: var(--md-default-fg-color--lighter); stroke-width: 0.5; } +.hg-edges line { stroke: var(--md-default-fg-color--light); stroke-width: 1.5; } +.hg-edges .hg-dash { stroke-dasharray: 4 4; } +.hg-hot line { stroke: var(--cg-brand); stroke-width: 3; } +.hg-nodes circle, .hg-nodes rect { fill: var(--md-default-bg-color); stroke: var(--md-default-fg-color); stroke-width: 2.5; } +.hg-nodes .hg-red { fill: var(--cg-brand); stroke: var(--cg-brand); } +.hg-labels text { font: 600 13px var(--md-code-font-family); fill: var(--md-default-fg-color); } +.hg-elabel text { font: 10px var(--md-code-font-family); fill: var(--md-default-fg-color--light); letter-spacing: 0.04em; } +.hg-elabel .hg-hot-t { fill: var(--cg-brand); font-weight: 600; } +.md-typeset .cg-hero-art figcaption { font-size: 0.65rem; color: var(--md-default-fg-color--light); } diff --git a/site/content/cli/auth-flags.md b/site/content/cli/auth-flags.md index 3d4542d..4d54d36 100644 --- a/site/content/cli/auth-flags.md +++ b/site/content/cli/auth-flags.md @@ -1,13 +1,16 @@ --- title: Auth flags -lede: "Every `--aws-*`, `--azure-*` and `--gcp-*` flag of `cloudg run`, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set." +description: Every --aws-*, --azure-* and --gcp-* flag of cloudg run, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set. --- -The credential flags exist on `cloudg run` only. Each one writes a single field of the loaded config and changes nothing else, so you can mix flags with values from `config.yaml`. `cloudg map` takes the same settings from `config.yaml` (it has `--profile` and nothing else). `cloudg collect` uses neither and works from the environment and default chains alone. The [Authentication](/guides/authentication/) guide covers the identities, roles and permissions each method needs; this page is about the flags. +Every `--aws-*`, `--azure-*` and `--gcp-*` flag of `cloudg run`, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set. +{ .lede } -:::warning Secrets on the command line -Values passed as flags can be seen in your shell history and, while cloudg runs, in the process list of the machine. Prefer the environment variables or a `config.yaml` with tight file permissions for secrets such as `--aws-secret` and `--azure-client-secret`. -::: +The credential flags exist on `cloudg run` only. Each one writes a single field of the loaded config and changes nothing else, so you can mix flags with values from `config.yaml`. `cloudg map` takes the same settings from `config.yaml` (it has `--profile` and nothing else). `cloudg collect` uses neither and works from the environment and default chains alone. The [Authentication](../guides/authentication.md) guide covers the identities, roles and permissions each method needs; this page is about the flags. + +!!! warning "Secrets on the command line" + + Values passed as flags can be seen in your shell history and, while cloudg runs, in the process list of the machine. Prefer the environment variables or a `config.yaml` with tight file permissions for secrets such as `--aws-secret` and `--azure-client-secret`. ## AWS @@ -23,7 +26,7 @@ Values passed as flags can be seen in your shell history and, while cloudg runs, cloudg builds the base session from the first method that is complete, then optionally assumes a role on top: -```mermaid caption="How cloudg picks AWS credentials" +```mermaid flowchart TD A{"Key and secret both set?"} -->|yes| K["Direct keys, plus session token if set"] A -->|no| B{"Role ARN and token file set?"} @@ -36,6 +39,8 @@ flowchart TD D --> R R -->|yes| S["sts:AssumeRole, with ExternalId if set"] ``` +How cloudg picks AWS credentials +{ .diagram-caption } 1. Direct keys need both `--aws-key` and `--aws-secret`. One without the other is ignored and the next method is tried. `--aws-session-token` is added when present, for temporary keys. 2. Web identity federation needs a role ARN and a token file. The token file comes from `--aws-web-identity-token-file`, or from `AWS_WEB_IDENTITY_TOKEN_FILE` when the flag is absent. This is the pattern for GitHub Actions, GitLab CI and EKS, with no stored keys. @@ -168,9 +173,22 @@ cloudg -c config.yaml map -p all --regions all ## Related -:::links -- [Authentication](/guides/authentication/) Identities, roles and least-privilege permissions per provider. -- [Provider and region flags](/cli/provider-flags/) `--profile`, `--subscription-id` and `--project-id`. -- [cloudg run](/cli/run/) The command these flags belong to. -- [AWS Organizations](/guides/aws-organizations/) Member-account roles for `cloudg map --org`. -::: +
    + +- [**Authentication**](../guides/authentication.md) + + Identities, roles and least-privilege permissions per provider. + +- [**Provider and region flags**](provider-flags.md) + + `--profile`, `--subscription-id` and `--project-id`. + +- [**cloudg run**](run.md) + + The command these flags belong to. + +- [**AWS Organizations**](../guides/aws-organizations.md) + + Member-account roles for `cloudg map --org`. + +
    diff --git a/site/content/cli/collect.md b/site/content/cli/collect.md index 8d4a008..bdedea5 100644 --- a/site/content/cli/collect.md +++ b/site/content/cli/collect.md @@ -1,15 +1,20 @@ --- +description: Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. command: collect -lede: Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. -intro: | - `cloudg collect` is phase 1 of `cloudg run` on its own, for one provider and, on AWS, one region. It is a quick way to test credentials and permissions before a long run. For a full inventory with relationships and dependency analysis, use [cloudg map](/cli/map/). --- +Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. +{ .lede } + +`cloudg collect` is phase 1 of `cloudg run` on its own, for one provider and, on AWS, one region. It is a quick way to test credentials and permissions before a long run. For a full inventory with relationships and dependency analysis, use [cloudg map](map.md). + + + ## How it works `-p/--provider` is required and takes exactly one of `aws`, `azure` or `gcp` (case-insensitive). There is no `all` here. -Credentials and scope come from the flags on this page, then from the provider's section of the config passed with `cloudg -c` (`aws`, `azure` or `gcp`), then from the environment and each provider's default chain, through `CredentialResolver`. Every method the config supports works here: direct keys, `role_arn` and web identity on AWS, a service principal, workload identity or managed identity on Azure, a credentials file or impersonation on GCP. See [Auth flags](/cli/auth-flags/) for the order within each provider. +Credentials and scope come from the flags on this page, then from the provider's section of the config passed with `cloudg -c` (`aws`, `azure` or `gcp`), then from the environment and each provider's default chain, through `CredentialResolver`. Every method the config supports works here: direct keys, `role_arn` and web identity on AWS, a service principal, workload identity or managed identity on Azure, a credentials file or impersonation on GCP. See [Auth flags](auth-flags.md) for the order within each provider. How each provider resolves its identity: @@ -91,9 +96,22 @@ The file is written to `-o/--output` (default `./reports`), and a second run for ## Related -:::links -- [cloudg map](/cli/map/) The full inventory map with relationships and dependencies. -- [cloudg run](/cli/run/) Collection as part of the complete pipeline. -- [Authentication](/guides/authentication/) Every supported identity per provider. -- [Provider and region flags](/cli/provider-flags/) How `--region` and `--regions` differ. -::: +
    + +- [**cloudg map**](map.md) + + The full inventory map with relationships and dependencies. + +- [**cloudg run**](run.md) + + Collection as part of the complete pipeline. + +- [**Authentication**](../guides/authentication.md) + + Every supported identity per provider. + +- [**Provider and region flags**](provider-flags.md) + + How `--region` and `--regions` differ. + +
    diff --git a/site/content/cli/deps.md b/site/content/cli/deps.md index 62a36f3..0c10576 100644 --- a/site/content/cli/deps.md +++ b/site/content/cli/deps.md @@ -1,17 +1,22 @@ --- +description: Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? command: deps -lede: Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? -intro: | - `cloudg deps` reads the files `cloudg map` wrote. It never calls a cloud API, so it runs offline and needs no credentials. --- +Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? +{ .lede } + +`cloudg deps` reads the files `cloudg map` wrote. It never calls a cloud API, so it runs offline and needs no credentials. + + + ## How it works The command loads `inventory-map.json` from `-m/--map`. You can pass the file itself or the folder that holds it; the default is `./reports`. If an `inventory-organization.json` sits next to the map, it is loaded too. From the assets and edges it builds a dependency graph in which every edge reads "dependent needs dependency". A Lambda function needs its role, its image, its KMS key and the queue that triggers it. A load balancer needs the WAF in front of it. A resource needs the CloudFormation stack that manages it. What happens next depends on whether you name an asset. -```mermaid caption="The two modes of cloudg deps" +```mermaid flowchart TD A["Load inventory-map.json"] --> B{"ASSET given?"} B -->|yes| C["Resolve the reference"] @@ -22,6 +27,8 @@ flowchart TD D -->|both| G["Both trees"] B -->|no| H["Overview: shared deps, blast radius, cross-account edges"] ``` +The two modes of cloudg deps +{ .diagram-caption } ### With an asset @@ -103,28 +110,32 @@ cloudg deps jobs-queue --direction down --depth 5 Feed a function's dependencies into another tool: -```bash tab="CLI" -cloudg deps api-handler --direction up --json > api-handler-deps.json -``` +=== "CLI" -```python tab="Python" title="deps_tree.py" -import json + ```bash + cloudg deps api-handler --direction up --json > api-handler-deps.json + ``` -from cloudg.inventory import InventoryResult +=== "Python" -result = InventoryResult.load("./reports") -graph = result.dependency_graph() + ```python title="deps_tree.py" + import json -asset = graph.find("api-handler") -if asset is None: - raise SystemExit("no unique asset matches 'api-handler'") + from cloudg.inventory import InventoryResult -tree = graph.tree(asset.id, direction="up", max_depth=3) -print(json.dumps(tree, indent=2, default=str)) + result = InventoryResult.load("./reports") + graph = result.dependency_graph() -for row in graph.blast_radius(top=5): - print(row["name"], row["transitive_dependents"]) -``` + asset = graph.find("api-handler") + if asset is None: + raise SystemExit("no unique asset matches 'api-handler'") + + tree = graph.tree(asset.id, direction="up", max_depth=3) + print(json.dumps(tree, indent=2, default=str)) + + for row in graph.blast_radius(top=5): + print(row["name"], row["transitive_dependents"]) + ``` ## Exit codes @@ -136,10 +147,26 @@ for row in graph.blast_radius(top=5): ## Related -:::links -- [Dependencies and blast radius](/guides/dependencies/) How to read the trees and what the edge types mean. -- [cloudg map](/cli/map/) Produces the inventory map this command reads. -- [DependencyGraph](/api/dependencygraph/) The Python class behind `deps`. -- [InventoryResult](/api/inventoryresult/) Loading and exporting saved maps. -- [MCP server](/mcp/) Ask the same questions from an AI assistant. -::: +
    + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + How to read the trees and what the edge types mean. + +- [**cloudg map**](map.md) + + Produces the inventory map this command reads. + +- [**DependencyGraph**](../api/dependencygraph.md) + + The Python class behind `deps`. + +- [**InventoryResult**](../api/inventoryresult.md) + + Loading and exporting saved maps. + +- [**MCP server**](../mcp/index.md) + + Ask the same questions from an AI assistant. + +
    diff --git a/site/content/cli/index.md b/site/content/cli/index.md new file mode 100644 index 0000000..38bcb4f --- /dev/null +++ b/site/content/cli/index.md @@ -0,0 +1,11 @@ +--- +title: CLI reference +description: Every cloudg command and the flags it takes, generated from the click definitions so the tables match --help. +--- + +Every `cloudg` command and the flags it takes. The tables are generated from the click definitions at build time, so they match `--help`. +{ .lede } + +Options that belong to the root command go before the subcommand: `cloudg -c config.yaml map ...`. Without `-c`, cloudg runs on its built-in defaults and does not look for a `config.yaml` on its own. The flags shared by several commands have pages of their own: [provider and region flags](provider-flags.md), [auth flags](auth-flags.md) and [output flags](output-flags.md). + + diff --git a/site/content/cli/ingest.md b/site/content/cli/ingest.md index 0b108d2..65a6860 100644 --- a/site/content/cli/ingest.md +++ b/site/content/cli/ingest.md @@ -1,15 +1,20 @@ --- +description: Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. command: ingest -lede: Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. -intro: | - Use `cloudg ingest` when Prowler, ScoutSuite, Checkov or Trivy already ran somewhere else: in a CI job, on another host, on a schedule. cloudg does the second half of the pipeline on their files: cross-scanner deduplication, scoring, compliance mapping and report rendering. --- +Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. +{ .lede } + +Use `cloudg ingest` when Prowler, ScoutSuite, Checkov or Trivy already ran somewhere else: in a CI job, on another host, on a schedule. cloudg does the second half of the pipeline on their files: cross-scanner deduplication, scoring, compliance mapping and report rendering. + + + ## How it works You pass each tool's native output with its own flag. Every flag is repeatable, and any mix of tools works, from one Trivy file to all four scanners at once. A path can be a single file or the directory the tool wrote. -```mermaid caption="From scanner files to reports" +```mermaid flowchart LR A["--prowler / --scoutsuite / --checkov / --trivy"] --> B["Parse each path"] B -->|"missing or unreadable"| W["Warn and skip"] @@ -18,6 +23,8 @@ flowchart LR D --> E["findings.json"] D --> F["report.html"] ``` +From scanner files to reports +{ .diagram-caption } 1. cloudg groups the paths by tool and prints them in an "Ingest Configuration" panel. With no input flag at all it stops with "No inputs" and exit code 1. 2. Each path goes through that tool's parser. A path that does not exist, or that the parser rejects, is skipped with a warning; the other paths still count. The per-tool totals are printed as they come in. @@ -33,7 +40,7 @@ What each flag accepts: | `--checkov` | A `checkov --output json` file, or a directory holding `results_json.json` | `checkov -d ./iac --output json > results_json.json` | | `--trivy` | A `trivy image` or `trivy fs` JSON file, or a directory of them | `trivy image --format json myrepo/app:latest > trivy-image.json` | -[Ingesting existing output](/guides/ingesting/) lists the exact fields cloudg reads from each format. +[Ingesting existing output](../guides/ingesting.md) lists the exact fields cloudg reads from each format. Prowler's two JSON formats both work. Each record is read as OCSF when it has OCSF keys (`finding_info`, `class_uid` and so on) and as ASFF otherwise, so one directory can hold both. When it does, Prowler reports the same check twice, and normalisation merges the pair into one finding. A muted OCSF finding comes in with `is_suppressed` set. The MCP server's `load_dataset` and `ingest_reports` tools read OCSF the same way. @@ -101,21 +108,25 @@ cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./repo The same aggregation in Python, with no files written: -```bash tab="CLI" -cloudg ingest --prowler ./prowler-output/ --trivy ./trivy-image.json --format json -``` +=== "CLI" -```python tab="Python" title="ingest_reports.py" -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser + ```bash + cloudg ingest --prowler ./prowler-output/ --trivy ./trivy-image.json --format json + ``` -findings = ingest_reports({ - "prowler": ["./prowler-output/"], - "trivy": ["./trivy-image.json"], -}) -result = FindingsNormaliser().normalise(findings) -print(len(findings), "parsed,", len(result.findings), "after deduplication") -``` +=== "Python" + + ```python title="ingest_reports.py" + from cloudg.ingest import ingest_reports + from cloudg.normaliser import FindingsNormaliser + + findings = ingest_reports({ + "prowler": ["./prowler-output/"], + "trivy": ["./trivy-image.json"], + }) + result = FindingsNormaliser().normalise(findings) + print(len(findings), "parsed,", len(result.findings), "after deduplication") + ``` ## Output files @@ -139,10 +150,26 @@ There is no SVG format here, because ingest has no inventory to draw. `raw-findi ## Related -:::links -- [Ingesting existing output](/guides/ingesting/) Input formats in detail and how deduplication works. -- [cloudg scan](/cli/scan/) Run the scanners from cloudg instead. -- [cloudg report](/cli/report/) Re-render reports from a `findings.json`. -- [parse_report](/api/parse-report/) The parser behind each flag. -- [FindingsNormaliser](/api/findingsnormaliser/) Deduplication, scoring and compliance mapping. -::: +
    + +- [**Ingesting existing output**](../guides/ingesting.md) + + Input formats in detail and how deduplication works. + +- [**cloudg scan**](scan.md) + + Run the scanners from cloudg instead. + +- [**cloudg report**](report.md) + + Re-render reports from a `findings.json`. + +- [**parse_report**](../api/parse-report.md) + + The parser behind each flag. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + Deduplication, scoring and compliance mapping. + +
    diff --git a/site/content/cli/map.md b/site/content/cli/map.md index 225934a..ea18767 100644 --- a/site/content/cli/map.md +++ b/site/content/cli/map.md @@ -1,10 +1,15 @@ --- +description: Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for cloudg deps, the MCP server or your own code. command: map -lede: Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for `cloudg deps`, the MCP server or your own code. -intro: | - `cloudg map` needs read access to the cloud APIs and nothing else. No scanner runs and none has to be installed. The [Inventory mapping](/guides/inventory-mapping/) guide explains what the map contains; this page covers the command itself. --- +Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for `cloudg deps`, the MCP server or your own code. +{ .lede } + +`cloudg map` needs read access to the cloud APIs and nothing else. No scanner runs and none has to be installed. The [Inventory mapping](../guides/inventory-mapping.md) guide explains what the map contains; this page covers the command itself. + + + ## How it works Like `cloudg run`, the command starts from the config loaded by the root command (`cloudg -c config.yaml map ...`), or from built-in defaults when there is no `-c`. It then applies three groups of flags. A flag you leave out never clears a value the config set. @@ -36,7 +41,7 @@ $ cloudg map -p aws --org --org-role cloudg-readonly --ou Workloads --regions al The mapping run itself goes through these steps: -```mermaid caption="What cloudg map does" +```mermaid flowchart TD A["Config + flags"] --> B{"--org?"} B -->|yes| C["Discover org, OUs, accounts, Control Tower"] @@ -51,6 +56,8 @@ flowchart TD J --> K{"--findings?"} K -->|yes| L["asset-map.json, compliance-map.json"] ``` +What cloudg map does +{ .diagram-caption } With `--org`, cloudg first lists the Organization from the account your credentials belong to (the management account, or a delegated administrator for the Organizations APIs). It picks the member accounts under `--ou`, drops `--exclude-account` entries, and, if you also passed `--accounts`, keeps only the accounts in both lists. The role assumed in each member account is `--org-role`, else `--role-name`, else `AWSControlTowerExecution`. If Control Tower is present and you asked for `--regions all`, only its governed regions are mapped (`aws.organization.use_governed_regions`, on by default). @@ -58,7 +65,7 @@ Without `--org`, `--accounts` plus `--role-name` gives you the same fan-out over Deep collection runs per account and region. Account-wide services (IAM, S3, CloudFront, Route 53, Shield and others) run once per account, in the primary region, so `--regions all` does not repeat them. After the deep collectors, the breadth sweeps fill gaps: Cloud Control lists every resource type with a list handler, and the Resource Groups Tagging API finds tagged resources. When the same resource turns up twice, the dedicated collector wins, then Cloud Control, then the tagging API. -The linker then resolves every identifier the collectors recorded into typed edges (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `PROTECTS`, `GOVERNS` and so on), adds an account node above each account's top-level resources, and computes the dependency analysis. The [Inventory mapping](/guides/inventory-mapping/) guide lists the edge types. +The linker then resolves every identifier the collectors recorded into typed edges (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `PROTECTS`, `GOVERNS` and so on), adds an account node above each account's top-level resources, and computes the dependency analysis. The [Inventory mapping](../guides/inventory-mapping.md) guide lists the edge types. The command prints an organization table (with `--org`), an inventory summary, the 15 services with the most assets, the 10 most shared dependencies, the 10 largest blast radii and up to 15 security service gaps. Here is the summary part for the synthetic estate in the test suite (`tests/mcp/fixtures/sample_estate.py`), so the numbers are small: @@ -86,11 +93,11 @@ If some collectors failed (an API you lack permission for, a region that is not `--findings` takes a cloudg findings file: a `raw-findings.json` from `cloudg scan` or `cloudg ingest`, or a `findings.json` report. Both shapes work, a bare list or an object with a `findings` key. The option is repeatable. Files that cannot be read or validated are skipped with a warning. Findings attach to assets by ARN, resource ID or name. -The merge happens after a fresh mapping run, because `--findings` belongs to `map`. To overlay findings on a map you already saved without calling the cloud again, use `InventoryMapper.export_merged()` from Python (see [InventoryMapper](/api/inventorymapper/)) or load both files into the MCP server. +The merge happens after a fresh mapping run, because `--findings` belongs to `map`. To overlay findings on a map you already saved without calling the cloud again, use `InventoryMapper.export_merged()` from Python (see [InventoryMapper](../api/inventorymapper.md)) or load both files into the MCP server. ### Credentials -`map` has no `--aws-key`, `--azure-*` or `--gcp-*` flags. It takes credentials from the config file, the standard environment variables, `--profile`, or the provider's default chain, in the order described under [Auth flags](/cli/auth-flags/). Put role ARNs, service principals and credential files in `config.yaml` when you map with something other than your default identity. +`map` has no `--aws-key`, `--azure-*` or `--gcp-*` flags. It takes credentials from the config file, the standard environment variables, `--profile`, or the provider's default chain, in the order described under [Auth flags](auth-flags.md). Put role ARNs, service principals and credential files in `config.yaml` when you map with something other than your default identity. ## Examples @@ -144,25 +151,29 @@ cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./repo The same mapping from Python: -```bash tab="CLI" -cloudg map -p aws --profile audit --regions all -o ./reports -``` +=== "CLI" -```python tab="Python" title="map_inventory.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg map -p aws --profile audit --regions all -o ./reports + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "audit" -config.aws.regions = ["ALL"] +=== "Python" -engine = CloudGEngine(config) -result = engine.map_inventory_sync("./reports") -print(result.summary["total_assets"], "assets") -``` + ```python title="map_inventory.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] + + engine = CloudGEngine(config) + result = engine.map_inventory_sync("./reports") + print(result.summary["total_assets"], "assets") + ``` ## Output files -Written to `-o/--output` (default `./reports`). The [output files reference](/reference/output-files/) describes each one in more detail. +Written to `-o/--output` (default `./reports`). The [output files reference](../reference/output-files.md) describes each one in more detail. | File | Written when | Contents | |---|---|---| @@ -186,11 +197,30 @@ Written to `-o/--output` (default `./reports`). The [output files reference](/re ## Related -:::links -- [Inventory mapping](/guides/inventory-mapping/) What the map collects and how relationships are built. -- [AWS Organizations](/guides/aws-organizations/) Roles and permissions for `--org`. -- [cloudg deps](/cli/deps/) Query the saved map: what depends on what. -- [Dependencies and blast radius](/guides/dependencies/) How to read dependency trees. -- [Provider and region flags](/cli/provider-flags/) `--regions all` and discovery. -- [InventoryMapper](/api/inventorymapper/) The Python class behind the command. -::: +
    + +- [**Inventory mapping**](../guides/inventory-mapping.md) + + What the map collects and how relationships are built. + +- [**AWS Organizations**](../guides/aws-organizations.md) + + Roles and permissions for `--org`. + +- [**cloudg deps**](deps.md) + + Query the saved map: what depends on what. + +- [**Dependencies and blast radius**](../guides/dependencies.md) + + How to read dependency trees. + +- [**Provider and region flags**](provider-flags.md) + + `--regions all` and discovery. + +- [**InventoryMapper**](../api/inventorymapper.md) + + The Python class behind the command. + +
    diff --git a/site/content/cli/mcp-call.md b/site/content/cli/mcp-call.md index 7b27f89..265b126 100644 --- a/site/content/cli/mcp-call.md +++ b/site/content/cli/mcp-call.md @@ -1,13 +1,18 @@ --- +description: Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. command: mcp call -lede: Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. -intro: | - `cloudg mcp call` is the quickest way to try a tool, debug a policy, or use a tool from a shell script. No server runs and no client is needed. --- +Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. +{ .lede } + +`cloudg mcp call` is the quickest way to try a tool, debug a policy, or use a tool from a shell script. No server runs and no client is needed. + + + ## How it works -Each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), calls `TOOL` once as the chosen principal, prints the result and exits. Nothing carries over between invocations. A `load_dataset` call in one command is gone by the next, so pass the data you need with `--dataset` every time. +Each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), calls `TOOL` once as the chosen principal, prints the result and exits. Nothing carries over between invocations. A `load_dataset` call in one command is gone by the next, so pass the data you need with `--dataset` every time. Arguments go in `--args` as a JSON object. You have three ways to supply it: @@ -19,7 +24,7 @@ Arguments go in `--args` as a JSON object. You have three ways to supply it: Anything that is not a JSON object is refused before the tool runs, with `Invalid value for --args: not valid JSON: ...` or `must be a JSON object` and exit code 2. Without `--args`, the tool is called with no arguments. -The principal is the local user unless you pass `--role` or `--principal-id`, as on [cloudg mcp tools](/cli/mcp-tools/). A tool the policy hides from that principal behaves as if it did not exist. +The principal is the local user unless you pass `--role` or `--principal-id`, as on [cloudg mcp tools](mcp-tools.md). A tool the policy hides from that principal behaves as if it did not exist. What gets printed depends on the result and on `--raw`: @@ -140,10 +145,26 @@ fi ## Related -:::links -- [Tool index](/mcp/tools/tool-index/) Every tool's arguments and results. -- [Conventions shared by every tool](/mcp/tools/conventions-shared-by-every-tool/) References, pagination, fields and errors. -- [Errors](/mcp/server/errors/) Error codes and what the error data holds. -- [cloudg mcp read](/cli/mcp-read/) The same thing for resources. -- [Live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/) What happens before a live tool calls the cloud. -::: +
    + +- [**Tool index**](../mcp/tools/tool-index.md) + + Every tool's arguments and results. + +- [**Conventions shared by every tool**](../mcp/tools/conventions-shared-by-every-tool.md) + + References, pagination, fields and errors. + +- [**Errors**](../mcp/server/errors.md) + + Error codes and what the error data holds. + +- [**cloudg mcp read**](mcp-read.md) + + The same thing for resources. + +- [**Live tools and the credential preflight**](../mcp/server/live-tools-and-the-credential-preflight.md) + + What happens before a live tool calls the cloud. + +
    diff --git a/site/content/cli/mcp-config.md b/site/content/cli/mcp-config.md index 427a03d..cda0412 100644 --- a/site/content/cli/mcp-config.md +++ b/site/content/cli/mcp-config.md @@ -1,10 +1,15 @@ --- +description: Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. command: mcp config -lede: Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. -intro: | - Each client stores MCP servers in a slightly different JSON shape. `cloudg mcp config` writes the right one, with the launch command and arguments filled in, so you don't have to work out paths by hand. --- +Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. +{ .lede } + +Each client stores MCP servers in a slightly different JSON shape. `cloudg mcp config` writes the right one, with the launch command and arguments filled in, so you don't have to work out paths by hand. + + + ## How it works The command builds a JSON snippet, prints it on stdout, and prints a one-line hint on stderr saying where the snippet goes. Because the hint is on stderr, you can redirect stdout straight into a file. @@ -134,9 +139,22 @@ The command exits 0 after printing. Click returns 2 for an unknown `--client` or ## Related -:::links -- [Connect a client](/mcp/connect/) Where each client keeps its configuration, step by step. -- [cloudg mcp serve](/cli/mcp-serve/) Every option the generated command can take. -- [Quick start with a desktop client](/mcp/server/quick-start-with-a-desktop-client/) From install to the first question. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) Which `--policy` to put in the launch arguments. -::: +
    + +- [**Connect a client**](../mcp/connect.md) + + Where each client keeps its configuration, step by step. + +- [**cloudg mcp serve**](mcp-serve.md) + + Every option the generated command can take. + +- [**Quick start with a desktop client**](../mcp/server/quick-start-with-a-desktop-client.md) + + From install to the first question. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + Which `--policy` to put in the launch arguments. + +
    diff --git a/site/content/cli/mcp-prompts.md b/site/content/cli/mcp-prompts.md index a9ea061..ec997ba 100644 --- a/site/content/cli/mcp-prompts.md +++ b/site/content/cli/mcp-prompts.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP prompts a server would expose, with the arguments each one takes. command: mcp prompts -lede: Lists the MCP prompts a server would expose, with the arguments each one takes. -intro: | - cloudg's prompts are ready-made workflows, such as a security posture review or an incident triage, that a client can offer its user as one-click tasks. `cloudg mcp prompts` shows which of them a principal can see and what arguments they expect. --- +Lists the MCP prompts a server would expose, with the arguments each one takes. +{ .lede } + +cloudg's prompts are ready-made workflows, such as a security posture review or an incident triage, that a client can offer its user as one-click tasks. `cloudg mcp prompts` shows which of them a principal can see and what arguments they expect. + + + ## How it works -The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)) and prints the prompts visible to one principal: the local user by default, or the one described by `--role` and `--principal-id` (see [cloudg mcp tools](/cli/mcp-tools/) for how those work). `--prefix` is applied to prompt names as well as tool names. +The command builds the layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)) and prints the prompts visible to one principal: the local user by default, or the one described by `--role` and `--principal-id` (see [cloudg mcp tools](mcp-tools.md) for how those work). `--prefix` is applied to prompt names as well as tool names. The table has four columns. Arguments are listed in order, and a `*` marks the required ones. In 0.6.0 there are 11 prompts, all in the `prompts` category, and every built-in policy lists all of them for the local user: @@ -110,9 +115,22 @@ A missing required argument raises `Missing required prompt arguments: ref`, and ## Related -:::links -- [Prompts](/mcp/tools/prompts/) Each prompt with its arguments and rendered output. -- [cloudg mcp tools](/cli/mcp-tools/) The tools the prompts tell the assistant to call. -- [In-process use](/mcp/server/in-process-use/) Driving the layer from Python. -- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. -::: +
    + +- [**Prompts**](../mcp/tools/prompts.md) + + Each prompt with its arguments and rendered output. + +- [**cloudg mcp tools**](mcp-tools.md) + + The tools the prompts tell the assistant to call. + +- [**In-process use**](../mcp/server/in-process-use.md) + + Driving the layer from Python. + +- [**cloudg mcp serve**](mcp-serve.md) + + The shared layer options in full. + +
    diff --git a/site/content/cli/mcp-read.md b/site/content/cli/mcp-read.md index 87fa26d..f920fd6 100644 --- a/site/content/cli/mcp-read.md +++ b/site/content/cli/mcp-read.md @@ -1,20 +1,25 @@ --- +description: Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. command: mcp read -lede: Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. -intro: | - Resources are fetched by URI, such as `cloudg://findings/summary` or `cloudg://assets/`. `cloudg mcp read` lets you look at one from the shell, which helps when you are writing a policy or checking what an assistant will be shown. --- +Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. +{ .lede } + +Resources are fetched by URI, such as `cloudg://findings/summary` or `cloudg://assets/`. `cloudg mcp read` lets you look at one from the shell, which helps when you are writing a policy or checking what an assistant will be shown. + + + ## How it works -Like [cloudg mcp call](/cli/mcp-call/), each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), reads `URI` once as the chosen principal, prints the contents and exits. Most resources read the active dataset, so load one with `--dataset` on the same command line. Without a dataset, those resources fail: +Like [cloudg mcp call](mcp-call.md), each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), reads `URI` once as the chosen principal, prints the contents and exits. Most resources read the active dataset, so load one with `--dataset` on the same command line. Without a dataset, those resources fail: ```console $ cloudg mcp read cloudg://findings/summary Error: No dataset is loaded. Call load_dataset(path=...) with an inventory-map.json / findings.json / scanner report, or map_inventory to collect live data. ``` -The URI can be a fixed resource or a filled-in template. [cloudg mcp resources](/cli/mcp-resources/) lists both. For the asset templates, the reference can be the internal ID, a unique name, or a full ARN, and the ARN can be written as is or URL-encoded. Quote the URI in the shell, since ARNs contain `:` and `/` and some templates contain characters the shell treats specially. +The URI can be a fixed resource or a filled-in template. [cloudg mcp resources](mcp-resources.md) lists both. For the asset templates, the reference can be the internal ID, a unique name, or a full ARN, and the ARN can be written as is or URL-encoded. Quote the URI in the shell, since ARNs contain `:` and `/` and some templates contain characters the shell treats specially. Text contents are printed as they are, which for most resources is indented JSON and for some is another format: `cloudg://ontology/turtle` prints Turtle, `cloudg://graph/graphml` prints GraphML XML, and `cloudg://docs/{topic}` prints Markdown. Binary contents, if a resource returns any, are printed as their wire JSON with the base64 blob. @@ -25,7 +30,7 @@ $ cloudg mcp read cloudg://ontology/turtle --policy strict --dataset prod=invent Error: Unknown resource: cloudg://ontology/turtle ``` -The principal options (`--role`, `--principal-id`) work as on [cloudg mcp tools](/cli/mcp-tools/). stdout carries only the contents, so the output can be piped or redirected. +The principal options (`--role`, `--principal-id`) work as on [cloudg mcp tools](mcp-tools.md). stdout carries only the contents, so the output can be piped or redirected. ## Examples @@ -101,9 +106,22 @@ cloudg mcp read cloudg://policy --policy ./mcp-policy.yaml --role analyst ## Related -:::links -- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with example contents. -- [cloudg mcp resources](/cli/mcp-resources/) List the URIs and templates. -- [cloudg mcp call](/cli/mcp-call/) Call a tool instead of reading a resource. -- [The workspace](/mcp/server/the-workspace/) How datasets are loaded and named. -::: +
    + +- [**Resources and resource templates**](../mcp/tools/resources-and-resource-templates.md) + + Every resource with example contents. + +- [**cloudg mcp resources**](mcp-resources.md) + + List the URIs and templates. + +- [**cloudg mcp call**](mcp-call.md) + + Call a tool instead of reading a resource. + +- [**The workspace**](../mcp/server/the-workspace.md) + + How datasets are loaded and named. + +
    diff --git a/site/content/cli/mcp-resources.md b/site/content/cli/mcp-resources.md index a4bbd91..4b8a707 100644 --- a/site/content/cli/mcp-resources.md +++ b/site/content/cli/mcp-resources.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. command: mcp resources -lede: Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. -intro: | - Resources are the read-only side of the MCP server: summaries, graphs, schemas and single assets that a client can fetch by URI without calling a tool. `cloudg mcp resources` shows which ones the active policy lets a principal see. --- +Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. +{ .lede } + +Resources are the read-only side of the MCP server: summaries, graphs, schemas and single assets that a client can fetch by URI without calling a tool. `cloudg mcp resources` shows which ones the active policy lets a principal see. + + + ## How it works -The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the resources and resource templates visible to one principal, and prints them. The principal options work as on [cloudg mcp tools](/cli/mcp-tools/): the local user by default, or `--role` and `--principal-id` to see what a token holder gets. +The command builds the layer from the shared options (see [cloudg mcp serve](mcp-serve.md#building-the-layer)), asks it for the resources and resource templates visible to one principal, and prints them. The principal options work as on [cloudg mcp tools](mcp-tools.md): the local user by default, or `--role` and `--principal-id` to see what a token holder gets. There are two kinds of entry. A resource has a fixed URI, such as `cloudg://findings/summary`. A resource template has placeholders in braces that the client fills in, such as `cloudg://assets/{+ref}`. The `{+ref}` form means the value may contain `/` and `:`, so a full ARN fits. The table lists resources first, then templates, under one "URI / template" column: @@ -33,7 +38,7 @@ In 0.6.0, the default policy shows the local user 14 resources and 11 templates: | `cloudg://docs` | `cloudg://docs/{topic}` | | `cloudg://policy`, `cloudg://privacy/detectors`, `cloudg://ratelimit`, `cloudg://metrics` | | -The list does not depend on loaded datasets. Resources that read data, like `cloudg://findings/summary`, use the active dataset when they are read, so preload one with `--dataset` before you [read](/cli/mcp-read/) them. +The list does not depend on loaded datasets. Resources that read data, like `cloudg://findings/summary`, use the active dataset when they are read, so preload one with `--dataset` before you [read](mcp-read.md) them. `--include-category` and `--exclude-category` apply to resources and templates as well as tools. `--include-tool` and `--exclude-tool` do not touch them. @@ -92,9 +97,22 @@ cloudg mcp resources --exclude-category ontology --exclude-category privacy ## Related -:::links -- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with an example of its content. -- [cloudg mcp read](/cli/mcp-read/) Fetch one of these URIs from the shell. -- [cloudg mcp tools](/cli/mcp-tools/) The same listing for tools. -- [The workspace](/mcp/server/the-workspace/) Datasets and which one is active. -::: +
    + +- [**Resources and resource templates**](../mcp/tools/resources-and-resource-templates.md) + + Every resource with an example of its content. + +- [**cloudg mcp read**](mcp-read.md) + + Fetch one of these URIs from the shell. + +- [**cloudg mcp tools**](mcp-tools.md) + + The same listing for tools. + +- [**The workspace**](../mcp/server/the-workspace.md) + + Datasets and which one is active. + +
    diff --git a/site/content/cli/mcp-serve.md b/site/content/cli/mcp-serve.md index 61424b5..777882d 100644 --- a/site/content/cli/mcp-serve.md +++ b/site/content/cli/mcp-serve.md @@ -1,15 +1,20 @@ --- +description: Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. command: mcp serve -lede: Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. -intro: | - Desktop clients (Claude Desktop, Claude Code, Cursor, VS Code) launch `cloudg mcp serve` themselves over stdio, and you rarely type it by hand. You do type it when you serve over HTTP, test a policy, or preload datasets. [cloudg mcp config](/cli/mcp-config/) writes the client snippet for you. The [MCP server](/mcp/) section covers the server in depth. --- +Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. +{ .lede } + +Desktop clients (Claude Desktop, Claude Code, Cursor, VS Code) launch `cloudg mcp serve` themselves over stdio, and you rarely type it by hand. You do type it when you serve over HTTP, test a policy, or preload datasets. [cloudg mcp config](mcp-config.md) writes the client snippet for you. The [MCP server](../mcp/index.md) section covers the server in depth. + + + ## How it works The command builds one MCP layer from its options, picks a server implementation, and serves until the transport closes: end of input on stdio, Ctrl+C on HTTP. -```mermaid caption="From the command line to a running server" +```mermaid flowchart TD A["Options and env vars"] --> B["Validate tokens and option combinations"] B -->|invalid| X["Error before start"] @@ -22,6 +27,8 @@ flowchart TD G --> I["Stop: save pseudonym vault"] H --> I ``` +From the command line to a running server +{ .diagram-caption } ### Building the layer @@ -29,7 +36,7 @@ These options are shared with `tools`, `resources`, `prompts`, `call` and `read` The registry is the built-in catalog unless `--registry MODULE:ATTR` names your own. `--read-only` then drops every tool that writes files, calls cloud APIs, runs scanners or is marked destructive. `--include-category`, `--exclude-category`, `--include-tool` and `--exclude-tool` narrow it further, and `--prefix` is put in front of every tool and prompt name (at most 64 characters from `A-Z a-z 0-9 _ . -`). -The policy comes from `--policy`, which accepts a profile name (`open`, `standard`, `strict`, `read_only`, `airgapped`, `audit`, `soc-analyst`), a YAML or JSON file, or inline JSON. Without the option, cloudg reads `CLOUDG_MCP_POLICY`, and without that it uses `standard`. The policy decides which tools each principal sees and which privacy transforms run on results. See [Built-in profiles](/mcp/privacy/built-in-profiles/). +The policy comes from `--policy`, which accepts a profile name (`open`, `standard`, `strict`, `read_only`, `airgapped`, `audit`, `soc-analyst`), a YAML or JSON file, or inline JSON. Without the option, cloudg reads `CLOUDG_MCP_POLICY`, and without that it uses `standard`. The policy decides which tools each principal sees and which privacy transforms run on results. See [Built-in profiles](../mcp/privacy/built-in-profiles.md). Middleware is added from the options. A metrics middleware is always on, which is why `cloudg://metrics` is always listed. `--audit-log` (or `CLOUDG_MCP_AUDIT_LOG`) appends a JSONL record per call with argument values hashed. `--max-concurrency` caps parallel tool calls, and `--cache-ttl` caches results of read-only, idempotent tools for that many seconds. `--timeout` is the default per-call limit, 300 seconds; tools with their own limit, such as the live collection tools, keep theirs. @@ -48,7 +55,7 @@ $ cloudg mcp serve --transport sse --flavor fastmcp --json-response Error: --json-response / --stateless apply to Streamable HTTP; the fastmcp flavor cannot combine them with --transport sse ``` -`--page-size` (items per page in list results) only applies to the native flavor. [Flavors](/mcp/server/flavors/) compares them. +`--page-size` (items per page in list results) only applies to the native flavor. [Flavors](../mcp/server/flavors.md) compares them. ### stdio @@ -94,9 +101,9 @@ $ cloudg mcp serve --transport http --auth-token env:CLOUDG_MCP_TOKEN Error: Invalid value for --auth-token: Environment variable 'CLOUDG_MCP_TOKEN' for an auth token is empty ``` -:::warning Binding beyond loopback -Keep `--host 127.0.0.1` unless you also set `--auth-token`. On a non-loopback bind the `Host` header is not checked unless you pass `--allowed-host`, so set that too. -::: +!!! warning "Binding beyond loopback" + + Keep `--host 127.0.0.1` unless you also set `--auth-token`. On a non-loopback bind the `Host` header is not checked unless you pass `--allowed-host`, so set that too. `--cors-origin` grants CORS to a browser origin. Only the native flavor supports it. @@ -172,11 +179,30 @@ cloudg mcp serve --log-level DEBUG 2> ./mcp-server.log ## Related -:::links -- [Connect a client](/mcp/connect/) Step-by-step setup for each desktop client. -- [cloudg mcp config](/cli/mcp-config/) Generate the client configuration snippet. -- [Transports and protocol versions](/mcp/server/transports-and-protocol-versions/) stdio, Streamable HTTP and SSE in detail. -- [Security model in brief](/mcp/server/security-model-in-brief/) What the policy, tokens and Origin checks protect. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each `--policy` profile shows and hides. -- [cloudg mcp tools](/cli/mcp-tools/) Check what a policy exposes before you serve it. -::: +
    + +- [**Connect a client**](../mcp/connect.md) + + Step-by-step setup for each desktop client. + +- [**cloudg mcp config**](mcp-config.md) + + Generate the client configuration snippet. + +- [**Transports and protocol versions**](../mcp/server/transports-and-protocol-versions.md) + + stdio, Streamable HTTP and SSE in detail. + +- [**Security model in brief**](../mcp/server/security-model-in-brief.md) + + What the policy, tokens and Origin checks protect. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each `--policy` profile shows and hides. + +- [**cloudg mcp tools**](mcp-tools.md) + + Check what a policy exposes before you serve it. + +
    diff --git a/site/content/cli/mcp-tools.md b/site/content/cli/mcp-tools.md index 9479e89..dd95454 100644 --- a/site/content/cli/mcp-tools.md +++ b/site/content/cli/mcp-tools.md @@ -1,13 +1,18 @@ --- +description: Lists the MCP tools a server would expose, as a table or as the exact tools/list JSON a client receives. command: mcp tools -lede: Lists the MCP tools a server would expose, as a table or as the exact `tools/list` JSON a client receives. -intro: | - Run `cloudg mcp tools` with the same options you plan to give `cloudg mcp serve`. What it prints is what the client will see, after the policy, the filters and `--read-only` have done their work. --- +Lists the MCP tools a server would expose, as a table or as the exact `tools/list` JSON a client receives. +{ .lede } + +Run `cloudg mcp tools` with the same options you plan to give `cloudg mcp serve`. What it prints is what the client will see, after the policy, the filters and `--read-only` have done their work. + + + ## How it works -The command builds the same layer `serve` would build from the shared options (`--policy`, `--read-only`, `--include-category` and the rest, listed on [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the tool list of one principal, and prints it. No server starts and no client is involved. +The command builds the same layer `serve` would build from the shared options (`--policy`, `--read-only`, `--include-category` and the rest, listed on [cloudg mcp serve](mcp-serve.md#building-the-layer)), asks it for the tool list of one principal, and prints it. No server starts and no client is involved. The principal is the local user unless you pass `--role` or `--principal-id`. The local user has the id `local` and the roles `default` and `local`. With either option, the caller becomes a principal with the id you gave (default `cli`) and the roles you gave (default `default`). This is how you check what a token holder will see before you hand out the token: give the same roles you put in its `--auth-token` spec. @@ -100,9 +105,22 @@ cloudg mcp tools --json --policy open > tools-0.6.0.json ## Related -:::links -- [Tool index](/mcp/tools/tool-index/) Every tool with its arguments and an example result. -- [cloudg mcp call](/cli/mcp-call/) Call one of the listed tools from the shell. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy hides and transforms. -- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. -::: +
    + +- [**Tool index**](../mcp/tools/tool-index.md) + + Every tool with its arguments and an example result. + +- [**cloudg mcp call**](mcp-call.md) + + Call one of the listed tools from the shell. + +- [**Built-in profiles**](../mcp/privacy/built-in-profiles.md) + + What each policy hides and transforms. + +- [**cloudg mcp serve**](mcp-serve.md) + + The shared layer options in full. + +
    diff --git a/site/content/cli/output-flags.md b/site/content/cli/output-flags.md index bc630ee..dd6d7cd 100644 --- a/site/content/cli/output-flags.md +++ b/site/content/cli/output-flags.md @@ -1,8 +1,11 @@ --- title: Output flags -lede: "Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: `-o`, `--format`, `--ontology`, `--rag-export`, `--terraform` and `--json`." +description: 'Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: -o, --format, --ontology, --rag-export, --terraform and --json.' --- +Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: `-o`, `--format`, `--ontology`, `--rag-export`, `--terraform` and `--json`. +{ .lede } + ## -o, --output Every command that writes files takes `-o/--output`, and it defaults to `./reports` everywhere. The directory is created if it doesn't exist, parents included. @@ -55,7 +58,7 @@ reports/ └── terraform/ provider.tf.json, variables.tf.json, main.tf.json, import_commands.sh (run, --terraform) ``` -The [output files reference](/reference/output-files/) describes each file's structure. +The [output files reference](../reference/output-files.md) describes each file's structure. The Terraform recreation goes to `/terraform`, so it follows `-o`. A `terraform.output_dir` in your config sends it somewhere else, except the old default value `./reports/terraform`, which counts as unset. @@ -98,7 +101,7 @@ Turn off the ontology and the RAG export when you only need findings: cloudg run -p aws --no-ontology --no-rag-export ``` -`--terraform` has a second effect: when no IaC directory is given by `--iac-dir` or `scanners.iac_directories`, Checkov and the Trivy filesystem scan target the generated Terraform. See [cloudg run](/cli/run/#how-it-works). +`--terraform` has a second effect: when no IaC directory is given by `--iac-dir` or `scanners.iac_directories`, Checkov and the Trivy filesystem scan target the generated Terraform. See [cloudg run](run.md#how-it-works). ## --json @@ -131,9 +134,22 @@ cloudg -v --log-file ./cloudg.log run -p aws ## Related -:::links -- [Output files](/reference/output-files/) The structure of every file in the output directory. -- [Reports](/guides/reports/) Reading the HTML report and `findings.json`. -- [Graph, ontology and RAG](/guides/graph-ontology-rag/) What the ontology and RAG exports are for. -- [Provider and region flags](/cli/provider-flags/) What gets collected in the first place. -::: +
    + +- [**Output files**](../reference/output-files.md) + + The structure of every file in the output directory. + +- [**Reports**](../guides/reports.md) + + Reading the HTML report and `findings.json`. + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + What the ontology and RAG exports are for. + +- [**Provider and region flags**](provider-flags.md) + + What gets collected in the first place. + +
    diff --git a/site/content/cli/provider-flags.md b/site/content/cli/provider-flags.md index 225ba78..be19cf6 100644 --- a/site/content/cli/provider-flags.md +++ b/site/content/cli/provider-flags.md @@ -1,8 +1,11 @@ --- title: Provider and region flags -lede: "`-p`, `--region`, `--regions`, `--subscription-id`, `--project-id` and `--profile` decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere." +description: -p, --region, --regions, --subscription-id, --project-id and --profile decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere. --- +`-p`, `--region`, `--regions`, `--subscription-id`, `--project-id` and `--profile` decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere. +{ .lede } + ## Which command takes which flag | Flag | `run` | `map` | `collect` | `scan` | @@ -43,7 +46,7 @@ Without either flag, `run` and `map` use the config: `aws.regions` defaults to ` ### How discovery works -```mermaid caption="Resolving the AWS region list" +```mermaid flowchart TD A["aws.regions after flags"] --> B{"Is it ALL?"} B -->|no| C["Use the list as given"] @@ -53,6 +56,8 @@ flowchart TD F -->|ok| G["Enabled regions, sorted"] F -->|"fails"| H["The 17 regions enabled by default"] ``` +Resolving the AWS region list +{ .diagram-caption } | Provider | API | Filter | Fallback | |---|---|---|---| @@ -89,7 +94,7 @@ To cover a whole GCP organization in one Cloud Asset Inventory listing, set `gcp ## --profile -An AWS CLI profile name, SSO profiles included. On `run` and `map` it sets `aws.profile`, which sits third in the AWS credential order: direct keys and OIDC federation win over it. See [Auth flags](/cli/auth-flags/). +An AWS CLI profile name, SSO profiles included. On `run` and `map` it sets `aws.profile`, which sits third in the AWS credential order: direct keys and OIDC federation win over it. See [Auth flags](auth-flags.md). On `run`, `collect` and `scan` it replaces `aws.profile` the same way. `run` and `scan` also hand the profile to the scanners, for AWS only: Prowler gets `-p ` and ScoutSuite gets `--profile `, unless direct keys or a `role_arn` win, in which case they get the resulting keys instead. On `collect`, with no profile from the flag or the config, `AWS_PROFILE` and then the default chain apply. @@ -127,9 +132,22 @@ cloudg collect -p aws --profile audit --region ap-southeast-2 ## Related -:::links -- [Auth flags](/cli/auth-flags/) Credentials for each provider and which method wins. -- [Output flags](/cli/output-flags/) Where results go. -- [cloudg map](/cli/map/) Organization-wide AWS mapping with `--org`. -- [Provider settings](/reference/config/providers/) The matching `config.yaml` keys. -::: +
    + +- [**Auth flags**](auth-flags.md) + + Credentials for each provider and which method wins. + +- [**Output flags**](output-flags.md) + + Where results go. + +- [**cloudg map**](map.md) + + Organization-wide AWS mapping with `--org`. + +- [**Provider settings**](../reference/config/providers.md) + + The matching `config.yaml` keys. + +
    diff --git a/site/content/cli/report.md b/site/content/cli/report.md index d8bd666..94da181 100644 --- a/site/content/cli/report.md +++ b/site/content/cli/report.md @@ -1,10 +1,15 @@ --- +description: Re-renders the HTML, JSON and SVG reports from a findings.json you already have. No cloud access and no scanners. command: report -lede: Re-renders the HTML, JSON and SVG reports from a `findings.json` you already have. No cloud access and no scanners. -intro: | - Use `cloudg report` after upgrading cloudg to get the newer HTML template, or to rebuild one format you skipped. It also turns a `raw-findings.json` into reports. --- +Re-renders the HTML, JSON and SVG reports from a `findings.json` you already have. No cloud access and no scanners. +{ .lede } + +Use `cloudg report` after upgrading cloudg to get the newer HTML template, or to rebuild one format you skipped. It also turns a `raw-findings.json` into reports. + + + ## How it works `-i/--input` names a `findings.json` written by `cloudg run`, `cloudg ingest` or `CloudGEngine`, or a `raw-findings.json` from `cloudg scan` or `cloudg ingest`. @@ -101,9 +106,22 @@ cloudg report -i ./reports/raw-findings.json -o ./reports/scan-report ## Related -:::links -- [Reports](/guides/reports/) What is in the HTML report and how to read it. -- [Output files](/reference/output-files/) The `findings.json` format. -- [cloudg ingest](/cli/ingest/) Build reports from scanner output. -- [Output flags](/cli/output-flags/) `-o` and `--format` across commands. -::: +
    + +- [**Reports**](../guides/reports.md) + + What is in the HTML report and how to read it. + +- [**Output files**](../reference/output-files.md) + + The `findings.json` format. + +- [**cloudg ingest**](ingest.md) + + Build reports from scanner output. + +- [**Output flags**](output-flags.md) + + `-o` and `--format` across commands. + +
    diff --git a/site/content/cli/run.md b/site/content/cli/run.md index 19fd512..615f258 100644 --- a/site/content/cli/run.md +++ b/site/content/cli/run.md @@ -1,10 +1,15 @@ --- +description: The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. command: run -lede: The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. -intro: | - `cloudg run` is the command most people start with. It is also the only command that takes the full set of credential flags, so it works the same way on a laptop, in CI and on cloud compute. If you only want to know what exists, [cloudg map](/cli/map/) is faster and needs no scanners. --- +The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. +{ .lede } + +`cloudg run` is the command most people start with. It is also the only command that takes the full set of credential flags, so it works the same way on a laptop, in CI and on cloud compute. If you only want to know what exists, [cloudg map](map.md) is faster and needs no scanners. + + + ## How it works `cloudg run` builds one `CloudGConfig` and runs a fixed sequence of phases against it. The config comes from the file you pass to the root command (`cloudg -c config.yaml run ...`). Without `-c`, cloudg uses its built-in defaults. It does not look for a `config.yaml` in the current directory on its own. @@ -14,11 +19,11 @@ On top of that config, the command applies its flags: 1. `-p/--provider` replaces `providers`. `all` expands to `aws`, `azure` and `gcp`. 2. `--regions` replaces the region list of all three providers at once. `all` becomes the `ALL` sentinel, which triggers region discovery. When `--regions` is absent, `--region` sets `aws.regions` to that single region and leaves Azure and GCP alone. 3. `--subscription-id`, `--project-id` and `--profile` set `azure.subscription_ids`, `gcp.project_ids` and `aws.profile`. -4. Every `--aws-*`, `--azure-*` and `--gcp-*` flag you give overwrites the matching credential field. Flags you leave out keep whatever the config file says. See [Auth flags](/cli/auth-flags/). +4. Every `--aws-*`, `--azure-*` and `--gcp-*` flag you give overwrites the matching credential field. Flags you leave out keep whatever the config file says. See [Auth flags](auth-flags.md). Then the phases run: -```mermaid caption="Phases of cloudg run" +```mermaid flowchart TD A["Config file + flags"] --> B["1: collect assets"] B --> C["2: graph and reachability"] @@ -30,6 +35,8 @@ flowchart TD F --> H["4: normalise"] H --> I["5: JSON, SVG, HTML"] ``` +Phases of cloudg run +{ .diagram-caption } Phase 1 is collection. The multi-provider collector walks every configured account, subscription or project, and every region, concurrently. An `aws.accounts` list in the config needs `aws.role_name` too (unless `aws.organization.enabled` is on); without it the command stops with exit status 2 before collecting anything. If collection throws, cloudg prints a "Collection failed" panel and carries on with an empty inventory, so the scanners still run. Each account, region or provider that could not be collected gets a "Not collected: ..." line at the end of the phase. @@ -39,7 +46,7 @@ Phase 2c, the RAG export, runs when `--rag-export` is on (the default) and `rag. Phase 2d, the Terraform recreation, runs when you pass `--terraform` or set `terraform.enabled: true`. Either one is enough. -Phase 3 runs the scanners. Every scanner job gets its own thread, so they all start at once. The scanner list comes from `--scanners`, or from `scanners.enabled` in config when the flag is absent. The default config enables all five: `prowler`, `scoutsuite`, `checkov`, `trivy` and `iam`. A [scanner plugin](/api/entry-points/) runs when its name is in the list, and a name that matches nothing is skipped with a warning. Prowler and ScoutSuite run once per provider. Checkov runs once per IaC directory. Trivy scans the images from `--images` (or `scanners.trivy_images`), and falls back to a filesystem scan of the IaC directories when there are no images. +Phase 3 runs the scanners. Every scanner job gets its own thread, so they all start at once. The scanner list comes from `--scanners`, or from `scanners.enabled` in config when the flag is absent. The default config enables all five: `prowler`, `scoutsuite`, `checkov`, `trivy` and `iam`. A [scanner plugin](../api/entry-points.md) runs when its name is in the list, and a name that matches nothing is skipped with a warning. Prowler and ScoutSuite run once per provider. Checkov runs once per IaC directory. Trivy scans the images from `--images` (or `scanners.trivy_images`), and falls back to a filesystem scan of the IaC directories when there are no images. The IaC directory is resolved in this order: `--iac-dir`, then `scanners.iac_directories`, then the Terraform recreation from phase 2d if it produced any `*.tf.json` files. There is no fallback to the current directory. A scan of whatever folder you happened to run cloudg from would report zero findings and look like a clean result, so cloudg warns "nothing to scan" instead. @@ -130,24 +137,28 @@ Skip the semantic exports when you only want findings and the HTML report: cloudg run -p aws --no-ontology --no-rag-export ``` -Drive the pipeline from Python when you want result objects as well as files. `CloudGEngine.run_pipeline_sync()` covers the same ground, though it orders the analysis steps a little differently and writes no `topology.graphml` (see [CloudGEngine](/api/cloudgengine/)): +Drive the pipeline from Python when you want result objects as well as files. `CloudGEngine.run_pipeline_sync()` covers the same ground, though it orders the analysis steps a little differently and writes no `topology.graphml` (see [CloudGEngine](../api/cloudgengine.md)): -```bash tab="CLI" -cloudg run -p aws --region us-east-1 --scanners prowler,checkov,iam -o ./reports -``` +=== "CLI" -```python tab="Python" title="run_pipeline.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg run -p aws --region us-east-1 --scanners prowler,checkov,iam -o ./reports + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] -config.scanners.enabled = ["prowler", "checkov", "iam"] +=== "Python" -engine = CloudGEngine(config) -result = engine.run_pipeline_sync(output_dir="./reports") -print(len(result.findings), "findings") -print(result.report_paths) -``` + ```python title="run_pipeline.py" + from cloudg import CloudGConfig, CloudGEngine + + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + config.scanners.enabled = ["prowler", "checkov", "iam"] + + engine = CloudGEngine(config) + result = engine.run_pipeline_sync(output_dir="./reports") + print(len(result.findings), "findings") + print(result.report_paths) + ``` ## Output files @@ -168,7 +179,7 @@ All paths are relative to `-o/--output` (default `./reports`). The Terraform files go to `/terraform`, so `cloudg run -o /tmp/scan --terraform` writes them to `/tmp/scan/terraform`. A `terraform.output_dir` in your config replaces that directory, except the old default `./reports/terraform`, which counts as unset. -`cloudg run` does not write `raw-findings.json`. Use [cloudg scan](/cli/scan/) or [cloudg ingest](/cli/ingest/) when you need the pre-normalisation findings. +`cloudg run` does not write `raw-findings.json`. Use [cloudg scan](scan.md) or [cloudg ingest](ingest.md) when you need the pre-normalisation findings. ## Exit codes @@ -181,15 +192,34 @@ The Terraform files go to `/terraform`, so `cloudg run -o /tmp/scan --te Exit 3 means no target could be collected at all: a profile that does not exist, a role that cannot be assumed, or no credentials anywhere in the default chain, where every service of the region fails on its own. A run where some services or regions worked exits 0; the coverage table shows what was missed. -To fail a CI job on findings, read `findings.json` after the run. [CI and automation](/guides/ci-automation/) has a severity gate. +To fail a CI job on findings, read `findings.json` after the run. [CI and automation](../guides/ci-automation.md) has a severity gate. ## Related -:::links -- [The pipeline](/guides/pipeline/) What each phase does and why it runs in that order. -- [Running scanners](/guides/running-scanners/) Installing Prowler, ScoutSuite, Checkov and Trivy. -- [Auth flags](/cli/auth-flags/) Every credential flag and which one wins. -- [Provider and region flags](/cli/provider-flags/) `-p`, `--region`, `--regions` and discovery. -- [Output flags](/cli/output-flags/) The output directory and what lands in it. -- [cloudg map](/cli/map/) Inventory only, no scanners. -::: +
    + +- [**The pipeline**](../guides/pipeline.md) + + What each phase does and why it runs in that order. + +- [**Running scanners**](../guides/running-scanners.md) + + Installing Prowler, ScoutSuite, Checkov and Trivy. + +- [**Auth flags**](auth-flags.md) + + Every credential flag and which one wins. + +- [**Provider and region flags**](provider-flags.md) + + `-p`, `--region`, `--regions` and discovery. + +- [**Output flags**](output-flags.md) + + The output directory and what lands in it. + +- [**cloudg map**](map.md) + + Inventory only, no scanners. + +
    diff --git a/site/content/cli/scan.md b/site/content/cli/scan.md index 55bd99a..db8af44 100644 --- a/site/content/cli/scan.md +++ b/site/content/cli/scan.md @@ -1,15 +1,20 @@ --- +description: Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. command: scan -lede: Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. -intro: | - `cloudg scan` is the scanner phase of `cloudg run` cut loose from the rest. It is handy for checking that your scanner binaries work, or for producing a `raw-findings.json` to merge into a map later. For a complete assessment with deduplication and reports, use [cloudg run](/cli/run/). --- +Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. +{ .lede } + +`cloudg scan` is the scanner phase of `cloudg run` cut loose from the rest. It is handy for checking that your scanner binaries work, or for producing a `raw-findings.json` to merge into a map later. For a complete assessment with deduplication and reports, use [cloudg run](run.md). + + + ## How it works `scan` reads the config loaded with `cloudg -c`, like `run` and `map`, and plans its scanner jobs with the same code as `cloudg run`. From the config it takes `scanners.enabled` (the default for `--scanners`), the `*_extra_args` lists, `checkov_frameworks`, `iac_directories`, `trivy_images` and `timeout_seconds`, plus the AWS credentials and regions (`aws.profile`, keys, `role_arn`, web identity, `aws.regions`). Flags replace the matching config values. Without `-c`, the built-in defaults apply: all five scanners, a 3600-second timeout. -`--scanners` is a comma-separated list of built-in names or [scanner plugin](/api/entry-points/) names; an unknown name is skipped with a warning. Each scanner becomes one or more jobs, and all jobs run in parallel threads with a progress display: +`--scanners` is a comma-separated list of built-in names or [scanner plugin](../api/entry-points.md) names; an unknown name is skipped with a warning. Each scanner becomes one or more jobs, and all jobs run in parallel threads with a progress display: | Scanner | What it runs | |---|---| @@ -104,7 +109,7 @@ cloudg map -p aws --profile audit --regions all --findings ./reports/raw-finding | `prowler//` | Prowler's native ASFF output, when Prowler ran | | `scoutsuite//` | ScoutSuite's native report, when ScoutSuite ran | -`scan` writes no `findings.json` and no HTML report. To get them, run `cloudg report -i ./reports/raw-findings.json -o ./reports/html`, which normalises the raw findings first, or feed the native files to [cloudg ingest](/cli/ingest/), for example `cloudg ingest --prowler ./reports/prowler/aws/`. +`scan` writes no `findings.json` and no HTML report. To get them, run `cloudg report -i ./reports/raw-findings.json -o ./reports/html`, which normalises the raw findings first, or feed the native files to [cloudg ingest](ingest.md), for example `cloudg ingest --prowler ./reports/prowler/aws/`. ## Exit codes @@ -118,9 +123,22 @@ A missing scanner next to one that worked still exits 0, so check the `not insta ## Related -:::links -- [Running scanners](/guides/running-scanners/) Install and configure Prowler, ScoutSuite, Checkov and Trivy. -- [cloudg run](/cli/run/) Scanners plus collection, graph, normalisation and reports. -- [cloudg ingest](/cli/ingest/) Normalise and report on scanner output. -- [Provider and region flags](/cli/provider-flags/) How `-p` differs between commands. -::: +
    + +- [**Running scanners**](../guides/running-scanners.md) + + Install and configure Prowler, ScoutSuite, Checkov and Trivy. + +- [**cloudg run**](run.md) + + Scanners plus collection, graph, normalisation and reports. + +- [**cloudg ingest**](ingest.md) + + Normalise and report on scanner output. + +- [**Provider and region flags**](provider-flags.md) + + How `-p` differs between commands. + +
    diff --git a/site/content/credits.md b/site/content/credits.md new file mode 100644 index 0000000..5be0927 --- /dev/null +++ b/site/content/credits.md @@ -0,0 +1,59 @@ +--- +title: Credits +description: The projects, data and standards cloudg builds on, and the licenses that come with them. +--- + +cloudg builds on other people's work, and some of it ships inside the package. This page lists where each piece comes from and under which license. +{ .lede } + +## At a glance + +| Project | License | What cloudg uses | +|---|---|---| +| [Prowler](https://github.com/prowler-cloud/prowler) | Apache-2.0 | 22 compliance framework files generated from its compliance JSON; its ASFF and OCSF output | +| [Checkov](https://github.com/bridgecrewio/checkov) | Apache-2.0 | Run as a separate program; reports parsed, `CKV_*` check IDs | +| [Trivy](https://github.com/aquasecurity/trivy) | Apache-2.0 | Run as a separate program; reports parsed, `AVD-*` check IDs | +| [ScoutSuite](https://github.com/nccgroup/ScoutSuite) | GPL-2.0 | Run as a separate program; reports parsed | +| [Cloud Custodian](https://github.com/cloud-custodian/cloud-custodian) | Apache-2.0 | Policy format and the `custodian` CLI for `cloudg/policies/` | +| [parliament](https://github.com/duo-labs/parliament) | BSD-3-Clause | IAM policy linting | +| [MITRE ATT&CK®](https://attack.mitre.org/) | Used with permission | Technique IDs and descriptions | +| [Chart.js](https://www.chartjs.org) 4.4.0 | MIT | Charts in report.html | +| [D3](https://d3js.org) 7.9.0 | ISC | The 2D graph view in report.html | +| [Anatomy](https://github.com/wheresryan22/anatomy) iso-kit | MIT | The camera, solids and shading of the isometric map in report.html | +| [Hairline](https://hairline.lucasmarkes.com) 0.5.1 kernel | MIT | The line figures in report.html's empty states | + +## Prowler + +[Prowler](https://github.com/prowler-cloud/prowler) (Apache-2.0). The 22 framework files in `cloudg/rules/frameworks/` are generated from Prowler's compliance JSON by `scripts/import_prowler_compliance.py`. That covers the control IDs, the control titles and descriptions, and the mapping from each control to Prowler check names. Every generated file says so in its header and in its `source` field. Prowler check names also appear in `cloudg/rules/check_equivalence.yaml`, and cloudg runs Prowler and parses its ASFF and OCSF output. + +## Scanners + +[Checkov](https://github.com/bridgecrewio/checkov) (Apache-2.0), [Trivy](https://github.com/aquasecurity/trivy) (Apache-2.0) and [ScoutSuite](https://github.com/nccgroup/ScoutSuite) (GPL-2.0) are run as separate programs, and their reports are parsed. Their check IDs (`CKV_*`, `AVD-*`) are used in the check-equivalence map. None of their code is part of cloudg. The Docker image installs Prowler, Checkov and Trivy, so their own licenses apply to those copies. + +## Cloud Custodian + +[Cloud Custodian](https://github.com/cloud-custodian/cloud-custodian) (Apache-2.0). The policy packs in `cloudg/policies/` are written in Cloud Custodian's policy format and run with its `custodian` CLI. + +## parliament + +[parliament](https://github.com/duo-labs/parliament) (BSD-3-Clause) from Duo Labs does the IAM policy linting. + +## Drawing engines + +The estate map in report.html is drawn with the iso-kit from [Anatomy](https://github.com/wheresryan22/anatomy) (MIT, Copyright (c) 2026 Ryan): its camera, its solids and their four-tone shading. The empty states use the kernel of [Hairline](https://hairline.lucasmarkes.com) 0.5.1 (MIT, Copyright (c) 2026 Lucas Marques), with a figure of an empty tray written for cloudg in Hairline's own format. Both files sit in `cloudg/templates/vendor/` and are embedded in every report, so the map works offline. The Hairline kernel is unmodified. The iso-kit is wrapped so it can sit inside a script element; [`NOTICE`](https://github.com/morpheuslord/cloudg/blob/main/cloudg/templates/vendor/NOTICE) lists exactly what changed. + +## MITRE ATT&CK + +[MITRE ATT&CK®](https://attack.mitre.org/). The `mitre_attack_*.yaml` files contain ATT&CK technique IDs and descriptions. © The MITRE Corporation. This work is reproduced and distributed with the permission of The MITRE Corporation. ATT&CK is a registered trademark of The MITRE Corporation. + +## Standards + +The hand-written rulesets in `cloudg/rules/` reference public control numbering from the CIS Benchmarks for AWS, Azure and GCP, NIST SP 800-53, NIST CSF, PCI DSS, the AICPA SOC 2 Trust Services Criteria, the HIPAA Security Rule, GDPR, ISO/IEC 27001:2022 and the AWS Foundational Security Best Practices. The standards belong to their publishers. CIS Benchmarks is a trademark of the Center for Internet Security, and cloudg is not affiliated with or endorsed by CIS or any other publisher named here. + +## Python libraries + +The Python libraries cloudg depends on (NetworkX, rdflib, pydantic, Click, Rich and the rest) are listed in [`pyproject.toml`](https://github.com/morpheuslord/cloudg/blob/main/pyproject.toml). + +## License + +cloudg itself is released under the [MIT License](https://github.com/morpheuslord/cloudg/blob/main/LICENSE). report.html embeds unmodified builds of Chart.js and D3 when `report.inline_js` is true (the default), and always embeds the Anatomy iso-kit and the Hairline kernel, so the report works offline. Their notices are in [`cloudg/templates/vendor/NOTICE`](https://github.com/morpheuslord/cloudg/blob/main/cloudg/templates/vendor/NOTICE). diff --git a/site/content/examples/errors.md b/site/content/examples/errors.md new file mode 100644 index 0000000..4a1bbc3 --- /dev/null +++ b/site/content/examples/errors.md @@ -0,0 +1,126 @@ +--- +title: Errors and strict mode +description: Get exceptions from the cloudg scanner wrappers instead of recorded failures, check credentials before a run, stop a pipeline at the first failed phase and reject unreadable reports. +--- + +cloudg is built to keep going: a scanner that times out, a service that refuses a call or a report that will not parse is recorded, and the rest of the run carries on. These scripts show how to turn those records into exceptions when your job should stop instead. +{ .lede } + +Every exception cloudg raises on purpose derives from `CloudGError`. [Exceptions](../api/exceptions.md) has the class tree and when each one is raised. + +## A scanner that raises + +**Offline.** With `strict=True` the first failure is raised at once. Without Checkov on `PATH`, that is a `ScannerNotInstalledError`. + +=== "Python" + + ```python title="strict_scanner.py" + --8<-- "errors/strict_scanner.py" + ``` + +=== "Output" + + ```console + $ python strict_scanner.py + checkov is missing. pip install checkov + ``` + +## Decide after the run + +`raise_for_errors()` raises what the last run recorded: one failure as it is, several as a `ScannerFailed` that lists each of them. Nothing recorded, nothing raised. + +=== "Python" + + ```python title="raise_for_errors.py" + --8<-- "errors/raise_for_errors.py" + ``` + +=== "Output" + + With a `trivy` on `PATH` that prints something other than JSON: + + ```console + $ python raise_for_errors.py + 0 findings, 2 failures recorded + - ScannerOutputError output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + - ScannerOutputError output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for nginx:1.27 is not JSON: Expecting value: line 1 column 1 (char 0) + failed: trivy: output for python:3.12-slim is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +## Check credentials before a run + +`CloudGEngine.collect()` records failures per service instead of raising. Build the session yourself first and a missing SDK or a refused role stops the job before anything runs. + +=== "Python" + + ```python title="engine_errors.py" + --8<-- "errors/engine_errors.py" + ``` + +=== "Output" + + ```console + $ pip uninstall -y boto3 + $ python engine_errors.py + boto3 is not installed: pip install 'cloudg[aws]' + ``` + +=== "CLI" + + On the CLI, check the exit status instead. `cloudg run` exits 3 when collection failed for every target, and 0 with a warning when only some failed: + + ```bash + cloudg run -p aws --profile audit --regions eu-west-1 + echo $? + ``` + +## Stop at the first failed phase + +Drive the phases yourself and check the `on_error` records between them. Exit status 2 means a phase failed, 0 a complete run. + +```python title="fail_fast.py" +--8<-- "errors/fail_fast.py" +``` + +Without credentials, every AWS service fails and the script stops after the collection with exit status 2, before any scanner starts. + +## Reject an unreadable report + +**Offline.** Each wrapper's `parse_report(path, strict=True)` raises `ReportParseError` for a file that is not a report, instead of skipping it with a warning. + +=== "Python" + + ```python title="strict_parse_report.py" + --8<-- "errors/strict_parse_report.py" + ``` + +=== "Output" + + ```console + $ echo "not json" > results_json.json + $ python strict_parse_report.py results_json.json + checkov report results_json.json is unreadable: output is not JSON: Expecting value: line 1 column 1 (char 0) + ``` + +## Related + +
    + +- [**Exceptions**](../api/exceptions.md) + + The `CloudGError` hierarchy, attribute by attribute. + +- [**Event hooks**](../api/event-hooks.md) + + `on_error` and the other callbacks. + +- [**cloudg.resilience**](../api/resilience.md) + + Circuit breakers, retry budgets and the errors they raise. + +- [**CI and automation**](../guides/ci-automation.md) + + Exit codes and gates in a pipeline. + +
    diff --git a/site/content/examples/graph-and-exports.md b/site/content/examples/graph-and-exports.md new file mode 100644 index 0000000..56c704a --- /dev/null +++ b/site/content/examples/graph-and-exports.md @@ -0,0 +1,188 @@ +--- +title: Graph, ontology and RAG +description: Build the cloudg graph from your own assets, find what the internet reaches, query an RDF ontology with SPARQL and cut text chunks for retrieval-augmented generation. +--- + +The graph, the ontology and the RAG export all start from the same two lists, `CloudAsset` and `NetworkEdge`. These scripts build a small estate by hand, so every one of them is **offline**: swap the hand-made lists for `collection.assets` and `collection.edges` from `CloudGEngine.collect()`, or for an `InventoryResult` you loaded, and nothing else changes. +{ .lede } + +```mermaid +flowchart LR + A["CloudAsset + NetworkEdge"] --> G["GraphBuilder"] + G --> P["attack paths, centrality"] + G --> R["ReachabilityAnalyzer"] + R --> F["exposure findings"] + A --> O["CloudOntology"] + O --> Q["SPARQL, Turtle, JSON-LD"] + A --> X["RAGExporter"] + X --> C["rag_chunks.jsonl"] +``` + +## Build and explore the graph + +`GraphBuilder` turns the lists into a NetworkX `DiGraph`, merges parallel rules between the same pair of nodes, and adds a placeholder node for `0.0.0.0/0`. + +=== "Python" + + ```python title="build_graph.py" + --8<-- "graph/build_graph.py" + ``` + +=== "Output" + + ```console + $ python build_graph.py + 9 nodes, 10 edges from 11 NetworkEdges + merged rule: 443,80 | HTTPS from anywhere; HTTP redirect + placeholder: EXTERNAL True + internet path: 0.0.0.0/0 -> sg-alb -> sg-web + identity path: web-1 -> app-role -> orders-db + lateral path: web-1 -> app-role + betweenness 0.080 web-1 + betweenness 0.071 web-tg + betweenness 0.054 app-role + community 0: ['0.0.0.0/0', 'sg-alb'] + community 1: ['sg-web', 'web-2'] + community 2: ['app-role', 'orders-db', 'web-1'] + community 3: ['web-alb', 'web-tg'] + wrote ['estate-cytoscape.json', 'estate-d3.json', 'estate.graphml'] + ``` + +=== "CLI" + + `cloudg run` builds the same graph from a live collection and writes it next to the reports: + + ```bash + cloudg run -p aws --regions eu-west-1 -o ./reports + ls reports/topology.graphml + ``` + + `graph.persist_graphml` (on by default) writes `topology.graphml`, and `graph.export_cytoscape` (off by default) adds `topology-cytoscape.json`. Both are in the [graph section](../reference/config/graph.md) of config.yaml. + +## What can the internet reach + +`ReachabilityAnalyzer` walks the graph from the internet along the edges traffic can actually take, and gives each finding an id that stays the same from run to run. + +=== "Python" + + ```python title="exposure.py" + --8<-- "graph/exposure.py" + ``` + +=== "Output" + + ```console + $ python exposure.py + entry points: ['0.0.0.0/0'] + exposed: ['orders-api', 'sg-alb', 'sg-web', 'web-1', 'web-alb', 'web-tg'] + hops out of sg-web: + -> web-1 via ATTACHED_TO (reversed) + HIGH Unexpected internet-exposed resource: orders-api id=b5fbe235 + HIGH Unexpected internet-exposed resource: web-1 id=4ec3a339 + CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 id=17cd5168 + web-1 exposure id: 4ec3a339 + flow edges: 10 of 14 + flow reversed: [('sg-alb', 'web-alb'), ('sg-db', 'orders-db'), ('sg-web', 'web-1')] + blast radius of web-1: ['0.0.0.0/0', 'app-role', 'orders-db', 'sg-db', 'sg-web'] depth 3 risk 4.5 + ``` + +## Query it with SPARQL + +`CloudOntology` writes the estate into an rdflib graph with typed relations, then exports it in four RDF serialisations and answers SPARQL queries. + +=== "Python" + + ```python title="ontology_demo.py" + --8<-- "graph/ontology_demo.py" + ``` + +=== "Output" + + ```console + $ python ontology_demo.py + 689 triples, 9 classes, 9 individuals + groups: {'GOVERNANCE': 2, 'NETWORK': 3, 'SECURITY': 4} + 0.0.0.0/0 INGRESS_ALLOWED sg-web + 0.0.0.0/0 INTERNET_REACHABLE sg-web + 0.0.0.0/0 ONLY_SSH sg-web + RDS instance has no deletion protection FINDING_AFFECTS orders-db + orders-db ENCRYPTED_BY_KMS orders-key + web-1 OWNED_BY https://cloudg.io/resource/tag_payments + web-1 PROTECTED_BY_SG sg-web + web-1 TAGGED_WITH https://cloudg.io/resource/tag_owner_payments + internet reachable: ['sg-web'] + security group: 4 triples + compliance gaps: [('HIGH', 'orders-db')] + reloaded 689 triples from ['ontology.jsonld', 'ontology.nt', 'ontology.rdf', 'ontology.ttl'] + ``` + +## Chunks for retrieval-augmented generation + +`RAGExporter` cuts the estate into one chunk per asset, per community and per relation group, each with metadata you can filter on before a similarity search. + +=== "Python" + + ```python title="rag_demo.py" + --8<-- "graph/rag_demo.py" + ``` + +=== "Output" + + ```console + $ python rag_demo.py + {'chunks': 'rag_chunks.jsonl', 'index': 'rag_metadata_index.json'} + {'total_chunks': 11, 'entity_chunks': 5, 'community_chunks': 2, 'relation_group_chunks': 4} + 11 records -> rag-out/vector-records.jsonl + Resource: web-1 + Type: EC2 + Provider: AWS + Region: eu-west-1 + ARN: arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1 + Account: 123456789012 + WARNING: INTERNET EXPOSED + + Relations (2): + -> PROTECTED_BY_SG: sg-web + -> RUNS_ON: app-role + + Findings (1): + [HIGH] Instance metadata service v1 enabled + {'chunk_type': 'entity', 'asset_type': 'EC2', 'provider': 'AWS', 'region': 'eu-west-1', 'account_id': '123456789012', 'is_internet_exposed': True, 'relation_types': 'PROTECTED_BY_SG,RUNS_ON', 'severity_max': 'HIGH', 'compliance_frameworks': 'CIS', 'neighbour_count': 2, 'finding_count': 1, 'arn': 'arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1'} + high or critical: ['entity::web-1', 'community::1'] + ``` + +=== "config.yaml" + + `cloudg run` writes the ontology and the chunks when these are on: + + ```yaml title="config.yaml" + ontology: + enabled: true + export_formats: [turtle, json-ld] + rag: + enabled: true + chunk_strategy: hybrid + max_chunk_tokens: 2000 + ``` + +## Related + +
    + +- [**Graph, ontology and RAG**](../guides/graph-ontology-rag.md) + + The guide: what each export contains and how to load it elsewhere. + +- [**GraphBuilder**](../api/graphbuilder.md) + + Every method, from `build()` to `save_graphml()`. + +- [**CloudOntology**](../api/cloudontology.md) + + The relations, the query helpers and the serialisations. + +- [**RAGExporter**](../api/ragexporter.md) + + Chunk types, metadata fields and size limits. + +
    diff --git a/site/content/examples/index.md b/site/content/examples/index.md new file mode 100644 index 0000000..9c2fdef --- /dev/null +++ b/site/content/examples/index.md @@ -0,0 +1,55 @@ +--- +title: Examples +description: Complete cloudg scripts for mapping, ingesting, graph exports and error handling, each shown next to the CLI command that does the same and the output it prints. +--- + +Complete scripts you can save and run, each next to the CLI command that does the same job and the output it printed when these pages were written. They are the files in [`site/examples/`](https://github.com/morpheuslord/cloudg/tree/main/site/examples), included into the pages as they are. +{ .lede } + +Most of them need no cloud account. Those marked **offline** build a small estate by hand or read sample scanner reports, so they run on any machine with `pip install cloudg`. The others say which credentials they need. + +## Sample reports to work with + +The ingest examples read native scanner output. If you have none at hand, this script writes one small report per scanner (Prowler ASFF, ScoutSuite, Checkov and two Trivy reports), shaped like the real tools' output: + +??? example "make_sample_reports.py" + + ```python title="make_sample_reports.py" + --8<-- "make_sample_reports.py" + ``` + +Run it in an empty directory, then run the examples from the same directory. + +## By task + +
    + +- :material-map-search-outline:{ .lg .middle } **[Map and query](map-and-query.md)** + + --- + + Map an account, save the map, load it again and ask what depends on what. + +- :material-file-import-outline:{ .lg .middle } **[Ingest and report](ingest-and-report.md)** + + --- + + Merge Prowler, Checkov and Trivy output, gate a CI job on severity and export the findings. + +- :material-graph-outline:{ .lg .middle } **[Graph, ontology and RAG](graph-and-exports.md)** + + --- + + Build the graph from your own assets, find what the internet reaches, query it with SPARQL, cut RAG chunks. + +- :material-alert-octagon-outline:{ .lg .middle } **[Errors and strict mode](errors.md)** + + --- + + Strict scanner wrappers, `raise_for_errors()`, credential checks and strict report parsing. + +
    + +## More examples + +Every page of the [Python API](../api/index.md) reference has its own examples above the generated reference, and [Python recipes](../guides/python-recipes.md) collects the longer integrations: streaming findings to a SIEM, combining a live inventory with ingested findings, and linking assets from a CMDB. diff --git a/site/content/examples/ingest-and-report.md b/site/content/examples/ingest-and-report.md new file mode 100644 index 0000000..f521012 --- /dev/null +++ b/site/content/examples/ingest-and-report.md @@ -0,0 +1,209 @@ +--- +title: Ingest and report +description: Merge Prowler, Checkov and Trivy output into one deduplicated result, render the reports, gate a CI job on severity and export the findings. +--- + +Already ran the scanners somewhere else? Feed cloudg their native output. No cloud credentials and no scanner binaries are needed, and the core package is enough. +{ .lede } + +Every script on this page is **offline**. Run [`make_sample_reports.py`](index.md#sample-reports-to-work-with) in an empty directory first, then copy its Trivy image report to the name the scripts read: + +```bash +python make_sample_reports.py +cp trivy-image.json trivy.json +``` + +## Merge reports and render them + +=== "CLI" + + ```bash + cloudg ingest --prowler ./prowler-output/ --checkov ./results_json.json --trivy ./trivy.json -o ./reports + ``` + +=== "Python" + + ```python title="ingest_only.py" + --8<-- "ingest/ingest_only.py" + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD:/app/input:ro" \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + ingest --prowler /app/input/prowler-output/ --checkov /app/input/results_json.json --trivy /app/input/trivy.json + ``` + +=== "Output" + + ```console + $ python ingest_only.py + phase: ingest + phase: normalisation + phase: reporting + 3 {'CRITICAL': 1, 'HIGH': 2} + {'json': PosixPath('reports/findings.json'), 'html': PosixPath('reports/report.html')} + CRITICAL 9.7 trivy [Trivy] CVE-2024-0001: openssl (alpine) + HIGH 7.5 prowler S3 bucket default encryption + HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled + ``` + +`reports/report.html` is the page shown in the [sample report](../sample-report.md), filled with these three findings. + +## Parse first, decide later + +`parse_report()` reads one report into `Finding` objects without normalising anything, so you can filter or enrich them before the merge. `ingest_reports()` does several at once and skips files it cannot read. + +=== "Python" + + ```python title="parse_first.py" + --8<-- "ingest/parse_first.py" + ``` + +=== "Output" + + ```console + $ python parse_first.py + INFO cloudg.scanners.prowler: Parsed 1 findings from Prowler (1 passed checks) + INFO cloudg.scanners.checkov: Parsed 1 findings from Checkov + INFO cloudg.scanners.trivy: Parsed 1 findings from Trivy for myrepo/app:latest + INFO cloudg.ingest: [trivy] 1 findings ingested from ./trivy.json + ERROR cloudg.ingest: Skipping trivy report ./missing.json: Report path does not exist: ./missing.json + ERROR cloudg.ingest: Skipping trivvy report ./trivy.json: Unsupported tool 'trivvy'. Supported: prowler, scoutsuite, checkov, trivy + INFO cloudg.normaliser: Normalising 3 total findings from 2 sources + INFO cloudg.normaliser: After deduplication: 3 findings + ('prowler', 'scoutsuite', 'checkov', 'trivy') + 2 findings from two reports + 1 findings from the batch + ValueError: Unsupported tool 'grype'. Supported: prowler, scoutsuite, checkov, trivy + {'CRITICAL': 1, 'HIGH': 2} 31 control results + ``` + +The unknown tool raises `ConfigurationError`, which is a `ValueError`, so the `except ValueError` in the script catches it. [Exceptions](../api/exceptions.md) has the strict variant that raises on an unreadable file as well. + +## Merge findings from your own tools + +`FindingsNormaliser` takes lists of `Finding` objects from any source. Findings from two scanners about the same check on the same resource are merged into one, with both tools listed. + +=== "Python" + + ```python title="normalise_demo.py" + --8<-- "ingest/normalise_demo.py" + ``` + +=== "Output" + + ```console + $ python normalise_demo.py + CRITICAL 9.7 trivy openssl: buffer overflow ['HIPAA', 'NIST-800-53', 'PCI-DSS', 'SOC2'] + HIGH 7.5 trivy, prowler [Trivy] S3 bucket default encryption ['GDPR-AWS', 'HIPAA-AWS', 'ISO27001-AWS', 'MITRE-ATTACK-AWS'] + LOW 2.5 custom CloudTrail logging disabled in eu-west-1 ['GDPR', 'HIPAA', 'ISO-27001', 'NIST-800-53'] + 38 compliance results + ``` + +## Gate a CI job on severity + +Write a summary the pipeline can archive, and exit non-zero when anything critical or high turns up. + +=== "Python" + + ```python title="save_result.py" + --8<-- "ingest/save_result.py" + ``` + +=== "Output" + + ```console + $ python save_result.py + { + "total_assets": 0, + "total_findings": 2, + "severity_breakdown": { + "CRITICAL": 1, + "HIGH": 1 + }, + "providers_scanned": [], + "regions_scanned": {}, + "graph_nodes": 0, + "graph_edges": 0, + "ontology_triples": 0, + "attack_paths_count": 0, + "duration_ms": 1219, + "errors": [] + } + 2 critical/high findings, failing the build + $ echo $? + 1 + ``` + +=== "GitHub Actions" + + ```yaml title=".github/workflows/scan-gate.yml" + jobs: + gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install cloudg + # results_json.json and trivy.json come from earlier steps or artifacts + - run: python save_result.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: cloudg-reports + path: reports/ + ``` + +[CI and automation](../guides/ci-automation.md) has complete workflows for GitHub Actions, GitLab and scheduled scans. + +## Re-render a report, or export the raw findings + +=== "CLI" + + ```bash + cloudg report -i ./reports/findings.json -o ./rendered --format all + cloudg report -i ./reports/raw-findings.json -o ./rendered + ``` + +=== "Python" + + ```python title="export_findings.py" + --8<-- "ingest/export_findings.py" + ``` + +=== "Output" + + ```console + $ python export_findings.py + wrote 3 findings to reports/raw-findings.json + ``` + +`cloudg report` reads either file: a `findings.json` keeps its scan ID, compliance results and edges, and a `raw-findings.json` is normalised first. + +## Related + +
    + +- [**Ingesting existing output**](../guides/ingesting.md) + + What each scanner's output must look like, and how deduplication works. + +- [**parse_report**](../api/parse-report.md) + + `parse_report()` and `ingest_reports()` in detail. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + Merging, CVSS rescoring and compliance mapping. + +- [**cloudg ingest**](../cli/ingest.md) + + Every flag of the command. + +
    diff --git a/site/content/examples/map-and-query.md b/site/content/examples/map-and-query.md new file mode 100644 index 0000000..34bdeed --- /dev/null +++ b/site/content/examples/map-and-query.md @@ -0,0 +1,162 @@ +--- +title: Map and query +description: Map an AWS account into an inventory, save it, load it again, ask what depends on what and overlay scanner findings later. +--- + +Map an account once, then ask questions of the saved map as often as you like: what faces the internet, what depends on a key, what breaks if a queue goes away. +{ .lede } + +```mermaid +flowchart LR + M["cloudg map / map_inventory()"] --> F["./inventory/inventory-map.json"] + F --> L["InventoryResult.load()"] + L --> Q["summary, exposure"] + L --> D["DependencyGraph: depends_on, blast_radius"] + L --> O["export_merged(): asset-map.json, compliance-map.json"] + S["scanner findings, any time later"] --> O +``` + +## Map an account + +Needs read-only AWS credentials and `pip install "cloudg[aws]"`. No scanner runs. + +=== "CLI" + + ```bash + cloudg map -p aws --regions all --no-sweep -o ./inventory + ``` + +=== "Python" + + ```python title="map_only.py" + --8<-- "map/map_only.py" + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/inventory:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --regions all --no-sweep + ``` + +`tagging_sweep=False` and `--no-sweep` both skip the Resource Groups Tagging API sweep. [InventoryMapper](../api/inventorymapper.md) has the other switches and [Inventory mapping](../guides/inventory-mapping.md) explains what each layer collects. + +## Save, load and summarise a map + +**Offline.** An `InventoryResult` built from hand-made assets, exported to disk and loaded back. The same `load()` reads a map that `cloudg map` wrote. + +=== "Python" + + ```python title="inventory_roundtrip.py" + --8<-- "map/inventory_roundtrip.py" + ``` + +=== "Output" + + ```console + $ python inventory_roundtrip.py + ['dependencies', 'graph', 'graphml', 'map'] + 5 6 {'INVOKES': 1, 'REFERENCES': 4, 'ASSUMES_ROLE': 1} + external accounts: 1 cross-account edges: 1 + ['shared_dependencies', 'largest_blast_radius', 'cross_account_edges', 'security_coverage', 'unresolved_references'] + [('orders-key', 3)] + [{'source': 'arn:aws:lambda:eu-west-1:123456789012:function:orders-worker', 'source_account': '123456789012', 'target': 'arn:aws:iam::999999999999:root', 'target_account': '999999999999', 'edge_type': 'ASSUMES_ROLE', 'relationship': None, 'external': True}] + ``` + +Then read the saved map without calling the cloud: + +```python title="whats_exposed.py" +--8<-- "map/whats_exposed.py" +``` + +## What breaks if this goes away + +**Offline.** The script builds its assets by hand. `DependencyGraph` answers "what does this need" and "what needs this", with the edge each step follows. + +=== "CLI" + + Against the map `inventory_roundtrip.py` saved in `./inventory`: + + ```bash + cloudg deps orders-key --direction down --map ./inventory + cloudg deps orders-worker --direction up --map ./inventory + ``` + +=== "Python" + + ```python title="blast_radius.py" + --8<-- "map/blast_radius.py" + ``` + +=== "Output" + + ```console + $ python blast_radius.py + If orders-key is disabled: + depth 1: orders-inbound (MESSAGE_QUEUE) needs orders-key via REFERENCES + depth 1: orders (DYNAMODB_TABLE) needs orders-key via REFERENCES + depth 2: orders-worker (LAMBDA_FUNCTION) needs orders-inbound via INVOKES + orders-worker needs: ['orders-worker-role', 'orders', 'orders-inbound', 'orders-api', 'orders-key'] + direct (needs, needed by): (4, 0) + orders-key 3 dependents, 0 internet-exposed + orders-api 1 dependents, 0 internet-exposed + orders 1 dependents, 0 internet-exposed + ``` + +[Dependencies and blast radius](../guides/dependencies.md) explains how direction works and how to read the tables `cloudg deps` prints. + +## Overlay findings on a saved map + +Scanner findings can arrive days after the map. Load both and write the overlay next to the map. The script reads the map in `./inventory` (from `inventory_roundtrip.py` above, or `cloudg map -o ./inventory`) and the sample reports: run [`make_sample_reports.py`](index.md#sample-reports-to-work-with) in the same directory first, and copy its `trivy-image.json` to `trivy.json`. + +=== "CLI" + + ```bash + cloudg map -p aws --regions all -o ./inventory --findings ./reports/raw-findings.json + ``` + + On the CLI the flag belongs to `cloudg map`, so it maps again. The Python call reuses the saved map. + +=== "Python" + + ```python title="overlay_findings.py" + --8<-- "map/overlay_findings.py" + ``` + +=== "Output" + + ```console + $ python overlay_findings.py + {'asset_map': 'inventory/asset-map.json', 'compliance_map': 'inventory/compliance-map.json'} + CIS 1 ['arn:aws:s3:::my-bucket'] + CVE 1 ['myrepo/app:latest'] + GDPR-AWS 1 ['arn:aws:s3:::my-bucket'] + 5 assets, 0 with findings + ``` + +The sample findings name resources that are not on this map, so no asset picks one up, but the compliance map still lists every affected resource. Against a map of the account the scanners looked at, each asset in `asset-map.json` carries its own findings. + +## Related + +
    + +- [**InventoryResult**](../api/inventoryresult.md) + + Every field and method of the saved map. + +- [**DependencyGraph**](../api/dependencygraph.md) + + `depends_on`, `dependents`, `tree`, `blast_radius` and the asset matcher. + +- [**cloudg deps**](../cli/deps.md) + + The same questions from the command line. + +- [**Inventory structures**](../reference/inventory/index.md) + + Every file the map writes, field by field. + +
    diff --git a/site/content/guides/authentication.md b/site/content/guides/authentication.md index faf849a..5e98686 100644 --- a/site/content/guides/authentication.md +++ b/site/content/guides/authentication.md @@ -1,25 +1,25 @@ --- title: Authentication -lede: "Every provider has several ways in, tried in a fixed order, so one `config.yaml` works on a laptop, in CI and on cloud compute." -meta: - - [Code, "`cloudg/credentials.py`"] - - [Providers, "AWS, Azure, GCP"] - - [Flags, "[Auth flags](/cli/auth-flags/)"] -source: cloudg/credentials.py -since: "0.6.0" +description: Every provider has several ways in, tried in a fixed order, so one config.yaml works on a laptop, in CI and on cloud compute. --- +Every provider has several ways in, tried in a fixed order, so one `config.yaml` works on a laptop, in CI and on cloud compute. +{ .lede } + +**Code** `cloudg/credentials.py` · **Providers** AWS, Azure, GCP · **Flags** [Auth flags](../cli/auth-flags.md) · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + All credential handling lives in three functions in `cloudg/credentials.py`: `build_aws_session`, `build_azure_credential` and `build_gcp_credentials`. `cloudg run`, `cloudg map`, `CloudGEngine`, `InventoryMapper` and the credential preflight of the MCP live tools all call them, so the precedence on this page holds everywhere. Each function walks its list from the top and stops at the first method whose settings are complete. It never tries the next method after a failure, which matters when you debug: if direct keys are set but wrong, cloudg will not fall back to your profile. You can supply each setting three ways. CLI flags (on `cloudg run`) are copied onto the loaded config before anything resolves, `config.yaml` holds the same keys under `aws`, `azure` and `gcp`, and a handful of environment variables are read directly. The tabs below show all three where they exist. -:::note Which commands take which flags -`cloudg run` has the full set of credential flags listed under [auth flags](/cli/auth-flags/). `cloudg map` has only `--profile`, `--accounts`, `--role-name` and `--org-role`; put everything else in `config.yaml` and pass it with `cloudg -c config.yaml map ...`. `cloudg collect` is the exception: it builds credentials from `--profile` (or `AWS_PROFILE`) and the environment only, and ignores the auth keys in `config.yaml`. -::: +!!! note "Which commands take which flags" + + `cloudg run` has the full set of credential flags listed under [auth flags](../cli/auth-flags.md). `cloudg map` has only `--profile`, `--accounts`, `--role-name` and `--org-role`; put everything else in `config.yaml` and pass it with `cloudg -c config.yaml map ...`. `cloudg collect` is the exception: it builds credentials from `--profile` (or `AWS_PROFILE`) and the environment only, and ignores the auth keys in `config.yaml`. ## AWS -```mermaid caption="How build_aws_session picks the base credentials, then the optional AssumeRole hop" +```mermaid flowchart TD A["access_key_id and secret_access_key set?"] -->|yes| K[Direct keys] A -->|no| B["role_arn and a web identity token file?"] @@ -38,6 +38,8 @@ flowchart TD M -->|no| Done[Session ready] T --> Done ``` +How build_aws_session picks the base credentials, then the optional AssumeRole hop +{ .diagram-caption } | Order | Method | Settings that select it | |---|---|---| @@ -47,33 +49,39 @@ flowchart TD | 4 | Default provider chain | nothing set: env vars, shared config, container credentials, instance role | | then | STS AssumeRole | `role_arn` (when not already used by step 2), then, for each account in `accounts`, `role_name` in that account | -Whatever the base method, the session is instrumented with cloudg's rate limiter before it is used, so STS, Organizations and every collector call are paced. See [rate limits](/guides/rate-limits/). +Whatever the base method, the session is instrumented with cloudg's rate limiter before it is used, so STS, Organizations and every collector call are paced. See [rate limits](rate-limits.md). ### Direct keys and session tokens The third-party auditor case: someone hands you an access key pair, or a temporary key pair with a session token. Keys win over everything else, including a profile set in the same config. -```bash tab="CLI" -cloudg run -p aws --regions us-east-1 \ - --aws-key AKIAIOSFODNN7EXAMPLE \ - --aws-secret "$AUDIT_SECRET" \ - --aws-session-token "$AUDIT_SESSION_TOKEN" -``` +=== "CLI" -```yaml tab="config.yaml" title="config.yaml" -aws: - regions: [us-east-1] - access_key_id: AKIAIOSFODNN7EXAMPLE - secret_access_key: REPLACE_ME # keep this file out of version control - session_token: null # only for temporary credentials -``` + ```bash + cloudg run -p aws --regions us-east-1 \ + --aws-key AKIAIOSFODNN7EXAMPLE \ + --aws-secret "$AUDIT_SECRET" \ + --aws-session-token "$AUDIT_SESSION_TOKEN" + ``` -```bash tab="Environment" -export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -export AWS_SECRET_ACCESS_KEY="$AUDIT_SECRET" -export AWS_SESSION_TOKEN="$AUDIT_SESSION_TOKEN" -cloudg run -p aws --regions us-east-1 -``` +=== "config.yaml" + + ```yaml title="config.yaml" + aws: + regions: [us-east-1] + access_key_id: AKIAIOSFODNN7EXAMPLE + secret_access_key: REPLACE_ME # keep this file out of version control + session_token: null # only for temporary credentials + ``` + +=== "Environment" + + ```bash + export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE + export AWS_SECRET_ACCESS_KEY="$AUDIT_SECRET" + export AWS_SESSION_TOKEN="$AUDIT_SESSION_TOKEN" + cloudg run -p aws --regions us-east-1 + ``` The environment tab works through a different door. cloudg does not copy `AWS_ACCESS_KEY_ID` into its config, so resolution falls through to step 4 and boto3's default chain picks the variables up. The effect is the same, and it is the variant to prefer: nothing secret lands in a file or in your shell history. @@ -81,31 +89,37 @@ The environment tab works through a different door. cloudg does not copy `AWS_AC For GitHub Actions, GitLab CI and EKS service accounts: no long-lived keys, only a short-lived token file exchanged for role credentials with `AssumeRoleWithWebIdentity`. cloudg reads the token file itself and names the session after `role_session_name` (default `cloudg-scan`). -```bash tab="CLI" -cloudg run -p aws --regions all \ - --aws-role-arn arn:aws:iam::123456789012:role/cloudg-readonly \ - --aws-web-identity-token-file /var/run/secrets/oidc/token -``` +=== "CLI" -```yaml tab="config.yaml" title="config.yaml" -aws: - role_arn: arn:aws:iam::123456789012:role/cloudg-readonly - web_identity_token_file: /var/run/secrets/oidc/token - role_session_name: cloudg-ci -``` + ```bash + cloudg run -p aws --regions all \ + --aws-role-arn arn:aws:iam::123456789012:role/cloudg-readonly \ + --aws-web-identity-token-file /var/run/secrets/oidc/token + ``` -```bash tab="Environment" -# Read by boto3's default chain (step 4), not by cloudg's step 2 -export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/cloudg-readonly -export AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/oidc/token -cloudg run -p aws --regions all -``` +=== "config.yaml" + + ```yaml title="config.yaml" + aws: + role_arn: arn:aws:iam::123456789012:role/cloudg-readonly + web_identity_token_file: /var/run/secrets/oidc/token + role_session_name: cloudg-ci + ``` + +=== "Environment" + + ```bash + # Read by boto3's default chain (step 4), not by cloudg's step 2 + export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/cloudg-readonly + export AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/oidc/token + cloudg run -p aws --regions all + ``` When step 2 succeeds, `role_arn` has been used up as the OIDC target and no second AssumeRole follows. A cross-account hop after federation needs `accounts` plus `role_name` instead. -:::warning The token file variable on EKS -cloudg also reads `$AWS_WEB_IDENTITY_TOKEN_FILE` from the environment, and EKS (IRSA) sets it in every pod. On such a pod, a `role_arn` meant as a second hop becomes the OIDC target instead, and the exchange fails unless that role trusts the cluster's OIDC provider. Either let the pod's own role do the work (no `role_arn`), or reach other accounts with `accounts` plus `role_name`. -::: +!!! warning "The token file variable on EKS" + + cloudg also reads `$AWS_WEB_IDENTITY_TOKEN_FILE` from the environment, and EKS (IRSA) sets it in every pod. On such a pod, a `role_arn` meant as a second hop becomes the OIDC target instead, and the exchange fails unless that role trusts the cluster's OIDC provider. Either let the pod's own role do the work (no `role_arn`), or reach other accounts with `accounts` plus `role_name`. A failed exchange raises `RuntimeError: AssumeRoleWithWebIdentity failed for : ...`. The usual causes are a trust policy whose `sub` condition does not match the branch or environment, or a token minted for the wrong audience (AWS expects `sts.amazonaws.com`). @@ -113,26 +127,32 @@ A failed exchange raises `RuntimeError: AssumeRoleWithWebIdentity failed for `, `AWS auth: default provider chain`, `AWS auth: assuming role `, `Azure auth: workload identity federation (client ...)`, `Azure auth: DefaultAzureCredential chain`, `GCP auth: file-based identity (...)`, `GCP auth: application default credentials` and `GCP auth: impersonating `. -:::links -- [Auth flags](/cli/auth-flags/) Every credential flag of `cloudg run`. -- [AWS Organizations](/guides/aws-organizations/) Mapping every account from the management account. -- [CI and automation](/guides/ci-automation/) OIDC from GitHub Actions and GitLab CI. -- [Docker](/guides/docker/) Mounting credentials into the container. -::: +
    + +- [**Auth flags**](../cli/auth-flags.md) + + Every credential flag of `cloudg run`. + +- [**AWS Organizations**](aws-organizations.md) + + Mapping every account from the management account. + +- [**CI and automation**](ci-automation.md) + + OIDC from GitHub Actions and GitLab CI. + +- [**Docker**](docker.md) + + Mounting credentials into the container. + +
    diff --git a/site/content/guides/aws-organizations.md b/site/content/guides/aws-organizations.md index 150ae0e..f8bdc93 100644 --- a/site/content/guides/aws-organizations.md +++ b/site/content/guides/aws-organizations.md @@ -1,19 +1,21 @@ --- title: AWS Organizations -lede: "With `--org`, `cloudg map` starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them." -meta: - - [Command, "`cloudg map --org`"] - - [Runs from, "Management account or delegated administrator"] -source: cloudg/inventory/organization.py +description: With --org, cloudg map starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them. --- -Mapping one account at a time misses what holds an organization together: which OU an account sits in, which SCPs restrict it, which Control Tower controls apply, and which roles in one account trust another. `cloudg map --org` reads the organization first, picks the accounts you asked for, assumes a role in each one, and maps all of them into a single [inventory map](/guides/inventory-mapping/). Accounts hang under their OUs, SCPs and controls point at what they govern, and cross-account trust becomes an edge you can query with [`cloudg deps`](/guides/dependencies/). +With `--org`, `cloudg map` starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them. +{ .lede } + +**Command** `cloudg map --org` · **Runs from** Management account or delegated administrator +{ .page-meta } + +Mapping one account at a time misses what holds an organization together: which OU an account sits in, which SCPs restrict it, which Control Tower controls apply, and which roles in one account trust another. `cloudg map --org` reads the organization first, picks the accounts you asked for, assumes a role in each one, and maps all of them into a single [inventory map](inventory-mapping.md). Accounts hang under their OUs, SCPs and controls point at what they govern, and cross-account trust becomes an edge you can query with [`cloudg deps`](dependencies.md). Every call cloudg makes here is read-only: `Describe*`, `List*` and `Get*` against Organizations and Control Tower, then the usual read calls inside each member account. ## What happens during discovery -```mermaid caption="Organization discovery, then one role assumption per member account and region" +```mermaid sequenceDiagram participant C as cloudg participant S as STS @@ -33,6 +35,8 @@ sequenceDiagram C->>M: Deep collectors (read-only) end ``` +Organization discovery, then one role assumption per member account and region +{ .diagram-caption } In order: @@ -80,13 +84,14 @@ The caller's own account is mapped too, with the caller's credentials and not th cloudg assumes one role, by name, in every member account. The default is `AWSControlTowerExecution`, because Control Tower creates it in every account it enrolls. -:::warning AWSControlTowerExecution has admin rights -`AWSControlTowerExecution` carries administrator permissions. cloudg only reads, but anything that can run cloudg with management-account credentials could do more with that role. The read-only role Control Tower creates, `aws-controltower-ReadOnlyExecutionRole`, can't be used instead: it only trusts a role in the audit account. For production, deploy your own read-only role to every account and pass it with `--org-role`. -::: +!!! warning "AWSControlTowerExecution has admin rights" + + `AWSControlTowerExecution` carries administrator permissions. cloudg only reads, but anything that can run cloudg with management-account credentials could do more with that role. The read-only role Control Tower creates, `aws-controltower-ReadOnlyExecutionRole`, can't be used instead: it only trusts a role in the audit account. For production, deploy your own read-only role to every account and pass it with `--org-role`. A service-managed CloudFormation StackSet deploys the role to every account in the OUs you target, and to accounts that join those OUs later. -::::steps +
    + ### Save the template ```yaml title="cloudg-readonly-role.yaml" @@ -157,48 +162,55 @@ aws cloudformation list-stack-instances --stack-set-name cloudg-readonly ``` Service-managed StackSets never deploy to the management account itself. cloudg doesn't need them to, since it maps the management account with the caller's own credentials. -:::: -To map Kubernetes workloads inside EKS clusters as well, each cluster needs an access entry for `cloudg-readonly` with the `AmazonEKSViewPolicy`. The [inventory mapping guide](/guides/inventory-mapping/#kubernetes-inside-eks) has the two `aws eks` commands. +
    + +To map Kubernetes workloads inside EKS clusters as well, each cluster needs an access entry for `cloudg-readonly` with the `AmazonEKSViewPolicy`. The [inventory mapping guide](inventory-mapping.md#kubernetes-inside-eks) has the two `aws eks` commands. ## Map the organization -```bash tab="CLI" -cloudg map -p aws --profile management --org --org-role cloudg-readonly --regions all -``` +=== "CLI" -```python tab="Python" title="map_org.py" -from cloudg import CloudGConfig -from cloudg.inventory import InventoryMapper - -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "management" -config.aws.regions = ["ALL"] - -org = config.aws.organization -org.enabled = True -org.role_name = "cloudg-readonly" -org.include_ous = ["Workloads"] -org.exclude_accounts = ["111122223333"] - -mapper = InventoryMapper(config) -inventory = mapper.map_inventory_sync() -paths = inventory.export("./reports") - -topology = mapper.organization # None when discovery failed -if topology is not None: - print(topology.organization_id, len(topology.accounts), "accounts") - print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) -print(paths.get("organization")) -``` + ```bash + cloudg map -p aws --profile management --org --org-role cloudg-readonly --regions all + ``` -```bash tab="Docker" -docker run --rm \ - -v ~/.aws:/home/cloudg/.aws:ro \ - -v "$PWD/reports:/app/reports" \ - ghcr.io/morpheuslord/cloudg:latest \ - map -p aws --profile management --org --org-role cloudg-readonly --regions all -``` +=== "Python" + + ```python title="map_org.py" + from cloudg import CloudGConfig + from cloudg.inventory import InventoryMapper + + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "management" + config.aws.regions = ["ALL"] + + org = config.aws.organization + org.enabled = True + org.role_name = "cloudg-readonly" + org.include_ous = ["Workloads"] + org.exclude_accounts = ["111122223333"] + + mapper = InventoryMapper(config) + inventory = mapper.map_inventory_sync() + paths = inventory.export("./reports") + + topology = mapper.organization # None when discovery failed + if topology is not None: + print(topology.organization_id, len(topology.accounts), "accounts") + print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) + print(paths.get("organization")) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile management --org --org-role cloudg-readonly --regions all + ``` With `--org`, the CLI's configuration panel shows "Organization: discover all accounts" (plus the OU filter, if any) and the member role it will use. After the run an "Organization" table lists the organization ID, the number of accounts and OUs, whether Control Tower was found and the governed regions, ahead of the usual inventory summary. @@ -385,9 +397,22 @@ It raises `RuntimeError` when the caller can't describe or list the organization | Far fewer accounts than expected | an `--ou` value matched a different OU of the same name | use OU IDs | | Fewer regions than `--regions all` would give | Control Tower governed regions replaced `all` | set `use_governed_regions: false`, or pass the regions explicitly | -:::links -- [Inventory mapping](/guides/inventory-mapping/) What a mapping run collects in each account. -- [Dependencies and blast radius](/guides/dependencies/) Query cross-account edges on the finished map. -- [Authentication](/guides/authentication/) Profiles, OIDC and role assumption for the base credentials. -- [cloudg map](/cli/map/) Every option of the command. -::: +
    + +- [**Inventory mapping**](inventory-mapping.md) + + What a mapping run collects in each account. + +- [**Dependencies and blast radius**](dependencies.md) + + Query cross-account edges on the finished map. + +- [**Authentication**](authentication.md) + + Profiles, OIDC and role assumption for the base credentials. + +- [**cloudg map**](../cli/map.md) + + Every option of the command. + +
    diff --git a/site/content/guides/ci-automation.md b/site/content/guides/ci-automation.md index 037d310..080117a 100644 --- a/site/content/guides/ci-automation.md +++ b/site/content/guides/ci-automation.md @@ -1,23 +1,43 @@ --- title: CI and automation -lede: "Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts." -meta: - - [Commands, "`cloudg run`, `cloudg map`, `cloudg ingest`"] - - [CI systems, "GitHub Actions, GitLab CI"] -source: .github/workflows/cloudg.yml -since: "0.6.0" +description: Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts. --- +Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts. +{ .lede } + +**Commands** `cloudg run`, `cloudg map`, `cloudg ingest` · **CI systems** GitHub Actions, GitLab CI · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + There are three useful shapes for cloudg in a pipeline, and most teams end up with two of them: -:::cells +
    + +
    + ### Scheduled posture run + `cloudg run` against the live account on a cron, with OIDC credentials, Prowler and the IAM linter. Slow, thorough, read-only. + +
    + +
    + ### Pull request gate + Checkov and Trivy over the IaC in the branch, aggregated with `cloudg ingest`. No cloud access at all, so it is safe for forks. + +
    + +
    + ### Inventory snapshot + `cloudg map` on a cron. No scanners, just the map, kept as an artifact for diffing and for the MCP server. -::: + +
    + +
    Before wiring any of them up, know what cloudg's exit code does and does not tell you. @@ -39,7 +59,7 @@ cloudg exits non-zero when it cannot do its job, never because of what it found. ## A severity gate -Every run writes `findings.json` (see [reports](/guides/reports/)). Its `findings` list holds the normalised, deduplicated findings, each with `severity`, `risk_score`, `source_tool`, `is_suppressed` and the affected resource, and `summary` holds the totals. Save this as `ci/severity_gate.py` in the repository that runs the job: +Every run writes `findings.json` (see [reports](reports.md)). Its `findings` list holds the normalised, deduplicated findings, each with `severity`, `risk_score`, `source_tool`, `is_suppressed` and the affected resource, and `summary` holds the totals. Save this as `ci/severity_gate.py` in the repository that runs the job: ```python title="ci/severity_gate.py" """Fail a CI job when a cloudg findings.json crosses a severity threshold. @@ -57,7 +77,6 @@ from collections import Counter ORDER = ["INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"] - def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("findings", help="findings.json written by cloudg run or cloudg ingest") @@ -96,7 +115,6 @@ def main() -> int: print("gate: passed") return 0 - if __name__ == "__main__": sys.exit(main()) ``` @@ -119,13 +137,14 @@ Use `--min-assets 1` (or a realistic floor for your account) on live runs, and l jq -e '[.findings[] | select(.severity == "CRITICAL" and (.is_suppressed | not))] | length == 0' reports/findings.json ``` -When the gate should run inside Python, for example in a job that already uses the engine, see the gate recipe in [Python recipes](/guides/python-recipes/). +When the gate should run inside Python, for example in a job that already uses the engine, see the gate recipe in [Python recipes](python-recipes.md). ## GitHub Actions with OIDC to AWS No access keys in repository secrets. GitHub mints an OIDC token per job, AWS exchanges it for a role session, and the role only trusts this repository. -::::steps +
    + ### Create the role In the target account, add GitHub as an identity provider (`token.actions.githubusercontent.com`, audience `sts.amazonaws.com`) once, then create a role with `SecurityAudit` and `ViewOnlyAccess` attached and this trust policy. Replace the account ID and repository: @@ -151,9 +170,10 @@ In the target account, add GitHub as an identity provider (`token.actions.github } ``` -:::warning Scope the subject -A `sub` of `repo:acme/cloud-posture:*` lets every branch and every pull request from a branch assume the role. Pin it to `main` or to a deployment environment (`repo:acme/cloud-posture:environment:cloud-audit`). -::: +!!! warning "Scope the subject" + + A `sub` of `repo:acme/cloud-posture:*` lets every branch and every pull request from a branch assume the role. Pin it to `main` or to a deployment environment (`repo:acme/cloud-posture:environment:cloud-audit`). + ### Add the config and the gate Commit `ci/severity_gate.py` from above and a config next to it: @@ -166,6 +186,7 @@ scanners: enabled: [prowler, iam] timeout_seconds: 3600 # per scanner ``` + ### Add the workflow ```yaml title=".github/workflows/cloud-posture.yml" @@ -240,9 +261,10 @@ jobs: path: reports/ retention-days: 30 ``` -:::: -Credentials go through `configure-aws-credentials` here, which exports `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` for the whole job; cloudg's collectors and Prowler pick them up through the default chain. cloudg's own `--aws-role-arn` with `--aws-web-identity-token-file` works as well: cloudg then assumes the role itself and hands Prowler and ScoutSuite the temporary keys. The action has the advantage that every other step of the job gets the same credentials. [Authentication](/guides/authentication/#what-the-scanners-receive) has the details. +
    + +Credentials go through `configure-aws-credentials` here, which exports `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` for the whole job; cloudg's collectors and Prowler pick them up through the default chain. cloudg's own `--aws-role-arn` with `--aws-web-identity-token-file` works as well: cloudg then assumes the role itself and hands Prowler and ScoutSuite the temporary keys. The action has the advantage that every other step of the job gets the same credentials. [Authentication](authentication.md#what-the-scanners-receive) has the details. The action's session lasts one hour by default. A long organization-wide run can outlive it; raise `role-duration-seconds` (and the role's maximum session duration) or split the run per account. @@ -341,7 +363,7 @@ Do not also pass `--aws-role-arn` here. With `AWS_WEB_IDENTITY_TOKEN_FILE` set i ## Scheduled inventory snapshots -A map is cheaper than a full run and changes less often than findings. A nightly `cloudg map` gives you `inventory-map.json`, `inventory-dependencies.json` and, with `--org`, `inventory-organization.json`, ready for `cloudg deps`, for diffing, or for an [MCP server](/cli/mcp-serve/) to load. +A map is cheaper than a full run and changes less often than findings. A nightly `cloudg map` gives you `inventory-map.json`, `inventory-dependencies.json` and, with `--org`, `inventory-organization.json`, ready for `cloudg deps`, for diffing, or for an [MCP server](../cli/mcp-serve.md) to load. ```yaml title=".github/workflows/inventory.yml" name: Inventory snapshot @@ -375,7 +397,7 @@ jobs: retention-days: 90 ``` -Throttling during a map is recorded in coverage and in the `throttling` block of `inventory-map.json` rather than failing the job, so check that block when a snapshot looks thin. Several pipelines mapping the same account at once share its API quota; stagger their schedules, or lower `ratelimit.aws.max_rps` for the CI config as described in [rate limits](/guides/rate-limits/). +Throttling during a map is recorded in coverage and in the `throttling` block of `inventory-map.json` rather than failing the job, so check that block when a snapshot looks thin. Several pipelines mapping the same account at once share its API quota; stagger their schedules, or lower `ratelimit.aws.max_rps` for the CI config as described in [rate limits](rate-limits.md). ## Caching and artifacts @@ -391,11 +413,24 @@ Do not cache `reports/`. Publish it as an artifact instead. A run's useful files | `inventory-map.json` | the map from `cloudg map`, loadable with `InventoryResult.load` | | `ontology.ttl`, `rag_chunks.jsonl` | feed downstream analysis or a retrieval index | -The reports describe your estate in detail: account IDs, ARNs, open ports, IAM grants. Keep artifact retention short and the repository private, and use the Docker image or a private runner when the reports must not leave your network. See [Docker](/guides/docker/) for the image, which bundles Prowler, Checkov and Trivy. +The reports describe your estate in detail: account IDs, ARNs, open ports, IAM grants. Keep artifact retention short and the repository private, and use the Docker image or a private runner when the reports must not leave your network. See [Docker](docker.md) for the image, which bundles Prowler, Checkov and Trivy. + +
    + +- [**Authentication**](authentication.md) + + Every credential method and its precedence. + +- [**Python recipes**](python-recipes.md) + + The gate and the pipeline from Python. + +- [**Ingesting existing output**](ingesting.md) + + What `cloudg ingest` accepts. + +- [**cloudg run**](../cli/run.md) + + Every flag of the full pipeline. -:::links -- [Authentication](/guides/authentication/) Every credential method and its precedence. -- [Python recipes](/guides/python-recipes/) The gate and the pipeline from Python. -- [Ingesting existing output](/guides/ingesting/) What `cloudg ingest` accepts. -- [cloudg run](/cli/run/) Every flag of the full pipeline. -::: +
    diff --git a/site/content/guides/compliance.md b/site/content/guides/compliance.md index 8873c84..5def4e2 100644 --- a/site/content/guides/compliance.md +++ b/site/content/guides/compliance.md @@ -1,14 +1,14 @@ --- title: Compliance frameworks -lede: "cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package." -meta: - - [Rulesets, "`cloudg/rules/`"] - - [Frameworks, "28, with 4,271 controls"] - - [Config, "`rulesets.rules_dir`"] -source: cloudg/normaliser.py -since: "0.6.0" +description: cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package. --- +cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package. +{ .lede } + +**Rulesets** `cloudg/rules/` · **Frameworks** 28, with 4,271 controls · **Config** `rulesets.rules_dir` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Compliance mapping is the last step of normalisation, so it runs wherever findings are normalised: phase 4 of `cloudg run`, `cloudg ingest`, `cloudg report` on raw findings, `CloudGEngine.normalise_findings()`, `scan()` followed by normalisation, `run_pipeline()`, `run_from_reports()` and the MCP server's dataset loaders. Its output is a list of `ComplianceResult` records in `findings.json` and the Compliance tab of `report.html`. One thing to understand before reading any of that output: a control only appears when cloudg has evidence about it. A control that a finding maps to is `FAIL`. A control that only a passing Prowler check maps to is `PASS`: Prowler reports the checks it ran and passed (ASFF `Compliance.Status: PASSED`, OCSF `status_code: PASS`), and cloudg keeps their names. Every other control was not assessed and gets no result at all. Checkov, Trivy, ScoutSuite and the IAM linter report failures only, so without Prowler the output is all `FAIL`. Read it as "these controls have open findings, and these passed Prowler's checks", not as a full audit score. @@ -56,13 +56,13 @@ Loaded together, that is 28 framework names and 4,271 controls. The README's fig The merged CIS frameworks mix two benchmark versions under one name: the generated controls use CIS 5.0 numbering (`1.4`) and the hand-written ones CIS 3.0 or 2.0 numbering with a prefix (`CIS-1.4`). They don't collide, but don't expect one numbering scheme inside `CIS-AWS`. -`rules/check_equivalence.yaml` sits in the same directory but isn't a ruleset. It tells the deduplicator which checks from different scanners mean the same thing; see [Ingesting existing output](/guides/ingesting/#normalisation-and-deduplication). +`rules/check_equivalence.yaml` sits in the same directory but isn't a ruleset. It tells the deduplicator which checks from different scanners mean the same thing; see [Ingesting existing output](ingesting.md#normalisation-and-deduplication). ## How a finding gets its controls Mapping works in four tiers, most precise first. Each finding carries a list of framework names, `compliance_frameworks`, which the tiers add to. -```mermaid caption="From a finding to framework controls" +```mermaid flowchart TD F["Finding"] --> T1["Tier 1: tags set by the scanner parser"] T1 --> T2["Tier 2: exact check-ID lookup in ruleset checks"] @@ -76,6 +76,8 @@ flowchart TD G --> CR["ComplianceResult per control: FAIL"] P["Prowler passing checks"] --> PC["PASS for ruleset controls no finding fails"] ``` +From a finding to framework controls +{ .diagram-caption } ### Tier 1: what the scanner said @@ -122,7 +124,7 @@ After tagging, cloudg groups findings by framework and control ID and writes one `ComplianceStatus` defines four values: `PASS`, `FAIL`, `NOT_APPLICABLE` and `MANUAL`. The normaliser produces `FAIL` and `PASS`. The other two are there for your own code, or for results you build yourself. -A `PASS` result needs an exact match: the passing check's name has to appear in a ruleset control's `checks` list (tier 2), and no finding may map to that control. Its `control_id` and `control_title` come from the ruleset. Passing checks travel from the Prowler parser to the normaliser on the parsed list (see [FindingsNormaliser](/api/findingsnormaliser/)); `cloudg run`, `cloudg ingest`, `CloudGEngine.scan()`, `run_pipeline()`, `run_from_reports()`, `ingest_reports()`, `parse_report()` and the MCP loaders all carry them through. +A `PASS` result needs an exact match: the passing check's name has to appear in a ruleset control's `checks` list (tier 2), and no finding may map to that control. Its `control_id` and `control_title` come from the ruleset. Passing checks travel from the Prowler parser to the normaliser on the parsed list (see [FindingsNormaliser](../api/findingsnormaliser.md)); `cloudg run`, `cloudg ingest`, `CloudGEngine.scan()`, `run_pipeline()`, `run_from_reports()`, `ingest_reports()`, `parse_report()` and the MCP loaders all carry them through. The `control_id` depends on how the framework was attached: @@ -167,7 +169,7 @@ for result in controls.get("CIS-AWS", []) + controls.get("NIST-CSF-AWS", []): print(" ", f["severity"], f["source_tool"], f["resource_arn"]) ``` -Against the [sample reports from the ingest guide](/guides/ingesting/#try-it-without-any-scanner), the start of the output is: +Against the [sample reports from the ingest guide](ingesting.md#try-it-without-any-scanner), the start of the output is: ```console $ python compliance_summary.py @@ -188,7 +190,7 @@ The coarse tier 1 tags and the ruleset frameworks appear side by side: `CIS` nex ### report.html {#report-html} -The Compliance tab shows one card per framework with Pass and Fail counts and a percentage bar. Fail is the number of distinct controls with findings, and Pass the number of controls that only passing Prowler checks map to. When the input had no passing-check data (no Prowler output, or only failing records), Pass is 0 and the bar reads 0%; treat such a card as a count of affected controls. The Analytics tab charts the same counts, and the detail panel of each finding lists its frameworks. [Reports](/guides/reports/) covers the rest of the page. +The Compliance tab shows one card per framework with Pass and Fail counts and a percentage bar. Fail is the number of distinct controls with findings, and Pass the number of controls that only passing Prowler checks map to. When the input had no passing-check data (no Prowler output, or only failing records), Pass is 0 and the bar reads 0%; treat such a card as a count of affected controls. The Analytics tab charts the same counts, and the detail panel of each finding lists its frameworks. [Reports](reports.md) covers the rest of the page. ### compliance-map.json @@ -267,28 +269,32 @@ cp acme_baseline.yaml ./my-rules/ Then point cloudg at it. -```yaml tab="CLI" title="config.yaml" -rulesets: - rules_dir: ./my-rules -``` +=== "CLI" -```python tab="Python" title="custom_rules.py" -from cloudg.ingest import ingest_reports -from cloudg.normaliser import FindingsNormaliser + ```yaml title="config.yaml" + rulesets: + rules_dir: ./my-rules + ``` -findings = ingest_reports( - { - "prowler": ["./prowler-output/"], - "checkov": ["./results_json.json"], - "scoutsuite": ["./scoutsuite-report/"], - } -) -scan_result = FindingsNormaliser(rules_dir="./my-rules").normalise(findings) +=== "Python" -for result in scan_result.compliance: - if result.framework == "ACME-BASELINE": - print(result.control_id, result.control_title, len(result.finding_ids)) -``` + ```python title="custom_rules.py" + from cloudg.ingest import ingest_reports + from cloudg.normaliser import FindingsNormaliser + + findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "scoutsuite": ["./scoutsuite-report/"], + } + ) + scan_result = FindingsNormaliser(rules_dir="./my-rules").normalise(findings) + + for result in scan_result.compliance: + if result.framework == "ACME-BASELINE": + print(result.control_id, result.control_title, len(result.finding_ids)) + ``` With the CLI, pass the config with `-c`: `cloudg -c config.yaml ingest ...` or `cloudg -c config.yaml run ...`. From `CloudGEngine`, set `config.rulesets.rules_dir`. On the sample reports, the result is: @@ -323,9 +329,22 @@ The list of frameworks it imports is the `FRAMEWORKS` constant at the top of the ## Next -:::links -- [Ingesting existing output](/guides/ingesting/) Deduplication and the check-equivalence map. -- [Reports](/guides/reports/) The Compliance tab and the rest of `report.html`. -- [Data models](/api/models/) `ComplianceResult`, `ComplianceStatus` and `ScanResult`. -- [FindingsNormaliser](/api/findingsnormaliser/) The class that does the mapping. -::: +
    + +- [**Ingesting existing output**](ingesting.md) + + Deduplication and the check-equivalence map. + +- [**Reports**](reports.md) + + The Compliance tab and the rest of `report.html`. + +- [**Data models**](../api/models.md) + + `ComplianceResult`, `ComplianceStatus` and `ScanResult`. + +- [**FindingsNormaliser**](../api/findingsnormaliser.md) + + The class that does the mapping. + +
    diff --git a/site/content/guides/dependencies.md b/site/content/guides/dependencies.md index bc6e872..8720015 100644 --- a/site/content/guides/dependencies.md +++ b/site/content/guides/dependencies.md @@ -1,13 +1,15 @@ --- title: Dependencies and blast radius -lede: "`cloudg deps` asks two questions of any asset on a saved map: what does it need, and what breaks with it?" -meta: - - [Command, "`cloudg deps`"] - - [Input, "`inventory-map.json` from `cloudg map`"] -source: cloudg/inventory/dependencies.py +description: 'cloudg deps asks two questions of any asset on a saved map: what does it need, and what breaks with it?' --- -A map from [`cloudg map`](/guides/inventory-mapping/) records edges the way you would describe them out loud: the queue invokes the function, the function assumes its role, the WAF protects the API. That is the right shape for drawing, but not for impact questions. To ask "what breaks if I rotate this KMS key?" you need every edge turned into one arrow that reads "dependent needs dependency". `cloudg deps` and the `DependencyGraph` class behind it do that conversion and then walk the result. +`cloudg deps` asks two questions of any asset on a saved map: what does it need, and what breaks with it? +{ .lede } + +**Command** `cloudg deps` · **Input** `inventory-map.json` from `cloudg map` +{ .page-meta } + +A map from [`cloudg map`](inventory-mapping.md) records edges the way you would describe them out loud: the queue invokes the function, the function assumes its role, the WAF protects the API. That is the right shape for drawing, but not for impact questions. To ask "what breaks if I rotate this KMS key?" you need every edge turned into one arrow that reads "dependent needs dependency". `cloudg deps` and the `DependencyGraph` class behind it do that conversion and then walk the result. `cloudg deps` reads a saved `inventory-map.json` and nothing else. It makes no cloud calls, so you can run it on a laptop against a map someone produced in CI last night. @@ -25,7 +27,7 @@ Two more rules shape the graph. `CONTAINS` edges whose container is an organizat Here is a small map as `cloudg map` would record it: -```mermaid caption="A slice of a map, with edges in their natural direction" +```mermaid flowchart LR WAF["orders-api-acl (WAF)"] -->|PROTECTS| API["orders-api (API Gateway)"] API -->|INVOKES| FN["orders-worker (Lambda)"] @@ -42,10 +44,12 @@ flowchart LR TABLE -->|REFERENCES| KEY EXT["account 999999999999"] -->|IAM_TRUST| ROLE ``` +A slice of a map, with edges in their natural direction +{ .diagram-caption } And the same slice after the conversion. Every arrow now points from the asset that needs something to the thing it needs. Note how the `INVOKES` and `PROTECTS` arrows flipped: -```mermaid caption="The same slice as dependency arrows. Walking with the arrows is depends_on (up); walking against them is dependents (down)." +```mermaid flowchart LR FN["orders-worker (Lambda)"] --> API["orders-api (API Gateway)"] API --> WAF["orders-api-acl (WAF)"] @@ -62,10 +66,12 @@ flowchart LR TABLE --> KEY EXT["account 999999999999"] --> ROLE ``` +The same slice as dependency arrows. Walking with the arrows is depends_on (up); walking against them is dependents (down). +{ .diagram-caption } Upstream of the function sits everything it needs to run, including the API and queue that feed it work. Downstream of the KMS key sits everything that stops working if the key is disabled. -This is an availability and change-impact view. Compromise runs the other way along identity edges: a compromised function exposes its role, not the reverse. For that question, walk the `IAM_TRUST`, `ASSUMES_ROLE` and `GRANTS_ACCESS` edges of the map directly, or use the attack path analysis in [GraphBuilder](/api/graphbuilder/). +This is an availability and change-impact view. Compromise runs the other way along identity edges: a compromised function exposes its role, not the reverse. For that question, walk the `IAM_TRUST`, `ASSUMES_ROLE` and `GRANTS_ACCESS` edges of the map directly, or use the attack path analysis in [GraphBuilder](../api/graphbuilder.md). ## Query one asset @@ -311,9 +317,22 @@ for ref in refs: Against the example map, the ambiguous `orders-worker` stays unresolved in both modes, `function:orders-worker` and `ORDERS-KEY` resolve only with the opt-in tiers, and the other two resolve either way. The opt-in tiers never pick one of several assets that share an exact name. `index.resolve_finding(finding)` tries a finding's `resource_id` and `resource_arn` through the same tiers. -:::links -- [Inventory mapping](/guides/inventory-mapping/) Produce the map that `cloudg deps` reads. -- [cloudg deps](/cli/deps/) The command reference. -- [DependencyGraph](/api/dependencygraph/) Every method and return type. -- [InventoryResult](/api/inventoryresult/) Loading, analysis and export. -::: +
    + +- [**Inventory mapping**](inventory-mapping.md) + + Produce the map that `cloudg deps` reads. + +- [**cloudg deps**](../cli/deps.md) + + The command reference. + +- [**DependencyGraph**](../api/dependencygraph.md) + + Every method and return type. + +- [**InventoryResult**](../api/inventoryresult.md) + + Loading, analysis and export. + +
    diff --git a/site/content/guides/docker.md b/site/content/guides/docker.md index 224ce3c..4a6976e 100644 --- a/site/content/guides/docker.md +++ b/site/content/guides/docker.md @@ -1,13 +1,14 @@ --- title: Docker -lede: "The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host." -meta: - - [Image, "`ghcr.io/morpheuslord/cloudg`"] - - [Base, "`python:3.12-slim`"] - - [Entrypoint, "`cloudg`"] -source: Dockerfile +description: The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host. --- +The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host. +{ .lede } + +**Image** `ghcr.io/morpheuslord/cloudg` · **Base** `python:3.12-slim` · **Entrypoint** `cloudg` +{ .page-meta } + The image is the quickest route to a full `cloudg run`, because the scanners are the hard part of a local install and the image has them preinstalled. It is also a clean way to run the MCP server on a host where you would rather not install Python packages. ## What is in the image @@ -42,16 +43,20 @@ At run time the container works like this: ## Get the image -```bash tab="Pull" -docker pull ghcr.io/morpheuslord/cloudg:latest -docker tag ghcr.io/morpheuslord/cloudg:latest cloudg:latest -``` +=== "Pull" -```bash tab="Build" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -docker build -t cloudg:latest . -``` + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + docker tag ghcr.io/morpheuslord/cloudg:latest cloudg:latest + ``` + +=== "Build" + + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + docker build -t cloudg:latest . + ``` Releases are tagged with their version (`ghcr.io/morpheuslord/cloudg:0.6.0`) as well as `latest`. The examples on this page use the local name `cloudg:latest`, which is what `docker compose` builds too. @@ -89,25 +94,31 @@ The container sees no credentials until you give it some. Each provider has a fe ### AWS -```bash tab="Profile" -docker run --rm \ - -v "$HOME/.aws:/home/cloudg/.aws:ro" \ - -e AWS_PROFILE=audit \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p aws --regions us-east-1 -``` +=== "Profile" -```bash tab="Environment" -docker run --rm \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p aws --regions us-east-1 -``` + ```bash + docker run --rm \ + -v "$HOME/.aws:/home/cloudg/.aws:ro" \ + -e AWS_PROFILE=audit \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 + ``` -```bash tab="Instance role" -# on EC2 or ECS: credentials come from the instance or task role -docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest run -p aws --regions all -``` +=== "Environment" + + ```bash + docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 + ``` + +=== "Instance role" + + ```bash + # on EC2 or ECS: credentials come from the instance or task role + docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest run -p aws --regions all + ``` Mount `~/.aws` at `/home/cloudg/.aws`, not `/root/.aws`; the container does not run as root. With SSO profiles, run `aws sso login` on the host first. A read-only mount can use the cached token but cannot store a refreshed one, so log in again on the host when it expires. @@ -119,26 +130,32 @@ On EC2 with IMDSv2, a container on Docker's default bridge network is one networ The usual container setup is a service principal in environment variables. cloudg reads `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` when the matching flags are not given: -```bash tab="Client secret" -docker run --rm \ - -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --subscription-id 00000000-0000-0000-0000-000000000000 -``` +=== "Client secret" -```bash tab="Certificate" -docker run --rm \ - -e AZURE_TENANT_ID -e AZURE_CLIENT_ID \ - -v "$PWD/sp-cert.pem:/secrets/sp-cert.pem:ro" \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --azure-cert-path /secrets/sp-cert.pem -``` + ```bash + docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --subscription-id 00000000-0000-0000-0000-000000000000 + ``` -```bash tab="Managed identity" -# on an Azure VM or container host with a managed identity -docker run --rm -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p azure --azure-managed-identity -``` +=== "Certificate" + + ```bash + docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID \ + -v "$PWD/sp-cert.pem:/secrets/sp-cert.pem:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-cert-path /secrets/sp-cert.pem + ``` + +=== "Managed identity" + + ```bash + # on an Azure VM or container host with a managed identity + docker run --rm -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-managed-identity + ``` Without `--subscription-id`, cloudg collects every enabled subscription the identity can see. @@ -146,21 +163,25 @@ Without `--subscription-id`, cloudg collects every enabled subscription the iden Mount a key file (or a workload identity federation config) and point `GOOGLE_APPLICATION_CREDENTIALS` at the path inside the container: -```bash tab="Key file" -docker run --rm \ - -v "$PWD/scanner-sa.json:/secrets/gcp.json:ro" \ - -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p gcp --project-id my-project -``` +=== "Key file" -```bash tab="gcloud ADC" -# after `gcloud auth application-default login` on the host -docker run --rm \ - -v "$HOME/.config/gcloud/application_default_credentials.json:/home/cloudg/.config/gcloud/application_default_credentials.json:ro" \ - -v "$PWD/reports:/app/reports" \ - cloudg:latest run -p gcp --project-id my-project -``` + ```bash + docker run --rm \ + -v "$PWD/scanner-sa.json:/secrets/gcp.json:ro" \ + -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project + ``` + +=== "gcloud ADC" + + ```bash + # after `gcloud auth application-default login` on the host + docker run --rm \ + -v "$HOME/.config/gcloud/application_default_credentials.json:/home/cloudg/.config/gcloud/application_default_credentials.json:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project + ``` google-auth finds the application default credentials file at its standard path under `HOME` without needing gcloud in the container. @@ -294,13 +315,15 @@ docker compose run --rm cloudg run -p aws -p gcp --project-id my-project --iac-d The MCP server runs from the same image. Over stdio, the AI client starts the container itself; over HTTP, the container runs as a long-lived service. -```mermaid caption="Two ways a client reaches cloudg in a container" +```mermaid flowchart LR D["Desktop client"] -->|"docker run -i (stdio)"| S1["cloudg mcp serve"] W["HTTP client"] -->|"127.0.0.1:8765/mcp, bearer token"| S2["cloudg mcp serve --transport http"] S1 --> V[("/app/reports")] S2 --> V ``` +Two ways a client reaches cloudg in a container +{ .diagram-caption } ### stdio for a desktop client @@ -358,4 +381,4 @@ curl -s http://127.0.0.1:8765/healthz The MCP live tools (`map_inventory`, `collect_assets`, `run_scanners`, `run_pipeline`) call cloud APIs with the server's credentials, whoever is asking. With no credentials mounted, their credential preflight (at most 25 seconds) fails and nothing is collected. If you do mount credentials, restrict who can use them: the `strict`, `read_only`, `airgapped`, `audit` and `soc-analyst` policies hide those tools, and `--read-only` drops them along with anything that writes files. A server that only answers questions about saved maps should run with `--read-only` and a read-only reports mount, as above. -[Connect a client](/mcp/connect/) has configuration for the other MCP clients, and [cloudg mcp serve](/cli/mcp-serve/) lists every server option. +[Connect a client](../mcp/connect.md) has configuration for the other MCP clients, and [cloudg mcp serve](../cli/mcp-serve.md) lists every server option. diff --git a/site/content/guides/graph-ontology-rag.md b/site/content/guides/graph-ontology-rag.md index 998da6f..31e69ff 100644 --- a/site/content/guides/graph-ontology-rag.md +++ b/site/content/guides/graph-ontology-rag.md @@ -1,17 +1,17 @@ --- title: Graph, ontology and RAG -lede: "Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM." -meta: - - [Modules, "`cloudg.graph`"] - - [Config, "`graph`, `ontology`, `rag`, `terraform`"] - - [Changed in, "0.6.0"] -source: cloudg/graph/reachability.py -since: "0.6.0" +description: Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM. --- +Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM. +{ .lede } + +**Modules** `cloudg.graph` · **Config** `graph`, `ontology`, `rag`, `terraform` · **Changed in** 0.6.0 · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every collection ends with two lists: `CloudAsset` objects and directed `NetworkEdge` objects between their ids. Everything on this page is derived from those two lists, plus findings when you have them. Nothing here calls a cloud API, so you can rebuild any of it offline from a saved map or a `findings.json`. -```mermaid caption="From collected data to the four analysis outputs" +```mermaid flowchart LR A["Assets and edges"] --> G["GraphBuilder: nx.DiGraph"] F[Findings] --> O @@ -26,6 +26,8 @@ flowchart LR C --> J["rag_chunks.jsonl"] A --> TF["TerraformExporter: .tf.json"] ``` +From collected data to the four analysis outputs +{ .diagram-caption } `cloudg run` does all of it in order: graph and reachability in phase 2, RAG export in 2c, Terraform in 2d, then after the scanners the ontology and a second RAG pass so both include the scanner findings. `CloudGEngine.analyze()` does the graph, ontology, RAG and Terraform steps for assets you collected yourself. The sections below use a small hand-built estate so every output is reproducible; the full script is at the end. @@ -103,7 +105,7 @@ Only `sg-web` admits the internet, yet the database is flagged. That is because blast radius of web-1: {'reachable_nodes': ['sg-web', 'role-app', 'sg-db', 'db-1'], 'depth': 2, 'risk_score': 4.0} ``` -The score is `0.5` per reachable node plus `2.0` per sensitive data store, capped at 10. It is a quick ranking signal. For dependency questions on a full inventory (what breaks if this KMS key goes, which shared resources have the most dependents) use `DependencyGraph` and `cloudg deps`, described in [dependencies and blast radius](/guides/dependencies/). +The score is `0.5` per reachable node plus `2.0` per sensitive data store, capped at 10. It is a quick ranking signal. For dependency questions on a full inventory (what breaks if this KMS key goes, which shared resources have the most dependents) use `DependencyGraph` and `cloudg deps`, described in [dependencies and blast radius](dependencies.md). ## The ontology @@ -345,7 +347,6 @@ from cloudg.schema.models import ( ACCOUNT, REGION = "123456789012", "eu-west-1" OUT = Path("./graph-out") - def aws(asset_id: str, name: str, kind: AssetType, arn_tail: str, **metadata) -> CloudAsset: service = arn_tail.split(":", 1)[0] return CloudAsset( @@ -359,11 +360,9 @@ def aws(asset_id: str, name: str, kind: AssetType, arn_tail: str, **metadata) -> metadata=metadata, ) - def edge(src: str, dst: str, kind: EdgeType, **attrs) -> NetworkEdge: return NetworkEdge(source_id=src, target_id=dst, edge_type=kind, **attrs) - assets = [ aws("vpc-1", "prod-vpc", AssetType.VPC, "ec2:vpc/vpc-1", vpc_id="vpc-1"), aws("subnet-a", "public-a", AssetType.SUBNET, "ec2:subnet/subnet-a", vpc_id="vpc-1"), @@ -444,9 +443,22 @@ print("terraform preview:", terraform.preview(assets)["resource_types"]) print({name: path.name for name, path in terraform.export(assets, edges).items()}) ``` -:::links -- [Dependencies and blast radius](/guides/dependencies/) `DependencyGraph` and `cloudg deps` on a full inventory. -- [GraphBuilder](/api/graphbuilder/) Every method of the builder. -- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) The walk and its helpers. -- [CloudOntology](/api/cloudontology/) and [RAGExporter](/api/ragexporter/) The exporters' APIs. -::: +
    + +- [**Dependencies and blast radius**](dependencies.md) + + `DependencyGraph` and `cloudg deps` on a full inventory. + +- [**GraphBuilder**](../api/graphbuilder.md) + + Every method of the builder. + +- [**ReachabilityAnalyzer**](../api/reachabilityanalyzer.md) + + The walk and its helpers. + +- [**CloudOntology**](../api/cloudontology.md) + + and [RAGExporter](../api/ragexporter.md) The exporters' APIs. + +
    diff --git a/site/content/guides/ingesting.md b/site/content/guides/ingesting.md index 7d011e5..d9b2a78 100644 --- a/site/content/guides/ingesting.md +++ b/site/content/guides/ingesting.md @@ -1,19 +1,19 @@ --- title: Ingesting existing output -lede: "`cloudg ingest` is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports." -meta: - - [Command, "`cloudg ingest`"] - - [Python, "`parse_report`, `ingest_reports`, `CloudGEngine.run_from_reports()`"] - - [Needs, "No credentials, no scanner binaries"] -source: cloudg/ingest.py -since: "0.6.0" +description: cloudg ingest is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports. --- +`cloudg ingest` is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports. +{ .lede } + +**Command** `cloudg ingest` · **Python** `parse_report`, `ingest_reports`, `CloudGEngine.run_from_reports()` · **Needs** No credentials, no scanner binaries · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Ingest exists for the common case where the scanners already run somewhere you don't control from cloudg: a CI job that runs Checkov on every pull request, a nightly Prowler container in the security account, a Trivy step in the image build. Those jobs produce files. `cloudg ingest` reads the files and does the rest. Nothing is executed and no cloud API is called, so it runs on a laptop with no AWS profile, in an air-gapped review environment, or in a CI step after the scan step. ## What ingest does -```mermaid caption="cloudg ingest, from native files to reports" +```mermaid flowchart LR P["--prowler files"] --> PR["parse_report"] S["--scoutsuite files"] --> PR @@ -24,6 +24,8 @@ flowchart LR N --> J["findings.json"] N --> H["report.html"] ``` +cloudg ingest, from native files to reports +{ .diagram-caption } Each path goes through `cloudg.ingest.parse_report(tool, path)`, which hands it to that scanner's own parser (the same code `cloudg run` uses on fresh output). Every parsed `Finding` is collected into one list and written, unchanged, to `raw-findings.json`. The list then goes through the same `FindingsNormaliser` that `cloudg run` uses: two-pass deduplication, CVSS-based rescoring and compliance mapping. The result is written as `findings.json` and `report.html`. @@ -33,44 +35,50 @@ What ingest doesn't do is anything that needs live assets. There is no collectio Every input flag is repeatable, and any combination of tools works. -```bash tab="CLI" -cloudg ingest \ - --prowler ./prowler-output/ \ - --scoutsuite ./scoutsuite-report/ \ - --checkov ./results_json.json \ - --trivy ./trivy-image.json \ - --trivy ./trivy-fs.json \ - -o ./reports -``` - -```python tab="Python" title="ingest_reports_demo.py" -from cloudg import CloudGConfig, CloudGEngine - -engine = CloudGEngine(CloudGConfig()) -result = engine.run_from_reports_sync( - { - "prowler": ["./prowler-output/"], - "scoutsuite": ["./scoutsuite-report/"], - "checkov": ["./results_json.json"], - "trivy": ["./trivy-image.json", "./trivy-fs.json"], - }, - output_dir="./reports", -) - -print(result.total_findings, result.severity_breakdown) -print(result.report_paths) -print(result.errors) -``` - -```bash tab="Docker" -docker run --rm \ - -v "$PWD:/data" \ - ghcr.io/morpheuslord/cloudg:latest ingest \ - --prowler /data/prowler-output/ \ - --checkov /data/results_json.json \ - --trivy /data/trivy-image.json \ - -o /data/reports -``` +=== "CLI" + + ```bash + cloudg ingest \ + --prowler ./prowler-output/ \ + --scoutsuite ./scoutsuite-report/ \ + --checkov ./results_json.json \ + --trivy ./trivy-image.json \ + --trivy ./trivy-fs.json \ + -o ./reports + ``` + +=== "Python" + + ```python title="ingest_reports_demo.py" + from cloudg import CloudGConfig, CloudGEngine + + engine = CloudGEngine(CloudGConfig()) + result = engine.run_from_reports_sync( + { + "prowler": ["./prowler-output/"], + "scoutsuite": ["./scoutsuite-report/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + }, + output_dir="./reports", + ) + + print(result.total_findings, result.severity_breakdown) + print(result.report_paths) + print(result.errors) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v "$PWD:/data" \ + ghcr.io/morpheuslord/cloudg:latest ingest \ + --prowler /data/prowler-output/ \ + --checkov /data/results_json.json \ + --trivy /data/trivy-image.json \ + -o /data/reports + ``` | Flag | Accepts | |---|---| @@ -81,7 +89,7 @@ docker run --rm \ | `--format` | `html`, `json` or `all` (default). `raw-findings.json` is written in every case. | | `-o, --output` | Output directory, `./reports` by default | -The global `-c` option matters here for one reason: `rulesets.rules_dir` decides which compliance rulesets the normaliser loads. See [Compliance frameworks](/guides/compliance/#adding-a-custom-ruleset). +The global `-c` option matters here for one reason: `rulesets.rules_dir` decides which compliance rulesets the normaliser loads. See [Compliance frameworks](compliance.md#adding-a-custom-ruleset). ```bash cloudg -c config.yaml ingest --prowler ./prowler-output/ -o ./reports @@ -92,86 +100,7 @@ cloudg -c config.yaml ingest --prowler ./prowler-output/ -o ./reports The script below writes one small file per tool in each tool's native shape, the same shapes the test suite uses. Run it in an empty directory, then run the `cloudg ingest` command above. ```python title="make_sample_reports.py" -"""Write one small native report per scanner, shaped like the real tools' output.""" -import json -from pathlib import Path - -Path("prowler-output").mkdir(exist_ok=True) -Path("prowler-output/prowler-output-123456789012.asff.json").write_text(json.dumps([ - { - "Id": "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-abc", - "Title": "S3 bucket default encryption", - "Description": "Bucket has no default encryption", - "Severity": {"Label": "HIGH"}, - "Resources": [{"Id": "arn:aws:s3:::my-bucket"}], - "Compliance": {"Status": "FAILED", "RelatedRequirements": ["CIS 2.1.1"]}, - "Remediation": {"Recommendation": {"Text": "Enable default encryption"}}, - } -])) - -results = Path("scoutsuite-report/scoutsuite-results") -results.mkdir(parents=True, exist_ok=True) -(results / "scoutsuite_results_aws-123456789012.js").write_text( - "scoutsuite_results =\n" + json.dumps({ - "services": {"s3": {"findings": {"s3-bucket-no-encryption": { - "description": "Bucket without encryption", - "rationale": "Data at rest should be encrypted", - "remediation": "Enable encryption", - "level": "danger", - "flagged_items": 1, - "items": ["s3.buckets.my-bucket"], - "references": [], - }}}} - }) -) - -Path("results_json.json").write_text(json.dumps({ - "check_type": "terraform", - "results": {"failed_checks": [{ - "check_id": "CKV_AWS_19", - "check_name": "Ensure S3 bucket has server-side encryption enabled", - "severity": "HIGH", - "resource": "aws_s3_bucket.data", - "file_path": "/main.tf", - "file_line_range": [1, 10], - "guideline": "https://docs.example/ckv-aws-19", - }]}, -})) - -Path("trivy-image.json").write_text(json.dumps({ - "ArtifactName": "myrepo/app:latest", - "ArtifactType": "container_image", - "Results": [{ - "Target": "myrepo/app:latest (alpine 3.19)", - "Type": "alpine", - "Vulnerabilities": [{ - "VulnerabilityID": "CVE-2024-0001", - "PkgName": "openssl", - "InstalledVersion": "3.1.0", - "FixedVersion": "3.1.1", - "Severity": "CRITICAL", - "Description": "Example vulnerability", - "CVSS": {"nvd": {"V3Score": 9.8}}, - }], - }], -})) - -Path("trivy-fs.json").write_text(json.dumps({ - "ArtifactName": "./iac", - "ArtifactType": "filesystem", - "Results": [{ - "Target": "main.tf", - "Type": "terraform", - "Misconfigurations": [{ - "ID": "AVD-AWS-0088", - "Title": "S3 bucket encryption not enabled", - "Description": "Bucket does not have encryption enabled", - "Severity": "HIGH", - "Resolution": "Enable encryption", - }], - }], -})) -print("sample reports written") +--8<-- "make_sample_reports.py" ``` ```console @@ -231,7 +160,7 @@ From each OCSF record: | `remediation.desc` | `remediation` | | `status_detail` (else `message`) | `evidence` | -The passing checks travel with the parsed findings (`passed_checks` on the list `parse_report` returns) into normalisation, where the compliance controls that only passing checks map to get a PASS result. See [Compliance frameworks](/guides/compliance/). +The passing checks travel with the parsed findings (`passed_checks` on the list `parse_report` returns) into normalisation, where the compliance controls that only passing checks map to get a PASS result. See [Compliance frameworks](compliance.md). ### ScoutSuite @@ -306,7 +235,7 @@ If you feed the same report twice, or two overlapping directories, the duplicate ## Normalisation and deduplication {#normalisation-and-deduplication} -The normaliser runs two deduplication passes, then rescoring, then compliance mapping. Compliance mapping has [its own page](/guides/compliance/); this is what happens to the findings themselves. +The normaliser runs two deduplication passes, then rescoring, then compliance mapping. Compliance mapping has [its own page](compliance.md); this is what happens to the findings themselves. Pass 1 works inside each scanner. The key is the tool, the check ID (or the normalised title when there is no check ID) and the resource. Two different checks that happen to share a title on the same bucket stay separate. @@ -390,7 +319,7 @@ HIGH 7.5 trivy [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enab The functions differ in how they fail. `parse_report` raises `ValueError` for a tool name it doesn't know (only `prowler`, `scoutsuite`, `checkov` and `trivy` are accepted, case-insensitive) and `FileNotFoundError` for a missing path. `ingest_reports` catches both, logs them and moves on. The per-scanner parsers are public too, as class methods: `ProwlerScanner.parse_report(path)`, `ScoutSuiteScanner.parse_report(path)`, `CheckovScanner.parse_report(path)` and `TrivyScanner.parse_report(path)`. -On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires the `on_phase_start("ingest")`, `on_finding` and `on_scan_complete` [event hooks](/api/event-hooks/). `run_from_reports()` and its sync wrapper go further and write `findings.json` and `report.html` as `report.formats` allows (not `raw-findings.json`), returning a `PipelineResult` whose `report_paths` holds both paths and whose `errors` lists any phase that failed. +On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires the `on_phase_start("ingest")`, `on_finding` and `on_scan_complete` [event hooks](../api/event-hooks.md). `run_from_reports()` and its sync wrapper go further and write `findings.json` and `report.html` as `report.formats` allows (not `raw-findings.json`), returning a `PipelineResult` whose `report_paths` holds both paths and whose `errors` lists any phase that failed. ## What comes out @@ -400,11 +329,11 @@ On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires | `findings.json` | `metadata`, `summary`, `findings`, `compliance`, and empty `assets`, `edges` and `graph` | | `report.html` | The interactive report: findings table, analytics, compliance cards | -With no assets, the report's Infrastructure Map tab falls back to a force graph of finding nodes linked to compliance framework nodes. The [Reports](/guides/reports/) guide covers both files in detail, and the field-level layout of `findings.json` is on [Output files](/reference/output-files/). +With no assets, the report's Infrastructure Map tab falls back to a force graph of finding nodes linked to compliance framework nodes. The [Reports](reports.md) guide covers both files in detail, and the field-level layout of `findings.json` is on [Output files](../reference/output-files.md). ## Combining with a live inventory {#combining-with-a-live-inventory} -Ingest needs no credentials; mapping needs read-only ones. Run them separately, at different times if you like, and merge at the end. [`cloudg map`](/guides/inventory-mapping/) accepts a cloudg findings file and writes `asset-map.json` (each asset with its findings) and `compliance-map.json` (each framework with its affected assets): +Ingest needs no credentials; mapping needs read-only ones. Run them separately, at different times if you like, and merge at the end. [`cloudg map`](inventory-mapping.md) accepts a cloudg findings file and writes `asset-map.json` (each asset with its findings) and `compliance-map.json` (each framework with its affected assets): ```bash cloudg ingest --prowler ./ci/prowler-output/ --checkov ./ci/results_json.json -o ./reports @@ -418,7 +347,6 @@ import asyncio from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -427,15 +355,27 @@ async def main() -> None: scan_result = engine.normalise_findings(ingested, assets=collection.assets) print(len(scan_result.assets), "assets,", len(scan_result.findings), "findings") - asyncio.run(main()) ``` ## Next -:::links -- [Compliance frameworks](/guides/compliance/) How findings are tagged with controls, and how to add your own. -- [Reports](/guides/reports/) Reading `findings.json` and `report.html`, and re-rendering them. -- [cloudg ingest](/cli/ingest/) The command reference. -- [parse_report](/api/parse-report/) The Python parser entry point. -::: +
    + +- [**Compliance frameworks**](compliance.md) + + How findings are tagged with controls, and how to add your own. + +- [**Reports**](reports.md) + + Reading `findings.json` and `report.html`, and re-rendering them. + +- [**cloudg ingest**](../cli/ingest.md) + + The command reference. + +- [**parse_report**](../api/parse-report.md) + + The Python parser entry point. + +
    diff --git a/site/content/guides/installation.md b/site/content/guides/installation.md index c0b2880..0de139a 100644 --- a/site/content/guides/installation.md +++ b/site/content/guides/installation.md @@ -1,13 +1,14 @@ --- title: Installation -lede: "cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image." -meta: - - [Python, "3.11 or newer"] - - [Package, "`cloudg` on PyPI"] - - [Image, "`ghcr.io/morpheuslord/cloudg`"] -source: pyproject.toml +description: cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image. --- +cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image. +{ .lede } + +**Python** 3.11 or newer · **Package** `cloudg` on PyPI · **Image** `ghcr.io/morpheuslord/cloudg` +{ .page-meta } + There are two things to install, and it helps to keep them apart. The first is cloudg itself: the CLI, the graph engine, the normaliser, the report renderers and, through extras, the cloud SDKs. The second is the set of scanners (Prowler, ScoutSuite, Checkov, Trivy) that `cloudg run` and `cloudg scan` call as subprocesses. cloudg works with any subset of them, including none. ## Requirements @@ -18,27 +19,33 @@ Credentials are not needed to install, and not needed at all for `cloudg ingest` ## Install cloudg -```bash tab="pip" -python3 -m venv .venv -source .venv/bin/activate -pip install "cloudg[all]" -``` +=== "pip" -```bash tab="uv" -# as a standalone CLI in its own environment -uv tool install "cloudg[all]" -# or into the current project's virtualenv -uv pip install "cloudg[all]" -``` + ```bash + python3 -m venv .venv + source .venv/bin/activate + pip install "cloudg[all]" + ``` -```bash tab="Docker" -docker pull ghcr.io/morpheuslord/cloudg:latest -docker run --rm ghcr.io/morpheuslord/cloudg:latest --version -``` +=== "uv" + + ```bash + # as a standalone CLI in its own environment + uv tool install "cloudg[all]" + # or into the current project's virtualenv + uv pip install "cloudg[all]" + ``` + +=== "Docker" + + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + docker run --rm ghcr.io/morpheuslord/cloudg:latest --version + ``` Quote the package spec. In zsh, unquoted square brackets are a glob pattern and `pip install cloudg[all]` fails with "no matches found". -The Docker image already contains cloudg, the AWS, Azure and GCP SDKs, Prowler, Checkov, Trivy and Parliament. It is the shortest way to the full pipeline; [Docker](/guides/docker/) covers running it with credentials and volumes. +The Docker image already contains cloudg, the AWS, Azure and GCP SDKs, Prowler, Checkov, Trivy and Parliament. It is the shortest way to the full pipeline; [Docker](docker.md) covers running it with credentials and volumes. ### From a source checkout @@ -71,9 +78,9 @@ The cloud SDKs are optional, so a machine that only aggregates scanner output do Extras combine: `pip install "cloudg[aws,gcp]"` gets exactly two providers. -:::note What the core package can do on its own -Without any extra you still get the graph builder, the ontology, the RAG exporter, the findings normaliser with every compliance ruleset, all four scanner parsers, the HTML, JSON and SVG renderers, and the MCP layer with its native server. That is enough for `cloudg ingest` on an air-gapped analysis host. -::: +!!! note "What the core package can do on its own" + + Without any extra you still get the graph builder, the ontology, the RAG exporter, the findings normaliser with every compliance ruleset, all four scanner parsers, the HTML, JSON and SVG renderers, and the MCP layer with its native server. That is enough for `cloudg ingest` on an air-gapped analysis host. About `full`: in 0.6.0 the IAM linter only checks whether `policy_sentry` can be imported, and nothing in the package imports `diagrams` or `weasyprint`. Installing `full` does no harm, but it changes no output today. @@ -103,17 +110,23 @@ Both tools put the executables in `~/.local/bin`, which must be on `PATH` for cl Trivy is a Go binary. Use your package manager or Aqua Security's install script: -```bash tab="macOS" -brew install trivy -``` +=== "macOS" -```bash tab="Linux" -curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin -``` + ```bash + brew install trivy + ``` -```bash tab="Windows" -winget install AquaSecurity.Trivy -``` +=== "Linux" + + ```bash + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin + ``` + +=== "Windows" + + ```bash + winget install AquaSecurity.Trivy + ``` The scanners authenticate on their own. cloudg passes Prowler the AWS profile (`-p`) and, when you gave direct keys, the key pair through the environment; ScoutSuite gets the AWS profile. For Azure and GCP they read the same environment and credential files the SDKs do. If a scanner works when you run it by hand, it works under cloudg. @@ -121,17 +134,21 @@ The scanners authenticate on their own. cloudg passes Prowler the AWS profile (` `install.sh` (Linux and macOS) and `install.bat` (Windows) set up a complete workstation in one go. Run them from the root of a source checkout, because they install cloudg in editable mode from the current directory: -```bash tab="Linux / macOS" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -./install.sh -``` +=== "Linux / macOS" -```bash tab="Windows" -git clone https://github.com/morpheuslord/cloudg.git -cd cloudg -install.bat -``` + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + ./install.sh + ``` + +=== "Windows" + + ```bash + git clone https://github.com/morpheuslord/cloudg.git + cd cloudg + install.bat + ``` `install.sh` detects the package manager (apt, dnf, yum, pacman, zypper or Homebrew) and then, in order: @@ -216,23 +233,29 @@ You should see "Ingested 1 findings from 1 tool(s), 1 after deduplication" and t ## Upgrading -```bash tab="pip" -pip install --upgrade "cloudg[all]" -``` +=== "pip" -```bash tab="uv" -uv tool upgrade cloudg -``` + ```bash + pip install --upgrade "cloudg[all]" + ``` -```bash tab="Docker" -docker pull ghcr.io/morpheuslord/cloudg:latest -``` +=== "uv" + + ```bash + uv tool upgrade cloudg + ``` + +=== "Docker" + + ```bash + docker pull ghcr.io/morpheuslord/cloudg:latest + ``` For a source checkout, `git pull` and then reinstall: `uv pip install -e ".[all,dev]"`. Reinstalling matters when the entry points change. The `cloudg-mcp` console script arrived in 0.6.0, so an editable install made before that does not have it until you reinstall. Image tags follow the release version (`ghcr.io/morpheuslord/cloudg:0.6.0`), and `latest` moves with each release. Pin the version tag in CI so an upgrade happens when you choose. -Configuration files carry over. A `config.yaml` that still uses the old single `provider:` key is migrated to `providers: [...]` when it loads. Read the [changelog](/changelog/) before upgrading across a minor version; it lists every changed default and output field. +Configuration files carry over. A `config.yaml` that still uses the old single `provider:` key is migrated to `providers: [...]` when it loads. Read the [changelog](../changelog.md) before upgrading across a minor version; it lists every changed default and output field. ## Troubleshooting @@ -242,4 +265,4 @@ If pip says no matching distribution was found, or mentions "a different Python" A scanner that works in your shell but is "not installed" under cloudg is on a different `PATH`. That happens with scanners installed in a second virtualenv that is not active. Use `uv tool` or `pipx`, or symlink the executable somewhere on `PATH`. -Next: run your first scan in the [quickstart](/guides/quickstart/). +Next: run your first scan in the [quickstart](quickstart.md). diff --git a/site/content/guides/introduction.md b/site/content/guides/introduction.md index 3b6dd9d..c76b2ef 100644 --- a/site/content/guides/introduction.md +++ b/site/content/guides/introduction.md @@ -1,14 +1,14 @@ --- title: Introduction -lede: "cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result." -meta: - - [Version, "0.6.0"] - - [Python, "3.11 or newer"] - - [Providers, "AWS, Azure, GCP"] - - [License, "MIT"] -source: README.md +description: cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result. --- +cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result. +{ .lede } + +**Version** 0.6.0 · **Python** 3.11 or newer · **Providers** AWS, Azure, GCP · **License** MIT +{ .page-meta } + cloudg is a Python command-line tool and a library. You point it at one or more cloud accounts and it builds an inventory of what is deployed there: instances, buckets, roles, security groups, load balancers, functions, key vaults, projects. Those assets and the network edges between them go into a directed graph. From the graph cloudg works out what the internet can reach, which paths an attacker could take sideways, and how much of the estate each node could touch if it were compromised. The same inventory then goes to the security scanners you already know (Prowler, ScoutSuite, Checkov and Trivy), plus a built-in IAM policy linter. Each scanner reports the same problem in its own words, so cloudg normalises every finding into one model, merges duplicates within and across tools, rescores them against CVSS, and maps them onto controls from 28 compliance frameworks. What you open at the end is a single HTML report, not four. @@ -19,16 +19,35 @@ None of that requires the whole pipeline. Each half of cloudg works without the Most people arrive with one of three questions, and cloudg has a command for each. -:::cells +
    + +
    + ### cloudg run + "What is wrong with my cloud?" Collects every configured provider, builds the graph, runs the scanners in parallel, normalises the findings and renders the reports. Needs credentials and, for full coverage, the scanner binaries. + +
    + +
    + ### cloudg ingest + "I already ran the scanners, can you make sense of the output?" Reads native Prowler, ScoutSuite, Checkov and Trivy files, then deduplicates, maps compliance and renders reports. No cloud credentials and no scanner binaries. + +
    + +
    + ### cloudg map + "What do I have, and how is it wired?" A scanner-free inventory mapper: deep collectors, a sweep across every service, typed relationships and dependency analysis. No scanner runs or needs to be installed. -::: -They are independent but they compose. You can `cloudg map` an account today, run Prowler in CI next week, and merge that output into the map later with `cloudg map --findings`. You can collect live assets from Python and normalise findings you ingested from someone else's pipeline in the same script. The [quickstart](/guides/quickstart/) walks through all three. +
    + +
    + +They are independent but they compose. You can `cloudg map` an account today, run Prowler in CI next week, and merge that output into the map later with `cloudg map --findings`. You can collect live assets from Python and normalise findings you ingested from someone else's pipeline in the same script. The [quickstart](quickstart.md) walks through all three. ### Choosing between them @@ -44,7 +63,7 @@ They are independent but they compose. You can `cloudg map` an account today, ru ## How the pieces fit -```mermaid caption="From cloud APIs to the files and tools you use" +```mermaid flowchart LR P["AWS, Azure, GCP APIs"] --> C[Collectors] C --> G["Graph (NetworkX)"] @@ -58,6 +77,8 @@ flowchart LR N --> M["MCP server"] G --> M ``` +From cloud APIs to the files and tools you use +{ .diagram-caption } Collectors talk to the provider APIs through a shared rate limiter, so a big account slows down instead of failing. In the NetworkX graph, edge endpoints that are not assets (the `0.0.0.0/0` on a security group rule, for example) become external nodes, and a breadth-first search from the internet sources `0.0.0.0/0` and `::/0` over network-flow edges is how cloudg decides what is internet-reachable. Scanners run as separate processes, each in its own worker thread, and anything not installed is skipped with a warning. The normaliser is the meeting point: scanner findings, the graph's reachability findings and the IAM linter's results all go through the same deduplication and compliance mapping. @@ -71,7 +92,7 @@ The MCP server sits beside all of this. It loads the files a run or a map produc | Azure | `cloudg[azure]` (azure-identity and the management SDKs) | Per subscription; every enabled subscription the credential can see when none is set | all locations | | GCP | `cloudg[gcp]` (google-cloud-asset, google-auth) | Cloud Asset Inventory, per project or once at organization scope | all regions | -Every provider supports several authentication methods, resolved in a fixed order: direct keys, OIDC federation, profiles and service principals, managed identities, and finally each SDK's default chain. The same configuration works on a laptop, in CI and on cloud compute. [Authentication](/guides/authentication/) has the details. +Every provider supports several authentication methods, resolved in a fixed order: direct keys, OIDC federation, profiles and service principals, managed identities, and finally each SDK's default chain. The same configuration works on a laptop, in CI and on cloud compute. [Authentication](authentication.md) has the details. ## Scanners @@ -83,7 +104,7 @@ Every provider supports several authentication methods, resolved in a fixed orde | Trivy | Container images, or a filesystem when no images are given | `trivy` | | IAM linter | IAM policies on collected roles and policies, using Parliament | built in | -The scanners are executables, not Python dependencies. cloudg checks `PATH` when it runs each one and skips the missing ones. [Installation](/guides/installation/#scanner-binaries) shows how to install each, and [Running scanners](/guides/running-scanners/) covers what cloudg passes to them. +The scanners are executables, not Python dependencies. cloudg checks `PATH` when it runs each one and skips the missing ones. [Installation](installation.md#scanner-binaries) shows how to install each, and [Running scanners](running-scanners.md) covers what cloudg passes to them. ## What you get @@ -98,7 +119,7 @@ A `cloudg run` writes everything into `./reports` (change it with `-o`): | `rag_chunks.jsonl`, `rag_metadata_index.json` | Retrieval-ready chunks of the infrastructure for LLM pipelines | | `terraform/*.tf.json` | Terraform recreation of the live infrastructure, with `--terraform` | -`cloudg ingest` writes `findings.json`, `report.html` and `raw-findings.json`. `cloudg map` writes `inventory-map.json`, `inventory-map.graphml`, `inventory-graph.json` and `inventory-dependencies.json`, plus `inventory-organization.json` with `--org` and `asset-map.json` and `compliance-map.json` with `--findings`. [Output files](/reference/output-files/) describes each one field by field. +`cloudg ingest` writes `findings.json`, `report.html` and `raw-findings.json`. `cloudg map` writes `inventory-map.json`, `inventory-map.graphml`, `inventory-graph.json` and `inventory-dependencies.json`, plus `inventory-organization.json` with `--org` and `asset-map.json` and `compliance-map.json` with `--findings`. [Output files](../reference/output-files.md) describes each one field by field. ## Using it from Python @@ -114,15 +135,34 @@ result = engine.run_pipeline_sync(output_dir="./reports") print(result.to_summary()) ``` -The graph builder, ontology, RAG exporter, scanner parsers, normaliser, inventory mapper and MCP layer are all importable on their own. [The pipeline](/guides/pipeline/) shows the phases from both the CLI and Python, and the [Python API](/api/) tab documents every class. +The graph builder, ontology, RAG exporter, scanner parsers, normaliser, inventory mapper and MCP layer are all importable on their own. [The pipeline](pipeline.md) shows the phases from both the CLI and Python, and the [Python API](../api/index.md) tab documents every class. ## Where to go next -:::links -- [Installation](/guides/installation/) pip, uv or Docker, the provider extras, and the scanner binaries. -- [Quickstart](/guides/quickstart/) A first report in a few minutes, with or without credentials. -- [The pipeline](/guides/pipeline/) What each phase of cloudg run reads, writes and does when something fails. -- [Inventory mapping](/guides/inventory-mapping/) The scanner-free map of everything deployed. -- [Ingesting existing output](/guides/ingesting/) Reports from scanner files you already have. -- [MCP server](/mcp/) Hand the inventory and findings to an AI assistant. -::: +
    + +- [**Installation**](installation.md) + + pip, uv or Docker, the provider extras, and the scanner binaries. + +- [**Quickstart**](quickstart.md) + + A first report in a few minutes, with or without credentials. + +- [**The pipeline**](pipeline.md) + + What each phase of cloudg run reads, writes and does when something fails. + +- [**Inventory mapping**](inventory-mapping.md) + + The scanner-free map of everything deployed. + +- [**Ingesting existing output**](ingesting.md) + + Reports from scanner files you already have. + +- [**MCP server**](../mcp/index.md) + + Hand the inventory and findings to an AI assistant. + +
    diff --git a/site/content/guides/inventory-mapping.md b/site/content/guides/inventory-mapping.md index 0e9358e..ac7bfa5 100644 --- a/site/content/guides/inventory-mapping.md +++ b/site/content/guides/inventory-mapping.md @@ -1,55 +1,63 @@ --- title: Inventory mapping -lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" -meta: - - [Command, "`cloudg map`"] - - [Providers, "AWS, Azure, GCP"] -source: docs/DOCUMENTATION.md +description: 'cloudg map asks a different question from cloudg run: what exists, and how is it wired together?' --- +`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together? +{ .lede } + +**Command** `cloudg map` · **Providers** AWS, Azure, GCP +{ .page-meta } + `cloudg map` reads your cloud accounts and writes down everything it finds, deployed resources and defaults alike, then links them into one graph of typed relationships. A Lambda function points at the role it runs as, the queue that triggers it, the image it was built from and the KMS key that encrypts its environment. A WAF points at the load balancer it protects. An SCP points at the accounts it restricts. No scanner runs during a map, and none needs to be installed. The command only needs read access to the cloud APIs. -:::note -The map doesn't depend on the scanners. You can map today and pull in next week's scan findings with `--findings ./reports/raw-findings.json`. On the CLI that flag belongs to `cloudg map`, so it maps again; to overlay findings onto a map you already saved without calling the cloud, use the Python call in [Overlay scanner findings](#overlay-scanner-findings). -::: +!!! note + + The map doesn't depend on the scanners. You can map today and pull in next week's scan findings with `--findings ./reports/raw-findings.json`. On the CLI that flag belongs to `cloudg map`, so it maps again; to overlay findings onto a map you already saved without calling the cloud, use the Python call in [Overlay scanner findings](#overlay-scanner-findings). ## Map an account The smallest useful run maps one AWS account in every enabled region: -```bash tab="CLI" -cloudg map -p aws --profile audit --regions all -``` +=== "CLI" -```python tab="Python" title="map_account.py" -from cloudg import CloudGConfig, CloudGEngine + ```bash + cloudg map -p aws --profile audit --regions all + ``` -config = CloudGConfig(providers=["aws"]) -config.aws.profile = "audit" -config.aws.regions = ["ALL"] +=== "Python" -engine = CloudGEngine(config) -inventory = engine.map_inventory_sync(output_dir="./reports") + ```python title="map_account.py" + from cloudg import CloudGConfig, CloudGEngine -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("internet-exposed:", summary["internet_exposed"]) -print("unlinked:", summary["unlinked_assets"]) -``` + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "audit" + config.aws.regions = ["ALL"] -```bash tab="Docker" -docker run --rm \ - -v ~/.aws:/home/cloudg/.aws:ro \ - -v "$PWD/reports:/app/reports" \ - ghcr.io/morpheuslord/cloudg:latest \ - map -p aws --profile audit --regions all -``` + engine = CloudGEngine(config) + inventory = engine.map_inventory_sync(output_dir="./reports") + + summary = inventory.summary + print(summary["total_assets"], "assets,", summary["total_edges"], "edges") + print("internet-exposed:", summary["internet_exposed"]) + print("unlinked:", summary["unlinked_assets"]) + ``` + +=== "Docker" + + ```bash + docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile audit --regions all + ``` The image runs as the user `cloudg` with its home at `/home/cloudg`, which is why the AWS config directory is mounted there. Reports land in `/app/reports`, the container's `./reports`. -While it runs, the CLI prints a "Map Configuration" panel (providers, regions, selected services, whether the Kubernetes, Cloud Control and catch-all sweeps are on) and then an "Inventory Summary" table: assets, interconnections, accounts, services, internet-exposed assets, cross-account edges, external accounts referenced, security service gaps, unlinked assets and unresolved references. Two ranked tables follow, "Most shared dependencies" and "Largest blast radius", and a warning line for each security service that is not enabled. [Dependencies and blast radius](/guides/dependencies/) explains how to read them. +While it runs, the CLI prints a "Map Configuration" panel (providers, regions, selected services, whether the Kubernetes, Cloud Control and catch-all sweeps are on) and then an "Inventory Summary" table: assets, interconnections, accounts, services, internet-exposed assets, cross-account edges, external accounts referenced, security service gaps, unlinked assets and unresolved references. Two ranked tables follow, "Most shared dependencies" and "Largest blast radius", and a warning line for each security service that is not enabled. [Dependencies and blast radius](dependencies.md) explains how to read them. Pass a config file with the group option, before the subcommand: `cloudg -c config.yaml map ...`. Without `-c`, cloudg uses its built-in defaults and does not look for a `config.yaml` on its own. @@ -59,7 +67,7 @@ Pass a config file with the group option, before the subcommand: `cloudg -c conf With `all`, cloudg discovers the enabled regions first and runs one collection unit per account and region. Account-wide services (IAM, Route 53, ECR Public, Direct Connect gateways and the other global tasks) run once per account, in the primary region: `us-east-1` when it is part of the run, otherwise the first region in the list. So `--regions all` doesn't collect IAM once per region. -With `--org` and Control Tower, `--regions all` is replaced by the landing zone's governed regions. An explicit list is always used as given. The [AWS Organizations](/guides/aws-organizations/) guide covers that case. +With `--org` and Control Tower, `--regions all` is replaced by the landing zone's governed regions. An explicit list is always used as given. The [AWS Organizations](aws-organizations.md) guide covers that case. ## Many accounts @@ -69,9 +77,9 @@ With `--org` and Control Tower, `--regions all` is replaced by the landing zone' cloudg map -p aws --org --org-role cloudg-readonly --regions all ``` -:::warning AWSControlTowerExecution has admin rights -Without `--org-role` (or `--role-name`), cloudg assumes `AWSControlTowerExecution` in every member account. That role exists wherever Control Tower enrolled an account, but it carries administrator permissions. Deploy a read-only role to every account and pass it with `--org-role`. [AWS Organizations](/guides/aws-organizations/) has a StackSet template for it. -::: +!!! warning "AWSControlTowerExecution has admin rights" + + Without `--org-role` (or `--role-name`), cloudg assumes `AWSControlTowerExecution` in every member account. That role exists wherever Control Tower enrolled an account, but it carries administrator permissions. Deploy a read-only role to every account and pass it with `--org-role`. [AWS Organizations](aws-organizations.md) has a StackSet template for it. Always pair `--accounts` with `--role-name`. Without a role name cloudg doesn't assume anything and maps your own account once per listed ID, labelled with each of those IDs. @@ -79,14 +87,33 @@ Always pair `--accounts` with `--role-name`. Without a role name cloudg doesn't Coverage comes in layers. Each one catches what the previous one missed. -:::cells +
    + +
    + ### Deep collectors + 131 dedicated AWS collector tasks in 19 service families, each reading one service in depth and declaring what every asset talks to. Azure goes through Resource Graph, GCP through Cloud Asset Inventory. + +
    + +
    + ### Breadth sweep + On AWS, a Cloud Control sweep lists every CloudFormation resource type that has a list handler, tagged or not, so services without a dedicated collector still show up. The tagging API sweep adds tagged resources on top. + +
    + +
    + ### Every account + Every account gets a `CLOUD_ACCOUNT` node that contains its top-level resources. With `--org`, accounts hang under the OU tree next to SCPs and Control Tower controls. -::: + +
    + +
    When the same resource turns up twice, the copy from a dedicated collector wins over a Cloud Control hit, which wins over a tagging API hit. Their declared relationships and aliases are merged into the copy that is kept. @@ -125,9 +152,9 @@ cloudg map -p aws --regions all --services containers,serverless,sweep cloudg map -p aws --regions all --exclude-services data,cloudformation ``` -:::warning The sweeps are a family too -The two sweeps belong to the family `sweep`. Once you pass `--services`, only what you listed runs, so `--services containers,serverless` also drops the Cloud Control and tagging API sweeps. Add `sweep` to the list if you want them. A family name that matches nothing (a typo, say) is not an error; it selects no tasks. -::: +!!! warning "The sweeps are a family too" + + The two sweeps belong to the family `sweep`. Once you pass `--services`, only what you listed runs, so `--services containers,serverless` also drops the Cloud Control and tagging API sweeps. Add `sweep` to the list if you want them. A family name that matches nothing (a typo, say) is not an error; it selects no tasks. Narrowing the run has a side effect on the links. A function whose role lives in the `identity` family can't be linked to that role when `identity` is excluded, so the reference shows up under `unresolved_references` instead of as an edge. @@ -158,9 +185,9 @@ Two sweeps catch what no deep collector reads. They are separate flags. `--sweep / --no-sweep` (on by default) controls the Resource Groups Tagging API sweep. AWS only returns resources that have been tagged at some point, so this sweep mostly enriches; the Cloud Control sweep is the one that finds untagged resources. Hits carry `discovered_via: "tagging-api"`. -:::note -`cloudg map` always hands the value of `--sweep` to the mapper, and that value defaults to on, so the CLI overrides `inventory.tagging_sweep: false` from your config file even when you don't type the flag. Use `--no-sweep` on the command line to turn the tagging sweep off. -::: +!!! note + + `cloudg map` always hands the value of `--sweep` to the mapper, and that value defaults to on, so the CLI overrides `inventory.tagging_sweep: false` from your config file even when you don't type the flag. Use `--no-sweep` on the command line to turn the tagging sweep off. ## Azure and GCP @@ -178,7 +205,7 @@ cloudg map -p all --regions all ## How a run fits together -```mermaid caption="One mapping run, from discovery to the files on disk" +```mermaid flowchart TD A["Org discovery (--org)"] --> B["Deep collectors, per account and region"] B --> C["Cloud Control and tagging sweeps"] @@ -188,6 +215,8 @@ flowchart TD F --> G["Dependency analysis"] G --> H["Exports in ./reports"] ``` +One mapping run, from discovery to the files on disk +{ .diagram-caption } Collectors never create edges across services themselves. They declare what each asset talks to, by any identifier they have (an ARN, a queue URL, an image URI, a bare `sg-` ID), and the relationship linker resolves those identifiers across services, regions and accounts. It runs three passes: declared relations, provider rules for older metadata conventions, and a generic pass that turns any identifier-looking string in an asset's metadata into a `REFERENCES` edge when it resolves to another asset. @@ -240,7 +269,7 @@ if inventory.throttling: print("throttled:", line) ``` -Throttling never fails a map. Calls are paced and retried; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` with an error starting `throttled:`. The run's throttling telemetry is attached as `inventory.throttling` and, when present, written under a top-level `throttling` key in `inventory-map.json`. The [rate limits guide](/guides/rate-limits/) covers tuning. +Throttling never fails a map. Calls are paced and retried; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` with an error starting `throttled:`. The run's throttling telemetry is attached as `inventory.throttling` and, when present, written under a top-level `throttling` key in `inventory-map.json`. The [rate limits guide](rate-limits.md) covers tuning. ## Output files @@ -256,7 +285,7 @@ Everything goes to `-o/--output`, `./reports` by default. | `asset-map.json` | with `--findings` | per-asset finding counts and severity breakdown, riskiest first | | `compliance-map.json` | with `--findings` | framework to findings and affected assets | -`inventory-map.json` is self-contained, and `cloudg deps` reads it back without cloud access. The graph files merge parallel rule edges, so count edges in `inventory-map.json`. The [output files reference](/reference/output-files/) has the field-level detail. +`inventory-map.json` is self-contained, and `cloudg deps` reads it back without cloud access. The graph files merge parallel rule edges, so count edges in `inventory-map.json`. The [output files reference](../reference/output-files.md) has the field-level detail. ## Overlay scanner findings @@ -333,11 +362,24 @@ graph = GraphBuilder().build(assets + linker.external_assets, edges) print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges in the graph") ``` -Your own collectors can declare relationships the same way the built-in ones do, by putting entries such as `{"target": "", "edge": "INVOKES", "relationship": "TRIGGERED_BY"}` in `metadata["relations"]`. The [RelationshipLinker](/api/relationshiplinker/) page has the details. +Your own collectors can declare relationships the same way the built-in ones do, by putting entries such as `{"target": "", "edge": "INVOKES", "relationship": "TRIGGERED_BY"}` in `metadata["relations"]`. The [RelationshipLinker](../api/relationshiplinker.md) page has the details. + +
    + +- [**Dependencies and blast radius**](dependencies.md) + + Query the saved map with `cloudg deps`. + +- [**AWS Organizations**](aws-organizations.md) + + Map every account under the management account. + +- [**cloudg map**](../cli/map.md) + + Every option of the command. + +- [**InventoryMapper**](../api/inventorymapper.md) + + The Python class behind it. -:::links -- [Dependencies and blast radius](/guides/dependencies/) Query the saved map with `cloudg deps`. -- [AWS Organizations](/guides/aws-organizations/) Map every account under the management account. -- [cloudg map](/cli/map/) Every option of the command. -- [InventoryMapper](/api/inventorymapper/) The Python class behind it. -::: +
    diff --git a/site/content/guides/pipeline.md b/site/content/guides/pipeline.md index d613554..b8f511f 100644 --- a/site/content/guides/pipeline.md +++ b/site/content/guides/pipeline.md @@ -1,16 +1,17 @@ --- title: The pipeline -lede: "`cloudg run` is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails." -meta: - - [Command, "`cloudg run`"] - - [Python, "`CloudGEngine.run_pipeline()`"] - - [Output, "`./reports` by default"] -source: cloudg/cli_run_helpers.py +description: 'cloudg run is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails.' --- +`cloudg run` is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails. +{ .lede } + +**Command** `cloudg run` · **Python** `CloudGEngine.run_pipeline()` · **Output** `./reports` by default +{ .page-meta } + Each phase hands its results to the next in memory. Files are written along the way, but no phase reads back what an earlier one wrote, so you can delete `./reports` between runs without breaking anything. The graph phase and the scan phase both start from the collected inventory; the normaliser is where their findings meet. -```mermaid caption="Phases of cloudg run and what each one writes" +```mermaid flowchart TD A["1 Collect: MultiAccountCollector"] --> B["2 Graph: GraphBuilder, reachability"] B --> B2["2c RAG chunks / 2d Terraform"] @@ -24,6 +25,8 @@ flowchart TD C2 -. "ontology.ttl, ontology.jsonld" .-> F E -. "findings.json, report.html" .-> F ``` +Phases of cloudg run and what each one writes +{ .diagram-caption } The CLI prints each phase as it starts, with the same numbering: @@ -52,7 +55,7 @@ Collection reads the configuration (providers, regions, accounts, credentials) a `--regions all` turns into `["ALL"]` for all three providers, and region discovery replaces it with the enabled regions before collection starts. Without `--regions`, AWS uses `us-east-1` (or `--region`), while Azure and GCP default to all. -Two limits keep collection polite. An `asyncio.Semaphore` sized by `concurrency_limit` (default 5) caps how many tasks run at once: AWS account and region pairs, Azure subscriptions and GCP projects or organizations, all providers together. Below that, every API call passes through the shared rate limiter configured under `ratelimit`, which paces calls per service, honours Retry-After, and opens a circuit breaker on an API that stays throttled. [Rate limits](/guides/rate-limits/) explains the knobs. +Two limits keep collection polite. An `asyncio.Semaphore` sized by `concurrency_limit` (default 5) caps how many tasks run at once: AWS account and region pairs, Azure subscriptions and GCP projects or organizations, all providers together. Below that, every API call passes through the shared rate limiter configured under `ratelimit`, which paces calls per service, honours Retry-After, and opens a circuit breaker on an API that stays throttled. [Rate limits](rate-limits.md) explains the knobs. ### When collection fails @@ -130,7 +133,7 @@ Three renderers write the final files: ## The whole run, step by step -```mermaid caption="Who calls whom during cloudg run" +```mermaid sequenceDiagram participant CLI as cloudg run participant Col as MultiAccountCollector @@ -149,6 +152,8 @@ sequenceDiagram N-->>CLI: ScanResult CLI->>R: findings.json, topology.svg, report.html ``` +Who calls whom during cloudg run +{ .diagram-caption } ## Running it from the CLI @@ -171,48 +176,50 @@ cloudg run -p aws --iac-dir ./infra --images nginx:1.27,myrepo/api:2.3 --no-onto cloudg -c config.yaml run -p all --regions all -o ./reports/2026-10-09 ``` -`config.yaml` is only read when you pass `-c`; there is no automatic lookup in the current directory. The [cloudg run reference](/cli/run/) lists every flag. +`config.yaml` is only read when you pass `-c`; there is no automatic lookup in the current directory. The [cloudg run reference](../cli/run.md) lists every flag. ## Running it from Python `CloudGEngine` implements the same phases. `run_pipeline()` is the async version and `run_pipeline_sync()` wraps it in `asyncio.run` for plain scripts. Inside a running event loop (Jupyter runs one) the sync wrapper raises `RuntimeError`; `await engine.run_pipeline()` there instead: -```python tab="Sync" title="run_pipeline.py" -from cloudg import CloudGConfig, CloudGEngine +=== "Sync" -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] -config.scanners.enabled = ["prowler", "iam"] + ```python title="run_pipeline.py" + from cloudg import CloudGConfig, CloudGEngine -engine = CloudGEngine(config) -engine.on_phase_start = lambda phase: print(f"-> {phase}") -engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + config.scanners.enabled = ["prowler", "iam"] -result = engine.run_pipeline_sync(output_dir="./reports") + engine = CloudGEngine(config) + engine.on_phase_start = lambda phase: print(f"-> {phase}") + engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") -print(result.to_summary()) -print(result.report_paths) # {'json': ..., 'html': ...} -for cov in result.coverage: - failed = [s.service for s in cov.services if s.status.value == "FAILED"] - if failed: - print(cov.provider, cov.region, "failed:", failed) -``` + result = engine.run_pipeline_sync(output_dir="./reports") -```python tab="Async" title="run_pipeline_async.py" -import asyncio + print(result.to_summary()) + print(result.report_paths) # {'json': ..., 'html': ...} + for cov in result.coverage: + failed = [s.service for s in cov.services if s.status.value == "FAILED"] + if failed: + print(cov.provider, cov.region, "failed:", failed) + ``` -from cloudg import CloudGConfig, CloudGEngine +=== "Async" + ```python title="run_pipeline_async.py" + import asyncio -async def main() -> None: - engine = CloudGEngine(CloudGConfig(providers=["aws", "gcp"])) - result = await engine.run_pipeline(output_dir="./reports") - print(result.total_assets, "assets,", result.total_findings, "findings") - print(result.severity_breakdown) + from cloudg import CloudGConfig, CloudGEngine + async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws", "gcp"])) + result = await engine.run_pipeline(output_dir="./reports") + print(result.total_assets, "assets,", result.total_findings, "findings") + print(result.severity_breakdown) -asyncio.run(main()) -``` + asyncio.run(main()) + ``` CLI flags become config fields or method arguments: @@ -258,7 +265,6 @@ import asyncio from cloudg import CloudGConfig, CloudGEngine - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -275,7 +281,6 @@ async def main() -> None: print(analysis.graph_nodes, "nodes,", len(analysis.attack_paths), "attack paths") print(scan_result.summary) - asyncio.run(main()) ``` @@ -317,7 +322,6 @@ edges = [ NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), ] - async def main() -> None: engine = CloudGEngine(CloudGConfig(providers=["aws"])) @@ -339,7 +343,6 @@ async def main() -> None: for finding in scan_result.findings: print(finding.severity.value, finding.title) - asyncio.run(main()) ``` @@ -353,4 +356,4 @@ findings.json ontology.jsonld ontology.ttl rag_chunks.jsonl rag_metadata_ind Three nodes for two assets: the third is the external `0.0.0.0/0` node the security group rule points from. -For scanner output that already exists, `run_from_reports()` replaces collection and scanning with ingest and runs normalisation and reporting on top; [Ingesting existing output](/guides/ingesting/) covers it. [Python recipes](/guides/python-recipes/) has more combinations, and [CloudGEngine](/api/cloudgengine/) documents every method. +For scanner output that already exists, `run_from_reports()` replaces collection and scanning with ingest and runs normalisation and reporting on top; [Ingesting existing output](ingesting.md) covers it. [Python recipes](python-recipes.md) has more combinations, and [CloudGEngine](../api/cloudgengine.md) documents every method. diff --git a/site/content/guides/python-recipes.md b/site/content/guides/python-recipes.md index aac266e..cec826e 100644 --- a/site/content/guides/python-recipes.md +++ b/site/content/guides/python-recipes.md @@ -1,18 +1,19 @@ --- title: Python recipes -lede: "Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI." -meta: - - [Entry point, "`cloudg.CloudGEngine`"] - - [Needs credentials, "recipes 3 to 6 only"] -source: docs/DOCUMENTATION.md -since: "0.6.0" +description: 'Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI.' --- -The CLI is a thin layer over a public API, so anything `cloudg run` does you can do from Python, in pieces, with your own decisions in between. Each recipe below is one file you can save and run. The ones that only read scanner output work on any machine with `pip install cloudg`; the ones that collect need the provider extra (`pip install "cloudg[aws]"`) and credentials resolved as described in [authentication](/guides/authentication/). +Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI. +{ .lede } -The sample inputs used in the offline recipes are ordinary scanner output: a Prowler output directory (OCSF or `-M json-asff`), a `checkov --output json` file and `trivy image` / `trivy fs` JSON. [Ingesting existing output](/guides/ingesting/) lists exactly which fields each parser reads. +**Entry point** `cloudg.CloudGEngine` · **Needs credentials** recipes 3 to 6 only · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } -```mermaid caption="Where each recipe enters the pipeline" +The CLI is a thin layer over a public API, so anything `cloudg run` does you can do from Python, in pieces, with your own decisions in between. Each recipe below is one file you can save and run. The ones that only read scanner output work on any machine with `pip install cloudg`; the ones that collect need the provider extra (`pip install "cloudg[aws]"`) and credentials resolved as described in [authentication](authentication.md). + +The sample inputs used in the offline recipes are ordinary scanner output: a Prowler output directory (OCSF or `-M json-asff`), a `checkov --output json` file and `trivy image` / `trivy fs` JSON. [Ingesting existing output](ingesting.md) lists exactly which fields each parser reads. + +```mermaid flowchart LR C["collect()"] --> S["scan()"] I["ingest_reports()"] --> N["normalise_findings()"] @@ -23,6 +24,8 @@ flowchart LR M["map_inventory()"] --> O["export_merged()"] I --> O ``` +Where each recipe enters the pipeline +{ .diagram-caption } ## 1. Aggregate existing scan results @@ -128,7 +131,7 @@ trivy image only: ['[Trivy] CVE-2024-0001: openssl (alpine)'] `on_finding` is called once per finding, as each scanner finishes during `scan()` (or once the reports are parsed, for ingest), with a copy of the finding. Hooks run inline in the engine and any exception they raise is swallowed, so the shipper below only puts findings on a queue and lets a worker thread batch them to an HTTP collector, keeping a local NDJSON spool either way. -```python title="stream_to_siem.py" hl="35-36,82" +```python title="stream_to_siem.py" hl_lines="35-36 82" """Ship every cloudg finding to a SIEM over HTTP while the pipeline runs.""" import asyncio @@ -148,7 +151,6 @@ SIEM_URL = os.environ.get("SIEM_URL") # e.g. https://siem.example.c SIEM_TOKEN = os.environ.get("SIEM_TOKEN", "") SPOOL_FILE = "cloudg-findings.ndjson" # local copy, also the fallback - class FindingShipper: """on_finding callback: queue the finding, send batches from a worker thread. @@ -203,7 +205,6 @@ class FindingShipper: except OSError as exc: # URLError and timeouts; the spool file keeps the batch log.warning("SIEM rejected a batch of %d: %s", len(batch), exc) - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1", "eu-west-1"] @@ -222,7 +223,6 @@ async def main() -> None: print(f"shipped {shipper.sent} raw findings; {result.total_findings} after normalisation") print("summary:", json.dumps(result.to_summary(), default=str)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") asyncio.run(main()) @@ -253,7 +253,6 @@ from cloudg.renderers.json_export import JSONExporter OUT = Path("./reports") - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1"] @@ -293,7 +292,6 @@ async def main() -> None: print(JSONExporter(output_dir=str(OUT)).export(scan_result)) print(HTMLReportGenerator(output_dir=str(OUT)).generate(scan_result)) - asyncio.run(main()) ``` @@ -328,7 +326,6 @@ from cloudg.renderers.json_export import JSONExporter OUT = Path("./reports") - async def main() -> None: config = CloudGConfig(providers=["aws"]) config.aws.regions = ["us-east-1"] @@ -356,7 +353,6 @@ async def main() -> None: print(f"{len(collection.assets)} assets, {analysis.ontology_triples} ontology triples") print("findings by tool:", by_tool) - asyncio.run(main()) ``` @@ -372,54 +368,60 @@ Passing the ingested findings to `analyze()` puts them into the ontology and the Inventory mapping needs read credentials and nothing else. Scanners can run elsewhere and on another schedule, and their output merges in whenever it arrives, even in a different process, because the map round-trips through `inventory-map.json`. -```python tab="Map" title="map_inventory.py" -"""Map the inventory now; no scanners involved.""" +=== "Map" -from cloudg import CloudGConfig, CloudGEngine + ```python title="map_inventory.py" + """Map the inventory now; no scanners involved.""" -config = CloudGConfig(providers=["aws"]) -config.aws.regions = ["us-east-1"] # or ["ALL"] + from cloudg import CloudGConfig, CloudGEngine -inventory = CloudGEngine(config).map_inventory_sync(output_dir="./map-reports") + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] # or ["ALL"] -summary = inventory.summary -print(summary["total_assets"], "assets,", summary["total_edges"], "edges") -print("by service:", summary["assets_by_service"]) -print("internet exposed:", summary["internet_exposed"]) -print("unlinked:", summary["unlinked_assets"]) -if inventory.throttling: - print("throttled:", inventory.throttling["messages"]) -``` + inventory = CloudGEngine(config).map_inventory_sync(output_dir="./map-reports") -```python tab="Overlay" title="overlay_findings.py" -"""Overlay scanner findings on an inventory map written earlier.""" + summary = inventory.summary + print(summary["total_assets"], "assets,", summary["total_edges"], "edges") + print("by service:", summary["assets_by_service"]) + print("internet exposed:", summary["internet_exposed"]) + print("unlinked:", summary["unlinked_assets"]) + if inventory.throttling: + print("throttled:", inventory.throttling["messages"]) + ``` -import json +=== "Overlay" -from cloudg import CloudGConfig, CloudGEngine -from cloudg.inventory import InventoryMapper, InventoryResult + ```python title="overlay_findings.py" + """Overlay scanner findings on an inventory map written earlier.""" -inventory = InventoryResult.load("./map-reports") # directory or inventory-map.json -print(len(inventory.assets), "assets in the saved map") + import json -engine = CloudGEngine(CloudGConfig()) -findings = engine.ingest_reports({"prowler": ["./prowler-output/"]}) + from cloudg import CloudGConfig, CloudGEngine + from cloudg.inventory import InventoryMapper, InventoryResult -mapper = InventoryMapper(CloudGConfig()) -paths = mapper.export_merged(inventory, findings, "./map-reports") -print({name: str(path) for name, path in paths.items()}) + inventory = InventoryResult.load("./map-reports") # directory or inventory-map.json + print(len(inventory.assets), "assets in the saved map") -asset_map = json.loads(paths["asset_map"].read_text()) -for entry in asset_map["assets"]: - if entry["finding_count"]: - print(entry["name"], entry["type"], entry["severity_breakdown"]) -``` + engine = CloudGEngine(CloudGConfig()) + findings = engine.ingest_reports({"prowler": ["./prowler-output/"]}) -```bash tab="CLI" -cloudg map -p aws --regions all -o ./map-reports -# later, with scanner output that cloudg already normalised: -cloudg map -p aws --regions all -o ./map-reports --findings ./reports/raw-findings.json -``` + mapper = InventoryMapper(CloudGConfig()) + paths = mapper.export_merged(inventory, findings, "./map-reports") + print({name: str(path) for name, path in paths.items()}) + + asset_map = json.loads(paths["asset_map"].read_text()) + for entry in asset_map["assets"]: + if entry["finding_count"]: + print(entry["name"], entry["type"], entry["severity_breakdown"]) + ``` + +=== "CLI" + + ```bash + cloudg map -p aws --regions all -o ./map-reports + # later, with scanner output that cloudg already normalised: + cloudg map -p aws --regions all -o ./map-reports --findings ./reports/raw-findings.json + ``` The overlay step needs no credentials. `asset-map.json` lists every asset with `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the worst first; `compliance-map.json` groups affected assets by framework. Both maps match a finding to an asset by asset id, then ARN, then a name or ARN tail that only one asset has, so scanner output that names resources by ARN (Prowler, ScoutSuite) overlays well, and IaC findings keyed by a Terraform address (Checkov) do not match a live asset at all. The CLI variant re-maps the account before merging; the Python one reuses the saved map. @@ -453,7 +455,6 @@ from cloudg.graph.builder import GraphBuilder from cloudg.inventory import RelationshipLinker from cloudg.schema.models import AssetType, CloudAsset, CloudProvider - def to_asset(row: dict) -> CloudAsset: relations = [] if row.get("role_arn"): @@ -478,7 +479,6 @@ def to_asset(row: dict) -> CloudAsset: metadata={"relations": relations}, ) - with open("cmdb_assets.json") as fh: assets = [to_asset(row) for row in json.load(fh)] @@ -509,7 +509,7 @@ Each relation is `{"target", "edge", "relationship", "reverse", "properties", "d ## 8. Gate CI on severity -For a CI job that already runs Python, the gate can live next to the aggregation. This version fails on HIGH and CRITICAL, honours suppressions and a list of accepted check IDs, and refuses to pass on partial data. For a gate that only reads `findings.json`, with no cloudg import, see [CI and automation](/guides/ci-automation/#a-severity-gate). +For a CI job that already runs Python, the gate can live next to the aggregation. This version fails on HIGH and CRITICAL, honours suppressions and a list of accepted check IDs, and refuses to pass on partial data. For a gate that only reads `findings.json`, with no cloudg import, see [CI and automation](ci-automation.md#a-severity-gate). ```python title="gate_on_severity.py" """Aggregate IaC scanner output and fail the build on blocking findings.""" @@ -557,9 +557,22 @@ exit 1 After a cross-scanner merge, `source_finding_id` holds one scanner's check ID and `source_tool` becomes a comma list such as `"prowler, checkov"`, so an allow-list keyed on one tool's ID can miss a merged finding. When that matters, filter on the parsed findings before normalising, as in recipe 2. -:::links -- [CloudGEngine](/api/cloudgengine/) Every method and its signature. -- [Event hooks](/api/event-hooks/) When each hook fires. -- [Result objects](/api/results/) `PipelineResult`, `CollectionResult`, `AnalysisResult`. -- [RelationshipLinker](/api/relationshiplinker/) Resolution rules for declared relations. -::: +
    + +- [**CloudGEngine**](../api/cloudgengine.md) + + Every method and its signature. + +- [**Event hooks**](../api/event-hooks.md) + + When each hook fires. + +- [**Result objects**](../api/results.md) + + `PipelineResult`, `CollectionResult`, `AnalysisResult`. + +- [**RelationshipLinker**](../api/relationshiplinker.md) + + Resolution rules for declared relations. + +
    diff --git a/site/content/guides/quickstart.md b/site/content/guides/quickstart.md index e9ad24b..77a675f 100644 --- a/site/content/guides/quickstart.md +++ b/site/content/guides/quickstart.md @@ -1,17 +1,20 @@ --- title: Quickstart -lede: "Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners." -meta: - - [Time, "about 10 minutes, plus scanner run time"] - - [Needs, "Python 3.11+, read-only AWS credentials"] -source: site/content/guides/quickstart.md +description: Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners. --- +Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners. +{ .lede } + +**Time** about 10 minutes, plus scanner run time · **Needs** Python 3.11+, read-only AWS credentials +{ .page-meta } + This page takes the shortest path to each of cloudg's three entry points. It sticks to AWS and one region so the first run is quick; every command takes `-p azure`, `-p gcp` or `-p all` the same way. ## Scan an AWS account -::::steps +
    + ### Install cloudg and a scanner ```bash @@ -19,7 +22,7 @@ pip install "cloudg[aws]" pip install prowler ``` -Prowler is enough for a first run. cloudg skips scanners it cannot find on `PATH`, so you will see warnings for ScoutSuite, Checkov and Trivy; that is expected. [Installation](/guides/installation/) covers the rest, and the [Docker image](/guides/docker/) has Prowler, Checkov and Trivy built in. +Prowler is enough for a first run. cloudg skips scanners it cannot find on `PATH`, so you will see warnings for ScoutSuite, Checkov and Trivy; that is expected. [Installation](installation.md) covers the rest, and the [Docker image](docker.md) has Prowler, Checkov and Trivy built in. ### Authenticate @@ -31,9 +34,9 @@ export AWS_PROFILE=audit aws sts get-caller-identity ``` -:::tip Use read-only credentials -cloudg only reads. The AWS managed policies `SecurityAudit` and `ViewOnlyAccess` are the usual starting point for the role or user you scan with. -::: +!!! tip "Use read-only credentials" + + cloudg only reads. The AWS managed policies `SecurityAudit` and `ViewOnlyAccess` are the usual starting point for the role or user you scan with. ### Run the pipeline @@ -53,7 +56,8 @@ xdg-open reports/report.html # Linux ``` `report.html` is a single file with the findings, assets and graph data embedded in it: the topology graph, the findings table and the compliance matrix. Chart.js and D3 are embedded too, so it opens without network access. -:::: + +
    ### What lands in ./reports @@ -84,7 +88,8 @@ cloudg report -i reports/findings.json --format html If someone else runs the scanners (in CI, on a schedule, on another network), you only need their output files. `cloudg ingest` needs no cloud access and no scanner binaries, and the core package without extras is enough. -::::steps +
    + ### Install the core package ```bash @@ -113,15 +118,17 @@ findings.json raw-findings.json report.html ``` `raw-findings.json` holds every parsed finding before deduplication. `findings.json` and `report.html` hold the merged, compliance-mapped set. The last line of the output tells you how many duplicates were merged, for example "Ingested 3 findings from 3 tool(s), 3 after deduplication". -:::: -There is no topology in an ingest report, because nothing was collected. [Ingesting existing output](/guides/ingesting/) covers the exact input formats and how to combine ingested findings with a live collection. +
    + +There is no topology in an ingest report, because nothing was collected. [Ingesting existing output](ingesting.md) covers the exact input formats and how to combine ingested findings with a live collection. ## No scanners: map the inventory `cloudg map` answers "what is deployed, and what depends on what". It needs read-only credentials and the provider extra, but no scanners. -::::steps +
    + ### Map one region ```bash @@ -146,15 +153,29 @@ cloudg deps arn:aws:iam::123456789012:role/app-role ``` `cloudg deps` reads the saved map from `./reports` and prints what that asset depends on and what would break without it. `-d down` shows only the blast radius. -:::: -When you do run scanners later, merge their findings into the map with `cloudg map --findings ./reports/findings.json` (or the `raw-findings.json` that `cloudg ingest` and `cloudg scan` write), which adds `asset-map.json` and `compliance-map.json`. [Inventory mapping](/guides/inventory-mapping/) goes through the map in depth. +
    + +When you do run scanners later, merge their findings into the map with `cloudg map --findings ./reports/findings.json` (or the `raw-findings.json` that `cloudg ingest` and `cloudg scan` write), which adds `asset-map.json` and `compliance-map.json`. [Inventory mapping](inventory-mapping.md) goes through the map in depth. ## Next steps -:::links -- [The pipeline](/guides/pipeline/) What each phase of cloudg run does, and how to drive it from Python. -- [Authentication](/guides/authentication/) Profiles, role assumption, OIDC in CI, Azure and GCP. -- [Running scanners](/guides/running-scanners/) IaC directories, container images and scanner arguments. -- [Reports](/guides/reports/) Reading report.html and findings.json. -::: +
    + +- [**The pipeline**](pipeline.md) + + What each phase of cloudg run does, and how to drive it from Python. + +- [**Authentication**](authentication.md) + + Profiles, role assumption, OIDC in CI, Azure and GCP. + +- [**Running scanners**](running-scanners.md) + + IaC directories, container images and scanner arguments. + +- [**Reports**](reports.md) + + Reading report.html and findings.json. + +
    diff --git a/site/content/guides/rate-limits.md b/site/content/guides/rate-limits.md index caaad8b..4a7d2b6 100644 --- a/site/content/guides/rate-limits.md +++ b/site/content/guides/rate-limits.md @@ -1,23 +1,23 @@ --- title: Rate limits and throttling -lede: "Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map." -meta: - - [Package, "`cloudg.resilience`"] - - [Config, "[`ratelimit`](/reference/config/ratelimit/)"] - - [Since, "0.6.0"] -source: docs/RESILIENCE.md -since: "0.6.0" +description: Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map. --- +Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map. +{ .lede } + +**Package** `cloudg.resilience` · **Config** [`ratelimit`](../reference/config/ratelimit.md) · **Since** 0.6.0 · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + A single `cloudg map -p aws --org --regions all` runs 133 deep collector tasks, a Cloud Control sweep and a tagging sweep in every region of every member account, and some collectors fan out per item (one `DescribeImageAttribute` per AMI). Those APIs have quotas, and the quotas are shared with your CI pipelines, Terraform runs and the console. Up to 0.5.x cloudg trusted each SDK's own retries. Forty EC2 clients in one account then backed off independently and kept the account's bucket empty, a service that stayed throttled looked the same in coverage as an access error, and nothing remembered that an API had just said no. -Since 0.6.0 the `cloudg.resilience` package sits between cloudg and every SDK. You rarely need to touch it. This page explains what it does so you can read its output and tune it when an estate needs it. Every key and default is in the [ratelimit reference](/reference/config/ratelimit/), and the Python side in the [cloudg.resilience API](/api/resilience/). +Since 0.6.0 the `cloudg.resilience` package sits between cloudg and every SDK. You rarely need to touch it. This page explains what it does so you can read its output and tune it when an estate needs it. Every key and default is in the [ratelimit reference](../reference/config/ratelimit.md), and the Python side in the [cloudg.resilience API](../api/resilience.md). ## How one call is paced Every call is attributed to a scope, `Scope(provider, account, region, service, operation)`, which prints as `aws/123456789012/us-east-1/ec2` (with `/DescribeSubnets` appended where the operation matters). Before the call goes out it passes four gates, and after it comes back the outcome feeds the same state. -```mermaid caption="call_with_resilience: the full sequence. The SDK integrations reuse the same parts and let the SDK do the retrying." +```mermaid flowchart TD A[Call arrives with a scope] --> B{"Breaker open or deadline spent?"} B -->|yes| X["Fail fast, nothing sent"] @@ -33,6 +33,8 @@ flowchart TD J -->|no| Z["Breaker counts a failure, record give-up"] G -->|FATAL| R[Re-raise at once] ``` +call_with_resilience: the full sequence. The SDK integrations reuse the same parts and let the SDK do the retrying. +{ .diagram-caption } Errors are sorted by `classify(exc)` into three kinds. THROTTLED means the provider asked cloudg to slow down: AWS `Throttling`, `RequestLimitExceeded`, `TooManyRequestsException` and the rest of botocore's list, Azure `SubscriptionRequestsThrottled`, GCP `ResourceExhausted`, or any bare HTTP 429 or 503. TRANSIENT covers 408, 500, 502, 504 and connection resets. FATAL is everything else with a provider error code, `AccessDenied` included. The classifier reads error codes, exception classes and HTTP statuses first and only falls back to message text when none of them says anything. @@ -83,7 +85,7 @@ A bulkhead caps requests in flight per provider and account: 128 for AWS, 16 for Each leaf scope has a breaker with the same granularity as its bucket: per EC2 action, per service elsewhere. A throttled `DescribeSubnets` therefore does not block `DescribeVpcs` in the same account and region. -```mermaid caption="One breaker. Rejected calls never reach the provider and never count as failures." +```mermaid stateDiagram-v2 [*] --> Closed Closed --> Closed: success resets the count @@ -93,6 +95,8 @@ stateDiagram-v2 HalfOpen --> Open: probe fails, cooldown doubled up to 600 s HalfOpen --> HalfOpen: probe lease expires, next call probes ``` +One breaker. Rejected calls never reach the provider and never count as failures. +{ .diagram-caption } A failure here means a call that still failed with throttling or a transient error after all its retries. The defaults are 5 consecutive failures (`breaker_threshold`) and a 60-second cooldown (`breaker_cooldown_seconds`). While open, calls fail at once with `circuit open for after repeated throttling; retry in s`. @@ -186,7 +190,7 @@ People rarely run `cloudg map` twice in a minute. An AI agent might. The MCP liv Scopes come from the config: `aws/` per configured account, else `aws/profile:`, else `aws`, and the equivalents for Azure subscriptions and GCP projects. A refusal is a tool error with code -31029 and `reason` `cooldown`, `busy` or `quota`, plus `retry_after_seconds` and a `use_dataset` pointer to the newest loaded live dataset covering the same providers. Principals with the `admin` or `operator` role can pass `force=true` to skip the cooldown, though not the concurrency caps. -The tool `rate_limit_status` and the resource `cloudg://ratelimit` show cooldowns, operations in flight and the governor's throttling state without calling any cloud API. Point agents at them before they reach for a live tool. These guard rails are separate from the MCP policy's per-principal rate limits, which decide how often someone may call a tool at all; see the [MCP server](/mcp/) section. +The tool `rate_limit_status` and the resource `cloudg://ratelimit` show cooldowns, operations in flight and the governor's throttling state without calling any cloud API. Point agents at them before they reach for a live tool. These guard rails are separate from the MCP policy's per-principal rate limits, which decide how often someone may call a tool at all; see the [MCP server](../mcp/index.md) section. ## Tuning the ratelimit block @@ -208,9 +212,9 @@ ratelimit: ec2.DescribeImageAttribute: {max_rps: 5, burst: 10} # AMI-heavy region ``` -:::warning An override replaces the whole built-in entry -`ec2: {max_rps: 10}` drops the built-in burst of 100 to 20 (twice the rate). Repeat the burst if you want to keep it: `ec2: {max_rps: 10, burst: 50}`. EC2 stays per action either way. -::: +!!! warning "An override replaces the whole built-in entry" + + `ec2: {max_rps: 10}` drops the built-in burst of 100 to 20 (twice the rate). Repeat the burst if you want to keep it: `ec2: {max_rps: 10, burst: 50}`. EC2 stays per action either way. ### A shared or large estate @@ -292,8 +296,18 @@ print(get_governor().budget("aws").remaining) Learned rates and breaker state live in memory, so each new `cloudg map` process starts from the configured rates. A few calls are not instrumented yet: the GCP organization-policy and access-context-manager clients outside the asset listing, the urllib fallback that lists Azure subscriptions, and the single region-discovery call. A blocking boto3 or Azure call made on an event-loop thread goes ahead without a bulkhead slot when none is free, so `max_concurrency` is not a hard cap on that path. And per-item fan-out is now held to the documented rates, which makes some maps slower than in 0.5.x: 1,000 `DescribeImageAttribute` calls in one region take about 50 seconds at 20 rps. -:::links -- [cloudg.resilience](/api/resilience/) `call_with_resilience`, `@resilient`, the governor and the guard from Python. -- [ratelimit reference](/reference/config/ratelimit/) Every key with its default. -- [Inventory mapping](/guides/inventory-mapping/) Where coverage records come from. -::: +
    + +- [**cloudg.resilience**](../api/resilience.md) + + `call_with_resilience`, `@resilient`, the governor and the guard from Python. + +- [**ratelimit reference**](../reference/config/ratelimit.md) + + Every key with its default. + +- [**Inventory mapping**](inventory-mapping.md) + + Where coverage records come from. + +
    diff --git a/site/content/guides/reports.md b/site/content/guides/reports.md index d9f2c97..a1f7220 100644 --- a/site/content/guides/reports.md +++ b/site/content/guides/reports.md @@ -1,19 +1,19 @@ --- title: Reports -lede: "Every path through cloudg ends in the same two files: `findings.json` for machines and `report.html` for people. `cloudg report` rebuilds them from an earlier run without touching the cloud." -meta: - - [Command, "`cloudg report`"] - - [Formats, "HTML, JSON, SVG"] - - [Python, "`JSONExporter`, `HTMLReportGenerator`, `SVGRenderer`"] -source: cloudg/renderers/html_report.py -since: "0.6.0" +description: 'Every path through cloudg ends in the same two files: findings.json for machines and report.html for people. cloudg report rebuilds them from an earlier run without touching the cloud.' --- +Every path through cloudg ends in the same two files: `findings.json` for machines and `report.html` for people. `cloudg report` rebuilds them from an earlier run without touching the cloud. +{ .lede } + +**Command** `cloudg report` · **Formats** HTML, JSON, SVG · **Python** `JSONExporter`, `HTMLReportGenerator`, `SVGRenderer` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + The renderers in `cloudg/renderers/` take a normalised `ScanResult` (assets, findings, compliance results) plus, when there is one, the D3 graph of the collected assets. They don't call any cloud API or scanner, which is why `cloudg report` can re-render an old run anywhere. ## Which command writes what -```mermaid caption="Report inputs and outputs" +```mermaid flowchart LR RUN["cloudg run"] --> SR["ScanResult and graph"] ING["cloudg ingest"] --> SR @@ -23,6 +23,8 @@ flowchart LR SR --> SVG["topology.svg"] SCAN["cloudg scan, cloudg ingest"] --> RAW["raw-findings.json"] ``` +Report inputs and outputs +{ .diagram-caption } | Command | `findings.json` | `report.html` | `topology.svg` | `raw-findings.json` | |---|---|---|---|---| @@ -35,7 +37,7 @@ flowchart LR `cloudg run`, `run_pipeline()` and `run_from_reports()` write the formats listed in `report.formats`, all three by default. `run_from_reports()` never writes `topology.svg`, because it has no collected assets to draw. `cloudg ingest` and `cloudg report` go by `--format` instead. -`cloudg run` writes more than these (GraphML, Cytoscape JSON when `graph.export_cytoscape` is on, the ontology, RAG chunks, Terraform); those come from the graph and analysis phases and are listed on [Output files](/reference/output-files/). +`cloudg run` writes more than these (GraphML, Cytoscape JSON when `graph.export_cytoscape` is on, the ontology, RAG chunks, Terraform); those come from the graph and analysis phases and are listed on [Output files](../reference/output-files.md). Those three are the only report formats. There is no CSV, SARIF or PDF writer in 0.6.0. `findings.json` is easy to convert, though; there's a [CSV recipe](#exporting-to-csv) further down. @@ -68,27 +70,44 @@ Those three are the only report formats. There is no CSV, SARIF or PDF writer in } ``` -`assets`, `findings`, `compliance` and `edges` are the Pydantic models from `cloudg.schema.models`, serialised in JSON mode. Assets leave out `raw_data`. Findings include the computed `risk_score`. `edges` holds the collected `NetworkEdge` objects, so `cloudg report` can draw the topology again. `graph` holds D3 nodes and links for `cloudg run`, and is empty for ingest. In the `metadata` block, the normaliser doesn't fill in `provider`, `account_id` or `region` in 0.6.0, so expect `null` there. Both timestamps are ISO 8601 strings set at normalisation time, and a `completed_at` that was never set is `null`. The field-by-field reference is on [Output files](/reference/output-files/), and the models are on [Data models](/api/models/). +`assets`, `findings`, `compliance` and `edges` are the Pydantic models from `cloudg.schema.models`, serialised in JSON mode. Assets leave out `raw_data`. Findings include the computed `risk_score`. `edges` holds the collected `NetworkEdge` objects, so `cloudg report` can draw the topology again. `graph` holds D3 nodes and links for `cloudg run`, and is empty for ingest. In the `metadata` block, the normaliser doesn't fill in `provider`, `account_id` or `region` in 0.6.0, so expect `null` there. Both timestamps are ISO 8601 strings set at normalisation time, and a `completed_at` that was never set is `null`. The field-by-field reference is on [Output files](../reference/output-files.md), and the models are on [Data models](../api/models.md). `raw-findings.json` is different: a bare JSON list of `Finding` objects as the parsers produced them, before deduplication, rescoring or compliance mapping. It has no `summary`, `compliance` or `graph`. ## report.html -The HTML report is one file rendered from the Jinja template `cloudg/templates/report.html.j2`. The data is embedded in the page as JSON, and so are Chart.js 4.4.0 and D3 7.9.0, so the page works offline. You can mail it, attach it to a ticket or archive it next to `findings.json`. +The HTML report is one file rendered from the Jinja template `cloudg/templates/report.html.j2`. The data is embedded in the page as JSON, and so are Chart.js 4.4.0, D3 7.9.0 and the two engines that draw the map, so the page works offline. You can mail it, attach it to a ticket or archive it next to `findings.json`. The [sample report](../sample-report.md) shows one built from a synthetic three-cloud estate. + +The overview at the top gives the scan ID, provider, account and start time, four counters (assets, findings, critical findings and internet-exposed assets) and a bar of findings by severity. Beside it, a ring shows the share of assets with no open findings, and the top risks list holds the six worst findings; clicking one opens it. The top bar switches between the dark and light themes (the choice is remembered in the browser), prints the page, and downloads the findings as CSV or the embedded data as JSON. Below are five tabs. + +### The estate map + +The Estate map tab draws the estate in three dimensions. Each provider is a base plate. Accounts and regions stand on it as platforms, VPCs and VNets as slabs and subnets as tiles, and every asset stands where it was found, as a small solid whose shape says what kind of asset it is. Identity and governance assets (roles, users, policies, organizations, accounts) sit on a control deck held on posts behind and above the plate. An asset with open findings carries a beacon in the colour of its worst one, an internet-exposed asset has a dashed red ring at its foot, and the internet itself is a mast in front of the plates. + +Placement comes from the collected `CONTAINS` edges first and from each asset's `vpc_id` / `subnet_id` metadata after that. Every other relation is a trace line: it leaves its source from a port of its own on the asset's top face, rises to a lane that no other trace uses, runs along the floor plan and drops onto its own port on the target. Trace lines are coloured by family: internet exposure, network paths, identity and access, data and invocation, and governance and protection. The chips above the map hide or show each family. Parallel edges between the same two assets collapse into one trace that counts them. -The header shows the scan ID, provider and account, a button that downloads the embedded data, and six counters: total assets, total findings, and the CRITICAL, HIGH, MEDIUM and LOW counts. Below it are four tabs. +- Click an asset to light its relations and dim everything else. A dot runs along each lit trace from source to target, and the inspector beside the map lists the asset's details and its relations. Click a relation there to follow that one alone. +- **Trace from internet** lights every path that starts at the internet mast, hop after hop, and says how many assets it reaches. +- Drag to pan, scroll to zoom, and use the turn buttons (or `[` and `]`) to turn the map a quarter at a time. Arrow keys step through the assets, `0` fits the map and Escape clears the selection. +- **Labels** names every asset; without it, names show for the asset under the pointer and the selected asset's neighbours. **Replay** plays the opening animation again, and **Find an asset** jumps to one by name or identifier. -The Infrastructure Map tab draws a D3 tree of account, region, VPC, subnet and resources, built from each asset's `region` and its `vpc_id` / `subnet_id` metadata. Assets with neither land under "Global / Ungrouped". Nodes with findings get a red ring and internet-exposed ones an orange ring. Toggle Layout switches to a force-directed graph of the collected graph plus one triangle node per finding (linked to its resource by a `FINDING_AFFECTS` edge) and one diamond per compliance framework (linked to its findings by `COMPLIANCE_GOVERNS`). With no assets, as after `cloudg ingest`, the tab goes straight to that force view, which then shows findings and frameworks only. +The layout is computed in Python, in `cloudg.renderers.iso_scene.build_scene()`, so it is the same in every browser and is covered by tests. The page projects it with the [Anatomy](https://github.com/wheresryan22/anatomy) iso-kit and animates it. Under reduced motion, nothing moves. -The Findings tab lists every finding, sorted by severity, with dropdown filters for severity and source tool and a text search over title, resource and description. Clicking a row, filtered or not, opens a side panel for that finding with the description, evidence, remediation, CVSS score, frameworks and risk score. +The **2D** switch shows the earlier views: a D3 tree of account, region, VPC, subnet and resources, and a force-directed graph of the collected graph plus one node per finding (linked to its resource by `FINDING_AFFECTS`) and one per compliance framework (linked to its findings by `COMPLIANCE_GOVERNS`). The report opens on the 2D view by itself when the scan has no assets (as after `cloudg ingest`), when it has more than 2500 assets, or when the map engines are missing from the package. -Analytics has four Chart.js charts: severity distribution, asset types, findings by source tool, and compliance per framework. +### The other tabs -Compliance shows one card per framework with Pass and Fail counts. As explained in [Compliance frameworks](/guides/compliance/#report-html), Fail counts the controls that have findings, and Pass counts the controls that only Prowler's passing checks map to. Without passing-check data in the input (no Prowler, or scanners that only report failures), Pass is 0 and the percentage bar reads 0%. +The Findings tab lists every finding with dropdown filters for severity and source tool and a text search over title, resource and description. Clicking a row, filtered or not, opens a side panel for that finding with the description, evidence, remediation, CVSS score, frameworks and risk score. A filter that matches nothing shows an empty tray, drawn with [Hairline](https://hairline.lucasmarkes.com). -The download button saves a file named `findings.json` that holds only `findings`, `graph` and the tree data. It isn't the same file cloudg writes, but `cloudg report -i` and the viewer both accept it. +Assets lists every asset with its type, provider, region, exposure and finding count, with a provider filter and a search. Clicking a row opens it on the map. -The two libraries come from `cloudg/templates/vendor/` in the package and add about 485 KB to the file. With `report.inline_js: false` the page loads the same pinned builds from `cdn.jsdelivr.net` instead, with Subresource Integrity hashes, which gives a smaller file that needs network access to show the map, charts and compliance cards. +Analytics has four Chart.js charts: severity distribution, the twelve most common asset types, findings by source tool, and compliance per framework. + +Compliance shows one card per framework with its pass rate and Pass and Fail counts. As explained in [Compliance frameworks](compliance.md#report-html), Fail counts the controls that have findings, and Pass counts the controls that only Prowler's passing checks map to. Without passing-check data in the input (no Prowler, or scanners that only report failures), Pass is 0 and the rate reads 0%. + +The JSON download saves a file named `findings.json` that holds `findings`, `assets`, `graph`, the tree data and the map's scene. It isn't the same file cloudg writes, but `cloudg report -i` and the viewer both accept it. The CSV download has one row per finding; a cell that starts with `=`, `+`, `-` or `@` gets a leading apostrophe so a spreadsheet does not run it as a formula. + +Chart.js and D3 come from `cloudg/templates/vendor/` in the package and add about 485 KB to the file. With `report.inline_js: false` the page loads the same pinned builds from `cdn.jsdelivr.net` instead, with Subresource Integrity hashes, which gives a smaller file that needs network access to show the charts and the 2D views. The map engines (about 64 KB) have no CDN build and are embedded either way. Everything that comes from scanner output or cloud metadata (titles, descriptions, evidence, resource ids, asset names, framework names) is HTML-escaped before it reaches the page, and the embedded JSON cannot close its ` diff --git a/site/content/mcp/connect.md b/site/content/mcp/connect.md index f0d5e4b..751b8f7 100644 --- a/site/content/mcp/connect.md +++ b/site/content/mcp/connect.md @@ -1,17 +1,17 @@ --- title: Connect a client -lede: "`cloudg mcp config` writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools." -meta: - - [Command, "`cloudg mcp config`"] - - [Clients, "Claude Desktop, Claude Code, Cursor, VS Code"] - - [Transports, "stdio, streamable HTTP"] -source: docs/MCP.md -since: "0.6.0" +description: cloudg mcp config writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools. --- +`cloudg mcp config` writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools. +{ .lede } + +**Command** `cloudg mcp config` · **Clients** Claude Desktop, Claude Code, Cursor, VS Code · **Transports** stdio, streamable HTTP · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Most people run cloudg over stdio: the client starts `cloudg mcp serve` as a child process when it opens and stops it when it closes. You don't keep a server running, and nothing listens on a port. HTTP is for the other cases: a server shared by several people, a client on a different machine, or a gateway in front of it. -```mermaid caption="Which setup to use" +```mermaid flowchart TD A{"Does the client run on the machine that holds the reports?"} -->|yes| B{"One user, one client?"} A -->|no| E["HTTP: cloudg mcp serve --transport http, with --auth-token"] @@ -21,26 +21,33 @@ flowchart TD F -->|yes| G["Bridge with npx mcp-remote"] F -->|no| H["Client connects to the URL directly"] ``` +Which setup to use +{ .diagram-caption } ## Before you start You need cloudg 0.6.0 or newer on Python 3.11 or newer. The base install includes the MCP layer and a dependency-free server; the `mcp` extra adds the official SDK, which only matters if you want `--flavor sdk`. -```bash tab="pip" -pip install cloudg -``` +=== "pip" -```bash tab="uv" -uv pip install cloudg -``` + ```bash + pip install cloudg + ``` + +=== "uv" + + ```bash + uv pip install cloudg + ``` -You also need something to look at: an `inventory-map.json` from [cloudg map](/cli/map/), a `findings.json` from [cloudg run](/cli/run/), or native scanner output that [cloudg ingest](/cli/ingest/) understands. The examples below assume `./reports/inventory-map.json`. +You also need something to look at: an `inventory-map.json` from [cloudg map](../cli/map.md), a `findings.json` from [cloudg run](../cli/run.md), or native scanner output that [cloudg ingest](../cli/ingest.md) understands. The examples below assume `./reports/inventory-map.json`. If you installed cloudg in editable mode before 0.6.0, run `pip install -e .` again so the `cloudg-mcp` console script gets created. ## Set up a stdio client -::::steps +
    + ### Check that the server loads your data Run one tool from a terminal before touching any client config. `cloudg mcp call` goes through the same layer, policy and transforms the server uses, so if this works, the server will too. @@ -57,66 +64,74 @@ The result lists the dataset with its asset, edge and finding counts, and the al The output below is from an install where `cloudg-mcp` was found on `PATH`; your path will differ. -```json tab="Claude Desktop" title="claude_desktop_config.json" -{ - "mcpServers": { - "cloudg": { - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] +=== "Claude Desktop" + + ```json title="claude_desktop_config.json" + { + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` - -```json tab="Claude Code" title=".mcp.json" -{ - "mcpServers": { - "cloudg": { - "type": "stdio", - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ], - "env": {} + ``` + +=== "Claude Code" + + ```json title=".mcp.json" + { + "mcpServers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ], + "env": {} + } + } } - } -} -``` - -```json tab="Cursor" title="~/.cursor/mcp.json" -{ - "mcpServers": { - "cloudg": { - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] + ``` + +=== "Cursor" + + ```json title="~/.cursor/mcp.json" + { + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` - -```json tab="VS Code" title=".vscode/mcp.json" -{ - "servers": { - "cloudg": { - "type": "stdio", - "command": "/home/you/.venv/bin/cloudg-mcp", - "args": [ - "serve" - ] + ``` + +=== "VS Code" + + ```json title=".vscode/mcp.json" + { + "servers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } } - } -} -``` + ``` The commands that produced them are `cloudg mcp config --client claude-desktop` (the default), `--client claude-code`, `--client cursor` and `--client vscode`. `--name` changes the server key from `cloudg`, and `--command "CMD ARGS"` replaces the detected launch command (it is split with `shlex`). -:::note Interpreter fallback -In a virtualenv where neither script is on `PATH`, the command is the interpreter and the arguments are `["-m", "cloudg.mcp", "serve"]`. That works just as well; it only looks different. -::: +!!! note "Interpreter fallback" + + In a virtualenv where neither script is on `PATH`, the command is the interpreter and the arguments are `["-m", "cloudg.mcp", "serve"]`. That works just as well; it only looks different. ### Put it where the client reads it @@ -138,7 +153,8 @@ claude mcp add-json cloudg '{"type": "stdio", "command": "/home/you/.venv/bin/cl ### Restart the client and start a conversation The server sends instructions on `initialize` that tell the model to begin with `workspace_status`. If you did not preload a dataset, ask the assistant to load one by absolute path, for example "Load /srv/cloudg/reports/inventory-map.json and show me the internet-exposed assets with HIGH findings." That path has to be inside an allowed root, which the next section explains. -:::: + +
    ## Load your reports with --dataset @@ -198,7 +214,7 @@ The separator is `os.pathsep`: `:` on macOS and Linux, `;` on Windows. Export to The counts are what `cloudg mcp tools --policy NAME` lists for the local user in 0.6.0. Run that command with your policy and, for HTTP, with `--role ROLE`, to see exactly what a client will get before you hand it out. -Under `strict`, pseudonyms come from a key that is random per process, so they change every time the client restarts the server and old ones stop resolving. Put `CLOUDG_MCP_VAULT_KEY` in the entry's `env` block if pseudonyms must stay stable across conversations, and keep that value away from the model. The [built-in profiles](/mcp/privacy/built-in-profiles/) page shows the same calls under every profile, and [loading and composing policies](/mcp/privacy/loading-and-composing-policies/) covers writing your own with `extends`. +Under `strict`, pseudonyms come from a key that is random per process, so they change every time the client restarts the server and old ones stop resolving. Put `CLOUDG_MCP_VAULT_KEY` in the entry's `env` block if pseudonyms must stay stable across conversations, and keep that value away from the model. The [built-in profiles](privacy/built-in-profiles.md) page shows the same calls under every profile, and [loading and composing policies](privacy/loading-and-composing-policies.md) covers writing your own with `extends`. ## Read-only mode @@ -222,67 +238,75 @@ It listens on `http://127.0.0.1:8765/mcp` (change with `--host`, `--port` and `- Then generate the client side with `--transport http` and `--token-env`, naming the variable that holds the token on the client's machine. `--policy`, `--dataset` and `--read-only` do nothing in this mode, since the server is already running with its own options. Use `--url` for anything other than the default address. -```json tab="Claude Desktop" title="claude_desktop_config.json" -{ - "mcpServers": { - "cloudg": { - "command": "npx", - "args": [ - "-y", - "mcp-remote", - "http://127.0.0.1:8765/mcp", - "--header", - "Authorization:Bearer ${CLOUDG_MCP_TOKEN}" - ] +=== "Claude Desktop" + + ```json title="claude_desktop_config.json" + { + "mcpServers": { + "cloudg": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "http://127.0.0.1:8765/mcp", + "--header", + "Authorization:Bearer ${CLOUDG_MCP_TOKEN}" + ] + } + } } - } -} -``` - -```json tab="Claude Code" title=".mcp.json" -{ - "mcpServers": { - "cloudg": { - "type": "http", - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + ``` + +=== "Claude Code" + + ```json title=".mcp.json" + { + "mcpServers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` - -```json tab="Cursor" title="~/.cursor/mcp.json" -{ - "mcpServers": { - "cloudg": { - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + ``` + +=== "Cursor" + + ```json title="~/.cursor/mcp.json" + { + "mcpServers": { + "cloudg": { + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` - -```json tab="VS Code" title=".vscode/mcp.json" -{ - "servers": { - "cloudg": { - "type": "http", - "url": "http://127.0.0.1:8765/mcp", - "headers": { - "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + ``` + +=== "VS Code" + + ```json title=".vscode/mcp.json" + { + "servers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } } } - } -} -``` + ``` Each client expands the variable in its own syntax: `${VAR}` in Claude Code, `${env:VAR}` in Cursor and VS Code. The token itself never appears in the file. Claude Desktop's config only launches local commands, so cloudg bridges to the URL with `npx mcp-remote` and passes the header on its command line. That entry has no `env` block, and whether `${CLOUDG_MCP_TOKEN}` gets a value depends on the environment Claude Desktop hands to `npx`, which the cloudg docs have not tested. If the server answers 401, add `"env": {"CLOUDG_MCP_TOKEN": "..."}` to the entry by hand. -To reach the server from another host, bind to a public address and keep the token: `--host 0.0.0.0` together with `--allowed-host 'mcp.internal.example:*'` (and `--allowed-origin` for browser clients). The [CLI reference](/mcp/server/cli-reference/) lists every serve option. +To reach the server from another host, bind to a public address and keep the token: `--host 0.0.0.0` together with `--allowed-host 'mcp.internal.example:*'` (and `--allowed-origin` for browser clients). The [CLI reference](server/cli-reference.md) lists every serve option. ## Troubleshooting @@ -317,13 +341,32 @@ The vault key is random per process unless `CLOUDG_MCP_VAULT_KEY` is set. Set it ### The client complains about a missing outputSchema -Under every profile except `open`, tools don't advertise `outputSchema`, because masking can change a field's type and a client must reject content that doesn't match an advertised schema. The structured content is still sent. This is deliberate; see [troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) for this and the other cases (timeouts, large results, "cooling down" from live tools). +Under every profile except `open`, tools don't advertise `outputSchema`, because masking can change a field's type and a client must reject content that doesn't match an advertised schema. The structured content is still sent. This is deliberate; see [troubleshooting and FAQ](server/troubleshooting-and-faq.md) for this and the other cases (timeouts, large results, "cooling down" from live tools). + +
    + +- [**Quick start**](server/quick-start-with-a-desktop-client.md) + + The same setup with captured output from the sample estate. + +- [**cloudg mcp config**](../cli/mcp-config.md) + + Every option of the config command. + +- [**cloudg mcp serve**](../cli/mcp-serve.md) + + Transports, flavors, tokens, Host and Origin checks. + +- [**Built-in profiles**](privacy/built-in-profiles.md) + + What each policy changes, call by call. + +- [**Principals and roles**](privacy/principals-and-roles.md) + + Map tokens to roles and give each role its own view. + +- [**Troubleshooting and FAQ**](server/troubleshooting-and-faq.md) + + The full list of known problems. -:::links -- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) The same setup with captured output from the sample estate. -- [cloudg mcp config](/cli/mcp-config/) Every option of the config command. -- [cloudg mcp serve](/cli/mcp-serve/) Transports, flavors, tokens, Host and Origin checks. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy changes, call by call. -- [Principals and roles](/mcp/privacy/principals-and-roles/) Map tokens to roles and give each role its own view. -- [Troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) The full list of known problems. -::: +
    diff --git a/site/content/mcp/index.md b/site/content/mcp/index.md index 495dbd0..663b06f 100644 --- a/site/content/mcp/index.md +++ b/site/content/mcp/index.md @@ -1,18 +1,17 @@ --- title: MCP server -lede: "cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request." -meta: - - [Command, "`cloudg mcp serve`"] - - [Python, "`cloudg.mcp.CloudGMCPLayer`"] - - [Catalog, "74 tools, 13 resources, 11 templates, 11 prompts"] - - [Default policy, "`standard`"] -source: docs/MCP.md -since: "0.6.0" +description: cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request. --- +cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request. +{ .lede } + +**Command** `cloudg mcp serve` · **Python** `cloudg.mcp.CloudGMCPLayer` · **Catalog** 74 tools, 13 resources, 11 templates, 11 prompts · **Default policy** `standard` · **Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + The MCP layer lives in `cloudg/mcp/` and ships with the base `pip install cloudg`. It takes the files cloudg already writes (`inventory-map.json`, `findings.json`, native scanner output) and turns them into tools an assistant can call: "which internet-exposed assets have HIGH findings?", "what breaks if this IAM role goes away?", "which CIS controls fail in the shared-services account?". The model asks, cloudg answers from its own graph and data model, and a policy decides what the model may see and do on the way. -It is written for three kinds of reader. People who run Claude Desktop, Claude Code, Cursor or VS Code and want to point it at a cloud estate should read [Connect a client](/mcp/connect/) next. Operators who already run an MCP server or gateway can mount cloudg into it. And Python authors (an agent framework, a test suite, a batch job) can call the same tools in-process, with no MCP traffic at all. +It is written for three kinds of reader. People who run Claude Desktop, Claude Code, Cursor or VS Code and want to point it at a cloud estate should read [Connect a client](connect.md) next. Operators who already run an MCP server or gateway can mount cloudg into it. And Python authors (an agent framework, a test suite, a batch job) can call the same tools in-process, with no MCP traffic at all. ## One layer, three ways in @@ -24,7 +23,7 @@ The center of the package is one class, `CloudGMCPLayer`. It holds a registry of | Mounted | `layer.register_into(server)` | You already run an MCP server (official `mcp` SDK 1.x or 2.x, or `fastmcp` 2, 3 or 4) and want cloudg's tools inside it | | In-process | `await layer.call_tool(...)` | You are in Python and do not need MCP on the wire | -```mermaid caption="How the pieces fit together" +```mermaid flowchart LR A["Desktop or IDE client"] -->|"stdio or HTTP"| B["cloudg mcp serve (native, sdk or fastmcp)"] C["Your MCP server"] -->|register_into| L[CloudGMCPLayer] @@ -37,8 +36,10 @@ flowchart LR W --> F["inventory-map.json, findings.json, scanner output"] W --> X["Cloud APIs (live tools only)"] ``` +How the pieces fit together +{ .diagram-caption } -The [server section](/mcp/server/what-the-layer-is/) covers the layer in depth, [mounting](/mcp/server/mounting-into-an-existing-mcp-server/) covers the second mode and [in-process use](/mcp/server/in-process-use/) the third. +The [server section](server/what-the-layer-is.md) covers the layer in depth, [mounting](server/mounting-into-an-existing-mcp-server.md) covers the second mode and [in-process use](server/in-process-use.md) the third. ## What an assistant can do with it @@ -57,9 +58,9 @@ The default catalog has 74 tools in ten categories. The counts below come from r | `meta` | 4 | What this caller may use, and cloudg's vocabulary of asset, edge and relation types | | `privacy` | 5 | The active policy, a transform preview, the detector list, the decision log, and `reveal_token` | -The default `standard` policy hides `reveal_token`, so `cloudg mcp tools` lists 73. The stricter profiles hide more (see [policies](#policies-and-privacy) below). Every tool is documented with its arguments, return fields and a captured example in the [tool index](/mcp/tools/tool-index/) and the per-category pages such as [graph](/mcp/tools/graph/) and [findings](/mcp/tools/findings/). +The default `standard` policy hides `reveal_token`, so `cloudg mcp tools` lists 73. The stricter profiles hide more (see [policies](#policies-and-privacy) below). Every tool is documented with its arguments, return fields and a captured example in the [tool index](tools/tool-index.md) and the per-category pages such as [graph](tools/graph/index.md) and [findings](tools/findings.md). -The four collecting tools in `live` are the only ones that touch a cloud provider. They check credentials before the first API call, refuse to re-collect a scope that was collected a couple of minutes ago, and are hidden by every built-in profile except `open` and `standard`. The details are in [live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/). +The four collecting tools in `live` are the only ones that touch a cloud provider. They check credentials before the first API call, refuse to re-collect a scope that was collected a couple of minutes ago, and are hidden by every built-in profile except `open` and `standard`. The details are in [live tools and the credential preflight](server/live-tools-and-the-credential-preflight.md). ### Resources and prompts @@ -76,7 +77,7 @@ The 11 prompts are packaged analysis workflows that a user picks from the client | `incident_triage` | `finding_id` | | `drift_review` | `base` | -The full list with arguments and rendered output is in [resources and resource templates](/mcp/tools/resources-and-resource-templates/) and [prompts](/mcp/tools/prompts/). +The full list with arguments and rendered output is in [resources and resource templates](tools/resources-and-resource-templates.md) and [prompts](tools/prompts.md). ## Workspaces and datasets @@ -86,13 +87,13 @@ Datasets get into the workspace in three ways. The operator can preload them wit The graph, dependency graph, centrality scores and RDF ontology are built the first time a tool needs them and cached until the dataset changes. On a large estate the first `attack_paths` or `sparql_query` call is the slow one. The workspace holds 16 datasets; loading a 17th evicts the oldest inactive one. -Two safety rules matter in practice. First, the model can only read and write inside the allowed roots: by default the directory the server was started from and the configured report directory, with `/` and your home directory left out. Set `CLOUDG_MCP_ALLOWED_ROOTS` to change them. Second, paths given with `--dataset` skip that check, because the operator typed them. `snapshot_dataset` and `diff_datasets` let a model compare two versions of an estate, which is what the `drift_review` prompt does. More in [the workspace](/mcp/server/the-workspace/). +Two safety rules matter in practice. First, the model can only read and write inside the allowed roots: by default the directory the server was started from and the configured report directory, with `/` and your home directory left out. Set `CLOUDG_MCP_ALLOWED_ROOTS` to change them. Second, paths given with `--dataset` skip that check, because the operator typed them. `snapshot_dataset` and `diff_datasets` let a model compare two versions of an estate, which is what the `drift_review` prompt does. More in [the workspace](server/the-workspace.md). ## The request lifecycle Every tool call takes the same route, whatever transport it arrived on. -```mermaid caption="What happens to one tools/call" +```mermaid flowchart TD A[MCP client] --> B["Transport: stdio line or HTTP POST"] B --> C["Adapter resolves the principal (token, OAuth or local user)"] @@ -106,10 +107,12 @@ flowchart TD I --> J["Output pipeline: sanitize, redact, project, annotate"] J --> L["Result with cloudg/transforms in _meta"] ``` +What happens to one tools/call +{ .diagram-caption } A few details are easy to miss. A tool the policy hides gives exactly the same "Unknown tool" error as a tool that does not exist, so a model cannot probe for what it is missing. Arguments are validated against a pydantic model built from the handler's signature, and unknown keys are rejected. Synchronous handlers run in a worker thread so a slow graph build does not block other requests, and every call has a timeout (300 seconds by default, longer for the live tools). The result is sent twice, as `structuredContent` and as indented JSON text for clients that ignore structured output; the text copy is capped at 200,000 characters. Errors from the handler pass through the same output pipeline as results, because exception messages tend to quote ARNs. -The step-by-step version, including resources, prompts and completions, is in [the request lifecycle](/mcp/server/the-request-lifecycle/). Error codes are listed in [errors](/mcp/server/errors/) and the middleware hooks in [middleware](/mcp/server/middleware/). +The step-by-step version, including resources, prompts and completions, is in [the request lifecycle](server/the-request-lifecycle.md). Error codes are listed in [errors](server/errors.md) and the middleware hooks in [middleware](server/middleware.md). ## Policies and privacy @@ -131,7 +134,7 @@ The output pipeline in `standard` runs four transforms in order. `sanitize` stri Under `strict` the redactor also pseudonymises identifiers through the token vault. Pseudonyms keep their format, so a 12-digit account id stays 12 digits and an ARN stays a valid ARN, and they are deterministic for the life of the vault key. The model can pass `res-8de954a6d9` back as a `ref` and the input pipeline resolves the real asset before the handler runs. Set `CLOUDG_MCP_VAULT_KEY` if pseudonyms must survive a server restart; without it a random key is generated per process. -One side effect surprises people: under every profile except `open`, tools do not advertise an `outputSchema`, since masking can change a field's type and clients reject structured content that does not match a schema. The data is still sent. The [privacy section](/mcp/privacy/threat-model-and-goals/) covers the threat model, [what the policy touches](/mcp/privacy/what-the-policy-touches/), the [built-in profiles](/mcp/privacy/built-in-profiles/) file by file, and [the token vault](/mcp/privacy/the-token-vault/). +One side effect surprises people: under every profile except `open`, tools do not advertise an `outputSchema`, since masking can change a field's type and clients reject structured content that does not match a schema. The data is still sent. The [privacy section](privacy/threat-model-and-goals.md) covers the threat model, [what the policy touches](privacy/what-the-policy-touches.md), the [built-in profiles](privacy/built-in-profiles.md) file by file, and [the token vault](privacy/the-token-vault.md). ## Transports and flavors @@ -148,13 +151,13 @@ The flavor picks which implementation speaks MCP. The layer and results are iden | `sdk` | the official SDK's low-level `Server`, uvicorn for HTTP | `pip install "cloudg[mcp]"` | | `fastmcp` | `fastmcp.FastMCP` with cloudg components | `pip install fastmcp` | -The native flavor supports every serve option, including `--cors-origin`, `--page-size` and `/healthz`. See [transports and protocol versions](/mcp/server/transports-and-protocol-versions/) and [flavors](/mcp/server/flavors/). +The native flavor supports every serve option, including `--cors-origin`, `--page-size` and `/healthz`. See [transports and protocol versions](server/transports-and-protocol-versions.md) and [flavors](server/flavors.md). ## Security model in brief Every transport binds to 127.0.0.1 by default. On HTTP, `--auth-token TOKEN[:ROLES[:ID]]` maps static bearer tokens to principals with roles, which the policy's `roles:` section uses; tokens are kept only as SHA-256 digests and compared in constant time. Every flavor checks `Origin` and `Host` headers against allow-lists to block DNS rebinding, and the native server warns at start-up when bound to a non-loopback address without tokens. On stdio, tokens are ignored and the caller is the local user. -File access is confined to the allowed roots and writes to the output directory. Error messages never repeat the caller's arguments. The JSONL audit log (`--audit-log`) records every call with hashed argument values, and every pseudonym reversal is logged at WARNING. `--registry` imports and runs Python code, so its value must never come from untrusted input. The longer list is in [security model in brief](/mcp/server/security-model-in-brief/). +File access is confined to the allowed roots and writes to the output directory. Error messages never repeat the caller's arguments. The JSONL audit log (`--audit-log`) records every call with hashed argument values, and every pseudonym reversal is logged at WARNING. `--registry` imports and runs Python code, so its value must never come from untrusted input. The longer list is in [security model in brief](server/security-model-in-brief.md). ## Examples @@ -204,7 +207,6 @@ from pathlib import Path from cloudg.config import CloudGConfig from cloudg.mcp import CloudGMCPLayer, NotFoundError, Principal, Workspace - async def main() -> None: reports = Path("reports").resolve() ws = Workspace(CloudGConfig(), allowed_roots=[reports], output_dir=reports / "mcp-out") @@ -232,7 +234,6 @@ async def main() -> None: except NotFoundError as exc: print("refused:", exc) - asyncio.run(main()) ``` @@ -250,15 +251,46 @@ The pseudonyms change between runs unless `CLOUDG_MCP_VAULT_KEY` is set. Note th ## Where to go next -:::links -- [Connect a client](/mcp/connect/) Claude Desktop, Claude Code, Cursor and VS Code, step by step. -- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) Check your data in a terminal before wiring a client. -- [CLI reference](/mcp/server/cli-reference/) Every option of serve, tools, call, read and config. -- [Tool index](/mcp/tools/tool-index/) All 74 tools with category, sensitivity and a one-line summary. -- [Conventions](/mcp/tools/conventions-shared-by-every-tool/) Paging, field selection, asset references and the result envelope. -- [Recipes](/mcp/tools/recipes/) Multi-step investigations as tool-call sequences. -- [Built-in profiles](/mcp/privacy/built-in-profiles/) The seven policies side by side, with the same calls under each. -- [Policy schema](/mcp/privacy/policy-schema-reference/) Write your own policy, roles and rules. -- [In-process use](/mcp/server/in-process-use/) CloudGMCPLayer, Principal and ToolContext from Python. -- [Internals](/mcp/internals/package-map/) How the package is built, for contributors. -::: +
    + +- [**Connect a client**](connect.md) + + Claude Desktop, Claude Code, Cursor and VS Code, step by step. + +- [**Quick start**](server/quick-start-with-a-desktop-client.md) + + Check your data in a terminal before wiring a client. + +- [**CLI reference**](server/cli-reference.md) + + Every option of serve, tools, call, read and config. + +- [**Tool index**](tools/tool-index.md) + + All 74 tools with category, sensitivity and a one-line summary. + +- [**Conventions**](tools/conventions-shared-by-every-tool.md) + + Paging, field selection, asset references and the result envelope. + +- [**Recipes**](tools/recipes.md) + + Multi-step investigations as tool-call sequences. + +- [**Built-in profiles**](privacy/built-in-profiles.md) + + The seven policies side by side, with the same calls under each. + +- [**Policy schema**](privacy/policy-schema-reference.md) + + Write your own policy, roles and rules. + +- [**In-process use**](server/in-process-use.md) + + CloudGMCPLayer, Principal and ToolContext from Python. + +- [**Internals**](internals/package-map.md) + + How the package is built, for contributors. + +
    diff --git a/site/content/reference/output-files.md b/site/content/reference/output-files.md index 3eff57a..3063cbe 100644 --- a/site/content/reference/output-files.md +++ b/site/content/reference/output-files.md @@ -1,15 +1,19 @@ --- title: Output files -lede: Every file cloudg writes, which command writes it, and what is inside. -source: docs/DOCUMENTATION.md -since: "0.6.0" +description: Every file cloudg writes, which command writes it, and what is inside. --- +Every file cloudg writes, which command writes it, and what is inside. +{ .lede } + +**Last changed** [v0.6.0](../changelog.md#0.6.0) +{ .page-meta } + Every command writes into one output directory, `-o` / `--output`, which defaults to `./reports` and is created when missing. Files with the same name are overwritten without a prompt, so give each run its own directory when you want to keep history. The exceptions to "everything goes into `-o`" are the Cloud Control type cache, the MCP server's audit log and vault, and a Terraform export sent elsewhere with `terraform.output_dir`, listed in their sections below. ## cloudg run -```mermaid caption="Where the files of cloudg run come from, phase by phase" +```mermaid flowchart LR A["Collect"] --> B["Graph"] B --> C["RAG export"] @@ -24,17 +28,19 @@ flowchart LR F -.-> F1["ontology.ttl, ontology.jsonld"] H -.-> H1["findings.json, topology.svg, report.html"] ``` +Where the files of cloudg run come from, phase by phase +{ .diagram-caption } | File | Written by | Contents | |---|---|---| -| `topology.graphml` | graph phase, unless `graph.persist_graphml: false` | The asset graph from [GraphBuilder](/api/graphbuilder/), for Gephi, yEd or `networkx.read_graphml`. Saved before the reachability walk, so `is_internet_exposed` is the collector's flag | +| `topology.graphml` | graph phase, unless `graph.persist_graphml: false` | The asset graph from [GraphBuilder](../api/graphbuilder.md), for Gephi, yEd or `networkx.read_graphml`. Saved before the reachability walk, so `is_internet_exposed` is the collector's flag | | `topology-cytoscape.json` | graph phase, with `graph.export_cytoscape: true` (off by default) | The same graph as Cytoscape.js `elements` | -| `rag_chunks.jsonl` | RAG export, unless `--no-rag-export` or `rag.enabled: false` | One retrieval chunk per line: entity, community and relation-group chunks ([RAGExporter](/api/ragexporter/)). Written once after the graph phase and rewritten after the scanners with every finding | +| `rag_chunks.jsonl` | RAG export, unless `--no-rag-export` or `rag.enabled: false` | One retrieval chunk per line: entity, community and relation-group chunks ([RAGExporter](../api/ragexporter.md)). Written once after the graph phase and rewritten after the scanners with every finding | | `rag_metadata_index.json` | RAG export | Chunk counts per kind, every `chunk_id` in file order, the chunk types | | `provider.tf.json`, `variables.tf.json`, `main.tf.json`, `import_commands.sh` | Terraform phase, with `--terraform` or `terraform.enabled: true` | Terraform JSON that recreates the collected resources, and a script of `terraform import` commands. Written to `/terraform`, so it follows `-o`. A `terraform.output_dir` set in `config.yaml` replaces that directory, except the old default `./reports/terraform`, which counts as unset | | `prowler//` | Prowler, when it runs | Prowler's own JSON-ASFF output files, which cloudg parses | | `scoutsuite//` | ScoutSuite, when it runs | ScoutSuite's report directory | -| `ontology.ttl`, `ontology.jsonld` | ontology phase, unless `--no-ontology` or `ontology.enabled: false` | The RDF graph with 64 relation types and the findings ([CloudOntology](/api/cloudontology/)). One file per `ontology.export_formats` entry; `xml` adds `ontology.rdf`, `nt` adds `ontology.nt` | +| `ontology.ttl`, `ontology.jsonld` | ontology phase, unless `--no-ontology` or `ontology.enabled: false` | The RDF graph with 64 relation types and the findings ([CloudOntology](../api/cloudontology.md)). One file per `ontology.export_formats` entry; `xml` adds `ontology.rdf`, `nt` adds `ontology.nt` | | `findings.json` | render phase | The normalised result: metadata, summary, assets, findings, compliance mappings, edges, D3 graph. Shape below | | `topology.svg` | render phase | Static topology diagram | | `report.html` | render phase | Interactive report: topology, findings table, compliance matrix. The data, Chart.js 4.4.0 and D3 7.9.0 are embedded, so it works offline. With `report.inline_js: false` the page loads the two libraries from `cdn.jsdelivr.net` instead | @@ -61,9 +67,9 @@ Checkov and Trivy write nothing to disk; cloudg reads their stdout. `cloudg run` | File | Written by | Contents | |---|---|---| -| `inventory-map.json` | every map | The self-contained map: `summary`, `providers`, `regions`, `assets`, `edges`, `coverage`, `unresolved_references`, and `throttling` when a cloud API pushed back. Read by `cloudg deps`, `InventoryResult.load()` and the MCP workspace. Field reference: [inventory-map.json](/reference/inventory/inventory-map-json/) | -| `inventory-map.graphml` | every map | The map's graph, with parallel rule edges merged ([field list](/reference/inventory/inventory-map-graphml/)) | -| `inventory-graph.json` | every map | D3 `nodes` and `links` for `docs/viewer.html` and other front ends ([field list](/reference/inventory/inventory-graph-json/)) | +| `inventory-map.json` | every map | The self-contained map: `summary`, `providers`, `regions`, `assets`, `edges`, `coverage`, `unresolved_references`, and `throttling` when a cloud API pushed back. Read by `cloudg deps`, `InventoryResult.load()` and the MCP workspace. Field reference: [inventory-map.json](inventory/inventory-map-json.md) | +| `inventory-map.graphml` | every map | The map's graph, with parallel rule edges merged ([field list](inventory/inventory-map-graphml.md)) | +| `inventory-graph.json` | every map | D3 `nodes` and `links` for `docs/viewer.html` and other front ends ([field list](inventory/inventory-graph-json.md)) | | `inventory-dependencies.json` | every map | Shared dependencies and largest blast radius (top 25 each), every cross-account edge, security service coverage, unresolved references | | `inventory-organization.json` | `--org`, or `aws.organization.enabled` | The AWS Organization and Control Tower topology: accounts, OU tree, SCPs, landing zone | | `asset-map.json` | `--findings`, when at least one finding loads | Every asset with its matched findings, sorted by finding count | @@ -87,7 +93,7 @@ cloudg deps arn:aws:iam::123456789012:role/orders-worker --json > role-deps.json cloudg deps --map ./reports --top 50 --json > overview.json ``` -The key-by-key shapes are in [cloudg deps --json](/reference/inventory/cloudg-deps-json/). +The key-by-key shapes are in [cloudg deps --json](inventory/cloudg-deps-json.md). ## cloudg ingest @@ -213,7 +219,7 @@ The global option `cloudg --log-file PATH ` (or `log_file` in `config.y |---|---| | `metadata` | Run identity. The timestamps are ISO 8601 strings, like the ones on assets and findings, and an unset `completed_at` is `null`. The CLI and the engine leave `provider`, `account_id` and `region` at `null` | | `summary` | Counts. `total_edges` counts the entries in `edges` | -| `assets` | `CloudAsset` in JSON mode, without `raw_data`, with the computed `display_id` (ARN, else id). Fields: [CloudAsset](/reference/inventory/cloudasset/) | +| `assets` | `CloudAsset` in JSON mode, without `raw_data`, with the computed `display_id` (ARN, else id). Fields: [CloudAsset](inventory/cloudasset.md) | | `findings` | `Finding` in JSON mode, sorted by severity, with the computed `risk_score` (severity base, averaged with `cvss_score` when present). Reachability findings have deterministic UUID5 ids; most scanner findings get a fresh UUID on every run. The normaliser can add frameworks the scanner did not set | | `compliance` | One `ComplianceResult` per framework control, with the ids of the findings that failed it. `-aggregate` collects findings mapped to a framework but to no specific control | | `edges` | Every collected `NetworkEdge` in JSON mode, parallel rules kept. `cloudg report` draws `topology.svg` from them | @@ -272,21 +278,37 @@ The map `cloudg map` writes, and `cloudg deps` and the MCP workspace read. The s | Key | Notes | |---|---| -| `summary` | Counts by type, service, region and account, edge counts by type and relationship, and health counters. `organization` and `throttling` blocks appear only when an organization was mapped or an API throttled. See [summary](/reference/inventory/summary/) | +| `summary` | Counts by type, service, region and account, edge counts by type and relationship, and health counters. `organization` and `throttling` blocks appear only when an organization was mapped or an API throttled. See [summary](inventory/summary.md) | | `assets` | Every asset, linked and deduplicated. Real collector assets carry rich `metadata`, including the `relations` the linker resolved | | `edges` | Every edge, parallel rules kept. An endpoint can be a CIDR or another non-asset string, so look endpoints up with `.get()` | | `unresolved_references` | Declared relations the linker could not resolve, with `source`, `source_name`, `target` and `edge_type` | | `coverage` | One record per provider, region and account collected: per service, `status` (`SUCCESS`, `PARTIAL`, `FAILED` or `SKIPPED`), `asset_count`, `error` and `duration_ms`. Trimmed to one service above | | `throttling` | Only when a cloud API throttled the run: the full throttling report of the run | -`edges` here and `links` in `inventory-graph.json` differ in length whenever parallel rules were merged for the graph. Count edges in this file. For every field of every structure, see the [inventory reference](/reference/inventory/inventory-map-json/). +`edges` here and `links` in `inventory-graph.json` differ in length whenever parallel rules were merged for the graph. Count edges in this file. For every field of every structure, see the [inventory reference](inventory/inventory-map-json.md). ## Related -:::links -- [cloudg run](/cli/run/) The flags that switch outputs on and off. -- [cloudg map](/cli/map/) The inventory map and its companions. -- [Reports](/guides/reports/) Reading `report.html` and `findings.json`. -- [Data models](/api/models/) The Pydantic models behind every JSON file. -- [CloudAsset reference](/reference/inventory/cloudasset/) Every asset field and identifier format. -::: +
    + +- [**cloudg run**](../cli/run.md) + + The flags that switch outputs on and off. + +- [**cloudg map**](../cli/map.md) + + The inventory map and its companions. + +- [**Reports**](../guides/reports.md) + + Reading `report.html` and `findings.json`. + +- [**Data models**](../api/models.md) + + The Pydantic models behind every JSON file. + +- [**CloudAsset reference**](inventory/cloudasset.md) + + Every asset field and identifier format. + +
    diff --git a/site/content/sample-report.md b/site/content/sample-report.md new file mode 100644 index 0000000..c4793e8 --- /dev/null +++ b/site/content/sample-report.md @@ -0,0 +1,53 @@ +--- +title: Sample report +description: The interactive report.html that cloudg writes, built from a synthetic three-cloud estate, so you can see it without an account. +hide: + - toc +--- + +This is the `report.html` cloudg writes at the end of `cloudg run`, `cloudg ingest` and `cloudg report`, built from a synthetic estate that spans AWS, Azure and GCP. Every account ID, ARN and finding in it is made up, so nothing here describes a real environment. +{ .lede } + +[Open full screen](sample-report.html){ .md-button .md-button--primary target="_blank" rel="noopener" } +[How to read a report](guides/reports.md){ .md-button } + + + +## Where it comes from + +The direct address of the file is [morpheuslord.github.io/cloudg/sample-report.html](https://morpheuslord.github.io/cloudg/sample-report.html). It is rebuilt from the fixtures the MCP test suite uses, with no cloud access, by a script in the repository: + +=== "Rebuild it" + + ```bash + python scripts/build_sample_report.py # writes site/static/sample-report.html + python scripts/build_sample_report.py --out ./tmp # or anywhere else + ``` + +=== "Python" + + The script is a few lines around the same classes you would use yourself. Run it from a clone of the repository, because the fixtures live in `tests/`: + + ```python title="build_sample_report.py" + from cloudg.graph.builder import GraphBuilder + from cloudg.renderers.html_report import HTMLReportGenerator + from cloudg.schema.models import ScanResult + from tests.mcp.fixtures import sample_estate + + inv = sample_estate.build_inventory() + result = ScanResult( + assets=inv.assets, + edges=inv.edges, + findings=sample_estate.findings(), + compliance=sample_estate.compliance(), + ) + graph = GraphBuilder() + graph.build(inv.assets, inv.edges) + + path = HTMLReportGenerator(output_dir="./tmp").generate( + result, graph_json=graph.to_d3_json(), filename="sample-report.html" + ) + print(path) + ``` + +The page is a single self-contained file: Chart.js and D3 are inlined (`report.inline_js`, on by default), so it works offline and can be mailed or attached to a ticket as it is. [Reports](guides/reports.md) explains each tab, and [Output files](reference/output-files.md) lists what else a run writes next to it. diff --git a/site/data/nav.yaml b/site/data/nav.yaml deleted file mode 100644 index ce90d23..0000000 --- a/site/data/nav.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Site map. Each tab has a sidebar made of groups; each group lists pages. -# -# page: hand-written page, read from site/content/.md -# split: docs/FILE.md one page per "##" section of a long reference document -# (sections over split_h3_kb are split again at "###") -# generated: built from code: cli-index, api-index, config, mcp-tools-index -# -# Paths are URLs under the site base, without slashes at either end. - -tabs: - - id: guides - label: Guides - root: guides/introduction - groups: - - title: Start - items: - - {title: Introduction, page: guides/introduction} - - {title: Installation, page: guides/installation} - - {title: Quickstart, page: guides/quickstart} - - {title: The pipeline, page: guides/pipeline} - - title: Mapping - items: - - {title: Inventory mapping, page: guides/inventory-mapping} - - {title: Dependencies & blast radius, page: guides/dependencies} - - {title: AWS Organizations, page: guides/aws-organizations} - - title: Scanning - items: - - {title: Running scanners, page: guides/running-scanners} - - {title: Ingesting existing output, page: guides/ingesting} - - {title: Compliance frameworks, page: guides/compliance} - - title: Analyse - items: - - {title: "Graph, ontology & RAG", page: guides/graph-ontology-rag} - - {title: Reports, page: guides/reports} - - title: Operate - items: - - {title: Authentication, page: guides/authentication} - - {title: Rate limits & throttling, page: guides/rate-limits} - - {title: Docker, page: guides/docker} - - {title: CI and automation, page: guides/ci-automation} - - {title: Python recipes, page: guides/python-recipes} - - - id: cli - label: CLI - root: cli - groups: - - title: Overview - items: - - {title: All commands, generated: cli-index, path: cli} - - title: Pipeline - items: - - {title: cloudg run, page: cli/run} - - {title: cloudg collect, page: cli/collect} - - {title: cloudg scan, page: cli/scan} - - {title: cloudg report, page: cli/report} - - title: Inventory - items: - - {title: cloudg map, page: cli/map} - - {title: cloudg deps, page: cli/deps} - - {title: cloudg ingest, page: cli/ingest} - - title: MCP - items: - - {title: cloudg mcp serve, page: cli/mcp-serve} - - {title: cloudg mcp config, page: cli/mcp-config} - - {title: cloudg mcp tools, page: cli/mcp-tools} - - {title: cloudg mcp resources, page: cli/mcp-resources} - - {title: cloudg mcp prompts, page: cli/mcp-prompts} - - {title: cloudg mcp call, page: cli/mcp-call} - - {title: cloudg mcp read, page: cli/mcp-read} - - title: Shared - items: - - {title: Provider & region flags, page: cli/provider-flags} - - {title: Auth flags, page: cli/auth-flags} - - {title: Output flags, page: cli/output-flags} - - - id: api - label: Python API - root: api - groups: - - title: Overview - items: - - {title: Python API, generated: api-index, path: api} - - title: Engine - items: - - {title: CloudGEngine, page: api/cloudgengine} - - {title: CloudGConfig, page: api/cloudgconfig} - - {title: Result objects, page: api/results} - - {title: Event hooks, page: api/event-hooks} - - title: Inventory - items: - - {title: InventoryMapper, page: api/inventorymapper} - - {title: InventoryResult, page: api/inventoryresult} - - {title: RelationshipLinker, page: api/relationshiplinker} - - {title: DependencyGraph, page: api/dependencygraph} - - title: Graph - items: - - {title: GraphBuilder, page: api/graphbuilder} - - {title: ReachabilityAnalyzer, page: api/reachabilityanalyzer} - - {title: CloudOntology, page: api/cloudontology} - - {title: RAGExporter, page: api/ragexporter} - - title: Findings - items: - - {title: FindingsNormaliser, page: api/findingsnormaliser} - - {title: parse_report, page: api/parse-report} - - {title: Data models, page: api/models} - - title: Extend - items: - - {title: Entry points, page: api/entry-points} - - {title: cloudg.resilience, page: api/resilience} - - {title: CloudGMCPLayer, page: api/mcp-layer} - - - id: mcp - label: MCP server - root: mcp - groups: - - title: Start - items: - - {title: Overview, page: mcp/index, path: mcp} - - {title: Connect a client, page: mcp/connect} - - title: Server - split: docs/MCP.md - prefix: mcp/server - skip: [Contents] - - title: Tools - split: docs/MCP_TOOLS.md - prefix: mcp/tools - skip: [Tool reference] - - title: Privacy & policy - split: docs/MCP_PRIVACY.md - prefix: mcp/privacy - skip: [Contents] - - title: Internals - split: docs/MCP_INTERNALS.md - prefix: mcp/internals - skip: [Contents] - - - id: reference - label: Reference - root: reference/config/providers - groups: - - title: config.yaml - generated: config - prefix: reference/config - - title: Outputs - items: - - {title: Output files, page: reference/output-files} - - title: Inventory structures - split: docs/INVENTORY_REFERENCE.md - prefix: reference/inventory - skip: [Contents] - - title: Catalogs - split: docs/INVENTORY_CATALOG.md - prefix: reference/catalog - skip: [Contents] - - title: Inventory internals - split: docs/INVENTORY_INTERNALS.md - prefix: reference/internals - skip: [Contents] - - title: Resilience internals - split: docs/RESILIENCE.md - prefix: reference/resilience - skip: [Contents] - - - id: changelog - label: Changelog - root: changelog - -# Links from the old one-page handbook (index.html#anchor) to their new homes. -redirects: - install: guides/installation - pipeline: guides/pipeline - run: cli/run - collect-scan: cli/collect - map: guides/inventory-mapping - deps: guides/dependencies - ingest: guides/ingesting - report: cli/report - inputs: guides/ingesting#input-requirements - outputs: reference/output-files - models: api/models - inventory-docs: reference/inventory - config: reference/config/providers - ratelimit: guides/rate-limits - api: api - recipes: guides/python-recipes - plugins: api/entry-points - auth: guides/authentication - docker: guides/docker - mcp: mcp - mcp-serve: cli/mcp-serve - mcp-embed: mcp/server/in-process-use - mcp-policies: mcp/privacy/built-in-profiles - mcp-catalog: mcp/tools/tool-index - mcp-cli: cli/mcp-serve - changelog: changelog diff --git a/site/data/stats.yaml b/site/data/stats.yaml deleted file mode 100644 index f76aabc..0000000 --- a/site/data/stats.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Figures on the docs home page. Entries with `compute` are counted from the -# code at build time; `value` is the fallback (and the only source for the -# framework count, which follows the README's way of counting). -stats: - - label: compliance frameworks - value: 28 - href: guides/compliance - - label: controls mapped - compute: controls - value: 4166 - href: guides/compliance - - label: asset types catalogued - compute: asset_types - value: 156 - href: reference/catalog - - label: MCP tools - compute: mcp_tools - value: 74 - href: mcp diff --git a/site/docsgen/__init__.py b/site/docsgen/__init__.py deleted file mode 100644 index 450dc9f..0000000 --- a/site/docsgen/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Static site generator for the cloudg documentation (see site/AUTHORING.md).""" diff --git a/site/docsgen/discover.py b/site/docsgen/discover.py deleted file mode 100644 index 0060357..0000000 --- a/site/docsgen/discover.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Page discovery: walks data/nav.yaml and creates every page before anything renders.""" - -from __future__ import annotations - -import yaml - -from docsgen import gen_config -from docsgen.page import REPO, SITE, Page -from docsgen.split import SplitDoc - -_GENERATED_SEARCH_TYPES = {"cli-index": "CLI", "api-index": "Python API"} -_TAB_KINDS = {"cli": "cli", "api": "api", "mcp": "guide", "reference": "guide"} -_TAB_SEARCH_TYPES = { - "guides": "Guides", - "cli": "CLI", - "api": "Python API", - "mcp": "MCP", - "reference": "Reference", -} - - -class DiscoverMixin: - """Nav walking for :class:`docsgen.site.Site`.""" - - nav: dict - tabs: list[dict] - sidebars: dict[str, list[dict]] - - def discover(self) -> None: - home = self.add( - Page("", "cloudg documentation", kind="home", search_type="Guides", in_sidebar=False) - ) - home.source = "site/templates/home.html" - for tab in self.nav["tabs"]: - self.tabs.append({"id": tab["id"], "label": tab["label"], "root": tab["root"]}) - self.sidebars[tab["id"]] = [ - { - "title": group["title"], - "num": f"{gi:02d}", - "pages": self._discover_group(tab["id"], group), - } - for gi, group in enumerate(tab.get("groups", []), start=1) - ] - self.add( - Page( - "changelog", - "Changelog", - tab="changelog", - kind="changelog", - search_type="Changelog", - source="CHANGELOG.md", - in_sidebar=False, - ) - ) - - def _discover_group(self, tab: str, group: dict) -> list[Page]: - if "split" in group: - return SplitDoc(self, tab, group).discover() - if group.get("generated") == "config": - return self._discover_config(tab, group) - return [self._discover_item(tab, group["title"], item) for item in group["items"]] - - def _discover_item(self, tab: str, group: str, item: dict) -> Page: - title = item["title"] - if "generated" in item: - page = Page(item["path"], title, tab=tab, group=group, kind=item["generated"]) - page.search_type = _GENERATED_SEARCH_TYPES.get(item["generated"], "Guides") - return self.add(page) - rel = item["page"] - file = SITE / "content" / f"{rel}.md" - page = Page( - item.get("path", rel), - title, - tab=tab, - group=group, - kind=_TAB_KINDS.get(tab, "guide"), - search_type=_TAB_SEARCH_TYPES[tab], - source=str(file.relative_to(REPO)), - nav_title=title, - ) - page.loader = file - return self.add(page) - - def _discover_config(self, tab: str, group: dict) -> list[Page]: - sections = gen_config.parse(REPO / "config.yaml") - gen_config.merge_model_descriptions(sections) - notes_file = SITE / "data" / "config-notes.yaml" - overrides = yaml.safe_load(notes_file.read_text()) or {} - pages = [] - for s in sections: - for k in s.keys: - note = overrides.get(f"{s.name}.{k.path}") - if note: - k.description = note - page = Page( - f"{group['prefix']}/{s.name}", - s.name, - tab=tab, - group=group["title"], - kind="config", - search_type="Config", - source="config.yaml", - mono_title=True, - ) - page.extra = {"section": s, "intro": overrides.get(s.name, "")} - pages.append(self.add(page)) - return pages diff --git a/site/docsgen/fences.py b/site/docsgen/fences.py deleted file mode 100644 index 782c50b..0000000 --- a/site/docsgen/fences.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Line-by-line tracking of fenced code blocks in markdown sources.""" - -from __future__ import annotations - -import re - -FENCE_RE = re.compile(r"^\s{0,3}(`{3,}|~{3,})(.*)$") - - -class FenceTracker: - """Follows ``` and ~~~ fences so callers can skip lines inside code. - - Feed every line in order; :meth:`feed` says whether that line belongs to a - fenced block (the opening and closing fence lines included). - """ - - def __init__(self) -> None: - self.fence: str | None = None - - def feed(self, line: str) -> bool: - m = FENCE_RE.match(line) - if self.fence is None: - if m: - self.fence = m.group(1) - return m is not None - if m and self._closes(m): - self.fence = None - return True - - def _closes(self, m: re.Match) -> bool: - mark = m.group(1) - return mark[0] == self.fence[0] and len(mark) >= len(self.fence) and not m.group(2).strip() diff --git a/site/docsgen/gen_api.py b/site/docsgen/gen_api.py deleted file mode 100644 index a3cb159..0000000 --- a/site/docsgen/gen_api.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Python API reference data, read from the live objects with ``inspect``.""" - -from __future__ import annotations - -import dataclasses -import enum -import inspect -import pkgutil -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -_SECTION_RE = re.compile( - r"^(Args|Arguments|Parameters|Params|Returns|Return|Yields|Raises|Example|Examples" - r"|Note|Notes|Attributes|Usage)\s*:\s*$" -) -_ITEM_RE = re.compile(r"^(\*{0,2}[\w.\[\], |]+?)\s*(\([^)]*\))?\s*:\s*(.*)$") -# objects documented on the API pages: dotted identifiers inside the cloudg package -_API_PATH_RE = re.compile(r"^cloudg(?:\.[A-Za-z_][A-Za-z0-9_]*)+$") - - -@dataclass -class Param: - name: str - type: str - default: str - description: str = "" - kind: str = "" - - -@dataclass -class Doc: - summary: str = "" - body: str = "" - params: dict[str, str] = field(default_factory=dict) - returns: str = "" - raises: list[tuple[str, str]] = field(default_factory=list) - examples: str = "" - attributes: dict[str, str] = field(default_factory=dict) - notes: str = "" - - -@dataclass -class Member: - name: str - kind: str # method, async, property, classmethod, staticmethod, attribute - signature: str - returns: str - doc: Doc - params: list[Param] - source_line: int = 0 - - -@dataclass -class ApiObject: - path: str - name: str - module: str - kind: str # class, function, module, enum, model, dataclass - signature: str - doc: Doc - params: list[Param] - returns: str - members: list[Member] = field(default_factory=list) - fields: list[Param] = field(default_factory=list) - enum_values: list[tuple[str, str]] = field(default_factory=list) - bases: list[str] = field(default_factory=list) - source_file: str = "" - source_line: int = 0 - - -def rst_to_md(text: str) -> str: - """Turn the reST bits docstrings use (``:func:`x```, double backticks) into markdown.""" - text = re.sub(r":[\w]+(?::[\w]+)?:`~?\.?([^`]+)`", r"`\1`", text) - return re.sub(r"``([^`]+)``", r"`\1`", text) - - -def _split_sections(text: str) -> dict[str, list[str]]: - """Group docstring lines under their Google-style section header ("" for the top).""" - sections: dict[str, list[str]] = {"": []} - current = "" - for line in text.split("\n"): - m = _SECTION_RE.match(line.strip()) if not line.startswith(" ") else None - if m: - current = m.group(1).lower() - sections.setdefault(current, []) - continue - sections.setdefault(current, []).append(line) - return sections - - -def _items(lines: list[str]) -> list[tuple[str, str]]: - """``name (type): text`` entries, with continuation lines folded in.""" - out: list[tuple[str, str]] = [] - lines = inspect.cleandoc("\n".join(lines)).split("\n") if lines else [] - for line in lines: - if not line.strip(): - continue - m = _ITEM_RE.match(line) if not line.startswith(" ") else None - if m: - out.append((m.group(1).strip("* "), m.group(3).strip())) - elif out: - out[-1] = (out[-1][0], (out[-1][1] + " " + line.strip()).strip()) - return out - - -def _take(sections: dict[str, list[str]], keys: tuple[str, ...]) -> list[list[str]]: - """Remove and return the sections named in ``keys``, in that order.""" - return [sections.pop(key) for key in keys if key in sections] - - -def _block(lines: list[str]) -> str: - return inspect.cleandoc("\n".join(lines)) - - -def parse_docstring(text: str | None) -> Doc: - doc = Doc() - if not text: - return doc - sections = _split_sections(rst_to_md(inspect.cleandoc(text))) - main = "\n".join(sections.pop("", [])).strip() - paras = re.split(r"\n\s*\n", main, maxsplit=1) - doc.summary = " ".join(paras[0].split()) - doc.body = paras[1].strip() if len(paras) > 1 else "" - for lines in _take(sections, ("args", "arguments", "parameters", "params")): - doc.params.update(dict(_items(lines))) - for lines in _take(sections, ("returns", "return", "yields")): - doc.returns = " ".join(" ".join(lines).split()) - for lines in _take(sections, ("raises",)): - doc.raises = _items(lines) - for lines in _take(sections, ("attributes",)): - doc.attributes = dict(_items(lines)) - for lines in _take(sections, ("example", "examples", "usage")): - doc.examples = _block(lines) - doc.notes = "\n\n".join(_block(lines) for lines in _take(sections, ("note", "notes"))) - return doc - - -def _ann(a: Any) -> str: - if a is inspect.Parameter.empty or a is inspect.Signature.empty: - return "" - if isinstance(a, str): - return a.strip("'\"") - if isinstance(a, type): - return a.__name__ if a.__module__ == "builtins" else a.__name__ - s = repr(a) - return re.sub(r"\b(?:[a-z_][\w]*\.)+([A-Z]\w*)", r"\1", s).replace("typing.", "") - - -def _default(d: Any) -> str: - if d is inspect.Parameter.empty: - return "" - if isinstance(d, enum.Enum): - return f"{type(d).__name__}.{d.name}" - r = repr(d) - return r if len(r) <= 60 else r[:57] + "..." - - -def _params(sig: inspect.Signature | None, doc: Doc, skip_self: bool) -> list[Param]: - if sig is None: - return [] - out = [] - for i, (name, p) in enumerate(sig.parameters.items()): - if skip_self and i == 0 and name in ("self", "cls"): - continue - prefix = "*" if p.kind is p.VAR_POSITIONAL else "**" if p.kind is p.VAR_KEYWORD else "" - desc = doc.params.get(name) or doc.params.get(prefix + name) or "" - out.append(Param(prefix + name, _ann(p.annotation), _default(p.default), desc, p.kind.name)) - return out - - -def _param_text(p: Param) -> str: - s = p.name - if p.type: - s += f": {p.type}" - if p.default: - s += f" = {p.default}" if p.type else f"={p.default}" - return s - - -def _has_var_positional(params: list[Param]) -> bool: - return any(q.name.startswith("*") and not q.name.startswith("**") for q in params) - - -def format_signature(name: str, params: list[Param], returns: str, width: int = 88) -> str: - pieces = [] - # a *args parameter already marks where the keyword-only ones start - star_written = _has_var_positional(params) - for p in params: - if p.kind == "KEYWORD_ONLY" and not star_written: - pieces.append("*") - star_written = True - pieces.append(_param_text(p)) - ret = f" -> {returns}" if returns else "" - one = f"{name}({', '.join(pieces)}){ret}" - if len(one) <= width: - return one - return f"{name}(\n" + "".join(f" {p},\n" for p in pieces) + f"){ret}" - - -def _signature(obj: Any) -> inspect.Signature | None: - try: - return inspect.signature(obj) - except (TypeError, ValueError): - return None - - -def _source(obj: Any, repo: Path) -> tuple[str, int]: - try: - target = inspect.unwrap(obj) - file = inspect.getsourcefile(target) or "" - line = inspect.getsourcelines(target)[1] - return str(Path(file).resolve().relative_to(repo.resolve())), line - except (TypeError, OSError, ValueError): - return "", 0 - - -def _is_pydantic(cls: type) -> bool: - return hasattr(cls, "model_fields") and hasattr(cls, "model_validate") - - -def _member(cls: type, name: str, repo: Path) -> Member | None: - raw = inspect.getattr_static(cls, name, None) - if raw is None: - return None - kind = "method" - target: Any = raw - if isinstance(raw, property): - kind, target = "property", raw.fget - elif isinstance(raw, classmethod): - kind, target = "classmethod", raw.__func__ - elif isinstance(raw, staticmethod): - kind, target = "staticmethod", raw.__func__ - elif inspect.iscoroutinefunction(raw): - kind = "async" - elif not callable(raw): - return Member(name, "attribute", name, _ann(type(raw)), Doc(), []) - doc = parse_docstring(inspect.getdoc(target)) - sig = _signature(target) - params = _params(sig, doc, skip_self=kind in ("method", "async", "classmethod", "property")) - returns = _ann(sig.return_annotation) if sig else "" - if kind == "property": - signature, params = name, [] - else: - signature = format_signature(name, params, returns) - _, line = _source(target, repo) - return Member(name, kind, signature, returns, doc, params, line) - - -def _public_members(cls: type) -> list[str]: - names = [] - base_names: set[str] = set() - for base in cls.__mro__[1:]: - if base.__module__.startswith(("pydantic", "builtins", "enum", "typing", "abc")): - base_names.update(vars(base)) - for name, value in vars(cls).items(): - if name.startswith("_") or name in base_names: - continue - if callable(value) or isinstance(value, (property, classmethod, staticmethod)): - names.append(name) - return names - - -def _resolve(path: str) -> tuple[Any, str, str]: - """Return (object, module name, attribute name) for a dotted ``cloudg.`` path. - - Only paths inside the cloudg package are accepted: the names come from the - front matter of the content pages. - """ - if not _API_PATH_RE.match(path): - raise ValueError(f"not a cloudg object path: {path!r}") - obj = pkgutil.resolve_name(path) - module_name, _, attr = path.rpartition(".") - if inspect.ismodule(obj): - module_name = path - return obj, module_name, attr - - -def _class_kind(cls: type) -> str: - if issubclass(cls, enum.Enum): - return "enum" - if _is_pydantic(cls): - return "model" - if dataclasses.is_dataclass(cls): - return "dataclass" - return "class" - - -def _factory_default(factory: Any) -> str: - name = getattr(factory, "__name__", "") - return f"{name}()" if name and name != "" else "generated" - - -def _model_field_default(finfo: Any) -> str: - if finfo.is_required(): - return "required" - if finfo.default_factory is not None: - return _factory_default(finfo.default_factory) - return _default(finfo.default) - - -def _dataclass_field_default(f: dataclasses.Field) -> str: - if f.default is not dataclasses.MISSING: - return _default(f.default) - if f.default_factory is not dataclasses.MISSING: # type: ignore[misc] - return _factory_default(f.default_factory) - return "required" - - -def _model_fields(cls: type, doc: Doc) -> list[Param]: - return [ - Param( - fname, - _ann(finfo.annotation), - _model_field_default(finfo), - finfo.description or doc.attributes.get(fname, ""), - ) - for fname, finfo in cls.model_fields.items() - ] - - -def _dataclass_fields(cls: type, doc: Doc) -> list[Param]: - return [ - Param(f.name, _ann(f.type), _dataclass_field_default(f), doc.attributes.get(f.name, "")) - for f in dataclasses.fields(cls) - ] - - -def _enum_values(cls: type) -> list[tuple[str, str]]: - return [(m.name, repr(m.value)) for m in cls] - - -def _member_names(cls: type, kind: str, members: list[str] | None) -> list[str]: - if members is not None: - return members - return [] if kind == "enum" else _public_members(cls) - - -def _fill_class(api: ApiObject, cls: type, repo: Path, members: list[str] | None) -> None: - kind = _class_kind(cls) - init_doc = parse_docstring(inspect.getdoc(cls.__init__)) if "__init__" in vars(cls) else Doc() - doc = api.doc - api.doc = Doc(**{**doc.__dict__, "params": {**init_doc.params, **doc.params}}) - api.name, api.kind, api.signature = cls.__name__, kind, cls.__name__ - if kind in ("class", "dataclass"): - init_sig = _signature(cls) - api.params = _params(init_sig, api.doc, skip_self=False) if init_sig else [] - api.signature = format_signature(cls.__name__, api.params, "") - api.bases = [b.__name__ for b in cls.__bases__ if b is not object] - if kind == "enum": - api.enum_values = _enum_values(cls) - fields_of = {"model": _model_fields, "dataclass": _dataclass_fields}.get(kind) - if fields_of: - api.fields = fields_of(cls, doc) - for name in _member_names(cls, kind, members): - m = _member(cls, name, repo) - if m is not None: - api.members.append(m) - - -def _fill_function(api: ApiObject, fn: Any) -> None: - sig = _signature(fn) - api.params = _params(sig, api.doc, skip_self=False) - api.returns = _ann(sig.return_annotation) if sig else "" - api.kind = "async" if inspect.iscoroutinefunction(fn) else "function" - api.signature = format_signature(api.name, api.params, api.returns) - - -def load(path: str, repo: Path, members: list[str] | None = None) -> ApiObject: - obj, module_name, attr = _resolve(path) - file, line = _source(obj, repo) - if inspect.ismodule(obj): - doc = parse_docstring(obj.__doc__) - return ApiObject( - path, attr, module_name, "module", path, doc, [], "", source_file=file, source_line=1 - ) - real_module = getattr(obj, "__module__", module_name) or module_name - doc = parse_docstring(inspect.getdoc(obj)) - api = ApiObject( - path, attr, real_module, "function", attr, doc, [], "", source_file=file, source_line=line - ) - if inspect.isclass(obj): - _fill_class(api, obj, repo, members) - else: - _fill_function(api, obj) - return api diff --git a/site/docsgen/gen_changelog.py b/site/docsgen/gen_changelog.py deleted file mode 100644 index 16de0a5..0000000 --- a/site/docsgen/gen_changelog.py +++ /dev/null @@ -1,54 +0,0 @@ -"""CHANGELOG.md split into releases and labelled blocks (Added, Changed, Fixed...).""" - -from __future__ import annotations - -import re -from dataclasses import dataclass, field -from pathlib import Path - -_RELEASE_RE = re.compile(r"^##\s+(.+?)\s*$") -_VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)\s*(?:\((\d{4}-\d{2}-\d{2})\))?") -_LABEL_RE = re.compile(r"^([A-Z][A-Za-z ]{2,30}):\s*$") - - -@dataclass -class Release: - title: str - version: str - date: str - anchor: str - intro: str - blocks: list[tuple[str, str]] = field(default_factory=list) - - -def _split_blocks(lines: list[str]) -> tuple[str, list[tuple[str, str]]]: - """Split a release body at ``Label:`` lines that sit outside code fences.""" - intro: list[str] = [] - blocks: list[tuple[str, list[str]]] = [] - fence = False - for line in lines: - if line.lstrip().startswith("```"): - fence = not fence - lm = _LABEL_RE.match(line) if not fence else None - if lm: - blocks.append((lm.group(1), [])) - continue - (blocks[-1][1] if blocks else intro).append(line) - return "\n".join(intro).strip(), [(k, "\n".join(v).strip()) for k, v in blocks] - - -def _release(part: str) -> Release: - lines = part.rstrip().split("\n") - title = _RELEASE_RE.match(lines[0]).group(1) - intro, blocks = _split_blocks(lines[1:]) - m = _VERSION_RE.match(title) - if m: - return Release(title, m.group(1), m.group(2) or "", m.group(1), intro, blocks) - anchor = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") - return Release(title, title, "", anchor, intro, blocks) - - -def parse(path: Path) -> tuple[str, list[Release]]: - parts = re.split(r"(?m)^(?=## )", path.read_text()) - head = re.sub(r"(?m)^#\s+.*$", "", parts[0]).strip() - return head, [_release(part) for part in parts[1:]] diff --git a/site/docsgen/highlight.py b/site/docsgen/highlight.py deleted file mode 100644 index ddc1dc4..0000000 --- a/site/docsgen/highlight.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Code blocks rendered as editor or terminal windows. - -Pygments does the tokenising; the HTML is built here, one ```` -per source line, so line numbers, highlighted lines and terminal prompts can be -drawn with CSS and stay out of the copied text. -""" - -from __future__ import annotations - -import re -from html import escape - -from pygments.lexers import get_lexer_by_name -from pygments.token import STANDARD_TYPES, Token -from pygments.util import ClassNotFound - -from docsgen.icons import icon - -TERMINAL_LANGS = {"bash", "sh", "shell", "zsh", "shell-session-commands"} -SESSION_LANGS = {"console", "shell-session"} -LANG_LABELS = { - "python": "Python", - "py": "Python", - "yaml": "YAML", - "yml": "YAML", - "json": "JSON", - "jsonc": "JSON", - "toml": "TOML", - "ini": "INI", - "dockerfile": "Dockerfile", - "docker": "Dockerfile", - "text": "Text", - "txt": "Text", - "sparql": "SPARQL", - "turtle": "Turtle", - "ttl": "Turtle", - "xml": "XML", - "html": "HTML", - "javascript": "JavaScript", - "js": "JavaScript", - "typescript": "TypeScript", - "hcl": "HCL", - "terraform": "HCL", - "sql": "SQL", - "powershell": "PowerShell", - "ps1": "PowerShell", - "diff": "Diff", - "jsonl": "JSON Lines", - "csv": "CSV", -} -_ATTR_RE = re.compile(r'(\w+)=("([^"]*)"|\'([^\']*)\'|(\S+))') - - -def parse_info(info: str) -> tuple[str, dict[str, str]]: - """Split a fence info string into language and attributes. - - ``python title="a.py" hl="2-4"`` gives ``("python", {"title": "a.py", "hl": "2-4"})``. - A bare ``{2-4}`` is accepted as ``hl`` too. - """ - info = info.strip() - if not info: - return "", {} - lang, _, rest = info.partition(" ") - attrs: dict[str, str] = {} - for m in _ATTR_RE.finditer(rest): - attrs[m.group(1)] = next(g for g in (m.group(3), m.group(4), m.group(5)) if g is not None) - brace = re.search(r"\{([\d,\-\s]+)\}", rest) - if brace and "hl" not in attrs: - attrs["hl"] = brace.group(1) - return lang.lower(), attrs - - -def parse_ranges(spec: str) -> set[int]: - out: set[int] = set() - for part in spec.replace(" ", "").split(","): - if not part: - continue - if "-" in part: - a, b = part.split("-", 1) - out.update(range(int(a), int(b) + 1)) - else: - out.add(int(part)) - return out - - -def _css_class(ttype) -> str: - while ttype not in STANDARD_TYPES and ttype is not Token: - ttype = ttype.parent - return STANDARD_TYPES.get(ttype, "") - - -def _lexer(lang: str): - alias = { - "jsonl": "json", - "yml": "yaml", - "docker": "dockerfile", - "ttl": "turtle", - "terraform": "hcl", - } - try: - return get_lexer_by_name(alias.get(lang, lang) or "text", stripnl=False, ensurenl=False) - except ClassNotFound: - return get_lexer_by_name("text", stripnl=False, ensurenl=False) - - -def tokenize_lines(code: str, lang: str) -> list[str]: - """Return one HTML string per source line.""" - lines: list[str] = [""] - for ttype, value in _lexer(lang).get_tokens(code): - cls = _css_class(ttype) - parts = value.split("\n") - for i, part in enumerate(parts): - if i: - lines.append("") - if part: - text = escape(part, quote=False) - lines[-1] += f'{text}' if cls else text - if lines and lines[-1] == "" and code.endswith("\n"): - lines.pop() - return lines - - -def _terminal_roles(raw_lines: list[str]) -> list[str]: - roles = [] - continued = False - in_heredoc: str | None = None - for line in raw_lines: - stripped = line.strip() - if in_heredoc is not None: - roles.append("out") - if stripped == in_heredoc: - in_heredoc = None - continue - if continued: - roles.append("cont") - elif not stripped: - roles.append("blank") - elif stripped.startswith("#"): - roles.append("cmt") - else: - roles.append("cmd") - m = re.search(r"<<-?\s*'?\"?(\w+)'?\"?\s*$", stripped) - if m and roles[-1] in ("cmd", "cont"): - in_heredoc = m.group(1) - continued = stripped.endswith("\\") and roles[-1] != "cmt" - return roles - - -def _session_roles(raw_lines: list[str]) -> list[tuple[str, str]]: - """For console blocks: (role, line without prompt).""" - out = [] - continued = False - for line in raw_lines: - if continued: - out.append(("cont", line)) - elif line.startswith("$ "): - out.append(("cmd", line[2:])) - elif line.startswith("# ") and not out: - out.append(("cmt", line)) - else: - out.append(("out", line)) - continued = out[-1][0] in ("cmd", "cont") and line.rstrip().endswith("\\") - return out - - -def code_bar_label(lang: str, attrs: dict[str, str]) -> tuple[str, str]: - """Icon name and label shown in the editor bar.""" - if lang in TERMINAL_LANGS or lang in SESSION_LANGS: - return "terminal", attrs.get("title", "Terminal") - return "file-code", attrs.get("title", LANG_LABELS.get(lang, lang.upper() if lang else "Text")) - - -def _session_lines(code: str, lang: str) -> list[tuple[str, str]]: # noqa: ARG001 - """Console transcript: commands highlighted as bash, output left plain.""" - lines = [] - for role, text in _session_roles(code.rstrip("\n").split("\n")): - if role in ("cmd", "cont"): - inner = tokenize_lines(text + "\n", "bash")[0] - else: - inner = escape(text, quote=False) - lines.append((role, inner)) - return lines - - -def _terminal_lines(code: str, lang: str) -> list[tuple[str, str]]: # noqa: ARG001 - raw = code.rstrip("\n").split("\n") - toks = tokenize_lines(code, "bash") - toks += [""] * (len(raw) - len(toks)) - return list(zip(_terminal_roles(raw), toks)) - - -def _editor_lines(code: str, lang: str) -> list[tuple[str, str]]: - return [("", t) for t in tokenize_lines(code, lang)] - - -def _line_span(n: int, role: str, inner: str, hl: set[int]) -> str: - classes = ["line"] - if role: - classes.append(role) - if n in hl: - classes.append("hl") - return f'{inner or " "}' - - -def render_code_body(code: str, lang: str, attrs: dict[str, str]) -> tuple[str, str]: - """Render the ``
    `` for a block. Returns (variant, html)."""
-    code = code.rstrip("\n") + "\n"
-    hl = parse_ranges(attrs["hl"]) if attrs.get("hl") else set()
-    if lang in SESSION_LANGS:
-        variant, lines_of = "terminal", _session_lines
-    elif lang in TERMINAL_LANGS:
-        variant, lines_of = "terminal", _terminal_lines
-    else:
-        variant, lines_of = "editor", _editor_lines
-    out = [
-        _line_span(n, role, inner, hl)
-        for n, (role, inner) in enumerate(lines_of(code, lang), start=1)
-    ]
-    lang_class = escape(lang or "text")
-    pre = (
-        f'
    ' + "\n".join(out) + "
    "
-    )
-    return variant, pre
-
-
-def render_code_block(code: str, info: str) -> str:
-    lang, attrs = parse_info(info)
-    variant, pre = render_code_body(code, lang, attrs)
-    ico, label = code_bar_label(lang, attrs)
-    lang_tag = (
-        LANG_LABELS.get(lang, lang) if lang and "title" in attrs and variant == "editor" else ""
-    )
-    bar = (
-        f'
    {icon(ico, 14)}{escape(label)}'
-        + (f'{escape(lang_tag)}' if lang_tag else "")
-        + copy_button()
-        + "
    "
-    )
-    return f'
    {bar}{pre}
    '
-
-
-def copy_button() -> str:
-    return (
-        f''
-    )
-
-
-def render_tab_group(blocks: list[tuple[str, str]]) -> str:
-    """Blocks are (code, info) pairs that all carry a ``tab`` attribute."""
-    tabs, panes = [], []
-    for i, (code, info) in enumerate(blocks):
-        lang, attrs = parse_info(info)
-        label = attrs.get("tab", f"Tab {i + 1}")
-        variant, pre = render_code_body(code, lang, attrs)
-        selected = "true" if i == 0 else "false"
-        ico = icon("terminal", 14) if variant == "terminal" else ""
-        tabs.append(
-            f''
-        )
-        sub = ""
-        if "title" in attrs:
-            sub_ico = "terminal" if variant == "terminal" else "file-code"
-            sub = f'
    {icon(sub_ico, 13)}{escape(attrs["title"])}
    '
-        hidden = "" if i == 0 else " hidden"
-        panes.append(
-            f'
    {sub}{pre}
    '
-        )
-    return (
-        '
    '
-        f'
    {"".join(tabs)}{copy_button()}
    '
-        f"{''.join(panes)}
    "
-    )
diff --git a/site/docsgen/icons.py b/site/docsgen/icons.py
deleted file mode 100644
index 44ef0c0..0000000
--- a/site/docsgen/icons.py
+++ /dev/null
@@ -1,63 +0,0 @@
-"""Lucide icons used by the site, inlined as SVG (stroke 2, currentColor)."""
-
-_PATHS = {
-    "search": '',
-    "chevron-down": '',
-    "github": (
-        ''
-    ),
-    "moon": '',
-    "sun": (
-        ''
-        ''
-        ''
-    ),
-    "pencil": (
-        ''
-    ),
-    "info": '',
-    "lightbulb": (
-        ''
-    ),
-    "triangle-alert": (
-        ''
-        ''
-    ),
-    "octagon-alert": (
-        ''
-    ),
-    "copy": '',
-    "check": '',
-    "arrow-left": '',
-    "arrow-right": '',
-    "terminal": '',
-    "corner-down-left": '',
-    "file-code": (
-        ''
-    ),
-    "menu": '',
-    "x": '',
-    "external": (
-        ''
-    ),
-    "list": (
-        ''
-        ''
-    ),
-}
-
-
-def icon(name: str, size: int = 16, cls: str = "") -> str:
-    attrs = f' class="{cls}"' if cls else ""
-    return (
-        f'