From 628bb8b29721123757f931bc4606185e6c2f8349 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 19 Jan 2026 15:51:14 +0500 Subject: [PATCH 001/640] Fix missing instance lock in delete_fleets (#3471) * Fix missing instance lock in delete_fleets * Handle terminating deleted instances * Fix comment * Fix log message --- .../background/tasks/process_instances.py | 12 ++-- .../_internal/server/services/fleets.py | 59 ++++++++++++------- .../tasks/test_process_instances.py | 28 +++++++++ 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/dstack/_internal/server/background/tasks/process_instances.py b/src/dstack/_internal/server/background/tasks/process_instances.py index 9a14bdc30d..454d6ee18a 100644 --- a/src/dstack/_internal/server/background/tasks/process_instances.py +++ b/src/dstack/_internal/server/background/tasks/process_instances.py @@ -11,7 +11,7 @@ from pydantic import ValidationError from sqlalchemy import and_, delete, func, not_, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import joinedload, with_loader_criteria +from sqlalchemy.orm import joinedload from dstack._internal import settings from dstack._internal.core.backends.base.compute import ( @@ -218,9 +218,8 @@ async def _process_instance(session: AsyncSession, instance: InstanceModel): .options(joinedload(InstanceModel.project).joinedload(ProjectModel.backends)) .options(joinedload(InstanceModel.jobs).load_only(JobModel.id, JobModel.status)) .options( - joinedload(InstanceModel.fleet).joinedload(FleetModel.instances), - with_loader_criteria( - InstanceModel, InstanceModel.deleted == False, include_aliases=True + joinedload(InstanceModel.fleet).joinedload( + FleetModel.instances.and_(InstanceModel.deleted == False) ), ) .execution_options(populate_existing=True) @@ -233,9 +232,8 @@ async def _process_instance(session: AsyncSession, instance: InstanceModel): .options(joinedload(InstanceModel.project)) .options(joinedload(InstanceModel.jobs).load_only(JobModel.id, JobModel.status)) .options( - joinedload(InstanceModel.fleet).joinedload(FleetModel.instances), - with_loader_criteria( - InstanceModel, InstanceModel.deleted == False, include_aliases=True + joinedload(InstanceModel.fleet).joinedload( + FleetModel.instances.and_(InstanceModel.deleted == False) ), ) .execution_options(populate_existing=True) diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 95ae519d07..588f34698d 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -42,7 +42,12 @@ ) from dstack._internal.core.models.projects import Project from dstack._internal.core.models.resources import ResourcesSpec -from dstack._internal.core.models.runs import JobProvisioningData, Requirements, get_policy_map +from dstack._internal.core.models.runs import ( + JobProvisioningData, + Requirements, + RunStatus, + get_policy_map, +) from dstack._internal.core.models.users import GlobalRole from dstack._internal.core.services import validate_dstack_resource_name from dstack._internal.core.services.diff import ModelDiff, copy_model, diff_models @@ -53,6 +58,7 @@ JobModel, MemberModel, ProjectModel, + RunModel, UserModel, ) from dstack._internal.server.services import events @@ -613,48 +619,61 @@ async def delete_fleets( instance_nums: Optional[List[int]] = None, ): res = await session.execute( - select(FleetModel) + select(FleetModel.id) .where( FleetModel.project_id == project.id, FleetModel.name.in_(names), FleetModel.deleted == False, ) - .options(joinedload(FleetModel.instances)) + .order_by(FleetModel.id) # take locks in order + .with_for_update(key_share=True) ) - fleet_models = res.scalars().unique().all() - fleets_ids = sorted([f.id for f in fleet_models]) - instances_ids = sorted([i.id for f in fleet_models for i in f.instances]) - await session.commit() - logger.info("Deleting fleets: %s", [v.name for v in fleet_models]) + fleets_ids = list(res.scalars().unique().all()) + res = await session.execute( + select(InstanceModel.id) + .where( + InstanceModel.fleet_id.in_(fleets_ids), + InstanceModel.deleted == False, + ) + .order_by(InstanceModel.id) # take locks in order + .with_for_update(key_share=True) + ) + instances_ids = list(res.scalars().unique().all()) + if is_db_sqlite(): + # Start new transaction to see committed changes after lock + await session.commit() async with ( get_locker(get_db().dialect_name).lock_ctx(FleetModel.__tablename__, fleets_ids), get_locker(get_db().dialect_name).lock_ctx(InstanceModel.__tablename__, instances_ids), ): - # Refetch after lock - # TODO: Lock instances with FOR UPDATE? - # TODO: Do not lock fleet when deleting only instances + # Refetch after lock. + # TODO: Do not lock fleet when deleting only instances. res = await session.execute( select(FleetModel) - .where( - FleetModel.project_id == project.id, - FleetModel.name.in_(names), - FleetModel.deleted == False, - ) + .where(FleetModel.id.in_(fleets_ids)) .options( - selectinload(FleetModel.instances) + joinedload(FleetModel.instances.and_(InstanceModel.id.in_(instances_ids))) .joinedload(InstanceModel.jobs) .load_only(JobModel.id) ) - .options(selectinload(FleetModel.runs)) + .options( + joinedload( + FleetModel.runs.and_(RunModel.status.not_in(RunStatus.finished_statuses())) + ) + ) .execution_options(populate_existing=True) - .order_by(FleetModel.id) # take locks in order - .with_for_update(key_share=True) ) fleet_models = res.scalars().unique().all() fleets = [fleet_model_to_fleet(m) for m in fleet_models] for fleet in fleets: if fleet.spec.configuration.ssh_config is not None: _check_can_manage_ssh_fleets(user=user, project=project) + if instance_nums is None: + logger.info("Deleting fleets: %s", [f.name for f in fleet_models]) + else: + logger.info( + "Deleting fleets %s instances %s", [f.name for f in fleet_models], instance_nums + ) for fleet_model in fleet_models: _terminate_fleet_instances(fleet_model=fleet_model, instance_nums=instance_nums) # TERMINATING fleets are deleted by process_fleets after instances are terminated diff --git a/src/tests/_internal/server/background/tasks/test_process_instances.py b/src/tests/_internal/server/background/tasks/test_process_instances.py index a72dc0c165..38bffc4421 100644 --- a/src/tests/_internal/server/background/tasks/test_process_instances.py +++ b/src/tests/_internal/server/background/tasks/test_process_instances.py @@ -597,6 +597,34 @@ async def test_terminate(self, test_db, session: AsyncSession): assert instance.deleted_at is not None assert instance.finished_at is not None + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_terminates_terminating_deleted_instance(self, test_db, session: AsyncSession): + # There was a race condition when instance could stay in Terminating while marked as deleted. + # TODO: Drop this after all such "bad" instances are processed. + project = await create_project(session=session) + instance = await create_instance( + session=session, project=project, status=InstanceStatus.TERMINATING + ) + instance.deleted = True + instance.termination_reason = InstanceTerminationReason.IDLE_TIMEOUT + instance.last_job_processed_at = instance.deleted_at = ( + get_current_datetime() + dt.timedelta(minutes=-19) + ) + await session.commit() + + with self.mock_terminate_in_backend() as mock: + await process_instances() + mock.assert_called_once() + + await session.refresh(instance) + + assert instance is not None + assert instance.status == InstanceStatus.TERMINATED + assert instance.deleted == True + assert instance.deleted_at is not None + assert instance.finished_at is not None + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( From a07ef352779fad5aa2fdc0334314becc940ddd47 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 19 Jan 2026 17:16:21 +0500 Subject: [PATCH 002/640] Optimize list and get fleets (#3472) * Do not include deleted instances when listing fleets * Do not include deleted instances when getting fleet * Optimize select in generate_volume_name --- src/dstack/_internal/server/routers/fleets.py | 2 ++ .../_internal/server/services/fleets.py | 27 ++++++++++++------- .../_internal/server/services/volumes.py | 9 +++++-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/dstack/_internal/server/routers/fleets.py b/src/dstack/_internal/server/routers/fleets.py index d423134675..a436d1123a 100644 --- a/src/dstack/_internal/server/routers/fleets.py +++ b/src/dstack/_internal/server/routers/fleets.py @@ -47,6 +47,7 @@ async def list_fleets( """ Returns all fleets and instances within them visible to user sorted by descending `created_at`. `project_name` and `only_active` can be specified as filters. + Includes only active fleet instances. To list all fleet instances, use `/api/instances/list`. The results are paginated. To get the next page, pass `created_at` and `id` of the last fleet from the previous page as `prev_created_at` and `prev_id`. @@ -72,6 +73,7 @@ async def list_project_fleets( ): """ Returns all fleets in the project. + Includes only active fleet instances. To list all fleet instances, use `/api/instances/list`. """ _, project = user_project return CustomORJSONResponse( diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 588f34698d..19e4a77e64 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -180,9 +180,7 @@ async def list_fleets( limit=limit, ascending=ascending, ) - return [ - fleet_model_to_fleet(v, include_deleted_instances=not only_active) for v in fleet_models - ] + return [fleet_model_to_fleet(v) for v in fleet_models] async def list_projects_fleet_models( @@ -227,7 +225,7 @@ async def list_projects_fleet_models( .where(*filters) .order_by(*order_by) .limit(limit) - .options(joinedload(FleetModel.instances)) + .options(joinedload(FleetModel.instances.and_(InstanceModel.deleted == False))) ) fleet_models = list(res.unique().scalars().all()) return fleet_models @@ -256,7 +254,9 @@ async def list_project_fleet_models( if not include_deleted: filters.append(FleetModel.deleted == False) res = await session.execute( - select(FleetModel).where(*filters).options(joinedload(FleetModel.instances)) + select(FleetModel) + .where(*filters) + .options(joinedload(FleetModel.instances.and_(InstanceModel.deleted == False))) ) return list(res.unique().scalars().all()) @@ -293,7 +293,9 @@ async def get_project_fleet_model_by_id( FleetModel.project_id == project.id, ] res = await session.execute( - select(FleetModel).where(*filters).options(joinedload(FleetModel.instances)) + select(FleetModel) + .where(*filters) + .options(joinedload(FleetModel.instances.and_(InstanceModel.deleted == False))) ) return res.unique().scalar_one_or_none() @@ -311,7 +313,9 @@ async def get_project_fleet_model_by_name( if not include_deleted: filters.append(FleetModel.deleted == False) res = await session.execute( - select(FleetModel).where(*filters).options(joinedload(FleetModel.instances)) + select(FleetModel) + .where(*filters) + .options(joinedload(FleetModel.instances.and_(InstanceModel.deleted == False))) ) return res.unique().scalar_one_or_none() @@ -717,8 +721,13 @@ def get_fleet_spec(fleet_model: FleetModel) -> FleetSpec: async def generate_fleet_name(session: AsyncSession, project: ProjectModel) -> str: - fleet_models = await list_project_fleet_models(session=session, project=project) - names = {v.name for v in fleet_models} + res = await session.execute( + select(FleetModel.name).where( + FleetModel.project_id == project.id, + FleetModel.deleted == False, + ) + ) + names = set(res.scalars().all()) while True: name = random_names.generate_name() if name not in names: diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index fa3471192d..eb8f4bab64 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -380,8 +380,13 @@ def instance_model_to_volume_instance(instance_model: InstanceModel) -> VolumeIn async def generate_volume_name(session: AsyncSession, project: ProjectModel) -> str: - volume_models = await list_project_volume_models(session=session, project=project) - names = {v.name for v in volume_models} + res = await session.execute( + select(VolumeModel.name).where( + VolumeModel.project_id == project.id, + VolumeModel.deleted == False, + ) + ) + names = set(res.scalars().all()) while True: name = random_names.generate_name() if name not in names: From 811643f5fa72916bef49faec9cb4e0ca61968863 Mon Sep 17 00:00:00 2001 From: Alexander <4584443+DragonStuff@users.noreply.github.com> Date: Tue, 20 Jan 2026 16:09:44 +0900 Subject: [PATCH 003/640] feat(logging): add fluent-bit log shipping (#3431) * feat(logging): add fluent-bit log shipping Implements #3430. This PR is partially implemented using Cursor. * Fix pyright errors by using try/except/else pattern for optional imports * refactor(fluentbit): cleanup protocol lambdas and address codex comments * feat(fluentbit): validate next_token format and raise ServerClientError for malformed tokens * chore(fluentbit): address quick comments * feat(fluentbit): add tag prefix support to HTTPFluentBitWriter --- docs/docs/guides/server-deployment.md | 80 ++- docs/docs/reference/environment-variables.md | 7 + pyproject.toml | 7 +- .../server/services/logs/__init__.py | 24 + .../server/services/logs/fluentbit.py | 338 +++++++++ src/dstack/_internal/server/settings.py | 9 + .../server/services/test_fluentbit_logs.py | 659 ++++++++++++++++++ 7 files changed, 1120 insertions(+), 4 deletions(-) create mode 100644 src/dstack/_internal/server/services/logs/fluentbit.py create mode 100644 src/tests/_internal/server/services/test_fluentbit_logs.py diff --git a/docs/docs/guides/server-deployment.md b/docs/docs/guides/server-deployment.md index f1d7546d77..dc5093f2f2 100644 --- a/docs/docs/guides/server-deployment.md +++ b/docs/docs/guides/server-deployment.md @@ -159,7 +159,7 @@ $ DSTACK_DATABASE_URL=postgresql+asyncpg://user:password@db-host:5432/dstack dst By default, `dstack` stores workload logs locally in `~/.dstack/server/projects//logs`. For multi-replica server deployments, it's required to store logs externally. -`dstack` supports storing logs using AWS CloudWatch or GCP Logging. +`dstack` supports storing logs using AWS CloudWatch, GCP Logging, or Fluent-bit with Elasticsearch / Opensearch. ### AWS CloudWatch @@ -222,6 +222,78 @@ To store logs using GCP Logging, set the `DSTACK_SERVER_GCP_LOGGING_PROJECT` env +### Fluent-bit + +To store logs using Fluent-bit, set the `DSTACK_SERVER_FLUENTBIT_HOST` environment variable. +Fluent-bit supports two modes depending on how you want to access logs. + +=== "Full mode" + + Logs are shipped to Fluent-bit and can be read back through the dstack UI and CLI via Elasticsearch or OpenSearch. + Use this mode when you want a complete integration with log viewing in dstack: + + ```shell + $ DSTACK_SERVER_FLUENTBIT_HOST=fluentbit.example.com \ + DSTACK_SERVER_ELASTICSEARCH_HOST=https://elasticsearch.example.com:9200 \ + dstack server + ``` + +=== "Ship-only mode" + + Logs are forwarded to Fluent-bit but cannot be read through `dstack`. + The dstack UI/CLI will show empty logs. Use this mode when: + + - You have an existing logging infrastructure (Kibana, Grafana, Datadog, etc.) + - You only need to forward logs without reading them back through dstack + - You want to reduce operational complexity by not running Elasticsearch/OpenSearch + + ```shell + $ DSTACK_SERVER_FLUENTBIT_HOST=fluentbit.example.com \ + dstack server + ``` + +??? info "Additional configuration" + The following optional environment variables can be used to customize the Fluent-bit integration: + + **Fluent-bit settings:** + + - `DSTACK_SERVER_FLUENTBIT_PORT` – The Fluent-bit port. Defaults to `24224`. + - `DSTACK_SERVER_FLUENTBIT_PROTOCOL` – The protocol to use: `forward` or `http`. Defaults to `forward`. + - `DSTACK_SERVER_FLUENTBIT_TAG_PREFIX` – The tag prefix for logs. Defaults to `dstack`. + + **Elasticsearch/OpenSearch settings (for full mode only):** + + - `DSTACK_SERVER_ELASTICSEARCH_HOST` – The Elasticsearch/OpenSearch host for reading logs. If not set, runs in ship-only mode. + - `DSTACK_SERVER_ELASTICSEARCH_INDEX` – The Elasticsearch/OpenSearch index pattern. Defaults to `dstack-logs`. + - `DSTACK_SERVER_ELASTICSEARCH_API_KEY` – The Elasticsearch/OpenSearch API key for authentication. + +??? info "Fluent-bit configuration" + Configure Fluent-bit to receive logs and forward them to Elasticsearch or OpenSearch. Example configuration: + + ```ini + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + + [OUTPUT] + Name es + Match dstack.* + Host elasticsearch.example.com + Port 9200 + Index dstack-logs + Suppress_Type_Name On + ``` + +??? info "Required dependencies" + To use Fluent-bit log storage, install the `fluentbit` extras: + + ```shell + $ pip install "dstack[all]" -U + # or + $ pip install "dstack[fluentbit]" -U + ``` + ## File storage When using [files](../concepts/dev-environments.md#files) or [repos](../concepts/dev-environments.md#repos), `dstack` uploads local files and diffs to the server so that you can have access to them within runs. By default, the files are stored in the DB and each upload is limited to 2MB. You can configure an object storage to be used for uploads and increase the default limit by setting the `DSTACK_SERVER_CODE_UPLOAD_LIMIT` environment variable @@ -426,8 +498,10 @@ If a deployment is stuck due to a deadlock when applying DB migrations, try scal ??? info "Can I run multiple replicas of dstack server?" - Yes, you can if you configure `dstack` to use [PostgreSQL](#postgresql) and [AWS CloudWatch](#aws-cloudwatch). + Yes, you can if you configure `dstack` to use [PostgreSQL](#postgresql) and an external log storage + such as [AWS CloudWatch](#aws-cloudwatch), [GCP Logging](#gcp-logging), or [Fluent-bit](#fluent-bit). ??? info "Does dstack server support blue-green or rolling deployments?" - Yes, it does if you configure `dstack` to use [PostgreSQL](#postgresql) and [AWS CloudWatch](#aws-cloudwatch). + Yes, it does if you configure `dstack` to use [PostgreSQL](#postgresql) and an external log storage + such as [AWS CloudWatch](#aws-cloudwatch), [GCP Logging](#gcp-logging), or [Fluent-bit](#fluent-bit). diff --git a/docs/docs/reference/environment-variables.md b/docs/docs/reference/environment-variables.md index 4575f1b8f8..62ce97cd12 100644 --- a/docs/docs/reference/environment-variables.md +++ b/docs/docs/reference/environment-variables.md @@ -113,6 +113,13 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_SERVER_CLOUDWATCH_LOG_GROUP`{ #DSTACK_SERVER_CLOUDWATCH_LOG_GROUP } – The CloudWatch Logs group for storing workloads logs. If not set, the default file-based log storage is used. - `DSTACK_SERVER_CLOUDWATCH_LOG_REGION`{ #DSTACK_SERVER_CLOUDWATCH_LOG_REGION } – The CloudWatch Logs region. Defaults to `None`. - `DSTACK_SERVER_GCP_LOGGING_PROJECT`{ #DSTACK_SERVER_GCP_LOGGING_PROJECT } – The GCP Logging project for storing workloads logs. If not set, the default file-based log storage is used. +- `DSTACK_SERVER_FLUENTBIT_HOST`{ #DSTACK_SERVER_FLUENTBIT_HOST } – The Fluent-bit host for log forwarding. If set, enables Fluent-bit log storage. +- `DSTACK_SERVER_FLUENTBIT_PORT`{ #DSTACK_SERVER_FLUENTBIT_PORT } – The Fluent-bit port. Defaults to `24224`. +- `DSTACK_SERVER_FLUENTBIT_PROTOCOL`{ #DSTACK_SERVER_FLUENTBIT_PROTOCOL } – The protocol to use: `forward` or `http`. Defaults to `forward`. +- `DSTACK_SERVER_FLUENTBIT_TAG_PREFIX`{ #DSTACK_SERVER_FLUENTBIT_TAG_PREFIX } – The tag prefix for logs. Defaults to `dstack`. +- `DSTACK_SERVER_ELASTICSEARCH_HOST`{ #DSTACK_SERVER_ELASTICSEARCH_HOST } – The Elasticsearch/OpenSearch host for reading logs back through dstack. Optional; if not set, Fluent-bit runs in ship-only mode (logs are forwarded but not readable through dstack UI/CLI). +- `DSTACK_SERVER_ELASTICSEARCH_INDEX`{ #DSTACK_SERVER_ELASTICSEARCH_INDEX } – The Elasticsearch/OpenSearch index pattern. Defaults to `dstack-logs`. +- `DSTACK_SERVER_ELASTICSEARCH_API_KEY`{ #DSTACK_SERVER_ELASTICSEARCH_API_KEY } – The Elasticsearch/OpenSearch API key for authentication. - `DSTACK_ENABLE_PROMETHEUS_METRICS`{ #DSTACK_ENABLE_PROMETHEUS_METRICS } — Enables Prometheus metrics collection and export. - `DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE`{ #DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE } – Request body size limit for services running with a gateway, in bytes. Defaults to 64 MiB. - `DSTACK_SERVICE_CLIENT_TIMEOUT`{ #DSTACK_SERVICE_CLIENT_TIMEOUT } – Timeout in seconds for HTTP requests sent from the in-server proxy and gateways to service replicas. Defaults to 60. diff --git a/pyproject.toml b/pyproject.toml index a8d635e7c9..2fe97f2cbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -215,6 +215,11 @@ nebius = [ "nebius>=0.3.4,<0.4; python_version >= '3.10'", "dstack[server]", ] +fluentbit = [ + "fluent-logger>=0.10.0", + "elasticsearch>=8.0.0", + "dstack[server]", +] all = [ - "dstack[gateway,server,aws,azure,gcp,verda,kubernetes,lambda,nebius,oci]", + "dstack[gateway,server,aws,azure,gcp,verda,kubernetes,lambda,nebius,oci,fluentbit]", ] diff --git a/src/dstack/_internal/server/services/logs/__init__.py b/src/dstack/_internal/server/services/logs/__init__.py index 5b06ff4ad2..1f8565d49c 100644 --- a/src/dstack/_internal/server/services/logs/__init__.py +++ b/src/dstack/_internal/server/services/logs/__init__.py @@ -8,6 +8,7 @@ from dstack._internal.server.schemas.logs import PollLogsRequest from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent from dstack._internal.server.services.logs import aws as aws_logs +from dstack._internal.server.services.logs import fluentbit as fluentbit_logs from dstack._internal.server.services.logs import gcp as gcp_logs from dstack._internal.server.services.logs.base import ( LogStorage, @@ -57,6 +58,29 @@ def get_log_storage() -> LogStorage: logger.debug("Using GCP Logs storage") else: logger.error("Cannot use GCP Logs storage: GCP deps are not installed") + elif settings.SERVER_FLUENTBIT_HOST: + if fluentbit_logs.FLUENTBIT_AVAILABLE: + try: + _log_storage = fluentbit_logs.FluentBitLogStorage( + host=settings.SERVER_FLUENTBIT_HOST, + port=settings.SERVER_FLUENTBIT_PORT, + protocol=settings.SERVER_FLUENTBIT_PROTOCOL, + tag_prefix=settings.SERVER_FLUENTBIT_TAG_PREFIX, + es_host=settings.SERVER_ELASTICSEARCH_HOST, + es_index=settings.SERVER_ELASTICSEARCH_INDEX, + es_api_key=settings.SERVER_ELASTICSEARCH_API_KEY, + ) + except LogStorageError as e: + logger.error("Failed to initialize Fluent-bit Logs storage: %s", e) + except Exception: + logger.exception("Got exception when initializing Fluent-bit Logs storage") + else: + if settings.SERVER_ELASTICSEARCH_HOST: + logger.debug("Using Fluent-bit Logs storage with Elasticsearch/OpenSearch") + else: + logger.debug("Using Fluent-bit Logs storage in ship-only mode") + else: + logger.error("Cannot use Fluent-bit Logs storage: fluent-logger is not installed") if _log_storage is None: _log_storage = FileLogStorage() logger.debug("Using file-based storage") diff --git a/src/dstack/_internal/server/services/logs/fluentbit.py b/src/dstack/_internal/server/services/logs/fluentbit.py new file mode 100644 index 0000000000..b45b2988d5 --- /dev/null +++ b/src/dstack/_internal/server/services/logs/fluentbit.py @@ -0,0 +1,338 @@ +from typing import List, Optional, Protocol +from uuid import UUID + +import httpx + +from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.logs import ( + JobSubmissionLogs, + LogEvent, + LogEventSource, + LogProducer, +) +from dstack._internal.server.models import ProjectModel +from dstack._internal.server.schemas.logs import PollLogsRequest +from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent +from dstack._internal.server.services.logs.base import ( + LogStorage, + LogStorageError, + unix_time_ms_to_datetime, +) +from dstack._internal.utils.common import batched +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +ELASTICSEARCH_AVAILABLE = True +try: + from elasticsearch import Elasticsearch + from elasticsearch.exceptions import ApiError, TransportError +except ImportError: + ELASTICSEARCH_AVAILABLE = False +else: + ElasticsearchError: tuple = (ApiError, TransportError) # type: ignore[misc] + + class ElasticsearchReader: + """Reads logs from Elasticsearch or OpenSearch.""" + + def __init__( + self, + host: str, + index: str, + api_key: Optional[str] = None, + ) -> None: + if api_key: + self._client = Elasticsearch(hosts=[host], api_key=api_key) + else: + self._client = Elasticsearch(hosts=[host]) + self._index = index + # Verify connection + try: + self._client.info() + except ElasticsearchError as e: + raise LogStorageError(f"Failed to connect to Elasticsearch/OpenSearch: {e}") from e + + def read( + self, + stream_name: str, + request: PollLogsRequest, + ) -> JobSubmissionLogs: + sort_order = "desc" if request.descending else "asc" + + query: dict = { + "bool": { + "must": [ + {"term": {"stream.keyword": stream_name}}, + ] + } + } + + if request.start_time: + query["bool"].setdefault("filter", []).append( + {"range": {"@timestamp": {"gt": request.start_time.isoformat()}}} + ) + if request.end_time: + query["bool"].setdefault("filter", []).append( + {"range": {"@timestamp": {"lt": request.end_time.isoformat()}}} + ) + + search_params: dict = { + "index": self._index, + "query": query, + "sort": [ + {"@timestamp": {"order": sort_order}}, + {"_id": {"order": sort_order}}, + ], + "size": request.limit, + } + + if request.next_token: + parts = request.next_token.split(":", 1) + if len(parts) != 2 or not parts[0] or not parts[1]: + raise ServerClientError( + f"Invalid next_token: {request.next_token}. " + "Must be in format 'timestamp:document_id'." + ) + search_params["search_after"] = [parts[0], parts[1]] + + try: + response = self._client.search(**search_params) + except ElasticsearchError as e: + logger.error("Elasticsearch/OpenSearch search error: %s", e) + raise LogStorageError(f"Elasticsearch/OpenSearch error: {e}") from e + + hits = response.get("hits", {}).get("hits", []) + logs = [] + last_sort_values = None + + for hit in hits: + source = hit.get("_source", {}) + timestamp_str = source.get("@timestamp") + message = source.get("message", "") + + if timestamp_str: + from datetime import datetime + + try: + timestamp = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00")) + except ValueError: + continue + else: + continue + + logs.append( + LogEvent( + timestamp=timestamp, + log_source=LogEventSource.STDOUT, + message=message, + ) + ) + + sort_values = hit.get("sort") + if sort_values and len(sort_values) >= 2: + last_sort_values = sort_values + + next_token = None + if len(logs) == request.limit and last_sort_values is not None: + next_token = f"{last_sort_values[0]}:{last_sort_values[1]}" + + return JobSubmissionLogs( + logs=logs, + next_token=next_token, + ) + + def close(self) -> None: + self._client.close() + + +FLUENTBIT_AVAILABLE = True +try: + from fluent import sender as fluent_sender +except ImportError: + FLUENTBIT_AVAILABLE = False +else: + + class FluentBitWriter(Protocol): + def write(self, tag: str, records: List[dict]) -> None: ... + def close(self) -> None: ... + + class LogReader(Protocol): + def read(self, stream_name: str, request: PollLogsRequest) -> JobSubmissionLogs: ... + def close(self) -> None: ... + + class HTTPFluentBitWriter: + """Writes logs to Fluent-bit via HTTP POST.""" + + def __init__(self, host: str, port: int, tag_prefix: str) -> None: + self._endpoint = f"http://{host}:{port}" + self._client = httpx.Client(timeout=30.0) + self._tag_prefix = tag_prefix + + def write(self, tag: str, records: List[dict]) -> None: + prefixed_tag = f"{self._tag_prefix}.{tag}" if self._tag_prefix else tag + for record in records: + try: + response = self._client.post( + f"{self._endpoint}/{prefixed_tag}", + json=record, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + except httpx.HTTPStatusError as e: + logger.error( + "Fluent-bit HTTP request failed with status %d: %s", + e.response.status_code, + e.response.text, + ) + raise LogStorageError( + f"Fluent-bit HTTP error: status {e.response.status_code}" + ) from e + except httpx.HTTPError as e: + logger.error("Failed to write log to Fluent-bit via HTTP: %s", e) + raise LogStorageError(f"Fluent-bit HTTP error: {e}") from e + + def close(self) -> None: + self._client.close() + + class ForwardFluentBitWriter: + """Writes logs to Fluent-bit using Forward protocol.""" + + def __init__(self, host: str, port: int, tag_prefix: str) -> None: + self._sender = fluent_sender.FluentSender(tag_prefix, host=host, port=port) + self._tag_prefix = tag_prefix + + def write(self, tag: str, records: List[dict]) -> None: + for record in records: + if not self._sender.emit(tag, record): + error = self._sender.last_error + logger.error("Failed to write log to Fluent-bit via Forward: %s", error) + self._sender.clear_last_error() + raise LogStorageError(f"Fluent-bit Forward error: {error}") + + def close(self) -> None: + self._sender.close() + + class NullLogReader: + """ + Null reader for ship-only mode (no Elasticsearch/OpenSearch configured). + + Returns empty logs. Useful when logs are shipped to an external system + that is accessed directly rather than through dstack. + """ + + def read(self, stream_name: str, request: PollLogsRequest) -> JobSubmissionLogs: + return JobSubmissionLogs(logs=[], next_token=None) + + def close(self) -> None: + pass + + class FluentBitLogStorage(LogStorage): + """ + Log storage using Fluent-bit for writing and optionally Elasticsearch/OpenSearch for reading. + + Supports two modes: + - Full mode: Writes to Fluent-bit and reads from Elasticsearch/OpenSearch + - Ship-only mode: Writes to Fluent-bit only (no reading, returns empty logs) + """ + + MAX_BATCH_SIZE = 100 + + def __init__( + self, + host: str, + port: int, + protocol: str, + tag_prefix: str, + es_host: Optional[str] = None, + es_index: str = "dstack-logs", + es_api_key: Optional[str] = None, + ) -> None: + self._tag_prefix = tag_prefix + + if protocol == "http": + self._writer: FluentBitWriter = HTTPFluentBitWriter( + host=host, port=port, tag_prefix=tag_prefix + ) + elif protocol == "forward": + self._writer = ForwardFluentBitWriter(host=host, port=port, tag_prefix=tag_prefix) + else: + raise LogStorageError(f"Unsupported Fluent-bit protocol: {protocol}") + + self._reader: LogReader + if es_host: + if not ELASTICSEARCH_AVAILABLE: + raise LogStorageError( + "Elasticsearch/OpenSearch host configured but elasticsearch package " + "is not installed. Install with: pip install elasticsearch" + ) + self._reader = ElasticsearchReader( + host=es_host, + index=es_index, + api_key=es_api_key, + ) + logger.debug( + "Fluent-bit log storage initialized with Elasticsearch/OpenSearch reader" + ) + else: + self._reader = NullLogReader() + logger.info( + "Fluent-bit log storage initialized in ship-only mode " + "(no Elasticsearch/OpenSearch configured for reading)" + ) + + def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs: + producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB + stream_name = self._get_stream_name( + project_name=project.name, + run_name=request.run_name, + job_submission_id=request.job_submission_id, + producer=producer, + ) + return self._reader.read(stream_name=stream_name, request=request) + + def write_logs( + self, + project: ProjectModel, + run_name: str, + job_submission_id: UUID, + runner_logs: List[RunnerLogEvent], + job_logs: List[RunnerLogEvent], + ) -> None: + producers_with_logs = [(LogProducer.RUNNER, runner_logs), (LogProducer.JOB, job_logs)] + for producer, producer_logs in producers_with_logs: + if not producer_logs: + continue + stream_name = self._get_stream_name( + project_name=project.name, + run_name=run_name, + job_submission_id=job_submission_id, + producer=producer, + ) + self._write_logs_to_stream(stream_name=stream_name, logs=producer_logs) + + def _write_logs_to_stream(self, stream_name: str, logs: List[RunnerLogEvent]) -> None: + for batch in batched(logs, self.MAX_BATCH_SIZE): + records = [] + for log in batch: + message = log.message.decode(errors="replace") + timestamp = unix_time_ms_to_datetime(log.timestamp) + records.append( + { + "message": message, + "@timestamp": timestamp.isoformat(), + "stream": stream_name, + } + ) + self._writer.write(tag=stream_name, records=records) + + def close(self) -> None: + try: + self._writer.close() + finally: + self._reader.close() + + def _get_stream_name( + self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer + ) -> str: + return f"{project_name}/{run_name}/{job_submission_id}/{producer.value}" diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 74d1d7b8d5..6e5c8e4bc1 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -78,6 +78,15 @@ SERVER_GCP_LOGGING_PROJECT = os.getenv("DSTACK_SERVER_GCP_LOGGING_PROJECT") +SERVER_FLUENTBIT_HOST = os.getenv("DSTACK_SERVER_FLUENTBIT_HOST") +SERVER_FLUENTBIT_PORT = int(os.getenv("DSTACK_SERVER_FLUENTBIT_PORT", "24224")) +SERVER_FLUENTBIT_PROTOCOL = os.getenv("DSTACK_SERVER_FLUENTBIT_PROTOCOL", "forward") +SERVER_FLUENTBIT_TAG_PREFIX = os.getenv("DSTACK_SERVER_FLUENTBIT_TAG_PREFIX", "dstack") + +SERVER_ELASTICSEARCH_HOST = os.getenv("DSTACK_SERVER_ELASTICSEARCH_HOST") +SERVER_ELASTICSEARCH_INDEX = os.getenv("DSTACK_SERVER_ELASTICSEARCH_INDEX", "dstack-logs") +SERVER_ELASTICSEARCH_API_KEY = os.getenv("DSTACK_SERVER_ELASTICSEARCH_API_KEY") + SERVER_METRICS_RUNNING_TTL_SECONDS = environ.get_int( "DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS", default=3600 ) diff --git a/src/tests/_internal/server/services/test_fluentbit_logs.py b/src/tests/_internal/server/services/test_fluentbit_logs.py new file mode 100644 index 0000000000..937838e016 --- /dev/null +++ b/src/tests/_internal/server/services/test_fluentbit_logs.py @@ -0,0 +1,659 @@ +from datetime import datetime, timezone +from unittest.mock import Mock, patch +from uuid import UUID + +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession + +from dstack._internal.core.errors import ServerClientError +from dstack._internal.server.models import ProjectModel +from dstack._internal.server.schemas.logs import PollLogsRequest +from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent +from dstack._internal.server.services.logs.base import LogStorageError +from dstack._internal.server.services.logs.fluentbit import ( + ELASTICSEARCH_AVAILABLE, + FLUENTBIT_AVAILABLE, +) +from dstack._internal.server.testing.common import create_project + +pytestmark = pytest.mark.skipif(not FLUENTBIT_AVAILABLE, reason="fluent-logger not installed") + +# Conditionally import classes that are only defined when FLUENTBIT_AVAILABLE is True +if FLUENTBIT_AVAILABLE: + from dstack._internal.server.services.logs.fluentbit import ( + FluentBitLogStorage, + ForwardFluentBitWriter, + HTTPFluentBitWriter, + NullLogReader, + ) + + if ELASTICSEARCH_AVAILABLE: + from dstack._internal.server.services.logs.fluentbit import ElasticsearchReader + + +class TestNullLogReader: + """Tests for the NullLogReader (ship-only mode).""" + + def test_read_returns_empty_logs(self): + reader = NullLogReader() + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + limit=100, + ) + result = reader.read("test-stream", request) + + assert result.logs == [] + assert result.next_token is None + + def test_close_does_nothing(self): + reader = NullLogReader() + reader.close() # Should not raise + + +class TestHTTPFluentBitWriter: + """Tests for the HTTPFluentBitWriter.""" + + @pytest.fixture + def mock_httpx_client(self): + with patch("dstack._internal.server.services.logs.fluentbit.httpx.Client") as mock: + yield mock.return_value + + def test_init_creates_client(self, mock_httpx_client): + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + assert writer._endpoint == "http://localhost:8080" + assert writer._tag_prefix == "dstack" + + def test_write_posts_records(self, mock_httpx_client): + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + records = [ + {"message": "Hello", "@timestamp": "2023-10-06T10:00:00+00:00"}, + {"message": "World", "@timestamp": "2023-10-06T10:00:01+00:00"}, + ] + writer.write(tag="test-tag", records=records) + + assert mock_httpx_client.post.call_count == 2 + mock_httpx_client.post.assert_any_call( + "http://localhost:8080/dstack.test-tag", + json=records[0], + headers={"Content-Type": "application/json"}, + ) + mock_httpx_client.post.assert_any_call( + "http://localhost:8080/dstack.test-tag", + json=records[1], + headers={"Content-Type": "application/json"}, + ) + + def test_write_calls_raise_for_status(self, mock_httpx_client): + """Test that response.raise_for_status() is called to detect non-2xx responses.""" + mock_response = Mock() + mock_httpx_client.post.return_value = mock_response + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + + writer.write(tag="test-tag", records=[{"message": "test"}]) + + mock_response.raise_for_status.assert_called_once() + + def test_write_raises_on_http_status_error(self, mock_httpx_client): + """Test that 4xx/5xx responses are properly detected and raise LogStorageError.""" + import httpx + + mock_response = Mock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + mock_httpx_client.post.return_value = mock_response + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Server Error", request=Mock(), response=mock_response + ) + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + + with pytest.raises(LogStorageError, match="Fluent-bit HTTP error: status 500"): + writer.write(tag="test-tag", records=[{"message": "test"}]) + + def test_write_raises_on_transport_error(self, mock_httpx_client): + import httpx + + mock_httpx_client.post.side_effect = httpx.HTTPError("Connection failed") + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + + with pytest.raises(LogStorageError, match="Fluent-bit HTTP error"): + writer.write(tag="test-tag", records=[{"message": "test"}]) + + def test_close_closes_client(self, mock_httpx_client): + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + writer.close() + mock_httpx_client.close.assert_called_once() + + def test_write_applies_tag_prefix(self, mock_httpx_client): + """Test that tag prefix is applied to tags in HTTP requests.""" + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="dstack") + records = [{"message": "test"}] + writer.write(tag="project/run/job", records=records) + + mock_httpx_client.post.assert_called_once_with( + "http://localhost:8080/dstack.project/run/job", + json=records[0], + headers={"Content-Type": "application/json"}, + ) + + def test_write_with_empty_tag_prefix(self, mock_httpx_client): + """Test that empty tag prefix doesn't break the tag.""" + writer = HTTPFluentBitWriter(host="localhost", port=8080, tag_prefix="") + records = [{"message": "test"}] + writer.write(tag="test-tag", records=records) + + mock_httpx_client.post.assert_called_once_with( + "http://localhost:8080/test-tag", + json=records[0], + headers={"Content-Type": "application/json"}, + ) + + +class TestForwardFluentBitWriter: + """Tests for the ForwardFluentBitWriter.""" + + @pytest.fixture + def mock_fluent_sender(self): + with patch( + "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" + ) as mock: + mock_instance = Mock() + mock_instance.emit.return_value = True + mock.return_value = mock_instance + yield mock_instance + + def test_init_creates_sender(self, mock_fluent_sender): + with patch( + "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" + ) as mock: + mock.return_value = mock_fluent_sender + ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + mock.assert_called_once_with("dstack", host="localhost", port=24224) + + def test_write_emits_records(self, mock_fluent_sender): + with patch( + "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" + ) as mock: + mock.return_value = mock_fluent_sender + writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + + records = [ + {"message": "Hello"}, + {"message": "World"}, + ] + writer.write(tag="test-tag", records=records) + + assert mock_fluent_sender.emit.call_count == 2 + + def test_write_raises_on_emit_failure(self, mock_fluent_sender): + mock_fluent_sender.emit.return_value = False + mock_fluent_sender.last_error = Exception("Connection refused") + + with patch( + "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" + ) as mock: + mock.return_value = mock_fluent_sender + writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + + with pytest.raises(LogStorageError, match="Fluent-bit Forward error"): + writer.write(tag="test-tag", records=[{"message": "test"}]) + + mock_fluent_sender.clear_last_error.assert_called_once() + + def test_close_closes_sender(self, mock_fluent_sender): + with patch( + "dstack._internal.server.services.logs.fluentbit.fluent_sender.FluentSender" + ) as mock: + mock.return_value = mock_fluent_sender + writer = ForwardFluentBitWriter(host="localhost", port=24224, tag_prefix="dstack") + writer.close() + mock_fluent_sender.close.assert_called_once() + + +class TestFluentBitLogStorage: + """Tests for the FluentBitLogStorage.""" + + @pytest_asyncio.fixture + async def project(self, test_db, session: AsyncSession) -> ProjectModel: + project = await create_project(session=session, name="test-proj") + return project + + @pytest.fixture + def mock_forward_writer(self): + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock_instance = Mock() + mock.return_value = mock_instance + yield mock_instance + + @pytest.fixture + def mock_http_writer(self): + with patch("dstack._internal.server.services.logs.fluentbit.HTTPFluentBitWriter") as mock: + mock_instance = Mock() + mock.return_value = mock_instance + yield mock_instance + + @pytest.fixture + def mock_es_reader(self): + with patch("dstack._internal.server.services.logs.fluentbit.ElasticsearchReader") as mock: + mock_instance = Mock() + mock.return_value = mock_instance + yield mock_instance + + def test_init_with_forward_protocol(self, mock_forward_writer): + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + mock.assert_called_once_with(host="localhost", port=24224, tag_prefix="dstack") + assert isinstance(storage._reader, NullLogReader) + + def test_init_with_http_protocol(self, mock_http_writer): + with patch("dstack._internal.server.services.logs.fluentbit.HTTPFluentBitWriter") as mock: + mock.return_value = mock_http_writer + FluentBitLogStorage( + host="localhost", + port=8080, + protocol="http", + tag_prefix="dstack", + ) + mock.assert_called_once_with(host="localhost", port=8080, tag_prefix="dstack") + + def test_init_with_unsupported_protocol_raises(self): + with pytest.raises(LogStorageError, match="Unsupported Fluent-bit protocol"): + FluentBitLogStorage( + host="localhost", + port=24224, + protocol="grpc", + tag_prefix="dstack", + ) + + def test_init_ship_only_mode(self, mock_forward_writer): + """Test initialization without Elasticsearch (ship-only mode).""" + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + es_host=None, + ) + assert isinstance(storage._reader, NullLogReader) + + @pytest.mark.skipif(not ELASTICSEARCH_AVAILABLE, reason="elasticsearch not installed") + def test_init_with_elasticsearch(self, mock_forward_writer, mock_es_reader): + """Test initialization with Elasticsearch configured.""" + with ( + patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as writer_mock, + patch( + "dstack._internal.server.services.logs.fluentbit.ElasticsearchReader" + ) as reader_mock, + ): + writer_mock.return_value = mock_forward_writer + reader_mock.return_value = mock_es_reader + + FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + es_host="http://elasticsearch:9200", + es_index="dstack-logs", + es_api_key="test-key", + ) + reader_mock.assert_called_once_with( + host="http://elasticsearch:9200", + index="dstack-logs", + api_key="test-key", + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_write_logs(self, test_db, project: ProjectModel, mock_forward_writer): + """Test writing logs to Fluent-bit.""" + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + + storage.write_logs( + project=project, + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + runner_logs=[ + RunnerLogEvent(timestamp=1696586513234, message=b"Runner log"), + ], + job_logs=[ + RunnerLogEvent(timestamp=1696586513235, message=b"Job log"), + ], + ) + + assert mock_forward_writer.write.call_count == 2 + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_write_logs_empty_logs_not_written( + self, test_db, project: ProjectModel, mock_forward_writer + ): + """Test that empty log lists are not written.""" + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + + storage.write_logs( + project=project, + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + runner_logs=[], + job_logs=[], + ) + + mock_forward_writer.write.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_poll_logs_ship_only_mode(self, test_db, project: ProjectModel): + """Test that ship-only mode returns empty logs.""" + with patch("dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter"): + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + limit=100, + ) + result = storage.poll_logs(project, request) + + assert result.logs == [] + assert result.next_token is None + + def test_close_closes_writer_and_reader(self, mock_forward_writer): + """Test that close() closes both writer and reader.""" + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + + storage.close() + + mock_forward_writer.close.assert_called_once() + + def test_close_closes_reader_even_if_writer_fails(self, mock_forward_writer): + """Test that reader is closed even if writer.close() raises an exception.""" + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock_forward_writer.close.side_effect = Exception("Writer close failed") + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + mock_reader = Mock() + storage._reader = mock_reader + + with pytest.raises(Exception, match="Writer close failed"): + storage.close() + + mock_reader.close.assert_called_once() + + def test_get_stream_name(self, mock_forward_writer): + """Test stream name generation.""" + from dstack._internal.core.models.logs import LogProducer + + with patch( + "dstack._internal.server.services.logs.fluentbit.ForwardFluentBitWriter" + ) as mock: + mock.return_value = mock_forward_writer + storage = FluentBitLogStorage( + host="localhost", + port=24224, + protocol="forward", + tag_prefix="dstack", + ) + + stream_name = storage._get_stream_name( + project_name="my-project", + run_name="my-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + producer=LogProducer.JOB, + ) + + assert stream_name == "my-project/my-run/1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e/job" + + +@pytest.mark.skipif( + not FLUENTBIT_AVAILABLE or not ELASTICSEARCH_AVAILABLE, + reason="fluent-logger or elasticsearch not installed", +) +class TestElasticsearchReader: + """Tests for the ElasticsearchReader.""" + + @pytest.fixture + def mock_es_client(self): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock_instance = Mock() + mock_instance.info.return_value = {"version": {"number": "8.0.0"}} + mock_instance.search.return_value = {"hits": {"hits": []}} + mock.return_value = mock_instance + yield mock_instance + + def test_init_verifies_connection(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + mock_es_client.info.assert_called_once() + + def test_init_with_api_key(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + api_key="test-api-key", + ) + mock.assert_called_once_with(hosts=["http://localhost:9200"], api_key="test-api-key") + + def test_init_connection_error_raises(self): + from elasticsearch.exceptions import ConnectionError as ESConnectionError + + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock_instance = Mock() + mock_instance.info.side_effect = ESConnectionError("Connection refused") + mock.return_value = mock_instance + + with pytest.raises(LogStorageError, match="Failed to connect"): + ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + def test_read_returns_logs(self, mock_es_client): + mock_es_client.search.return_value = { + "hits": { + "hits": [ + { + "_source": { + "@timestamp": "2023-10-06T10:01:53.234000+00:00", + "message": "Hello", + "stream": "test-stream", + }, + "sort": [1696586513234, "doc1"], + }, + { + "_source": { + "@timestamp": "2023-10-06T10:01:53.235000+00:00", + "message": "World", + "stream": "test-stream", + }, + "sort": [1696586513235, "doc2"], + }, + ] + } + } + + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + limit=2, + ) + result = reader.read("test-stream", request) + + assert len(result.logs) == 2 + assert result.logs[0].message == "Hello" + assert result.logs[1].message == "World" + assert result.next_token == "1696586513235:doc2" + + def test_read_with_time_filtering(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + start_time=datetime(2023, 10, 6, 10, 0, 0, tzinfo=timezone.utc), + end_time=datetime(2023, 10, 6, 11, 0, 0, tzinfo=timezone.utc), + limit=100, + ) + reader.read("test-stream", request) + + call_args = mock_es_client.search.call_args + query = call_args.kwargs["query"] + assert "filter" in query["bool"] + assert len(query["bool"]["filter"]) == 2 + + def test_read_descending_order(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + limit=100, + descending=True, + ) + reader.read("test-stream", request) + + call_args = mock_es_client.search.call_args + assert call_args.kwargs["sort"] == [ + {"@timestamp": {"order": "desc"}}, + {"_id": {"order": "desc"}}, + ] + + def test_read_with_next_token(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + next_token="1696586513234:doc1", + limit=100, + ) + reader.read("test-stream", request) + + call_args = mock_es_client.search.call_args + assert call_args.kwargs["search_after"] == ["1696586513234", "doc1"] + + def test_read_with_malformed_next_token_raises_client_error(self, mock_es_client): + """Test that malformed next_token raises ServerClientError (400) instead of IndexError (500).""" + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + + request = PollLogsRequest( + run_name="test-run", + job_submission_id=UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e"), + next_token="invalid_token_no_colon", + limit=100, + ) + with pytest.raises(ServerClientError, match="Invalid next_token"): + reader.read("test-stream", request) + + request.next_token = ":" + with pytest.raises(ServerClientError, match="Invalid next_token"): + reader.read("test-stream", request) + + request.next_token = ":doc1" + with pytest.raises(ServerClientError, match="Invalid next_token"): + reader.read("test-stream", request) + + request.next_token = "1696586513234:" + with pytest.raises(ServerClientError, match="Invalid next_token"): + reader.read("test-stream", request) + + mock_es_client.search.assert_not_called() + + def test_close_closes_client(self, mock_es_client): + with patch("dstack._internal.server.services.logs.fluentbit.Elasticsearch") as mock: + mock.return_value = mock_es_client + reader = ElasticsearchReader( + host="http://localhost:9200", + index="dstack-logs", + ) + reader.close() + mock_es_client.close.assert_called_once() From 29076ba178c97096b88ac07159b4f96171931453 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 20 Jan 2026 13:07:22 +0500 Subject: [PATCH 004/640] Adjust fluent-bit logging integration (#3478) * Move import to the top * Fix double error logging * Add missing backticks --- docs/docs/guides/server-deployment.md | 6 +++--- .../_internal/server/services/logs/__init__.py | 11 ++++++++--- .../_internal/server/services/logs/fluentbit.py | 13 ++----------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/docs/guides/server-deployment.md b/docs/docs/guides/server-deployment.md index dc5093f2f2..42365452aa 100644 --- a/docs/docs/guides/server-deployment.md +++ b/docs/docs/guides/server-deployment.md @@ -229,8 +229,8 @@ Fluent-bit supports two modes depending on how you want to access logs. === "Full mode" - Logs are shipped to Fluent-bit and can be read back through the dstack UI and CLI via Elasticsearch or OpenSearch. - Use this mode when you want a complete integration with log viewing in dstack: + Logs are shipped to Fluent-bit and can be read back through the `dstack` UI and CLI via Elasticsearch or OpenSearch. + Use this mode when you want a complete integration with log viewing in `dstack`: ```shell $ DSTACK_SERVER_FLUENTBIT_HOST=fluentbit.example.com \ @@ -244,7 +244,7 @@ Fluent-bit supports two modes depending on how you want to access logs. The dstack UI/CLI will show empty logs. Use this mode when: - You have an existing logging infrastructure (Kibana, Grafana, Datadog, etc.) - - You only need to forward logs without reading them back through dstack + - You only need to forward logs without reading them back through `dstack` - You want to reduce operational complexity by not running Elasticsearch/OpenSearch ```shell diff --git a/src/dstack/_internal/server/services/logs/__init__.py b/src/dstack/_internal/server/services/logs/__init__.py index 1f8565d49c..bc601688bc 100644 --- a/src/dstack/_internal/server/services/logs/__init__.py +++ b/src/dstack/_internal/server/services/logs/__init__.py @@ -2,6 +2,7 @@ from typing import List, Optional from uuid import UUID +from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.logs import JobSubmissionLogs from dstack._internal.server import settings from dstack._internal.server.models import ProjectModel @@ -105,9 +106,13 @@ def write_logs( async def poll_logs_async(project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs: - job_submission_logs = await run_async( - get_log_storage().poll_logs, project=project, request=request - ) + try: + job_submission_logs = await run_async( + get_log_storage().poll_logs, project=project, request=request + ) + except LogStorageError as e: + logger.error("Failed to poll logs from log storage: %s", repr(e)) + raise ServerClientError("Failed to poll logs from log storage") # Logs are stored in plaintext but transmitted in base64 for API/CLI backward compatibility. # Old logs stored in base64 are encoded twice for transmission and shown as base64 in CLI/UI. # We live with that. diff --git a/src/dstack/_internal/server/services/logs/fluentbit.py b/src/dstack/_internal/server/services/logs/fluentbit.py index b45b2988d5..bb97e21f09 100644 --- a/src/dstack/_internal/server/services/logs/fluentbit.py +++ b/src/dstack/_internal/server/services/logs/fluentbit.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import List, Optional, Protocol from uuid import UUID @@ -99,7 +100,6 @@ def read( try: response = self._client.search(**search_params) except ElasticsearchError as e: - logger.error("Elasticsearch/OpenSearch search error: %s", e) raise LogStorageError(f"Elasticsearch/OpenSearch error: {e}") from e hits = response.get("hits", {}).get("hits", []) @@ -112,8 +112,6 @@ def read( message = source.get("message", "") if timestamp_str: - from datetime import datetime - try: timestamp = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00")) except ValueError: @@ -180,16 +178,10 @@ def write(self, tag: str, records: List[dict]) -> None: ) response.raise_for_status() except httpx.HTTPStatusError as e: - logger.error( - "Fluent-bit HTTP request failed with status %d: %s", - e.response.status_code, - e.response.text, - ) raise LogStorageError( f"Fluent-bit HTTP error: status {e.response.status_code}" ) from e except httpx.HTTPError as e: - logger.error("Failed to write log to Fluent-bit via HTTP: %s", e) raise LogStorageError(f"Fluent-bit HTTP error: {e}") from e def close(self) -> None: @@ -206,7 +198,6 @@ def write(self, tag: str, records: List[dict]) -> None: for record in records: if not self._sender.emit(tag, record): error = self._sender.last_error - logger.error("Failed to write log to Fluent-bit via Forward: %s", error) self._sender.clear_last_error() raise LogStorageError(f"Fluent-bit Forward error: {error}") @@ -271,7 +262,7 @@ def __init__( index=es_index, api_key=es_api_key, ) - logger.debug( + logger.info( "Fluent-bit log storage initialized with Elasticsearch/OpenSearch reader" ) else: From 54d2d0aa09a54e980529e9c4464a0d1d14db48d2 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Tue, 20 Jan 2026 10:03:55 +0000 Subject: [PATCH 005/640] Emit events for instance status changes (#3477) - Emit an event on every instance status change - To make events more informative, set termination reasons whenever terminating instances - Add `terminated_by_user` termination reason - Remove redundant logging now covered by events - Refactor runtime-only status changes that were not persisted and did not affect logic - For event readability, only include the busy blocks count in job assigned/unassigned events, which is the only place where the count can change --- src/dstack/_internal/core/models/instances.py | 1 + .../tasks/process_compute_groups.py | 9 +- .../server/background/tasks/process_fleets.py | 11 +- .../background/tasks/process_instances.py | 208 +++++------------- .../tasks/process_submitted_jobs.py | 9 +- src/dstack/_internal/server/models.py | 1 + .../_internal/server/services/fleets.py | 33 ++- .../_internal/server/services/instances.py | 56 ++++- .../server/services/jobs/__init__.py | 27 ++- .../_internal/server/routers/test_fleets.py | 30 ++- .../server/services/test_instances.py | 49 ++++- 11 files changed, 214 insertions(+), 220 deletions(-) diff --git a/src/dstack/_internal/core/models/instances.py b/src/dstack/_internal/core/models/instances.py index bf1696758d..012916f97e 100644 --- a/src/dstack/_internal/core/models/instances.py +++ b/src/dstack/_internal/core/models/instances.py @@ -256,6 +256,7 @@ def finished_statuses(cls) -> List["InstanceStatus"]: class InstanceTerminationReason(str, Enum): + TERMINATED_BY_USER = "terminated_by_user" IDLE_TIMEOUT = "idle_timeout" PROVISIONING_TIMEOUT = "provisioning_timeout" ERROR = "error" diff --git a/src/dstack/_internal/server/background/tasks/process_compute_groups.py b/src/dstack/_internal/server/background/tasks/process_compute_groups.py index 5f7b6820a4..6b449efab4 100644 --- a/src/dstack/_internal/server/background/tasks/process_compute_groups.py +++ b/src/dstack/_internal/server/background/tasks/process_compute_groups.py @@ -17,6 +17,7 @@ ) from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services.compute_groups import compute_group_model_to_compute_group +from dstack._internal.server.services.instances import switch_instance_status from dstack._internal.server.services.locking import get_locker from dstack._internal.server.utils import sentry_utils from dstack._internal.utils.common import get_current_datetime, run_async @@ -83,12 +84,14 @@ async def _process_compute_group(session: AsyncSession, compute_group_model: Com ) compute_group_model = res.unique().scalar_one() if all(i.status == InstanceStatus.TERMINATING for i in compute_group_model.instances): - await _terminate_compute_group(compute_group_model) + await _terminate_compute_group(session, compute_group_model) compute_group_model.last_processed_at = get_current_datetime() await session.commit() -async def _terminate_compute_group(compute_group_model: ComputeGroupModel) -> None: +async def _terminate_compute_group( + session: AsyncSession, compute_group_model: ComputeGroupModel +) -> None: if ( compute_group_model.last_termination_retry_at is not None and _next_termination_retry_at(compute_group_model) > get_current_datetime() @@ -147,7 +150,7 @@ async def _terminate_compute_group(compute_group_model: ComputeGroupModel) -> No instance_model.deleted = True instance_model.deleted_at = get_current_datetime() instance_model.finished_at = get_current_datetime() - instance_model.status = InstanceStatus.TERMINATED + switch_instance_status(session, instance_model, InstanceStatus.TERMINATED) logger.info( "Terminated compute group %s", compute_group.name, diff --git a/src/dstack/_internal/server/background/tasks/process_fleets.py b/src/dstack/_internal/server/background/tasks/process_fleets.py index d369c7d242..50c3dcfe2a 100644 --- a/src/dstack/_internal/server/background/tasks/process_fleets.py +++ b/src/dstack/_internal/server/background/tasks/process_fleets.py @@ -26,7 +26,7 @@ is_fleet_in_use, switch_fleet_status, ) -from dstack._internal.server.services.instances import format_instance_status_for_event +from dstack._internal.server.services.instances import switch_instance_status from dstack._internal.server.services.locking import get_locker from dstack._internal.server.utils import sentry_utils from dstack._internal.utils.common import get_current_datetime @@ -219,15 +219,10 @@ def _maintain_fleet_nodes_in_min_max_range( if nodes_redundant == 0: break if instance.status in [InstanceStatus.IDLE]: - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.MAX_INSTANCES_LIMIT instance.termination_reason_message = "Fleet has too many instances" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) nodes_redundant -= 1 - logger.info( - "Terminating instance %s: %s", - instance.name, - instance.termination_reason, - ) return True nodes_missing = fleet_spec.configuration.nodes.min - active_instances_num for i in range(nodes_missing): @@ -243,7 +238,7 @@ def _maintain_fleet_nodes_in_min_max_range( session, ( "Instance created to meet target fleet node count." - f" Status: {format_instance_status_for_event(instance_model)}" + f" Status: {instance_model.status.upper()}" ), actor=events.SystemActor(), targets=[events.Target.from_model(instance_model)], diff --git a/src/dstack/_internal/server/background/tasks/process_instances.py b/src/dstack/_internal/server/background/tasks/process_instances.py index 454d6ee18a..c2bc27ee85 100644 --- a/src/dstack/_internal/server/background/tasks/process_instances.py +++ b/src/dstack/_internal/server/background/tasks/process_instances.py @@ -87,6 +87,7 @@ get_instance_requirements, get_instance_ssh_private_keys, remove_dangling_tasks_from_instance, + switch_instance_status, ) from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.logging import fmt @@ -242,7 +243,7 @@ async def _process_instance(session: AsyncSession, instance: InstanceModel): if instance.status == InstanceStatus.PENDING: if instance.remote_connection_info is not None: - await _add_remote(instance) + await _add_remote(session, instance) else: await _create_instance( session=session, @@ -253,17 +254,21 @@ async def _process_instance(session: AsyncSession, instance: InstanceModel): InstanceStatus.IDLE, InstanceStatus.BUSY, ): - idle_duration_expired = _check_and_mark_terminating_if_idle_duration_expired(instance) + idle_duration_expired = _check_and_mark_terminating_if_idle_duration_expired( + session, instance + ) if not idle_duration_expired: await _check_instance(session, instance) elif instance.status == InstanceStatus.TERMINATING: - await _terminate(instance) + await _terminate(session, instance) instance.last_processed_at = get_current_datetime() await session.commit() -def _check_and_mark_terminating_if_idle_duration_expired(instance: InstanceModel): +def _check_and_mark_terminating_if_idle_duration_expired( + session: AsyncSession, instance: InstanceModel +): if not ( instance.status == InstanceStatus.IDLE and instance.termination_policy == TerminationPolicy.DESTROY_AFTER_IDLE @@ -282,17 +287,9 @@ def _check_and_mark_terminating_if_idle_duration_expired(instance: InstanceModel idle_seconds = instance.termination_idle_time delta = datetime.timedelta(seconds=idle_seconds) if idle_duration > delta: - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.IDLE_TIMEOUT - logger.info( - "Instance %s idle duration expired: idle time %ss. Terminating", - instance.name, - str(idle_duration.seconds), - extra={ - "instance_name": instance.name, - "instance_status": instance.status.value, - }, - ) + instance.termination_reason_message = f"Instance idle for {idle_duration.seconds}s" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) return True return False @@ -311,24 +308,16 @@ def _can_terminate_fleet_instances_on_idle_duration(fleet_model: FleetModel) -> return active_instances_num > fleet.spec.configuration.nodes.min -async def _add_remote(instance: InstanceModel) -> None: +async def _add_remote(session: AsyncSession, instance: InstanceModel) -> None: logger.info("Adding ssh instance %s...", instance.name) - if instance.status == InstanceStatus.PENDING: - instance.status = InstanceStatus.PROVISIONING retry_duration_deadline = instance.created_at + timedelta(seconds=PROVISIONING_TIMEOUT_SECONDS) if retry_duration_deadline < get_current_datetime(): - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.PROVISIONING_TIMEOUT - logger.warning( - "Failed to start instance %s in %d seconds. Terminating...", - instance.name, - PROVISIONING_TIMEOUT_SECONDS, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, + instance.termination_reason_message = ( + f"Failed to add SSH instance in {PROVISIONING_TIMEOUT_SECONDS}s" ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) return try: @@ -341,17 +330,9 @@ async def _add_remote(instance: InstanceModel) -> None: else: ssh_proxy_pkeys = None except (ValueError, PasswordRequiredException): - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = "Unsupported private SSH key type" - logger.warning( - "Failed to add instance %s: unsupported private SSH key type", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) return authorized_keys = [pk.public.strip() for pk in remote_details.ssh_keys] @@ -368,19 +349,13 @@ async def _add_remote(instance: InstanceModel) -> None: raise ProvisioningError(f"Deploy timeout: {e}") from e except Exception as e: raise ProvisioningError(f"Deploy instance raised an error: {e}") from e - else: - logger.info( - "The instance %s (%s) was successfully added", - instance.name, - remote_details.host, - ) except ProvisioningError as e: logger.warning( "Provisioning instance %s could not be completed because of the error: %s", instance.name, e, ) - instance.status = InstanceStatus.PENDING + # Stays in PENDING, may retry later return instance_type = host_info_to_instance_type(host_info, arch) @@ -400,35 +375,19 @@ async def _add_remote(instance: InstanceModel) -> None: addresses=host_network_addresses, ) if instance_network is not None and internal_ip is None: - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = ( "Failed to locate internal IP address on the given network" ) - logger.warning( - "Failed to add instance %s: failed to locate internal IP address on the given network", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) return if internal_ip is not None: if not is_ip_among_addresses(ip_address=internal_ip, addresses=host_network_addresses): - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = ( "Specified internal IP not found among instance interfaces" ) - logger.warning( - "Failed to add instance %s: specified internal IP not found among instance interfaces", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) return divisible, blocks = is_divisible_into_blocks( @@ -439,17 +398,9 @@ async def _add_remote(instance: InstanceModel) -> None: if divisible: instance.total_blocks = blocks else: - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = "Cannot split into blocks" - logger.warning( - "Failed to add instance %s: cannot split into blocks", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) return region = instance.region @@ -470,7 +421,9 @@ async def _add_remote(instance: InstanceModel) -> None: ssh_proxy=remote_details.ssh_proxy, ) - instance.status = InstanceStatus.IDLE if health else InstanceStatus.PROVISIONING + switch_instance_status( + session, instance, InstanceStatus.IDLE if health else InstanceStatus.PROVISIONING + ) instance.backend = BackendType.REMOTE instance_offer = InstanceOfferWithAvailability( backend=BackendType.REMOTE, @@ -562,18 +515,13 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No profile = get_instance_profile(instance) requirements = get_instance_requirements(instance) except ValidationError as e: - instance.status = InstanceStatus.TERMINATED instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = ( f"Error to parse profile, requirements or instance_configuration: {e}" ) - logger.warning( - "Error to parse profile, requirements or instance_configuration. Terminate instance: %s", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, + switch_instance_status(session, instance, InstanceStatus.TERMINATED) + logger.exception( + "%s: error parsing profile, requirements or instance configuration", fmt(instance) ) return @@ -664,7 +612,7 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No ) continue - instance.status = InstanceStatus.PROVISIONING + switch_instance_status(session, instance, InstanceStatus.PROVISIONING) instance.backend = backend.TYPE instance.region = instance_offer.region instance.price = instance_offer.price @@ -674,14 +622,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No instance.total_blocks = instance_offer.total_blocks instance.started_at = get_current_datetime() - logger.info( - "Created instance %s", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.PROVISIONING.value, - }, - ) if instance.fleet_id and instance.id == master_instance.id: # Clean up placement groups that did not end up being used. # Flush to update still uncommitted placement groups. @@ -695,18 +635,17 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No ) return - _mark_terminated( - instance, - InstanceTerminationReason.NO_OFFERS, - "All offers failed" if offers else "No offers found", - ) + instance.termination_reason = InstanceTerminationReason.NO_OFFERS + instance.termination_reason_message = "All offers failed" if offers else "No offers found" + switch_instance_status(session, instance, InstanceStatus.TERMINATED) if instance.fleet and instance.id == master_instance.id and is_cloud_cluster(instance.fleet): # Do not attempt to deploy other instances, as they won't determine the correct cluster # backend, region, and placement group without a successfully deployed master instance for sibling_instance in instance.fleet.instances: if sibling_instance.id == instance.id: continue - _mark_terminated(sibling_instance, InstanceTerminationReason.MASTER_FAILED) + sibling_instance.termination_reason = InstanceTerminationReason.MASTER_FAILED + switch_instance_status(session, sibling_instance, InstanceStatus.TERMINATED) async def _get_fleet_master_instance( @@ -723,25 +662,6 @@ async def _get_fleet_master_instance( return res.scalar_one() -def _mark_terminated( - instance: InstanceModel, - termination_reason: InstanceTerminationReason, - termination_reason_message: Optional[str] = None, -) -> None: - instance.status = InstanceStatus.TERMINATED - instance.termination_reason = termination_reason - instance.termination_reason_message = termination_reason_message - logger.info( - "Terminated instance %s: %s", - instance.name, - instance.termination_reason, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) - - async def _check_instance(session: AsyncSession, instance: InstanceModel) -> None: if ( instance.status == InstanceStatus.BUSY @@ -749,9 +669,9 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non and all(job.status.is_finished() for job in instance.jobs) ): # A busy instance could have no active jobs due to this bug: https://github.com/dstackai/dstack/issues/2068 - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.JOB_FINISHED - logger.info( + switch_instance_status(session, instance, InstanceStatus.TERMINATING) + logger.warning( "Detected busy instance %s with finished job. Marked as TERMINATING", instance.name, extra={ @@ -770,6 +690,7 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non ) project = res.unique().scalar_one() await _wait_for_instance_provisioning_data( + session=session, project=project, instance=instance, job_provisioning_data=job_provisioning_data, @@ -778,7 +699,7 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non if not job_provisioning_data.dockerized: if instance.status == InstanceStatus.PROVISIONING: - instance.status = InstanceStatus.BUSY + switch_instance_status(session, instance, InstanceStatus.BUSY) return ssh_private_keys = get_instance_ssh_private_keys(instance) @@ -845,15 +766,10 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non instance.termination_deadline = None if instance.status == InstanceStatus.PROVISIONING: - instance.status = InstanceStatus.IDLE if not instance.jobs else InstanceStatus.BUSY - logger.info( - "Instance %s has switched to %s status", - instance.name, - instance.status.value, - extra={ - "instance_name": instance.name, - "instance_status": instance.status.value, - }, + switch_instance_status( + session, + instance, + InstanceStatus.IDLE if not instance.jobs else InstanceStatus.BUSY, ) return @@ -866,31 +782,18 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non job_provisioning_data=job_provisioning_data, ) if get_current_datetime() > provisioning_deadline: - instance.status = InstanceStatus.TERMINATING - logger.warning( - "Instance %s has not started in time. Marked as TERMINATING", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATING.value, - }, - ) + instance.termination_reason = InstanceTerminationReason.PROVISIONING_TIMEOUT + instance.termination_reason_message = "Instance did not become reachable in time" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) elif instance.status.is_available(): deadline = instance.termination_deadline if get_current_datetime() > deadline: - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.UNREACHABLE - logger.warning( - "Instance %s shim waiting timeout. Marked as TERMINATING", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATING.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATING) async def _wait_for_instance_provisioning_data( + session: AsyncSession, project: ProjectModel, instance: InstanceModel, job_provisioning_data: JobProvisioningData, @@ -904,12 +807,9 @@ async def _wait_for_instance_provisioning_data( job_provisioning_data=job_provisioning_data, ) if get_current_datetime() > provisioning_deadline: - logger.warning( - "Instance %s failed because instance has not become running in time", instance.name - ) - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.PROVISIONING_TIMEOUT instance.termination_reason_message = "Backend did not complete provisioning in time" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) return backend = await backends_services.get_project_backend_by_type( @@ -921,9 +821,9 @@ async def _wait_for_instance_provisioning_data( "Instance %s failed because instance's backend is not available", instance.name, ) - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = "Backend not available" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) return try: await run_async( @@ -939,9 +839,9 @@ async def _wait_for_instance_provisioning_data( instance.name, repr(e), ) - instance.status = InstanceStatus.TERMINATING instance.termination_reason = InstanceTerminationReason.ERROR instance.termination_reason_message = "Error while waiting for instance to become running" + switch_instance_status(session, instance, InstanceStatus.TERMINATING) except Exception: logger.exception( "Got exception when updating instance %s provisioning data", instance.name @@ -1137,7 +1037,7 @@ def _get_instance_cpu_arch(instance: InstanceModel) -> Optional[gpuhunt.CPUArchi return jpd.instance_type.resources.cpu_arch -async def _terminate(instance: InstanceModel) -> None: +async def _terminate(session: AsyncSession, instance: InstanceModel) -> None: if ( instance.last_termination_retry_at is not None and _next_termination_retry_at(instance) > get_current_datetime() @@ -1190,15 +1090,7 @@ async def _terminate(instance: InstanceModel) -> None: instance.deleted = True instance.deleted_at = get_current_datetime() instance.finished_at = get_current_datetime() - instance.status = InstanceStatus.TERMINATED - logger.info( - "Instance %s terminated", - instance.name, - extra={ - "instance_name": instance.name, - "instance_status": InstanceStatus.TERMINATED.value, - }, - ) + switch_instance_status(session, instance, InstanceStatus.TERMINATED) def _next_termination_retry_at(instance: InstanceModel) -> datetime.datetime: diff --git a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py index e132f83a49..2320394436 100644 --- a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py +++ b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py @@ -87,8 +87,9 @@ is_cloud_cluster, ) from dstack._internal.server.services.instances import ( - format_instance_status_for_event, + format_instance_blocks_for_event, get_instance_provisioning_data, + switch_instance_status, ) from dstack._internal.server.services.jobs import ( check_can_attach_job_volumes, @@ -507,7 +508,7 @@ async def _process_submitted_job( session.add(instance) events.emit( session, - f"Instance created for job. Instance status: {format_instance_status_for_event(instance)}", + f"Instance created for job. Instance status: {instance.status.upper()}", actor=events.SystemActor(), targets=[ events.Target.from_model(instance), @@ -646,7 +647,7 @@ async def _assign_job_to_fleet_instance( .options(joinedload(InstanceModel.volume_attachments)) ) instance = res.unique().scalar_one() - instance.status = InstanceStatus.BUSY + switch_instance_status(session, instance, InstanceStatus.BUSY) instance.busy_blocks += offer.blocks job_model.instance = instance @@ -657,7 +658,7 @@ async def _assign_job_to_fleet_instance( session, ( "Job assigned to instance." - f" Instance status: {format_instance_status_for_event(instance)}" + f" Instance blocks: {format_instance_blocks_for_event(instance)}" ), actor=events.SystemActor(), targets=[ diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 5274d9ebfd..6a8aa41eb4 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -632,6 +632,7 @@ class InstanceModel(BaseModel): compute_group_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("compute_groups.id")) compute_group: Mapped[Optional["ComputeGroupModel"]] = relationship(back_populates="instances") + # NOTE: `status` must be changed only via `switch_instance_status()` status: Mapped[InstanceStatus] = mapped_column(EnumAsString(InstanceStatus, 100), index=True) unreachable: Mapped[bool] = mapped_column(Boolean) diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 19e4a77e64..9b877475f7 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -31,6 +31,7 @@ from dstack._internal.core.models.instances import ( InstanceOfferWithAvailability, InstanceStatus, + InstanceTerminationReason, RemoteConnectionInfo, SSHConnectionParams, SSHKey, @@ -65,9 +66,9 @@ from dstack._internal.server.services import instances as instances_services from dstack._internal.server.services import offers as offers_services from dstack._internal.server.services.instances import ( - format_instance_status_for_event, get_instance_remote_connection_info, list_active_remote_instances, + switch_instance_status, ) from dstack._internal.server.services.locking import ( get_locker, @@ -679,7 +680,9 @@ async def delete_fleets( "Deleting fleets %s instances %s", [f.name for f in fleet_models], instance_nums ) for fleet_model in fleet_models: - _terminate_fleet_instances(fleet_model=fleet_model, instance_nums=instance_nums) + _terminate_fleet_instances( + session=session, fleet_model=fleet_model, instance_nums=instance_nums, actor=user + ) # TERMINATING fleets are deleted by process_fleets after instances are terminated if instance_nums is None: switch_fleet_status( @@ -873,7 +876,7 @@ async def _create_fleet( session, ( "Instance created on fleet submission." - f" Status: {format_instance_status_for_event(instance_model)}" + f" Status: {instance_model.status.upper()}" ), actor=events.UserActor.from_user(user), targets=[events.Target.from_model(instance_model)], @@ -892,7 +895,7 @@ async def _create_fleet( session, ( "Instance created on fleet submission." - f" Status: {format_instance_status_for_event(instance_model)}" + f" Status: {instance_model.status.upper()}" ), # Set `SystemActor` for consistency with other places where cloud instances can be # created (fleet spec consolidation, job provisioning, etc). Think of the fleet as being @@ -978,17 +981,14 @@ async def _update_fleet( ) events.emit( session, - ( - "Instance created on fleet update." - f" Status: {format_instance_status_for_event(instance_model)}" - ), + f"Instance created on fleet update. Status: {instance_model.status.upper()}", actor=events.UserActor.from_user(user), targets=[events.Target.from_model(instance_model)], ) fleet_model.instances.append(instance_model) active_instance_nums.add(instance_num) if removed_instance_nums: - _terminate_fleet_instances(fleet_model, removed_instance_nums) + _terminate_fleet_instances(session, fleet_model, removed_instance_nums, actor=user) await session.commit() return fleet_model_to_fleet(fleet_model) @@ -1197,7 +1197,12 @@ def _get_fleet_nodes_to_provision(spec: FleetSpec) -> int: return spec.configuration.nodes.target -def _terminate_fleet_instances(fleet_model: FleetModel, instance_nums: Optional[List[int]]): +def _terminate_fleet_instances( + session: AsyncSession, + fleet_model: FleetModel, + instance_nums: Optional[List[int]], + actor: UserModel, +): if is_fleet_in_use(fleet_model, instance_nums=instance_nums): if instance_nums is not None: raise ServerClientError( @@ -1210,4 +1215,10 @@ def _terminate_fleet_instances(fleet_model: FleetModel, instance_nums: Optional[ if instance.status == InstanceStatus.TERMINATED: instance.deleted = True else: - instance.status = InstanceStatus.TERMINATING + instance.termination_reason = InstanceTerminationReason.TERMINATED_BY_USER + switch_instance_status( + session, + instance, + InstanceStatus.TERMINATING, + actor=events.UserActor.from_user(actor), + ) diff --git a/src/dstack/_internal/server/services/instances.py b/src/dstack/_internal/server/services/instances.py index bf837469d0..14f26cc3f0 100644 --- a/src/dstack/_internal/server/services/instances.py +++ b/src/dstack/_internal/server/services/instances.py @@ -25,6 +25,7 @@ InstanceOffer, InstanceOfferWithAvailability, InstanceStatus, + InstanceTerminationReason, InstanceType, RemoteConnectionInfo, Resources, @@ -49,6 +50,7 @@ ) from dstack._internal.server.schemas.health.dcgm import DCGMHealthResponse from dstack._internal.server.schemas.runner import InstanceHealthResponse, TaskStatus +from dstack._internal.server.services import events from dstack._internal.server.services.logging import fmt from dstack._internal.server.services.offers import generate_shared_offer from dstack._internal.server.services.projects import list_user_project_models @@ -59,11 +61,55 @@ logger = get_logger(__name__) -def format_instance_status_for_event(instance_model: InstanceModel) -> str: - msg = instance_model.status.upper() - if instance_model.total_blocks is not None: - msg += f" ({instance_model.busy_blocks}/{instance_model.total_blocks} blocks busy)" - return msg +def switch_instance_status( + session: AsyncSession, + instance_model: InstanceModel, + new_status: InstanceStatus, + actor: events.AnyActor = events.SystemActor(), +): + """ + Switch instance status. + + **Usage notes**: + + - When switching to `TERMINATING` or `TERMINATED`, + `instance_model.termination_reason` must be set + + - When `instance_model.termination_reason` is set to `ERROR`, + the error must be further explained in `instance_model.termination_reason_message` + """ + + old_status = instance_model.status + if old_status == new_status: + return + + instance_model.status = new_status + + msg = f"Instance status changed {old_status.upper()} -> {new_status.upper()}" + if ( + new_status == InstanceStatus.TERMINATING + or new_status == InstanceStatus.TERMINATED + and old_status != InstanceStatus.TERMINATING + ): + if instance_model.termination_reason is None: + raise ValueError( + f"termination_reason must be set when switching to {new_status.upper()} status" + ) + if ( + instance_model.termination_reason == InstanceTerminationReason.ERROR + and not instance_model.termination_reason_message + ): + raise ValueError( + "termination_reason_message must be set when termination_reason is ERROR" + ) + msg += f". Termination reason: {instance_model.termination_reason.upper()}" + if instance_model.termination_reason_message: + msg += f" ({instance_model.termination_reason_message})" + events.emit(session, msg, actor=actor, targets=[events.Target.from_model(instance_model)]) + + +def format_instance_blocks_for_event(instance_model: InstanceModel) -> str: + return f"{instance_model.busy_blocks}/{instance_model.total_blocks} busy" async def get_instance_health_checks( diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index b86bd09643..18d410c133 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -21,7 +21,7 @@ ) from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import RunConfigurationType -from dstack._internal.core.models.instances import InstanceStatus +from dstack._internal.core.models.instances import InstanceStatus, InstanceTerminationReason from dstack._internal.core.models.runs import ( Job, JobProvisioningData, @@ -44,8 +44,9 @@ from dstack._internal.server.services import events, services from dstack._internal.server.services import volumes as volumes_services from dstack._internal.server.services.instances import ( - format_instance_status_for_event, + format_instance_blocks_for_event, get_instance_ssh_private_keys, + switch_instance_status, ) from dstack._internal.server.services.jobs.configurators.base import ( JobConfigurator, @@ -352,18 +353,16 @@ async def process_terminating_job( blocks = 1 instance_model.busy_blocks -= blocks - if instance_model.status == InstanceStatus.BUSY: + if instance_model.status != InstanceStatus.BUSY or jpd is None or not jpd.dockerized: + # Terminate instances that: + # - have not finished provisioning yet + # - belong to container-based backends, and hence cannot be reused + if instance_model.status not in InstanceStatus.finished_statuses(): + instance_model.termination_reason = InstanceTerminationReason.JOB_FINISHED + switch_instance_status(session, instance_model, InstanceStatus.TERMINATING) + elif not [j for j in instance_model.jobs if j.id != job_model.id]: # no other jobs besides this one - if not [j for j in instance_model.jobs if j.id != job_model.id]: - instance_model.status = InstanceStatus.IDLE - elif instance_model.status != InstanceStatus.TERMINATED: - # instance was PROVISIONING (specially for the job) - # schedule for termination - instance_model.status = InstanceStatus.TERMINATING - - if jpd is None or not jpd.dockerized: - # do not reuse vastai/k8s instances - instance_model.status = InstanceStatus.TERMINATING + switch_instance_status(session, instance_model, InstanceStatus.IDLE) # The instance should be released even if detach fails # so that stuck volumes don't prevent the instance from terminating. @@ -374,7 +373,7 @@ async def process_terminating_job( session, ( "Job unassigned from instance." - f" Instance status: {format_instance_status_for_event(instance_model)}" + f" Instance blocks: {format_instance_blocks_for_event(instance_model)}" ), actor=events.SystemActor(), targets=[ diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index afa68b788d..b00d6ccf57 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from typing import Optional from unittest.mock import Mock, patch -from uuid import UUID, uuid4 +from uuid import uuid4 import pytest from freezegun import freeze_time @@ -603,19 +603,17 @@ async def test_updates_ssh_fleet(self, test_db, session: AsyncSession, client: A remote_connection_info=get_remote_connection_info(host="10.0.0.100"), ) - with patch("uuid.uuid4") as m: - m.return_value = UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e") - response = await client.post( - f"/api/project/{project.name}/fleets/apply", - headers=get_auth_headers(user.token), - json={ - "plan": { - "spec": spec.dict(), - "current_resource": _fleet_model_to_json_dict(fleet), - }, - "force": False, + response = await client.post( + f"/api/project/{project.name}/fleets/apply", + headers=get_auth_headers(user.token), + json={ + "plan": { + "spec": spec.dict(), + "current_resource": _fleet_model_to_json_dict(fleet), }, - ) + "force": False, + }, + ) assert response.status_code == 200, response.json() assert response.json() == { @@ -711,7 +709,7 @@ async def test_updates_ssh_fleet(self, test_db, session: AsyncSession, client: A "status": "terminating", "unreachable": False, "health_status": "healthy", - "termination_reason": None, + "termination_reason": "terminated_by_user", "termination_reason_message": None, "created": "2023-01-02T03:04:00+00:00", "region": "remote", @@ -721,7 +719,7 @@ async def test_updates_ssh_fleet(self, test_db, session: AsyncSession, client: A "busy_blocks": 0, }, { - "id": "1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e", + "id": SomeUUID4Str(), "project_name": project.name, "backend": "remote", "instance_type": { @@ -761,7 +759,7 @@ async def test_updates_ssh_fleet(self, test_db, session: AsyncSession, client: A await session.refresh(instance) assert instance.status == InstanceStatus.TERMINATING res = await session.execute( - select(InstanceModel).where(InstanceModel.id == "1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e") + select(InstanceModel).where(InstanceModel.id == response.json()["instances"][1]["id"]) ) instance = res.unique().scalar_one() assert instance.status == InstanceStatus.PENDING diff --git a/src/tests/_internal/server/services/test_instances.py b/src/tests/_internal/server/services/test_instances.py index aa248aa485..9e4cb02e3a 100644 --- a/src/tests/_internal/server/services/test_instances.py +++ b/src/tests/_internal/server/services/test_instances.py @@ -1,6 +1,7 @@ import uuid import pytest +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession import dstack._internal.server.services.instances as instances_services @@ -9,11 +10,12 @@ from dstack._internal.core.models.instances import ( Instance, InstanceStatus, + InstanceTerminationReason, InstanceType, Resources, ) from dstack._internal.core.models.profiles import Profile -from dstack._internal.server.models import InstanceModel +from dstack._internal.server.models import EventModel, InstanceModel from dstack._internal.server.testing.common import ( create_instance, create_project, @@ -24,6 +26,51 @@ from dstack._internal.utils.common import get_current_datetime +class TestSwitchInstanceStatus: + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_includes_termination_reason_in_event_messages_only_once( + self, test_db, session: AsyncSession + ) -> None: + project = await create_project(session=session) + instance = await create_instance( + session=session, project=project, status=InstanceStatus.PENDING + ) + instance.termination_reason = InstanceTerminationReason.ERROR + instance.termination_reason_message = "Some err" + instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATING) + instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATED) + + res = await session.execute(select(EventModel)) + events = res.scalars().all() + assert len(events) == 2 + assert {e.message for e in events} == { + "Instance status changed PENDING -> TERMINATING. Termination reason: ERROR (Some err)", + # Do not duplicate the termination reason in the second event + "Instance status changed TERMINATING -> TERMINATED", + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_includes_termination_reason_in_event_message_when_switching_directly_to_terminated( + self, test_db, session: AsyncSession + ) -> None: + project = await create_project(session=session) + instance = await create_instance( + session=session, project=project, status=InstanceStatus.PENDING + ) + instance.termination_reason = InstanceTerminationReason.ERROR + instance.termination_reason_message = "Some err" + instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATED) + + res = await session.execute(select(EventModel)) + events = res.scalars().all() + assert len(events) == 1 + assert events[0].message == ( + "Instance status changed PENDING -> TERMINATED. Termination reason: ERROR (Some err)" + ) + + class TestFilterPoolInstances: # TODO: Refactor filter_pool_instances to not depend on InstanceModel and simplify tests @pytest.mark.asyncio From c01b022a0a6611c272626540234f0f4fe7148fee Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 20 Jan 2026 11:06:01 +0000 Subject: [PATCH 006/640] [runner] Restore `--home-dir` option as no-op (#3480) Fixes: https://github.com/dstackai/dstack/issues/3474 --- runner/cmd/runner/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runner/cmd/runner/main.go b/runner/cmd/runner/main.go index c2ed94f0eb..c8125dc848 100644 --- a/runner/cmd/runner/main.go +++ b/runner/cmd/runner/main.go @@ -78,6 +78,12 @@ func mainInner() int { Usage: "dstack server or user authorized key. May be specified multiple times", Destination: &sshAuthorizedKeys, }, + // --home-dir is not used since 0.20.4, but the flag was retained as no-op + // for compatibility with pre-0.20.4 shims; remove the flag eventually + &cli.StringFlag{ + Name: "home-dir", + Hidden: true, + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { return start(ctx, tempDir, httpPort, sshPort, sshAuthorizedKeys, logLevel, Version) From 65eacc7796f81aeb3dcff34e64b9e6ab4fa6fa7a Mon Sep 17 00:00:00 2001 From: Oleg Date: Tue, 20 Jan 2026 23:45:48 +0300 Subject: [PATCH 007/640] [UI] Default fleet in project wizard (#3464) * [UI] Default fleet in project wizard #373 * Minor cosmetic changes * Fixes after review * Was added create project wizard for oss * Cosmetical changes + help info * Was added create fleet wizard * Fixes after review * Refactoring after review * Fixes after review * Fixes after review * Cosmetics * Fixes after review --------- Co-authored-by: peterschmidt85 --- frontend/src/api.ts | 1 + .../ButtonWithConfirmation/index.tsx | 20 +- .../components/ConfirmationDialog/index.tsx | 5 +- .../components/ConfirmationDialog/slice.ts | 34 ++ .../components/form/Toogle/index.module.scss | 17 + frontend/src/components/form/Toogle/index.tsx | 78 +++++ frontend/src/components/form/Toogle/types.ts | 13 + frontend/src/components/index.ts | 1 + frontend/src/hooks/index.ts | 1 + frontend/src/hooks/useConfirmationDialog.ts | 27 ++ frontend/src/hooks/useNotifications.ts | 1 + frontend/src/layouts/AppLayout/index.tsx | 7 + frontend/src/locale/en.json | 40 ++- .../Fleets/Add/FleetFormFields/constants.tsx | 115 +++++++ .../Fleets/Add/FleetFormFields/index.tsx | 79 +++++ .../pages/Fleets/Add/FleetFormFields/type.ts | 15 + frontend/src/pages/Fleets/Add/index.tsx | 254 ++++++++++++++ frontend/src/pages/Fleets/Add/types.ts | 5 + frontend/src/pages/Fleets/index.ts | 1 + frontend/src/pages/Project/Add/index.tsx | 311 +++++++++++++++++- .../src/pages/Project/CreateWizard/index.tsx | 297 +++++++++-------- .../src/pages/Project/CreateWizard/types.ts | 9 +- frontend/src/pages/Project/Form/types.ts | 10 +- frontend/src/pages/Project/List/index.tsx | 4 +- .../components/NoFleetProjectAlert/index.tsx | 8 +- frontend/src/pages/Project/constants.tsx | 32 ++ .../Project/hooks/useYupValidationResolver.ts | 38 +++ frontend/src/pages/User/Details/index.tsx | 8 +- frontend/src/pages/User/List/index.tsx | 2 + frontend/src/router.tsx | 6 +- frontend/src/routes.ts | 4 + frontend/src/services/fleet.ts | 20 +- frontend/src/services/project.ts | 2 +- frontend/src/store.ts | 2 + frontend/src/types/fleet.d.ts | 16 +- frontend/src/types/project.d.ts | 4 + 36 files changed, 1282 insertions(+), 205 deletions(-) create mode 100644 frontend/src/components/ConfirmationDialog/slice.ts create mode 100644 frontend/src/components/form/Toogle/index.module.scss create mode 100644 frontend/src/components/form/Toogle/index.tsx create mode 100644 frontend/src/components/form/Toogle/types.ts create mode 100644 frontend/src/hooks/useConfirmationDialog.ts create mode 100644 frontend/src/pages/Fleets/Add/FleetFormFields/constants.tsx create mode 100644 frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx create mode 100644 frontend/src/pages/Fleets/Add/FleetFormFields/type.ts create mode 100644 frontend/src/pages/Fleets/Add/index.tsx create mode 100644 frontend/src/pages/Fleets/Add/types.ts create mode 100644 frontend/src/pages/Project/constants.tsx create mode 100644 frontend/src/pages/Project/hooks/useYupValidationResolver.ts diff --git a/frontend/src/api.ts b/frontend/src/api.ts index d58dbc7d38..144a21bc86 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -99,6 +99,7 @@ export const API = { // Fleets FLEETS: (projectName: IProject['project_name']) => `${API.BASE()}/project/${projectName}/fleets/list`, FLEETS_DETAILS: (projectName: IProject['project_name']) => `${API.BASE()}/project/${projectName}/fleets/get`, + FLEETS_APPLY: (projectName: IProject['project_name']) => `${API.BASE()}/project/${projectName}/fleets/apply`, FLEETS_DELETE: (projectName: IProject['project_name']) => `${API.BASE()}/project/${projectName}/fleets/delete`, FLEET_INSTANCES_DELETE: (projectName: IProject['project_name']) => `${API.BASE()}/project/${projectName}/fleets/delete_instances`, diff --git a/frontend/src/components/ButtonWithConfirmation/index.tsx b/frontend/src/components/ButtonWithConfirmation/index.tsx index 78c2793d9c..56ae78ad59 100644 --- a/frontend/src/components/ButtonWithConfirmation/index.tsx +++ b/frontend/src/components/ButtonWithConfirmation/index.tsx @@ -1,4 +1,5 @@ import React, { useState } from 'react'; +import { useTranslation } from 'react-i18next'; import Box from '@cloudscape-design/components/box'; import { Button } from '../Button'; @@ -13,20 +14,31 @@ export const ButtonWithConfirmation: React.FC = ({ confirmButtonLabel, ...props }) => { + const { t } = useTranslation(); const [showDeleteConfirm, setShowConfirmDelete] = useState(false); const toggleDeleteConfirm = () => { setShowConfirmDelete((val) => !val); }; - const content = typeof confirmContent === 'string' ? {confirmContent} : confirmContent; - const onConfirm = () => { if (onClick) onClick(); setShowConfirmDelete(false); }; + const getContent = () => { + if (!confirmContent) { + return {t('confirm_dialog.message')}; + } + + if (typeof confirmContent === 'string') { + return {confirmContent}; + } + + return confirmContent; + }; + return ( <> } + {isAvailableProjectManaging && } ); }; @@ -137,7 +137,7 @@ export const ProjectList: React.FC = () => { {t('common.delete')} - + } diff --git a/frontend/src/pages/Project/constants.tsx b/frontend/src/pages/Project/constants.tsx new file mode 100644 index 0000000000..151740116b --- /dev/null +++ b/frontend/src/pages/Project/constants.tsx @@ -0,0 +1,32 @@ +import React from 'react'; + +export const DEFAULT_FLEET_INFO = { + header:

Default fleet

, + body: ( + <> +

+ Fleets act both as pools of instances and as templates for how those instances are provisioned. When you submit + a dev environment, task, or service, dstack reuses idle instances or provisions new + ones based on the fleet configuration. +

+ +

+ If you set Min number of instances to 0, dstack will provision instances + only when you run a dev environment, task, or service. +

+ +

+ At least one fleet is required to run dev environments, tasks, or services. Create it here, or create it using + the dstack apply command via the CLI. +

+ +

+ To learn more about fleets, see the{' '} + + documentation + + . +

+ + ), +}; diff --git a/frontend/src/pages/Project/hooks/useYupValidationResolver.ts b/frontend/src/pages/Project/hooks/useYupValidationResolver.ts new file mode 100644 index 0000000000..2cd694c63d --- /dev/null +++ b/frontend/src/pages/Project/hooks/useYupValidationResolver.ts @@ -0,0 +1,38 @@ +import { useCallback } from 'react'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-expect-error +export function useYupValidationResolver(validationSchema) { + return useCallback( + async (data: TData) => { + try { + const values = await validationSchema.validate(data, { + abortEarly: false, + }); + + return { + values, + errors: {}, + }; + } catch (errors) { + return { + values: {}, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + errors: errors.inner.reduce( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + (allErrors, currentError) => ({ + ...allErrors, + [currentError.path]: { + type: currentError.type ?? 'validation', + message: currentError.message, + }, + }), + {}, + ), + }; + } + }, + [validationSchema], + ); +} diff --git a/frontend/src/pages/User/Details/index.tsx b/frontend/src/pages/User/Details/index.tsx index 805b2efc98..1ee131094c 100644 --- a/frontend/src/pages/User/Details/index.tsx +++ b/frontend/src/pages/User/Details/index.tsx @@ -95,7 +95,13 @@ export const UserDetails: React.FC = () => { - + {t('confirm_dialog.message')}} + onDiscard={toggleDeleteConfirm} + onConfirm={deleteUserHandler} + confirmButtonLabel={t('common.delete')} + /> ); }; diff --git a/frontend/src/pages/User/List/index.tsx b/frontend/src/pages/User/List/index.tsx index 17bc417e94..5831d03383 100644 --- a/frontend/src/pages/User/List/index.tsx +++ b/frontend/src/pages/User/List/index.tsx @@ -208,8 +208,10 @@ export const UserList: React.FC = () => { {t('confirm_dialog.message')}} onDiscard={toggleDeleteConfirm} onConfirm={deleteSelectedUserHandler} + confirmButtonLabel={t('common.delete')} /> ); diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index fbdeca2942..34a8abaaf0 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -10,7 +10,7 @@ import { LoginByGoogleCallback } from 'App/Login/LoginByGoogleCallback'; import { LoginByOktaCallback } from 'App/Login/LoginByOktaCallback'; import { TokenLogin } from 'App/Login/TokenLogin'; import { Logout } from 'App/Logout'; -import { FleetDetails, FleetList } from 'pages/Fleets'; +import { FleetAdd, FleetDetails, FleetList } from 'pages/Fleets'; import { EventsList as FleetEventsList } from 'pages/Fleets/Details/Events'; import { FleetDetails as FleetDetailsGeneral } from 'pages/Fleets/Details/FleetDetails'; import { FleetInspect } from 'pages/Fleets/Details/Inspect'; @@ -202,6 +202,10 @@ export const router = createBrowserRouter([ path: ROUTES.FLEETS.LIST, element: , }, + { + path: ROUTES.FLEETS.ADD.TEMPLATE, + element: , + }, { path: ROUTES.FLEETS.DETAILS.TEMPLATE, element: , diff --git a/frontend/src/routes.ts b/frontend/src/routes.ts index fea2f978a4..288cef72fc 100644 --- a/frontend/src/routes.ts +++ b/frontend/src/routes.ts @@ -137,6 +137,10 @@ export const ROUTES = { FLEETS: { LIST: '/fleets', + ADD: { + TEMPLATE: `/projects/:projectName/fleets/add`, + FORMAT: (projectName: string) => buildRoute(ROUTES.FLEETS.ADD.TEMPLATE, { projectName }), + }, DETAILS: { TEMPLATE: `/projects/:projectName/fleets/:fleetId`, FORMAT: (projectName: string, fleetId: string) => diff --git a/frontend/src/services/fleet.ts b/frontend/src/services/fleet.ts index 3405a18b8b..fa723d7d2d 100644 --- a/frontend/src/services/fleet.ts +++ b/frontend/src/services/fleet.ts @@ -66,7 +66,25 @@ export const fleetApi = createApi({ invalidatesTags: ['Fleets'], }), + + applyFleet: builder.mutation({ + query: ({ projectName, ...body }) => { + return { + url: API.PROJECTS.FLEETS_APPLY(projectName), + method: 'POST', + body, + }; + }, + + invalidatesTags: ['Fleets'], + }), }), }); -export const { useGetFleetsQuery, useLazyGetFleetsQuery, useDeleteFleetMutation, useGetFleetDetailsQuery } = fleetApi; +export const { + useGetFleetsQuery, + useLazyGetFleetsQuery, + useDeleteFleetMutation, + useGetFleetDetailsQuery, + useApplyFleetMutation, +} = fleetApi; diff --git a/frontend/src/services/project.ts b/frontend/src/services/project.ts index 2f0a4bd6b5..8875c48df6 100644 --- a/frontend/src/services/project.ts +++ b/frontend/src/services/project.ts @@ -74,7 +74,7 @@ export const projectApi = createApi({ providesTags: (result) => (result ? [{ type: 'Projects' as const, id: result.project_name }] : []), }), - createProject: builder.mutation({ + createProject: builder.mutation({ query: (project) => ({ url: API.PROJECTS.CREATE(), method: 'POST', diff --git a/frontend/src/store.ts b/frontend/src/store.ts index ca19b1206d..03d2c820e7 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -1,5 +1,6 @@ import { configureStore } from '@reduxjs/toolkit'; +import confirmationReducer from 'components/ConfirmationDialog/slice'; import notificationsReducer from 'components/Notifications/slice'; import { artifactApi } from 'services/artifact'; @@ -25,6 +26,7 @@ export const store = configureStore({ reducer: { app: appReducer, notifications: notificationsReducer, + confirmation: confirmationReducer, [projectApi.reducerPath]: projectApi.reducer, [runApi.reducerPath]: runApi.reducer, [artifactApi.reducerPath]: artifactApi.reducer, diff --git a/frontend/src/types/fleet.d.ts b/frontend/src/types/fleet.d.ts index 892acf41fa..2813cd4023 100644 --- a/frontend/src/types/fleet.d.ts +++ b/frontend/src/types/fleet.d.ts @@ -45,9 +45,12 @@ declare interface IFleetConfigurationRequest { max?: number; }; placement?: 'any' | 'cluster'; + reservation?: string; resources?: IFleetConfigurationResource[]; + blocks?: string | number; backends?: TBackendType[]; regions?: string[]; + availability_zones?: string[]; instance_types?: string[]; spot_policy?: TSpotPolicy; retry?: @@ -76,13 +79,14 @@ declare interface IProfileRequest { instance_name?: string; creation_policy?: 'reuse' | 'reuse-or-create'; idle_duration?: number | string; - name: string; + name?: string; default?: boolean; } declare interface IFleetSpec { - autocreated: boolean; + autocreated?: boolean; configuration: IFleetConfigurationRequest; + configuration_path?: string; profile: IProfileRequest; } @@ -96,3 +100,11 @@ declare interface IFleet { status: 'submitted' | 'active' | 'terminating' | 'terminated' | 'failed'; status_message: string; } + +declare interface IApplyFleetPlanRequestRequest { + plan: { + spec: IFleetSpec; + }; + + force: boolean; +} diff --git a/frontend/src/types/project.d.ts b/frontend/src/types/project.d.ts index cf24c84d03..babb4dab7a 100644 --- a/frontend/src/types/project.d.ts +++ b/frontend/src/types/project.d.ts @@ -46,3 +46,7 @@ declare interface IProjectSecret { name: string; value?: string; } + +declare type IProjectCreateRequestParams = Pick & { + is_public: boolean; +}; From 196459136466fe89ede9c318075b4289507a6375 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 21 Jan 2026 12:25:40 +0500 Subject: [PATCH 008/640] Support shared AWS compute caches (#3483) * Log get_offers times * Request aws quotas and zones in parallel * Revert "Request aws quotas and zones in parallel" This reverts commit a0f365e4a662087824266e5e5dccd7e7a8028bee. * Add AWSQuotasSharedCache * Refactor compute caches --- .../_internal/core/backends/aws/backend.py | 9 ++- .../_internal/core/backends/aws/compute.py | 78 ++++++++++--------- .../_internal/core/backends/base/compute.py | 15 +++- .../_internal/core/backends/gcp/compute.py | 18 ++--- .../server/services/backends/__init__.py | 13 +++- 5 files changed, 84 insertions(+), 49 deletions(-) diff --git a/src/dstack/_internal/core/backends/aws/backend.py b/src/dstack/_internal/core/backends/aws/backend.py index 3dfd4f4093..1169227cc7 100644 --- a/src/dstack/_internal/core/backends/aws/backend.py +++ b/src/dstack/_internal/core/backends/aws/backend.py @@ -1,3 +1,5 @@ +from typing import Optional + import botocore.exceptions from dstack._internal.core.backends.aws.compute import AWSCompute @@ -11,9 +13,12 @@ class AWSBackend(Backend): TYPE = BackendType.AWS COMPUTE_CLASS = AWSCompute - def __init__(self, config: AWSConfig): + def __init__(self, config: AWSConfig, compute: Optional[AWSCompute] = None): self.config = config - self._compute = AWSCompute(self.config) + if compute is not None: + self._compute = compute + else: + self._compute = AWSCompute(self.config) self._check_credentials() def compute(self) -> AWSCompute: diff --git a/src/dstack/_internal/core/backends/aws/compute.py b/src/dstack/_internal/core/backends/aws/compute.py index 48720bb316..be3133456c 100644 --- a/src/dstack/_internal/core/backends/aws/compute.py +++ b/src/dstack/_internal/core/backends/aws/compute.py @@ -1,6 +1,7 @@ import threading from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Tuple import boto3 @@ -19,6 +20,8 @@ ) from dstack._internal.core.backends.base.compute import ( Compute, + ComputeCache, + ComputeTTLCache, ComputeWithAllOffersCached, ComputeWithCreateInstanceSupport, ComputeWithGatewaySupport, @@ -94,6 +97,11 @@ def _ec2client_cache_methodkey(self, ec2_client, *args, **kwargs): return hashkey(*args, **kwargs) +@dataclass +class AWSQuotasCache(ComputeTTLCache): + execution_lock: threading.Lock = field(default_factory=threading.Lock) + + class AWSCompute( ComputeWithAllOffersCached, ComputeWithCreateInstanceSupport, @@ -106,7 +114,12 @@ class AWSCompute( ComputeWithVolumeSupport, Compute, ): - def __init__(self, config: AWSConfig): + def __init__( + self, + config: AWSConfig, + quotas_cache: Optional[AWSQuotasCache] = None, + zones_cache: Optional[ComputeCache] = None, + ): super().__init__() self.config = config if isinstance(config.creds, AWSAccessKeyCreds): @@ -119,23 +132,18 @@ def __init__(self, config: AWSConfig): # Caches to avoid redundant API calls when provisioning many instances # get_offers is already cached but we still cache its sub-functions # with more aggressive/longer caches. - self._offers_post_filter_cache_lock = threading.Lock() - self._offers_post_filter_cache = TTLCache(maxsize=10, ttl=180) - self._get_regions_to_quotas_cache_lock = threading.Lock() - self._get_regions_to_quotas_execution_lock = threading.Lock() - self._get_regions_to_quotas_cache = TTLCache(maxsize=10, ttl=300) - self._get_regions_to_zones_cache_lock = threading.Lock() - self._get_regions_to_zones_cache = Cache(maxsize=10) - self._get_vpc_id_subnet_id_or_error_cache_lock = threading.Lock() - self._get_vpc_id_subnet_id_or_error_cache = TTLCache(maxsize=100, ttl=600) - self._get_maximum_efa_interfaces_cache_lock = threading.Lock() - self._get_maximum_efa_interfaces_cache = Cache(maxsize=100) - self._get_subnets_availability_zones_cache_lock = threading.Lock() - self._get_subnets_availability_zones_cache = Cache(maxsize=100) - self._create_security_group_cache_lock = threading.Lock() - self._create_security_group_cache = TTLCache(maxsize=100, ttl=600) - self._get_image_id_and_username_cache_lock = threading.Lock() - self._get_image_id_and_username_cache = TTLCache(maxsize=100, ttl=600) + self._offers_post_filter_cache = ComputeTTLCache(cache=TTLCache(maxsize=10, ttl=180)) + if quotas_cache is None: + quotas_cache = AWSQuotasCache(cache=TTLCache(maxsize=10, ttl=600)) + self._regions_to_quotas_cache = quotas_cache + if zones_cache is None: + zones_cache = ComputeCache(cache=Cache(maxsize=10)) + self._regions_to_zones_cache = zones_cache + self._vpc_id_subnet_id_cache = ComputeTTLCache(cache=TTLCache(maxsize=100, ttl=600)) + self._maximum_efa_interfaces_cache = ComputeCache(cache=Cache(maxsize=100)) + self._subnets_availability_zones_cache = ComputeCache(cache=Cache(maxsize=100)) + self._security_group_cache = ComputeTTLCache(cache=TTLCache(maxsize=100, ttl=600)) + self._image_id_and_username_cache = ComputeTTLCache(cache=TTLCache(maxsize=100, ttl=600)) def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]: offers = get_catalog_offers( @@ -144,7 +152,7 @@ def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability extra_filter=_supported_instances, ) regions = list(set(i.region for i in offers)) - with self._get_regions_to_quotas_execution_lock: + with self._regions_to_quotas_cache.execution_lock: # Cache lock does not prevent concurrent execution. # We use a separate lock to avoid requesting quotas in parallel and hitting rate limits. regions_to_quotas = self._get_regions_to_quotas(self.session, regions) @@ -173,9 +181,9 @@ def _get_offers_cached_key(self, requirements: Requirements) -> int: return hash(requirements.json()) @cachedmethod( - cache=lambda self: self._offers_post_filter_cache, + cache=lambda self: self._offers_post_filter_cache.cache, key=_get_offers_cached_key, - lock=lambda self: self._offers_post_filter_cache_lock, + lock=lambda self: self._offers_post_filter_cache.lock, ) def get_offers_post_filter( self, requirements: Requirements @@ -789,9 +797,9 @@ def _get_regions_to_quotas_key( return hashkey(tuple(regions)) @cachedmethod( - cache=lambda self: self._get_regions_to_quotas_cache, + cache=lambda self: self._regions_to_quotas_cache.cache, key=_get_regions_to_quotas_key, - lock=lambda self: self._get_regions_to_quotas_cache_lock, + lock=lambda self: self._regions_to_quotas_cache.lock, ) def _get_regions_to_quotas( self, @@ -808,9 +816,9 @@ def _get_regions_to_zones_key( return hashkey(tuple(regions)) @cachedmethod( - cache=lambda self: self._get_regions_to_zones_cache, + cache=lambda self: self._regions_to_zones_cache.cache, key=_get_regions_to_zones_key, - lock=lambda self: self._get_regions_to_zones_cache_lock, + lock=lambda self: self._regions_to_zones_cache.lock, ) def _get_regions_to_zones( self, @@ -832,9 +840,9 @@ def _get_vpc_id_subnet_id_or_error_cache_key( ) @cachedmethod( - cache=lambda self: self._get_vpc_id_subnet_id_or_error_cache, + cache=lambda self: self._vpc_id_subnet_id_cache.cache, key=_get_vpc_id_subnet_id_or_error_cache_key, - lock=lambda self: self._get_vpc_id_subnet_id_or_error_cache_lock, + lock=lambda self: self._vpc_id_subnet_id_cache.lock, ) def _get_vpc_id_subnet_id_or_error( self, @@ -853,9 +861,9 @@ def _get_vpc_id_subnet_id_or_error( ) @cachedmethod( - cache=lambda self: self._get_maximum_efa_interfaces_cache, + cache=lambda self: self._maximum_efa_interfaces_cache.cache, key=_ec2client_cache_methodkey, - lock=lambda self: self._get_maximum_efa_interfaces_cache_lock, + lock=lambda self: self._maximum_efa_interfaces_cache.lock, ) def _get_maximum_efa_interfaces( self, @@ -877,9 +885,9 @@ def _get_subnets_availability_zones_key( return hashkey(region, tuple(subnet_ids)) @cachedmethod( - cache=lambda self: self._get_subnets_availability_zones_cache, + cache=lambda self: self._subnets_availability_zones_cache.cache, key=_get_subnets_availability_zones_key, - lock=lambda self: self._get_subnets_availability_zones_cache_lock, + lock=lambda self: self._subnets_availability_zones_cache.lock, ) def _get_subnets_availability_zones( self, @@ -893,9 +901,9 @@ def _get_subnets_availability_zones( ) @cachedmethod( - cache=lambda self: self._create_security_group_cache, + cache=lambda self: self._security_group_cache.cache, key=_ec2client_cache_methodkey, - lock=lambda self: self._create_security_group_cache_lock, + lock=lambda self: self._security_group_cache.lock, ) def _create_security_group( self, @@ -923,9 +931,9 @@ def _get_image_id_and_username_cache_key( ) @cachedmethod( - cache=lambda self: self._get_image_id_and_username_cache, + cache=lambda self: self._image_id_and_username_cache.cache, key=_get_image_id_and_username_cache_key, - lock=lambda self: self._get_image_id_and_username_cache_lock, + lock=lambda self: self._image_id_and_username_cache.lock, ) def _get_image_id_and_username( self, diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 75a68e77ff..49513e3211 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -6,6 +6,7 @@ import threading from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field from enum import Enum from functools import lru_cache from pathlib import Path @@ -14,7 +15,7 @@ import git import requests import yaml -from cachetools import TTLCache, cachedmethod +from cachetools import Cache, TTLCache, cachedmethod from gpuhunt import CPUArchitecture from dstack._internal import settings @@ -89,6 +90,18 @@ def to_cpu_architecture(self) -> CPUArchitecture: assert False, self +@dataclass +class ComputeCache: + cache: Cache + lock: threading.Lock = field(default_factory=threading.Lock) + + +@dataclass +class ComputeTTLCache: + cache: TTLCache + lock: threading.Lock = field(default_factory=threading.Lock) + + class Compute(ABC): """ A base class for all compute implementations with minimal features. diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index c2c18e3d9f..cd5ecb829f 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -1,7 +1,6 @@ import concurrent.futures import json import re -import threading from collections import defaultdict from collections.abc import Iterable from dataclasses import dataclass @@ -19,6 +18,7 @@ from dstack import version from dstack._internal.core.backends.base.compute import ( Compute, + ComputeTTLCache, ComputeWithAllOffersCached, ComputeWithCreateInstanceSupport, ComputeWithGatewaySupport, @@ -127,11 +127,9 @@ def __init__(self, config: GCPConfig): credentials=self.credentials ) self.reservations_client = compute_v1.ReservationsClient(credentials=self.credentials) - self._usable_subnets_cache_lock = threading.Lock() - self._usable_subnets_cache = TTLCache(maxsize=1, ttl=120) - self._find_reservation_cache_lock = threading.Lock() - # smaller TTL, since we check the reservation's in_use_count, which can change often - self._find_reservation_cache = TTLCache(maxsize=8, ttl=20) + self._usable_subnets_cache = ComputeTTLCache(cache=TTLCache(maxsize=1, ttl=120)) + # Smaller TTL since we check the reservation's in_use_count, which can change often + self._reservation_cache = ComputeTTLCache(cache=TTLCache(maxsize=8, ttl=20)) def get_all_offers_with_availability(self) -> List[InstanceOfferWithAvailability]: regions = get_or_error(self.config.regions) @@ -948,8 +946,8 @@ def _get_roce_subnets( return nic_subnets @cachedmethod( - cache=lambda self: self._usable_subnets_cache, - lock=lambda self: self._usable_subnets_cache_lock, + cache=lambda self: self._usable_subnets_cache.cache, + lock=lambda self: self._usable_subnets_cache.lock, ) def _list_usable_subnets(self) -> list[compute_v1.UsableSubnetwork]: # To avoid hitting the `ListUsable requests per minute` system limit, we fetch all subnets @@ -969,8 +967,8 @@ def _get_vpc_subnet(self, region: str) -> Optional[str]: ) @cachedmethod( - cache=lambda self: self._find_reservation_cache, - lock=lambda self: self._find_reservation_cache_lock, + cache=lambda self: self._reservation_cache.cache, + lock=lambda self: self._reservation_cache.lock, ) def _find_reservation(self, configured_name: str) -> dict[str, compute_v1.Reservation]: if match := RESERVATION_PATTERN.fullmatch(configured_name): diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 53284e6175..ce0f17bde5 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,5 +1,6 @@ import asyncio import heapq +import time from collections.abc import Iterable, Iterator from typing import Callable, Coroutine, Dict, List, Optional, Tuple from uuid import UUID @@ -361,7 +362,7 @@ def get_filtered_offers_with_backends( yield (backend, offer) logger.info("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends]) - tasks = [run_async(backend.compute().get_offers, requirements) for backend in backends] + tasks = [run_async(get_offers_tracked, backend, requirements) for backend in backends] offers_by_backend = [] for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)): if isinstance(result, BackendError): @@ -391,3 +392,13 @@ def check_backend_type_available(backend_type: BackendType): " Ensure that backend dependencies are installed." f" Available backends: {[b.value for b in list_available_backend_types()]}." ) + + +def get_offers_tracked( + backend: Backend, requirements: Requirements +) -> Iterator[InstanceOfferWithAvailability]: + start = time.time() + res = backend.compute().get_offers(requirements) + duration = time.time() - start + logger.debug("Got offers from %s in %.6fs", backend.TYPE.value, duration) + return res From 32fbc02816e2ea5f10302bbf372c1142d952de0a Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Wed, 21 Jan 2026 10:07:53 +0100 Subject: [PATCH 009/640] [UI] Minor re-order in the sidebar (#3484) --- frontend/src/layouts/AppLayout/hooks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/layouts/AppLayout/hooks.ts b/frontend/src/layouts/AppLayout/hooks.ts index a305317d50..f46366fcd6 100644 --- a/frontend/src/layouts/AppLayout/hooks.ts +++ b/frontend/src/layouts/AppLayout/hooks.ts @@ -25,11 +25,11 @@ export const useSideNavigation = () => { const generalLinks = [ { type: 'link', text: t('navigation.runs'), href: ROUTES.RUNS.LIST }, { type: 'link', text: t('navigation.offers'), href: ROUTES.OFFERS.LIST }, - { type: 'link', text: t('navigation.models'), href: ROUTES.MODELS.LIST }, { type: 'link', text: t('navigation.fleets'), href: ROUTES.FLEETS.LIST }, { type: 'link', text: t('navigation.instances'), href: ROUTES.INSTANCES.LIST }, { type: 'link', text: t('navigation.volumes'), href: ROUTES.VOLUMES.LIST }, { type: 'link', text: t('navigation.events'), href: ROUTES.EVENTS.LIST }, + { type: 'link', text: t('navigation.models'), href: ROUTES.MODELS.LIST }, { type: 'link', text: t('navigation.project_other'), href: ROUTES.PROJECT.LIST }, isGlobalAdmin && { From 6d14aadcb1ee7309f7b7f97d2c2239a5f2766470 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 21 Jan 2026 16:14:37 +0500 Subject: [PATCH 010/640] Add missing Box imports (#3485) --- frontend/src/pages/User/Details/index.tsx | 2 +- frontend/src/pages/User/List/index.tsx | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/User/Details/index.tsx b/frontend/src/pages/User/Details/index.tsx index 1ee131094c..8f1b2d393d 100644 --- a/frontend/src/pages/User/Details/index.tsx +++ b/frontend/src/pages/User/Details/index.tsx @@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Outlet, useNavigate, useParams } from 'react-router-dom'; -import { ConfirmationDialog, ContentLayout, SpaceBetween, Tabs } from 'components'; +import { Box, ConfirmationDialog, ContentLayout, SpaceBetween, Tabs } from 'components'; import { DetailsHeader } from 'components'; import { useNotifications /* usePermissionGuard*/ } from 'hooks'; diff --git a/frontend/src/pages/User/List/index.tsx b/frontend/src/pages/User/List/index.tsx index 5831d03383..61d14e83fc 100644 --- a/frontend/src/pages/User/List/index.tsx +++ b/frontend/src/pages/User/List/index.tsx @@ -4,6 +4,7 @@ import { useNavigate } from 'react-router-dom'; import { format } from 'date-fns'; import { + Box, Button, ConfirmationDialog, Header, From f09d06180cca3ac93933840f7153161e60444471 Mon Sep 17 00:00:00 2001 From: Oleg Date: Wed, 21 Jan 2026 15:00:48 +0300 Subject: [PATCH 011/640] Hotfix. Fixed generation fleet fields in project forms (#3486) --- frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx | 4 ++-- frontend/src/pages/Project/Add/index.tsx | 2 +- frontend/src/pages/Project/CreateWizard/index.tsx | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx b/frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx index 96a12ef4c8..a19e701366 100644 --- a/frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx +++ b/frontend/src/pages/Fleets/Add/FleetFormFields/index.tsx @@ -18,12 +18,12 @@ export function FleetFormFields({ const { t } = useTranslation(); const [openHelpPanel] = useHelpPanel(); - const getFieldNameWitPrefix = (name: string) => { + const getFieldNameWitPrefix = (name: string): string => { if (!fieldNamePrefix) { return name; } - [fieldNamePrefix, name].join('.'); + return [fieldNamePrefix, name].join('.'); }; return ( diff --git a/frontend/src/pages/Project/Add/index.tsx b/frontend/src/pages/Project/Add/index.tsx index 14bbcc71fe..23a9eb0a19 100644 --- a/frontend/src/pages/Project/Add/index.tsx +++ b/frontend/src/pages/Project/Add/index.tsx @@ -302,7 +302,7 @@ export const ProjectAdd: React.FC = () => { control={control} disabledAllFields={loading} - fieldNamePrefix="fleet." + fieldNamePrefix="fleet" /> )} diff --git a/frontend/src/pages/Project/CreateWizard/index.tsx b/frontend/src/pages/Project/CreateWizard/index.tsx index 4981efb6f1..83cf3804d3 100644 --- a/frontend/src/pages/Project/CreateWizard/index.tsx +++ b/frontend/src/pages/Project/CreateWizard/index.tsx @@ -442,7 +442,7 @@ export const CreateProjectWizard: React.FC = () => { control={control} disabledAllFields={loading} - fieldNamePrefix="fleet." + fieldNamePrefix="fleet" /> )} From 2ad526cdcd0588f1657b0c3481295252c5a67ea1 Mon Sep 17 00:00:00 2001 From: Oleg Vavilov Date: Wed, 21 Jan 2026 20:33:03 +0300 Subject: [PATCH 012/640] Small fix --- frontend/src/locale/en.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 5cd0808be4..c821abf317 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -586,6 +586,9 @@ "terminating": "Terminating", "terminated": "Terminated" }, + "create": { + "success_notification": "The fleet is created!" + }, "instances": { "active_only": "Active instances", "filter_property_placeholder": "Filter by properties", From d3beebc926cd68ff2f9364f35c52f66aa6665b5b Mon Sep 17 00:00:00 2001 From: Oleg Vavilov Date: Wed, 21 Jan 2026 22:11:21 +0300 Subject: [PATCH 013/640] Added react rules for eslint --- frontend/eslint.config.cjs | 8 +- frontend/package-lock.json | 2181 ++++++++++++++++++++---- frontend/package.json | 3 +- frontend/src/components/Code/index.tsx | 2 + 4 files changed, 1858 insertions(+), 336 deletions(-) diff --git a/frontend/eslint.config.cjs b/frontend/eslint.config.cjs index 3cfe1c41e4..9750398241 100644 --- a/frontend/eslint.config.cjs +++ b/frontend/eslint.config.cjs @@ -1,6 +1,7 @@ const { defineConfig, globalIgnores } = require('eslint/config'); const i18N = require('eslint-plugin-i18n'); const simpleImportSort = require('eslint-plugin-simple-import-sort'); +const react = require('eslint-plugin-react'); const { FlatCompat } = require('@eslint/eslintrc'); const js = require('@eslint/js'); const typescriptEslint = require('@typescript-eslint/eslint-plugin'); @@ -18,20 +19,25 @@ const BASE_CONFIG = { 'plugin:@typescript-eslint/eslint-recommended', 'plugin:@typescript-eslint/recommended', 'prettier', - 'plugin:prettier/recommended' + 'plugin:prettier/recommended', + 'plugin:react/recommended', ), plugins: { '@typescript-eslint': typescriptEslint, i18n: i18N, 'simple-import-sort': simpleImportSort, + react: react, }, languageOptions: { parser: tsParser, }, + settings: {}, rules: { + 'react/jsx-no-target-blank': 'off', + 'react/no-unescaped-entities': 'off', 'i18n/no-russian-character': 1, 'simple-import-sort/imports': [ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e66eba969c..a3d3f29d93 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -81,10 +81,13 @@ "cross-env": "^7.0.3", "css-loader": "^6.7.3", "enzyme": "^3.11.0", - "eslint": "^9.32.0", + "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.5", "eslint-plugin-i18n": "^2.4.0", + "eslint-plugin-import": "^2.32.0", + "eslint-plugin-import-x": "^4.16.1", "eslint-plugin-prettier": "^5.4.1", + "eslint-plugin-react": "^7.37.5", "eslint-plugin-simple-import-sort": "^12.1.1", "favicons": "^7.2.0", "favicons-webpack-plugin": "^6.0.1", @@ -2548,11 +2551,35 @@ "react": ">=16.8.0" } }, + "node_modules/@emnapi/core": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", + "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/runtime": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", - "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", + "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", "dev": true, + "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" @@ -2583,9 +2610,9 @@ "peer": true }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", - "integrity": "sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2611,13 +2638,13 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.0", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.0.tgz", - "integrity": "sha512-ENIdc4iLu0d93HeYirvKmrzshzofPw6VkZRKQGe9Nv46ZnWUzcF1xV01dcvEg/1wXUR61OmmlSfyeyO7EvjLxQ==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", + "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^2.1.6", + "@eslint/object-schema": "^2.1.7", "debug": "^4.3.1", "minimatch": "^3.1.2" }, @@ -2626,19 +2653,22 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.0.tgz", - "integrity": "sha512-ViuymvFmcJi04qdZeDc2whTHryouGcDlaxPqarTD0ZE10ISpxGUVZGZDx4w01upyIynL3iu6IXH2bS1NhclQMw==", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@eslint/core": { - "version": "0.15.1", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.15.1.tgz", - "integrity": "sha512-bkOp+iumZCCbt1K1CmWf0R9pM5yKpDv+ZXtvSyQpudrI9kuFLp+bM2WOPXImuD/ceQuaa8f5pj93Y7zyECIGNA==", + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2710,9 +2740,9 @@ "license": "MIT" }, "node_modules/@eslint/js": { - "version": "9.32.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.32.0.tgz", - "integrity": "sha512-BBpRFZK3eX6uMLKz8WxFOBIFFcGFJ/g8XuwjTHCqHROSIsopI+ddn/d5Cfh36+7+e5edVS8dbSHnBNhrLEX0zg==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", + "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", "dev": true, "license": "MIT", "engines": { @@ -2723,9 +2753,9 @@ } }, "node_modules/@eslint/object-schema": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.6.tgz", - "integrity": "sha512-RBMg5FRL0I0gs51M/guSAj5/e14VQ4tpZnQNWwuDT66P14I43ItmPfIZRhO9fUVIPOAQXU47atlywZ/czoqFPA==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", "dev": true, "license": "Apache-2.0", "engines": { @@ -2733,13 +2763,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.3.4.tgz", - "integrity": "sha512-Ul5l+lHEcw3L5+k8POx6r74mxEYKG5kOb6Xpy2gCRW6zweT6TEhAf8vhxGgjhqrd/VO/Dirhsb+1hNpD1ue9hw==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.15.1", + "@eslint/core": "^0.17.0", "levn": "^0.4.1" }, "engines": { @@ -3231,6 +3261,29 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -3820,6 +3873,19 @@ "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", "dev": true }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", + "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.10.0" + } + }, "node_modules/@nicolo-ribaudo/chokidar-2": { "version": "2.1.8-no-fsevents.3", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz", @@ -4296,6 +4362,13 @@ "node": ">=10" } }, + "node_modules/@rtsao/scc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", + "integrity": "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==", + "dev": true, + "license": "MIT" + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -4707,6 +4780,17 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -5016,6 +5100,13 @@ "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, + "node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/lodash": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", @@ -5534,6 +5625,275 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@unrs/resolver-binding-android-arm-eabi": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", + "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@unrs/resolver-binding-android-arm64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", + "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@unrs/resolver-binding-darwin-arm64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", + "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@unrs/resolver-binding-darwin-x64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", + "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@unrs/resolver-binding-freebsd-x64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", + "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", + "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", + "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", + "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", + "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", + "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", + "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", + "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", + "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", + "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", + "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", + "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^0.2.11" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", + "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", + "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-x64-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", + "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@webassemblyjs/ast": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", @@ -5982,13 +6342,14 @@ } }, "node_modules/array-buffer-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz", - "integrity": "sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", + "integrity": "sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", - "is-array-buffer": "^3.0.4" + "call-bound": "^1.0.3", + "is-array-buffer": "^3.0.5" }, "engines": { "node": ">= 0.4" @@ -6003,8 +6364,31 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "dev": true }, - "node_modules/array-union": { - "version": "2.1.0", + "node_modules/array-includes": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.9.tgz", + "integrity": "sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "define-properties": "^1.2.1", + "es-abstract": "^1.24.0", + "es-object-atoms": "^1.1.1", + "get-intrinsic": "^1.3.0", + "is-string": "^1.1.1", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array-union": { + "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true, @@ -6032,16 +6416,79 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/array.prototype.findlast": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz", + "integrity": "sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.7", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.2", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", + "es-shim-unscopables": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.findlastindex": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.6.tgz", + "integrity": "sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.9", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "es-shim-unscopables": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/array.prototype.flat": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", - "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.3.tgz", + "integrity": "sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.5", + "es-shim-unscopables": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.flatmap": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.3.tgz", + "integrity": "sha512-Y7Wt51eKJSyi80hFrJCePGGNo5ktJCslFuboqJsbf57CCPcm5zztluPlc4/aD8sWsKvlwatezpV4U1efk8kpjg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "es-shim-unscopables": "^1.0.0" + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.5", + "es-shim-unscopables": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -6050,20 +6497,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/array.prototype.tosorted": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.4.tgz", + "integrity": "sha512-p6Fx8B7b7ZhL/gmUsAy0D15WhvDccw3mnGNbZpi3pmeJdxtWsj2jEaI4Y6oo3XiHfzuSgPwKc04MYt6KgvC/wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.7", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.3", + "es-errors": "^1.3.0", + "es-shim-unscopables": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/arraybuffer.prototype.slice": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz", - "integrity": "sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.4.tgz", + "integrity": "sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==", "dev": true, + "license": "MIT", "dependencies": { "array-buffer-byte-length": "^1.0.1", - "call-bind": "^1.0.5", + "call-bind": "^1.0.8", "define-properties": "^1.2.1", - "es-abstract": "^1.22.3", - "es-errors": "^1.2.1", - "get-intrinsic": "^1.2.3", - "is-array-buffer": "^3.0.4", - "is-shared-array-buffer": "^1.0.2" + "es-abstract": "^1.23.5", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "is-array-buffer": "^3.0.4" }, "engines": { "node": ">= 0.4" @@ -6078,6 +6542,16 @@ "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "dev": true }, + "node_modules/async-function": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", + "integrity": "sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -6143,6 +6617,7 @@ "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", "dev": true, + "license": "MIT", "dependencies": { "possible-typed-array-names": "^1.0.0" }, @@ -6902,16 +7377,47 @@ } }, "node_modules/call-bind": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", "dev": true, + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" }, "engines": { "node": ">= 0.4" @@ -7370,6 +7876,16 @@ "node": ">= 6" } }, + "node_modules/comment-parser": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.4.4.tgz", + "integrity": "sha512-0D6qSQ5IkeRrGJFHRClzaMOenMeT0gErz3zIw3AprKMqhRN6LNU2jQOdkPG/FZ+8bCgXE1VidrgSzlBBDZRr8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/common-path-prefix": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", @@ -8043,14 +8559,15 @@ } }, "node_modules/data-view-buffer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", - "integrity": "sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.2.tgz", + "integrity": "sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-data-view": "^1.0.1" + "is-data-view": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -8060,29 +8577,31 @@ } }, "node_modules/data-view-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz", - "integrity": "sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.2.tgz", + "integrity": "sha512-tuhGbE6CfTM9+5ANGf+oQb72Ky/0+s3xKUpHvShfiz2RxMFgFPjsXuRLBVMtvMs15awe45SRb83D6wH4ew6wlQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-data-view": "^1.0.1" + "is-data-view": "^1.0.2" }, "engines": { "node": ">= 0.4" }, "funding": { - "url": "https://github.com/sponsors/ljharb" + "url": "https://github.com/sponsors/inspect-js" } }, "node_modules/data-view-byte-offset": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.0.tgz", - "integrity": "sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.1.tgz", + "integrity": "sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.2", "es-errors": "^1.3.0", "is-data-view": "^1.0.1" }, @@ -8370,6 +8889,19 @@ "node": ">=6" } }, + "node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/dom-accessibility-api": { "version": "0.5.16", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", @@ -8464,6 +8996,21 @@ "tslib": "^2.0.3" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -8659,57 +9206,66 @@ } }, "node_modules/es-abstract": { - "version": "1.23.3", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.3.tgz", - "integrity": "sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A==", + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.24.1.tgz", + "integrity": "sha512-zHXBLhP+QehSSbsS9Pt23Gg964240DPd6QCf8WpkqEXxQ7fhdZzYsocOr5u7apWonsS5EjZDmTF+/slGMyasvw==", "dev": true, + "license": "MIT", "dependencies": { - "array-buffer-byte-length": "^1.0.1", - "arraybuffer.prototype.slice": "^1.0.3", + "array-buffer-byte-length": "^1.0.2", + "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "data-view-buffer": "^1.0.1", - "data-view-byte-length": "^1.0.1", - "data-view-byte-offset": "^1.0.0", - "es-define-property": "^1.0.0", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "data-view-buffer": "^1.0.2", + "data-view-byte-length": "^1.0.2", + "data-view-byte-offset": "^1.0.1", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", - "es-object-atoms": "^1.0.0", - "es-set-tostringtag": "^2.0.3", - "es-to-primitive": "^1.2.1", - "function.prototype.name": "^1.1.6", - "get-intrinsic": "^1.2.4", - "get-symbol-description": "^1.0.2", - "globalthis": "^1.0.3", - "gopd": "^1.0.1", + "es-object-atoms": "^1.1.1", + "es-set-tostringtag": "^2.1.0", + "es-to-primitive": "^1.3.0", + "function.prototype.name": "^1.1.8", + "get-intrinsic": "^1.3.0", + "get-proto": "^1.0.1", + "get-symbol-description": "^1.1.0", + "globalthis": "^1.0.4", + "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", - "has-proto": "^1.0.3", - "has-symbols": "^1.0.3", + "has-proto": "^1.2.0", + "has-symbols": "^1.1.0", "hasown": "^2.0.2", - "internal-slot": "^1.0.7", - "is-array-buffer": "^3.0.4", + "internal-slot": "^1.1.0", + "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", - "is-data-view": "^1.0.1", + "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", - "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.3", - "is-string": "^1.0.7", - "is-typed-array": "^1.1.13", - "is-weakref": "^1.0.2", - "object-inspect": "^1.13.1", + "is-regex": "^1.2.1", + "is-set": "^2.0.3", + "is-shared-array-buffer": "^1.0.4", + "is-string": "^1.1.1", + "is-typed-array": "^1.1.15", + "is-weakref": "^1.1.1", + "math-intrinsics": "^1.1.0", + "object-inspect": "^1.13.4", "object-keys": "^1.1.1", - "object.assign": "^4.1.5", - "regexp.prototype.flags": "^1.5.2", - "safe-array-concat": "^1.1.2", - "safe-regex-test": "^1.0.3", - "string.prototype.trim": "^1.2.9", - "string.prototype.trimend": "^1.0.8", + "object.assign": "^4.1.7", + "own-keys": "^1.0.1", + "regexp.prototype.flags": "^1.5.4", + "safe-array-concat": "^1.1.3", + "safe-push-apply": "^1.0.0", + "safe-regex-test": "^1.1.0", + "set-proto": "^1.0.0", + "stop-iteration-iterator": "^1.1.0", + "string.prototype.trim": "^1.2.10", + "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", - "typed-array-buffer": "^1.0.2", - "typed-array-byte-length": "^1.0.1", - "typed-array-byte-offset": "^1.0.2", - "typed-array-length": "^1.0.6", - "unbox-primitive": "^1.0.2", - "which-typed-array": "^1.1.15" + "typed-array-buffer": "^1.0.3", + "typed-array-byte-length": "^1.0.3", + "typed-array-byte-offset": "^1.0.4", + "typed-array-length": "^1.0.7", + "unbox-primitive": "^1.1.0", + "which-typed-array": "^1.1.19" }, "engines": { "node": ">= 0.4" @@ -8725,13 +9281,11 @@ "dev": true }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "dev": true, - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -8745,16 +9299,45 @@ "node": ">= 0.4" } }, + "node_modules/es-iterator-helpers": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.2.2.tgz", + "integrity": "sha512-BrUQ0cPTB/IwXj23HtwHjS9n7O4h9FX94b4xc5zlTHxeLgTAdzYUDyy6KdExAl9lbN5rtfe44xpjpmj9grxs5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "define-properties": "^1.2.1", + "es-abstract": "^1.24.1", + "es-errors": "^1.3.0", + "es-set-tostringtag": "^2.1.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.3.0", + "globalthis": "^1.0.4", + "gopd": "^1.2.0", + "has-property-descriptors": "^1.0.2", + "has-proto": "^1.2.0", + "has-symbols": "^1.1.0", + "internal-slot": "^1.1.0", + "iterator.prototype": "^1.1.5", + "safe-array-concat": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==" }, "node_modules/es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "dev": true, + "license": "MIT", "dependencies": { "es-errors": "^1.3.0" }, @@ -8763,37 +9346,44 @@ } }, "node_modules/es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "dev": true, + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" } }, "node_modules/es-shim-unscopables": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz", - "integrity": "sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.1.0.tgz", + "integrity": "sha512-d9T8ucsEhh8Bi1woXCf+TIKDIROLG5WCkxg8geBCbvk22kzwC5G2OnXVMO6FUsvQlgUUXQ2itephWDLqDzbeCw==", "dev": true, + "license": "MIT", "dependencies": { - "hasown": "^2.0.0" + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" } }, "node_modules/es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.3.0.tgz", + "integrity": "sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==", "dev": true, + "license": "MIT", "dependencies": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" + "is-callable": "^1.2.7", + "is-date-object": "^1.0.5", + "is-symbol": "^1.0.4" }, "engines": { "node": ">= 0.4" @@ -8835,25 +9425,24 @@ } }, "node_modules/eslint": { - "version": "9.32.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.32.0.tgz", - "integrity": "sha512-LSehfdpgMeWcTZkWZVIJl+tkZ2nuSkyyB9C27MZqFWXuph7DvaowgcTvKqxvpLW1JZIk8PN7hFY3Rj9LQ7m7lg==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.0", - "@eslint/config-helpers": "^0.3.0", - "@eslint/core": "^0.15.0", + "@eslint/config-array": "^0.21.1", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.32.0", - "@eslint/plugin-kit": "^0.3.4", + "@eslint/js": "9.39.2", + "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", - "@types/json-schema": "^7.0.15", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.6", @@ -8911,6 +9500,81 @@ "eslint": ">=7.0.0" } }, + "node_modules/eslint-import-context": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/eslint-import-context/-/eslint-import-context-0.1.9.tgz", + "integrity": "sha512-K9Hb+yRaGAGUbwjhFNHvSmmkZs9+zbuoe3kFQ4V1wYjrepUFYM2dZAfNtjbbj3qsPfUfsA68Bx/ICWQMi+C8Eg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-tsconfig": "^4.10.1", + "stable-hash-x": "^0.2.0" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-import-context" + }, + "peerDependencies": { + "unrs-resolver": "^1.0.0" + }, + "peerDependenciesMeta": { + "unrs-resolver": { + "optional": true + } + } + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", + "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^3.2.7", + "is-core-module": "^2.13.0", + "resolve": "^1.22.4" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint-module-utils": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.12.1.tgz", + "integrity": "sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^3.2.7" + }, + "engines": { + "node": ">=4" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/eslint-module-utils/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, "node_modules/eslint-plugin-i18n": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/eslint-plugin-i18n/-/eslint-plugin-i18n-2.4.0.tgz", @@ -8920,6 +9584,130 @@ "node": ">=12.0.0" } }, + "node_modules/eslint-plugin-import": { + "version": "2.32.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", + "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rtsao/scc": "^1.1.0", + "array-includes": "^3.1.9", + "array.prototype.findlastindex": "^1.2.6", + "array.prototype.flat": "^1.3.3", + "array.prototype.flatmap": "^1.3.3", + "debug": "^3.2.7", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.9", + "eslint-module-utils": "^2.12.1", + "hasown": "^2.0.2", + "is-core-module": "^2.16.1", + "is-glob": "^4.0.3", + "minimatch": "^3.1.2", + "object.fromentries": "^2.0.8", + "object.groupby": "^1.0.3", + "object.values": "^1.2.1", + "semver": "^6.3.1", + "string.prototype.trimend": "^1.0.9", + "tsconfig-paths": "^3.15.0" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9" + } + }, + "node_modules/eslint-plugin-import-x": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import-x/-/eslint-plugin-import-x-4.16.1.tgz", + "integrity": "sha512-vPZZsiOKaBAIATpFE2uMI4w5IRwdv/FpQ+qZZMR4E+PeOcM4OeoEbqxRMnywdxP19TyB/3h6QBB0EWon7letSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "^8.35.0", + "comment-parser": "^1.4.1", + "debug": "^4.4.1", + "eslint-import-context": "^0.1.9", + "is-glob": "^4.0.3", + "minimatch": "^9.0.3 || ^10.0.1", + "semver": "^7.7.2", + "stable-hash-x": "^0.2.0", + "unrs-resolver": "^1.9.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-import-x" + }, + "peerDependencies": { + "@typescript-eslint/utils": "^8.0.0", + "eslint": "^8.57.0 || ^9.0.0", + "eslint-import-resolver-node": "*" + }, + "peerDependenciesMeta": { + "@typescript-eslint/utils": { + "optional": true + }, + "eslint-import-resolver-node": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-import-x/node_modules/@typescript-eslint/types": { + "version": "8.53.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.53.1.tgz", + "integrity": "sha512-jr/swrr2aRmUAUjW5/zQHbMaui//vQlsZcJKijZf3M26bnmLj8LyZUpj8/Rd6uzaek06OWsqdofN/Thenm5O8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/eslint-plugin-import-x/node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/eslint-plugin-import-x/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint-plugin-import/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, "node_modules/eslint-plugin-prettier": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.4.1.tgz", @@ -8951,6 +9739,67 @@ } } }, + "node_modules/eslint-plugin-react": { + "version": "7.37.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.5.tgz", + "integrity": "sha512-Qteup0SqU15kdocexFNAJMvCJEfa2xUKNV4CC1xsVMrIIqEy3SQ/rqyxCWNzfrd3/ldy6HMlD2e0JDVpDg2qIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "array-includes": "^3.1.8", + "array.prototype.findlast": "^1.2.5", + "array.prototype.flatmap": "^1.3.3", + "array.prototype.tosorted": "^1.1.4", + "doctrine": "^2.1.0", + "es-iterator-helpers": "^1.2.1", + "estraverse": "^5.3.0", + "hasown": "^2.0.2", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.1.2", + "object.entries": "^1.1.9", + "object.fromentries": "^2.0.8", + "object.values": "^1.2.1", + "prop-types": "^15.8.1", + "resolve": "^2.0.0-next.5", + "semver": "^6.3.1", + "string.prototype.matchall": "^4.0.12", + "string.prototype.repeat": "^1.0.0" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9.7" + } + }, + "node_modules/eslint-plugin-react/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/eslint-plugin-react/node_modules/resolve": { + "version": "2.0.0-next.5", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", + "integrity": "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/eslint-plugin-simple-import-sort": { "version": "12.1.1", "resolved": "https://registry.npmjs.org/eslint-plugin-simple-import-sort/-/eslint-plugin-simple-import-sort-12.1.1.tgz", @@ -9692,12 +10541,19 @@ } }, "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "dev": true, + "license": "MIT", "dependencies": { - "is-callable": "^1.1.3" + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/fork-ts-checker-webpack-plugin": { @@ -9943,15 +10799,18 @@ } }, "node_modules/function.prototype.name": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz", - "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.8.tgz", + "integrity": "sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.22.1", - "functions-have-names": "^1.2.3" + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "define-properties": "^1.2.1", + "functions-have-names": "^1.2.3", + "hasown": "^2.0.2", + "is-callable": "^1.2.7" }, "engines": { "node": ">= 0.4" @@ -9969,6 +10828,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10001,16 +10870,22 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dev": true, + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -10025,7 +10900,21 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true, "engines": { - "node": ">=8.0.0" + "node": ">=8.0.0" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" } }, "node_modules/get-stream": { @@ -10041,14 +10930,15 @@ } }, "node_modules/get-symbol-description": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.2.tgz", - "integrity": "sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", + "integrity": "sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4" + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -10057,6 +10947,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", + "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -10188,12 +11091,13 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "dev": true, - "dependencies": { - "get-intrinsic": "^1.1.3" + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -10247,10 +11151,14 @@ } }, "node_modules/has-bigints": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", - "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", + "integrity": "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==", "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -10276,10 +11184,14 @@ } }, "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.2.0.tgz", + "integrity": "sha512-KIL7eQPfHQRC8+XluaIw7BHUwwqL19bQn4hzNgdr+1wXoU0KKj6rufu47lhY7KbJR2C6T6+PfyN0Ea7wkSS+qQ==", "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.0" + }, "engines": { "node": ">= 0.4" }, @@ -10288,10 +11200,11 @@ } }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -10849,14 +11762,15 @@ "dev": true }, "node_modules/internal-slot": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz", - "integrity": "sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", + "integrity": "sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==", "dev": true, + "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "hasown": "^2.0.0", - "side-channel": "^1.0.4" + "hasown": "^2.0.2", + "side-channel": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -10892,13 +11806,15 @@ } }, "node_modules/is-array-buffer": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", - "integrity": "sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", + "integrity": "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.1" + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -10913,13 +11829,37 @@ "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, + "node_modules/is-async-function": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.1.1.tgz", + "integrity": "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "async-function": "^1.0.0", + "call-bound": "^1.0.3", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-bigint": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", - "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.1.0.tgz", + "integrity": "sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==", "dev": true, + "license": "MIT", "dependencies": { - "has-bigints": "^1.0.1" + "has-bigints": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -10938,13 +11878,14 @@ } }, "node_modules/is-boolean-object": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", - "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.2.2.tgz", + "integrity": "sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -10971,10 +11912,11 @@ } }, "node_modules/is-core-module": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", - "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", "dev": true, + "license": "MIT", "dependencies": { "hasown": "^2.0.2" }, @@ -10986,11 +11928,14 @@ } }, "node_modules/is-data-view": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.1.tgz", - "integrity": "sha512-AHkaJrsUVW6wq6JS8y3JnM/GJF/9cf+k20+iDzlSaJrinEo5+7vRiteOSwBhHRiAyQATN1AmY4hwzxJKPmYf+w==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.2.tgz", + "integrity": "sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw==", "dev": true, + "license": "MIT", "dependencies": { + "call-bound": "^1.0.2", + "get-intrinsic": "^1.2.6", "is-typed-array": "^1.1.13" }, "engines": { @@ -11001,12 +11946,14 @@ } }, "node_modules/is-date-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", - "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", + "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -11039,6 +11986,22 @@ "node": ">=0.10.0" } }, + "node_modules/is-finalizationregistry": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.1.1.tgz", + "integrity": "sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -11057,6 +12020,26 @@ "node": ">=6" } }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -11102,6 +12085,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-map": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", + "integrity": "sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-negative-zero": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.3.tgz", @@ -11136,12 +12132,14 @@ } }, "node_modules/is-number-object": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", - "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.1.1.tgz", + "integrity": "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -11175,13 +12173,16 @@ } }, "node_modules/is-regex": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", - "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -11204,13 +12205,27 @@ "node": ">=6" } }, + "node_modules/is-set": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz", + "integrity": "sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-shared-array-buffer": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz", - "integrity": "sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.4.tgz", + "integrity": "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7" + "call-bound": "^1.0.3" }, "engines": { "node": ">= 0.4" @@ -11232,12 +12247,14 @@ } }, "node_modules/is-string": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", - "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", + "integrity": "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==", "dev": true, + "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -11253,12 +12270,15 @@ "dev": true }, "node_modules/is-symbol": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", - "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.1.1.tgz", + "integrity": "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w==", "dev": true, + "license": "MIT", "dependencies": { - "has-symbols": "^1.0.2" + "call-bound": "^1.0.2", + "has-symbols": "^1.1.0", + "safe-regex-test": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -11268,12 +12288,13 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", - "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "dev": true, + "license": "MIT", "dependencies": { - "which-typed-array": "^1.1.14" + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -11282,13 +12303,47 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-weakmap": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", + "integrity": "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-weakref": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", - "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.1.1.tgz", + "integrity": "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2" + "call-bound": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-weakset": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.4.tgz", + "integrity": "sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "get-intrinsic": "^1.2.6" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -11310,7 +12365,8 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", @@ -11441,6 +12497,24 @@ "node": ">=8" } }, + "node_modules/iterator.prototype": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz", + "integrity": "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-object-atoms": "^1.0.0", + "get-intrinsic": "^1.2.6", + "get-proto": "^1.0.0", + "has-symbols": "^1.1.0", + "set-function-name": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/jake": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", @@ -12409,6 +13483,22 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsx-ast-utils": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", + "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "array-includes": "^3.1.6", + "array.prototype.flat": "^1.3.1", + "object.assign": "^4.1.4", + "object.values": "^1.1.6" + }, + "engines": { + "node": ">=4.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -12988,6 +14078,16 @@ "tmpl": "1.0.5" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -13248,6 +14348,22 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-postinstall": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", + "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", + "dev": true, + "license": "MIT", + "bin": { + "napi-postinstall": "lib/cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/napi-postinstall" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -18584,10 +19700,11 @@ } }, "node_modules/object-inspect": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", - "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -18621,14 +19738,17 @@ } }, "node_modules/object.assign": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", - "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.7.tgz", + "integrity": "sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.5", + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", "define-properties": "^1.2.1", - "has-symbols": "^1.0.3", + "es-object-atoms": "^1.0.0", + "has-symbols": "^1.1.0", "object-keys": "^1.1.1" }, "engines": { @@ -18639,27 +19759,65 @@ } }, "node_modules/object.entries": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.8.tgz", - "integrity": "sha512-cmopxi8VwRIAw/fkijJohSfpef5PdN0pMQJN6VC/ZKvn0LIknWD8KtgY6KlQdEc4tIjcQ3HxSMmnvtzIscdaYQ==", + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.9.tgz", + "integrity": "sha512-8u/hfXFRBD1O0hPUjioLhoWFHRmt6tKA4/vZPyckBr18l1KE9uHrFaFaUi8MDRTpi4uak2goyPTSNJLXX2k2Hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "define-properties": "^1.2.1", + "es-object-atoms": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.fromentries": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.8.tgz", + "integrity": "sha512-k6E21FzySsSK5a21KRADBd/NGneRegFO5pLHfdQLpRDETUNJueLXs3WCzyQ3tFRDYgbq3KHGXfTbi2bs8WQ6rQ==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", + "es-abstract": "^1.23.2", "es-object-atoms": "^1.0.0" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object.values": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.0.tgz", - "integrity": "sha512-yBYjY9QX2hnRmZHAjG/f13MzmBzxzYgQhFrke06TTyKY5zSTEqkOeukBzIdVA3j3ulu8Qa3MbVFShV7T2RmGtQ==", + "node_modules/object.groupby": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.3.tgz", + "integrity": "sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", + "es-abstract": "^1.23.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.values": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.1.tgz", + "integrity": "sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" }, "engines": { @@ -18799,6 +19957,24 @@ "node": ">= 0.8.0" } }, + "node_modules/own-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz", + "integrity": "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.6", + "object-keys": "^1.1.1", + "safe-push-apply": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -19211,10 +20387,11 @@ } }, "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -21020,6 +22197,29 @@ "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==" }, + "node_modules/reflect.getprototypeof": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", + "integrity": "sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.9", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", + "get-intrinsic": "^1.2.7", + "get-proto": "^1.0.1", + "which-builtin-type": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/reftools": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/reftools/-/reftools-1.1.9.tgz", @@ -21068,14 +22268,17 @@ "dev": true }, "node_modules/regexp.prototype.flags": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.3.tgz", - "integrity": "sha512-vqlC04+RQoFalODCbCumG2xIOvapzVMHwsyIGM/SIE8fRhFFsXeH8/QQ+s0T0kDAhKc4k30s73/0ydkHQz6HlQ==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", + "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-errors": "^1.3.0", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "set-function-name": "^2.0.2" }, "engines": { @@ -21309,6 +22512,16 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/resolve-url-loader": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-5.0.0.tgz", @@ -21490,14 +22703,16 @@ } }, "node_modules/safe-array-concat": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", - "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz", + "integrity": "sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", - "get-intrinsic": "^1.2.4", - "has-symbols": "^1.0.3", + "call-bind": "^1.0.8", + "call-bound": "^1.0.2", + "get-intrinsic": "^1.2.6", + "has-symbols": "^1.1.0", "isarray": "^2.0.5" }, "engines": { @@ -21526,15 +22741,33 @@ } ] }, + "node_modules/safe-push-apply": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", + "integrity": "sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "isarray": "^2.0.5" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safe-regex-test": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz", - "integrity": "sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bound": "^1.0.2", "es-errors": "^1.3.0", - "is-regex": "^1.1.4" + "is-regex": "^1.2.1" }, "engines": { "node": ">= 0.4" @@ -21868,6 +23101,7 @@ "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", "dev": true, + "license": "MIT", "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -21878,6 +23112,21 @@ "node": ">= 0.4" } }, + "node_modules/set-proto": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/set-proto/-/set-proto-1.0.0.tgz", + "integrity": "sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -22048,15 +23297,73 @@ "dev": true }, "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" }, "engines": { "node": ">= 0.4" @@ -22232,6 +23539,16 @@ "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", "deprecated": "Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility" }, + "node_modules/stable-hash-x": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/stable-hash-x/-/stable-hash-x-0.2.0.tgz", + "integrity": "sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -22268,6 +23585,20 @@ "node": ">= 0.8" } }, + "node_modules/stop-iteration-iterator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", + "integrity": "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "internal-slot": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -22314,16 +23645,59 @@ "node": ">=8" } }, + "node_modules/string.prototype.matchall": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz", + "integrity": "sha512-6CC9uyBL+/48dYizRf7H7VAYCMCNTBeM78x/VTUe9bFEaxBepPJDa1Ow99LqI/1yF7kuy7Q3cQsYMrcjGUcskA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.3", + "define-properties": "^1.2.1", + "es-abstract": "^1.23.6", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", + "get-intrinsic": "^1.2.6", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "internal-slot": "^1.1.0", + "regexp.prototype.flags": "^1.5.3", + "set-function-name": "^2.0.2", + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.repeat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-1.0.0.tgz", + "integrity": "sha512-0u/TldDbKD8bFCQ/4f5+mNRrXwZ8hg2w7ZR8wa16e8z9XpePWl3eGEcUD0OXpEH/VJH/2G3gjUtR3ZOiBe2S/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, "node_modules/string.prototype.trim": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz", - "integrity": "sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw==", + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz", + "integrity": "sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.2", + "define-data-property": "^1.1.4", "define-properties": "^1.2.1", - "es-abstract": "^1.23.0", - "es-object-atoms": "^1.0.0" + "es-abstract": "^1.23.5", + "es-object-atoms": "^1.0.0", + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -22333,15 +23707,20 @@ } }, "node_modules/string.prototype.trimend": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz", - "integrity": "sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.9.tgz", + "integrity": "sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.2", "define-properties": "^1.2.1", "es-object-atoms": "^1.0.0" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -23025,6 +24404,42 @@ } } }, + "node_modules/tsconfig-paths": { + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", + "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, + "node_modules/tsconfig-paths/node_modules/json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/tsconfig-paths/node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/tslib": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz", @@ -23077,30 +24492,32 @@ } }, "node_modules/typed-array-buffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz", - "integrity": "sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bound": "^1.0.3", "es-errors": "^1.3.0", - "is-typed-array": "^1.1.13" + "is-typed-array": "^1.1.14" }, "engines": { "node": ">= 0.4" } }, "node_modules/typed-array-byte-length": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz", - "integrity": "sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.3.tgz", + "integrity": "sha512-BaXgOuIxz8n8pIq3e7Atg/7s+DpiYrxn4vdot3w9KbnBhcRQq6o3xemQdIfynqSeXeDrF32x+WvfzmOjPiY9lg==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-proto": "^1.0.3", - "is-typed-array": "^1.1.13" + "gopd": "^1.2.0", + "has-proto": "^1.2.0", + "is-typed-array": "^1.1.14" }, "engines": { "node": ">= 0.4" @@ -23110,17 +24527,19 @@ } }, "node_modules/typed-array-byte-offset": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz", - "integrity": "sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.4.tgz", + "integrity": "sha512-bTlAFB/FBYMcuX81gbL4OcpH5PmlFHqlCCpAl8AlEzMz5k53oNDvN8p1PNOWLEmI2x4orp3raOFB51tv9X+MFQ==", "dev": true, + "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", + "call-bind": "^1.0.8", "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-proto": "^1.0.3", - "is-typed-array": "^1.1.13" + "gopd": "^1.2.0", + "has-proto": "^1.2.0", + "is-typed-array": "^1.1.15", + "reflect.getprototypeof": "^1.0.9" }, "engines": { "node": ">= 0.4" @@ -23130,17 +24549,18 @@ } }, "node_modules/typed-array-length": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.6.tgz", - "integrity": "sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.7.tgz", + "integrity": "sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg==", "dev": true, + "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "for-each": "^0.3.3", "gopd": "^1.0.1", - "has-proto": "^1.0.3", "is-typed-array": "^1.1.13", - "possible-typed-array-names": "^1.0.0" + "possible-typed-array-names": "^1.0.0", + "reflect.getprototypeof": "^1.0.6" }, "engines": { "node": ">= 0.4" @@ -23163,15 +24583,19 @@ } }, "node_modules/unbox-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", - "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", + "integrity": "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==", "dev": true, + "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", + "call-bound": "^1.0.3", "has-bigints": "^1.0.2", - "has-symbols": "^1.0.3", - "which-boxed-primitive": "^1.0.2" + "has-symbols": "^1.1.0", + "which-boxed-primitive": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -23249,6 +24673,41 @@ "node": ">= 0.8" } }, + "node_modules/unrs-resolver": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", + "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "napi-postinstall": "^0.3.0" + }, + "funding": { + "url": "https://opencollective.com/unrs-resolver" + }, + "optionalDependencies": { + "@unrs/resolver-binding-android-arm-eabi": "1.11.1", + "@unrs/resolver-binding-android-arm64": "1.11.1", + "@unrs/resolver-binding-darwin-arm64": "1.11.1", + "@unrs/resolver-binding-darwin-x64": "1.11.1", + "@unrs/resolver-binding-freebsd-x64": "1.11.1", + "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", + "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", + "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", + "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", + "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", + "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", + "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", + "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", + "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", + "@unrs/resolver-binding-linux-x64-musl": "1.11.1", + "@unrs/resolver-binding-wasm32-wasi": "1.11.1", + "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", + "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", + "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", @@ -23849,31 +25308,85 @@ } }, "node_modules/which-boxed-primitive": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz", + "integrity": "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-bigint": "^1.1.0", + "is-boolean-object": "^1.2.1", + "is-number-object": "^1.1.1", + "is-string": "^1.1.1", + "is-symbol": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-builtin-type": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.2.1.tgz", + "integrity": "sha512-6iBczoX+kDQ7a3+YJBnh3T+KZRxM/iYNPXicqk66/Qfm1b93iu+yOImkg0zHbj5LNOcNv1TEADiZ0xa34B4q6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "function.prototype.name": "^1.1.6", + "has-tostringtag": "^1.0.2", + "is-async-function": "^2.0.0", + "is-date-object": "^1.1.0", + "is-finalizationregistry": "^1.1.0", + "is-generator-function": "^1.0.10", + "is-regex": "^1.2.1", + "is-weakref": "^1.0.2", + "isarray": "^2.0.5", + "which-boxed-primitive": "^1.1.0", + "which-collection": "^1.0.2", + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-collection": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", - "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.2.tgz", + "integrity": "sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==", "dev": true, + "license": "MIT", "dependencies": { - "is-bigint": "^1.0.1", - "is-boolean-object": "^1.1.0", - "is-number-object": "^1.0.4", - "is-string": "^1.0.5", - "is-symbol": "^1.0.3" + "is-map": "^2.0.3", + "is-set": "^2.0.3", + "is-weakmap": "^2.0.2", + "is-weakset": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", - "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", + "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", "dev": true, + "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" }, "engines": { diff --git a/frontend/package.json b/frontend/package.json index 9d581114df..f2ef0d9ca1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -59,10 +59,11 @@ "cross-env": "^7.0.3", "css-loader": "^6.7.3", "enzyme": "^3.11.0", - "eslint": "^9.32.0", + "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.5", "eslint-plugin-i18n": "^2.4.0", "eslint-plugin-prettier": "^5.4.1", + "eslint-plugin-react": "^7.37.5", "eslint-plugin-simple-import-sort": "^12.1.1", "favicons": "^7.2.0", "favicons-webpack-plugin": "^6.0.1", diff --git a/frontend/src/components/Code/index.tsx b/frontend/src/components/Code/index.tsx index 180d4586a7..db16c4fcb1 100644 --- a/frontend/src/components/Code/index.tsx +++ b/frontend/src/components/Code/index.tsx @@ -17,3 +17,5 @@ export const Code = forwardRef(({ children, className }, ); }); + +Code.displayName = 'Code'; From 28797f65e322aa316ce6b0adff815925d45abf96 Mon Sep 17 00:00:00 2001 From: Oleg Vavilov Date: Thu, 22 Jan 2026 16:58:56 +0300 Subject: [PATCH 014/640] Fix CI errors --- frontend/package-lock.json | 1677 ++++++++++++++---------------------- 1 file changed, 626 insertions(+), 1051 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a3d3f29d93..445c61daa9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -84,8 +84,6 @@ "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.5", "eslint-plugin-i18n": "^2.4.0", - "eslint-plugin-import": "^2.32.0", - "eslint-plugin-import-x": "^4.16.1", "eslint-plugin-prettier": "^5.4.1", "eslint-plugin-react": "^7.37.5", "eslint-plugin-simple-import-sort": "^12.1.1", @@ -163,9 +161,9 @@ } }, "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { @@ -239,14 +237,15 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.0.tgz", - "integrity": "sha512-INCKxTtbXtcNbUZ3YXutwMpEleqttcswhAdee7dhuoVrD2cnuc3PqtERBtxkX5nziX9vnBL8WXmSGwv8CuPV6g==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.28.6.tgz", + "integrity": "sha512-JYgintcMjRiCvS8mMECzaEn+m3PfoQiyqukOMCCVQtoJGYJw8j/8LBJEiqkHLkfwCcs74E3pbAUFNg7d9VNJ+Q==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.25.9", + "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" + "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" @@ -527,19 +526,21 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", - "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", "dev": true, + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", - "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=6.9.0" } @@ -568,25 +569,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", - "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/template": "^7.25.9", - "@babel/types": "^7.26.0" + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.26.1", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.1.tgz", - "integrity": "sha512-reoQYNiAJreZNsJzyrDNzFQ+IQ5JFiIzAHJg9bn94S3l+4++J7RsIhNMoB+lgP/9tpmiAQqspv+xfdxTSzREOw==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.6.tgz", + "integrity": "sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/types": "^7.26.0" + "@babel/types": "^7.28.6" }, "bin": { "parser": "bin/babel-parser.js" @@ -2020,25 +2023,24 @@ } }, "node_modules/@babel/runtime": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", - "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", - "dependencies": { - "regenerator-runtime": "^0.14.0" - }, + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", + "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/template": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", - "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/types": "^7.25.9" + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" }, "engines": { "node": ">=6.9.0" @@ -2063,13 +2065,14 @@ } }, "node_modules/@babel/types": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", - "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.6.tgz", + "integrity": "sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==", "dev": true, + "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9" + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" @@ -2551,18 +2554,6 @@ "react": ">=16.8.0" } }, - "node_modules/@emnapi/core": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", - "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.1.0", - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/runtime": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", @@ -2574,17 +2565,6 @@ "tslib": "^2.4.0" } }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emotion/is-prop-valid": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.2.2.tgz", @@ -3261,29 +3241,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -3333,9 +3290,9 @@ } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { @@ -3873,19 +3830,6 @@ "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", "dev": true }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" - } - }, "node_modules/@nicolo-ribaudo/chokidar-2": { "version": "2.1.8-no-fsevents.3", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz", @@ -3893,6 +3837,19 @@ "dev": true, "optional": true }, + "node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4209,6 +4166,165 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/@peculiar/asn1-cms": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-cms/-/asn1-cms-2.6.0.tgz", + "integrity": "sha512-2uZqP+ggSncESeUF/9Su8rWqGclEfEiz1SyU02WX5fUONFfkjzS2Z/F1Li0ofSmf4JqYXIOdCAZqIXAIBAT1OA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "@peculiar/asn1-x509-attr": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-csr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-csr/-/asn1-csr-2.6.0.tgz", + "integrity": "sha512-BeWIu5VpTIhfRysfEp73SGbwjjoLL/JWXhJ/9mo4vXnz3tRGm+NGm3KNcRzQ9VMVqwYS2RHlolz21svzRXIHPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-ecc": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-ecc/-/asn1-ecc-2.6.0.tgz", + "integrity": "sha512-FF3LMGq6SfAOwUG2sKpPXblibn6XnEIKa+SryvUl5Pik+WR9rmRA3OCiwz8R3lVXnYnyRkSZsSLdml8H3UiOcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pfx": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pfx/-/asn1-pfx-2.6.0.tgz", + "integrity": "sha512-rtUvtf+tyKGgokHHmZzeUojRZJYPxoD/jaN1+VAB4kKR7tXrnDCA/RAWXAIhMJJC+7W27IIRGe9djvxKgsldCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.0", + "@peculiar/asn1-pkcs8": "^2.6.0", + "@peculiar/asn1-rsa": "^2.6.0", + "@peculiar/asn1-schema": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs8": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs8/-/asn1-pkcs8-2.6.0.tgz", + "integrity": "sha512-KyQ4D8G/NrS7Fw3XCJrngxmjwO/3htnA0lL9gDICvEQ+GJ+EPFqldcJQTwPIdvx98Tua+WjkdKHSC0/Km7T+lA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs9": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs9/-/asn1-pkcs9-2.6.0.tgz", + "integrity": "sha512-b78OQ6OciW0aqZxdzliXGYHASeCvvw5caqidbpQRYW2mBtXIX2WhofNXTEe7NyxTb0P6J62kAAWLwn0HuMF1Fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.0", + "@peculiar/asn1-pfx": "^2.6.0", + "@peculiar/asn1-pkcs8": "^2.6.0", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "@peculiar/asn1-x509-attr": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-rsa": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-rsa/-/asn1-rsa-2.6.0.tgz", + "integrity": "sha512-Nu4C19tsrTsCp9fDrH+sdcOKoVfdfoQQ7S3VqjJU6vedR7tY3RLkQ5oguOIB3zFW33USDUuYZnPEQYySlgha4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.6.0.tgz", + "integrity": "sha512-xNLYLBFTBKkCzEZIw842BxytQQATQv+lDTCEMZ8C196iJcJJMBUZxrhSTxLaohMyKK8QlzRNTRkUmanucnDSqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "asn1js": "^3.0.6", + "pvtsutils": "^1.3.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.6.0.tgz", + "integrity": "sha512-uzYbPEpoQiBoTq0/+jZtpM6Gq6zADBx+JNFP3yqRgziWBxQ/Dt/HcuvRfm9zJTPdRcBqPNdaRHTVwpyiq6iNMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "asn1js": "^3.0.6", + "pvtsutils": "^1.3.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509-attr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-attr/-/asn1-x509-attr-2.6.0.tgz", + "integrity": "sha512-MuIAXFX3/dc8gmoZBkwJWxUWOSvG4MMDntXhrOZpJVMkYX+MYc/rUAU2uJOved9iJEoiUx7//3D8oG83a78UJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/x509": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@peculiar/x509/-/x509-1.14.3.tgz", + "integrity": "sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.0", + "@peculiar/asn1-csr": "^2.6.0", + "@peculiar/asn1-ecc": "^2.6.0", + "@peculiar/asn1-pkcs9": "^2.6.0", + "@peculiar/asn1-rsa": "^2.6.0", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "pvtsutils": "^1.3.6", + "reflect-metadata": "^0.2.2", + "tslib": "^2.8.1", + "tsyringe": "^4.10.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@pkgr/core": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.7.tgz", @@ -4324,9 +4440,10 @@ } }, "node_modules/@remix-run/router": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.20.0.tgz", - "integrity": "sha512-mUnk8rPJBI9loFDZ+YzPGdeniYK+FTmRD1TMCz7ev2SNIozyKKpnGgsxO34u6Z4z/t0ITuu7voi/AshfsGsgFg==", + "version": "1.23.2", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz", + "integrity": "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w==", + "license": "MIT", "engines": { "node": ">=14.0.0" } @@ -4362,13 +4479,6 @@ "node": ">=10" } }, - "node_modules/@rtsao/scc": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", - "integrity": "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==", - "dev": true, - "license": "MIT" - }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -4780,17 +4890,6 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -4951,15 +5050,16 @@ "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" }, "node_modules/@types/express": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", - "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", "dev": true, + "license": "MIT", "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", "@types/qs": "*", - "@types/serve-static": "*" + "@types/serve-static": "^1" } }, "node_modules/@types/express-serve-static-core": { @@ -5100,13 +5200,6 @@ "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, - "node_modules/@types/json5": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", - "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/lodash": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", @@ -5135,15 +5228,6 @@ "form-data": "^4.0.0" } }, - "node_modules/@types/node-forge": { - "version": "1.3.11", - "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", - "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -5532,9 +5616,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5625,324 +5709,55 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@unrs/resolver-binding-android-arm-eabi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", - "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] + "node_modules/@webassemblyjs/ast": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", + "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6" + } }, - "node_modules/@unrs/resolver-binding-android-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", - "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", + "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==" }, - "node_modules/@unrs/resolver-binding-darwin-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", - "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", + "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==" }, - "node_modules/@unrs/resolver-binding-darwin-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", - "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz", + "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==" }, - "node_modules/@unrs/resolver-binding-freebsd-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", - "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", + "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", + "@xtuc/long": "4.2.2" + } }, - "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", - "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", + "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==" }, - "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", - "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", - "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", - "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", - "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", - "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", - "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", - "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", - "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", - "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-wasm32-wasi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", - "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^0.2.11" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", - "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", - "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-x64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", - "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@webassemblyjs/ast": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", - "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==", - "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.6", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6" - } - }, - "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", - "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==" - }, - "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", - "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==" - }, - "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz", - "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==" - }, - "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", - "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", - "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.6", - "@webassemblyjs/helper-api-error": "1.11.6", - "@xtuc/long": "4.2.2" - } - }, - "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", - "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==" - }, - "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz", - "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==", - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/wasm-gen": "1.12.1" - } + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz", + "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==", + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-buffer": "1.12.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/wasm-gen": "1.12.1" + } }, "node_modules/@webassemblyjs/ieee754": { "version": "1.11.6", @@ -6362,7 +6177,8 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/array-includes": { "version": "3.1.9", @@ -6437,28 +6253,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array.prototype.findlastindex": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.6.tgz", - "integrity": "sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "call-bound": "^1.0.4", - "define-properties": "^1.2.1", - "es-abstract": "^1.23.9", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "es-shim-unscopables": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/array.prototype.flat": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.3.tgz", @@ -6536,6 +6330,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asn1js": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.7.tgz", + "integrity": "sha512-uLvq6KJu04qoQM6gvBfKFjlh6Gl0vOKQuR5cJMDHQkmwfMOQeN3F3SHCv9SNYSL+CRoHvOGFfllDlVz03GQjvQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.3", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -6629,13 +6438,14 @@ } }, "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", + "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", "dev": true, + "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", + "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, @@ -7199,43 +7009,36 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", "dev": true, + "license": "MIT", "dependencies": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/body-parser/node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/body-parser/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -7245,6 +7048,7 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, + "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -7256,7 +7060,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/bonjour-service": { "version": "1.2.1", @@ -7274,10 +7079,11 @@ "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -7368,14 +7174,25 @@ } }, "node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } }, + "node_modules/bytestreamjs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bytestreamjs/-/bytestreamjs-2.0.1.tgz", + "integrity": "sha512-U1Z/ob71V/bXfVABvNr/Kumf5VyeQRBEm6Txb0PQ6S7V5GpBM3w4Cbqz/xPDicR5tN0uvDifng8C+5qECeGwyQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -7399,7 +7216,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -7876,16 +7692,6 @@ "node": ">= 6" } }, - "node_modules/comment-parser": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.4.4.tgz", - "integrity": "sha512-0D6qSQ5IkeRrGJFHRClzaMOenMeT0gErz3zIw3AprKMqhRN6LNU2jQOdkPG/FZ+8bCgXE1VidrgSzlBBDZRr8A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 12.0.0" - } - }, "node_modules/common-path-prefix": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", @@ -7903,6 +7709,7 @@ "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", "dev": true, + "license": "MIT", "dependencies": { "mime-db": ">= 1.43.0 < 2" }, @@ -7911,17 +7718,18 @@ } }, "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", "dev": true, + "license": "MIT", "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", + "bytes": "3.1.2", + "compressible": "~2.0.18", "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", "vary": "~1.1.2" }, "engines": { @@ -7933,6 +7741,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -7941,13 +7750,18 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true + "dev": true, + "license": "MIT" }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } }, "node_modules/concat-map": { "version": "0.0.1", @@ -7969,6 +7783,7 @@ "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "dev": true, + "license": "MIT", "dependencies": { "safe-buffer": "5.2.1" }, @@ -7981,6 +7796,7 @@ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -7992,19 +7808,21 @@ "dev": true }, "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "dev": true + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "dev": true, + "license": "MIT" }, "node_modules/copy-webpack-plugin": { "version": "11.0.0", @@ -8758,6 +8576,7 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -8776,6 +8595,7 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -8842,10 +8662,11 @@ "dev": true }, "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -9000,7 +8821,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -9075,6 +8895,7 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -9284,7 +9105,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -9294,7 +9114,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -9336,7 +9155,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -9349,7 +9167,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -9484,228 +9301,29 @@ } } }, - "node_modules/eslint-config-prettier": { - "version": "10.1.5", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.5.tgz", - "integrity": "sha512-zc1UmCpNltmVY34vuLRV61r1K27sWuX39E+uyUnY8xS2Bex88VV9cugG+UZbRSRGtGyFboj+D8JODyme1plMpw==", - "dev": true, - "license": "MIT", - "bin": { - "eslint-config-prettier": "bin/cli.js" - }, - "funding": { - "url": "https://opencollective.com/eslint-config-prettier" - }, - "peerDependencies": { - "eslint": ">=7.0.0" - } - }, - "node_modules/eslint-import-context": { - "version": "0.1.9", - "resolved": "https://registry.npmjs.org/eslint-import-context/-/eslint-import-context-0.1.9.tgz", - "integrity": "sha512-K9Hb+yRaGAGUbwjhFNHvSmmkZs9+zbuoe3kFQ4V1wYjrepUFYM2dZAfNtjbbj3qsPfUfsA68Bx/ICWQMi+C8Eg==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-tsconfig": "^4.10.1", - "stable-hash-x": "^0.2.0" - }, - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint-import-context" - }, - "peerDependencies": { - "unrs-resolver": "^1.0.0" - }, - "peerDependenciesMeta": { - "unrs-resolver": { - "optional": true - } - } - }, - "node_modules/eslint-import-resolver-node": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", - "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^3.2.7", - "is-core-module": "^2.13.0", - "resolve": "^1.22.4" - } - }, - "node_modules/eslint-import-resolver-node/node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/eslint-module-utils": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.12.1.tgz", - "integrity": "sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^3.2.7" - }, - "engines": { - "node": ">=4" - }, - "peerDependenciesMeta": { - "eslint": { - "optional": true - } - } - }, - "node_modules/eslint-module-utils/node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/eslint-plugin-i18n": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-i18n/-/eslint-plugin-i18n-2.4.0.tgz", - "integrity": "sha512-6RpPoj+lr0xk6SNljziOjGfDtuQSN6cw/gdds248N5MvCQUrPxo5+0s7b7TQsEl1qLr5OVnCMxsaRBy/4T62cg==", - "dev": true, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/eslint-plugin-import": { - "version": "2.32.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", - "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rtsao/scc": "^1.1.0", - "array-includes": "^3.1.9", - "array.prototype.findlastindex": "^1.2.6", - "array.prototype.flat": "^1.3.3", - "array.prototype.flatmap": "^1.3.3", - "debug": "^3.2.7", - "doctrine": "^2.1.0", - "eslint-import-resolver-node": "^0.3.9", - "eslint-module-utils": "^2.12.1", - "hasown": "^2.0.2", - "is-core-module": "^2.16.1", - "is-glob": "^4.0.3", - "minimatch": "^3.1.2", - "object.fromentries": "^2.0.8", - "object.groupby": "^1.0.3", - "object.values": "^1.2.1", - "semver": "^6.3.1", - "string.prototype.trimend": "^1.0.9", - "tsconfig-paths": "^3.15.0" - }, - "engines": { - "node": ">=4" - }, - "peerDependencies": { - "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9" - } - }, - "node_modules/eslint-plugin-import-x": { - "version": "4.16.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-import-x/-/eslint-plugin-import-x-4.16.1.tgz", - "integrity": "sha512-vPZZsiOKaBAIATpFE2uMI4w5IRwdv/FpQ+qZZMR4E+PeOcM4OeoEbqxRMnywdxP19TyB/3h6QBB0EWon7letSQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "^8.35.0", - "comment-parser": "^1.4.1", - "debug": "^4.4.1", - "eslint-import-context": "^0.1.9", - "is-glob": "^4.0.3", - "minimatch": "^9.0.3 || ^10.0.1", - "semver": "^7.7.2", - "stable-hash-x": "^0.2.0", - "unrs-resolver": "^1.9.2" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint-plugin-import-x" - }, - "peerDependencies": { - "@typescript-eslint/utils": "^8.0.0", - "eslint": "^8.57.0 || ^9.0.0", - "eslint-import-resolver-node": "*" - }, - "peerDependenciesMeta": { - "@typescript-eslint/utils": { - "optional": true - }, - "eslint-import-resolver-node": { - "optional": true - } - } - }, - "node_modules/eslint-plugin-import-x/node_modules/@typescript-eslint/types": { - "version": "8.53.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.53.1.tgz", - "integrity": "sha512-jr/swrr2aRmUAUjW5/zQHbMaui//vQlsZcJKijZf3M26bnmLj8LyZUpj8/Rd6uzaek06OWsqdofN/Thenm5O8A==", + "node_modules/eslint-config-prettier": { + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.5.tgz", + "integrity": "sha512-zc1UmCpNltmVY34vuLRV61r1K27sWuX39E+uyUnY8xS2Bex88VV9cugG+UZbRSRGtGyFboj+D8JODyme1plMpw==", "dev": true, "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "bin": { + "eslint-config-prettier": "bin/cli.js" }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/eslint-plugin-import-x/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" + "url": "https://opencollective.com/eslint-config-prettier" }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "peerDependencies": { + "eslint": ">=7.0.0" } }, - "node_modules/eslint-plugin-import-x/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "node_modules/eslint-plugin-i18n": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-i18n/-/eslint-plugin-i18n-2.4.0.tgz", + "integrity": "sha512-6RpPoj+lr0xk6SNljziOjGfDtuQSN6cw/gdds248N5MvCQUrPxo5+0s7b7TQsEl1qLr5OVnCMxsaRBy/4T62cg==", "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, "engines": { - "node": ">=10" - } - }, - "node_modules/eslint-plugin-import/node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" + "node": ">=12.0.0" } }, "node_modules/eslint-plugin-prettier": { @@ -10015,6 +9633,7 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -10090,45 +9709,50 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "dev": true, + "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", - "on-finished": "2.4.1", + "on-finished": "~2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "6.13.0", + "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", + "send": "~0.19.0", + "serve-static": "~1.16.2", "setprototypeof": "1.2.0", - "statuses": "2.0.1", + "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/express/node_modules/debug": { @@ -10136,6 +9760,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -10144,7 +9769,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-deep-equal": { "version": "3.1.3", @@ -10380,10 +10006,11 @@ } }, "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -10422,17 +10049,18 @@ } }, "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", "dev": true, + "license": "MIT", "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", - "on-finished": "2.4.1", + "on-finished": "~2.4.1", "parseurl": "~1.3.3", - "statuses": "2.0.1", + "statuses": "~2.0.2", "unpipe": "~1.0.0" }, "engines": { @@ -10444,6 +10072,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -10452,7 +10081,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/find-cache-dir": { "version": "2.1.0", @@ -10682,12 +10312,15 @@ } }, "node_modules/form-data": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", - "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", "mime-types": "^2.1.12" }, "engines": { @@ -10716,6 +10349,7 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -10738,6 +10372,7 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -10793,7 +10428,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -10873,7 +10507,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -10907,7 +10540,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -10947,19 +10579,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-tsconfig": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", - "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -11094,7 +10713,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11203,7 +10821,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11216,7 +10833,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "dependencies": { "has-symbols": "^1.0.3" }, @@ -11231,7 +10847,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -11440,19 +11055,24 @@ "dev": true }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "dev": true, + "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/http-parser-js": { @@ -11476,10 +11096,11 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", - "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", "dev": true, + "license": "MIT", "dependencies": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", @@ -13413,9 +13034,9 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -13793,14 +13414,16 @@ } }, "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "license": "MIT" }, "node_modules/lodash-es": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", - "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.23.tgz", + "integrity": "sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==", + "license": "MIT" }, "node_modules/lodash.camelcase": { "version": "4.3.0", @@ -14082,7 +13705,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -14108,6 +13730,7 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -14129,6 +13752,7 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", "dev": true, + "license": "MIT", "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -14152,6 +13776,7 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -14174,6 +13799,7 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", "dev": true, + "license": "MIT", "bin": { "mime": "cli.js" }, @@ -14332,15 +13958,16 @@ "integrity": "sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA==" }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -14348,22 +13975,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/napi-postinstall": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", - "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", - "dev": true, - "license": "MIT", - "bin": { - "napi-postinstall": "lib/cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/napi-postinstall" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -14478,15 +14089,6 @@ "node": "4.x || >=6.0.0" } }, - "node_modules/node-forge": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", - "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", - "dev": true, - "engines": { - "node": ">= 6.13.0" - } - }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -19793,21 +19395,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object.groupby": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.3.tgz", - "integrity": "sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "define-properties": "^1.2.1", - "es-abstract": "^1.23.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/object.values": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.1.tgz", @@ -19846,10 +19433,11 @@ } }, "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -20173,10 +19761,11 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "dev": true + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" }, "node_modules/path-type": { "version": "4.0.0", @@ -20386,6 +19975,24 @@ "node": ">=4" } }, + "node_modules/pkijs": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.3.3.tgz", + "integrity": "sha512-+KD8hJtqQMYoTuL1bbGOqxb4z+nZkTAwVdNtWwe8Tc2xNbEmdJYIYoc6Qt0uF55e6YW6KuTHw1DjQ18gMhzepw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@noble/hashes": "1.4.0", + "asn1js": "^3.0.6", + "bytestreamjs": "^2.0.1", + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.3", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -21604,6 +21211,7 @@ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dev": true, + "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" @@ -21617,6 +21225,7 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.10" } @@ -21651,13 +21260,34 @@ } ] }, + "node_modules/pvtsutils": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", + "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/pvutils": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.1.5.tgz", + "integrity": "sha512-KTqnxsgGiQ6ZAzZCVlJH5eOjSnvlyEgx1m8bkRJfOhmGRqfo5KLvmAlACQkrjEtOQ4B7wF9TdSLIs9O90MX9xA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", "dev": true, + "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -21732,34 +21362,27 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, + "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8" } }, - "node_modules/raw-body/node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/raw-body/node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, + "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -22042,11 +21665,12 @@ } }, "node_modules/react-router": { - "version": "6.27.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.27.0.tgz", - "integrity": "sha512-YA+HGZXz4jaAkVoYBE98VQl+nVzI+cVI2Oj/06F5ZM+0u3TgedN9Y9kmMRo2mnkSK2nCpNQn0DVob4HCsY/WLw==", + "version": "6.30.3", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.3.tgz", + "integrity": "sha512-XRnlbKMTmktBkjCLE8/XcZFlnHvr2Ltdr1eJX4idL55/9BbORzyZEaIkBFDhFGCEWBBItsVrDxwx3gnisMitdw==", + "license": "MIT", "dependencies": { - "@remix-run/router": "1.20.0" + "@remix-run/router": "1.23.2" }, "engines": { "node": ">=14.0.0" @@ -22056,12 +21680,13 @@ } }, "node_modules/react-router-dom": { - "version": "6.27.0", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.27.0.tgz", - "integrity": "sha512-+bvtFWMC0DgAFrfKXKG9Fc+BcXWRUO1aJIihbB79xaeq0v5UzfvnM5houGUm1Y461WVRcgAQ+Clh5rdb1eCx4g==", + "version": "6.30.3", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.3.tgz", + "integrity": "sha512-pxPcv1AczD4vso7G4Z3TKcvlxK7g7TNt3/FNGMhfqyntocvYKj+GCatfigGDjbLozC4baguJ0ReCigoDJXb0ag==", + "license": "MIT", "dependencies": { - "@remix-run/router": "1.20.0", - "react-router": "6.27.0" + "@remix-run/router": "1.23.2", + "react-router": "6.30.3" }, "engines": { "node": ">=14.0.0" @@ -22197,6 +21822,13 @@ "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==" }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -22247,11 +21879,6 @@ "node": ">=4" } }, - "node_modules/regenerator-runtime": { - "version": "0.14.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", - "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" - }, "node_modules/regenerator-transform": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.2.tgz", @@ -22512,16 +22139,6 @@ "node": ">=4" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/resolve-url-loader": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-5.0.0.tgz", @@ -22909,16 +22526,17 @@ "dev": true }, "node_modules/selfsigned": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", - "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", + "integrity": "sha512-ftnu3TW4+3eBfLRFnDEkzGxSF/10BJBkaLJuBHZX0kiPS7bRdlpZGu6YGt4KngMkdTwJE6MbjavFpqHvqVt+Ew==", "dev": true, + "license": "MIT", "dependencies": { - "@types/node-forge": "^1.3.0", - "node-forge": "^1" + "@peculiar/x509": "^1.14.2", + "pkijs": "^3.3.3" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/semver": { @@ -22931,24 +22549,25 @@ } }, "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", "dev": true, + "license": "MIT", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", "mime": "1.6.0", "ms": "2.1.3", - "on-finished": "2.4.1", + "on-finished": "~2.4.1", "range-parser": "~1.2.1", - "statuses": "2.0.1" + "statuses": "~2.0.2" }, "engines": { "node": ">= 0.8.0" @@ -22959,6 +22578,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -22967,16 +22587,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true - }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "dev": true, - "engines": { - "node": ">= 0.8" - } + "license": "MIT" }, "node_modules/serialize-javascript": { "version": "6.0.2", @@ -23065,15 +22677,16 @@ } }, "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", "dev": true, + "license": "MIT", "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.19.0" + "send": "~0.19.1" }, "engines": { "node": ">= 0.8.0" @@ -23131,7 +22744,8 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/shallow-clone": { "version": "3.0.1", @@ -23539,16 +23153,6 @@ "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", "deprecated": "Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility" }, - "node_modules/stable-hash-x": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/stable-hash-x/-/stable-hash-x-0.2.0.tgz", - "integrity": "sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -23577,10 +23181,11 @@ "dev": true }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -24258,6 +23863,7 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.6" } @@ -24404,46 +24010,31 @@ } } }, - "node_modules/tsconfig-paths": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", - "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/json5": "^0.0.29", - "json5": "^1.0.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - } + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, - "node_modules/tsconfig-paths/node_modules/json5": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", - "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "node_modules/tsyringe": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/tsyringe/-/tsyringe-4.10.0.tgz", + "integrity": "sha512-axr3IdNuVIxnaK5XGEUFTu3YmAQ6lllgrvqfEoR16g/HGnYY/6We4oWENtAnzK6/LpJ2ur9PAb80RBt7/U4ugw==", "dev": true, "license": "MIT", "dependencies": { - "minimist": "^1.2.0" + "tslib": "^1.9.3" }, - "bin": { - "json5": "lib/cli.js" - } - }, - "node_modules/tsconfig-paths/node_modules/strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", - "dev": true, - "license": "MIT", "engines": { - "node": ">=4" + "node": ">= 6.0.0" } }, - "node_modules/tslib": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz", - "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==" + "node_modules/tsyringe/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true, + "license": "0BSD" }, "node_modules/type-check": { "version": "0.4.0", @@ -24483,6 +24074,7 @@ "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "dev": true, + "license": "MIT", "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" @@ -24602,10 +24194,11 @@ } }, "node_modules/undici": { - "version": "6.20.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.20.1.tgz", - "integrity": "sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==", + "version": "6.23.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz", + "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==", "dev": true, + "license": "MIT", "engines": { "node": ">=18.17" } @@ -24669,45 +24262,11 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/unrs-resolver": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", - "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "napi-postinstall": "^0.3.0" - }, - "funding": { - "url": "https://opencollective.com/unrs-resolver" - }, - "optionalDependencies": { - "@unrs/resolver-binding-android-arm-eabi": "1.11.1", - "@unrs/resolver-binding-android-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-x64": "1.11.1", - "@unrs/resolver-binding-freebsd-x64": "1.11.1", - "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", - "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", - "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-musl": "1.11.1", - "@unrs/resolver-binding-wasm32-wasi": "1.11.1", - "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", - "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", - "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" - } - }, "node_modules/update-browserslist-db": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", @@ -24769,6 +24328,7 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4.0" } @@ -24807,6 +24367,7 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -25024,14 +24585,16 @@ } }, "node_modules/webpack-dev-server": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.1.0.tgz", - "integrity": "sha512-aQpaN81X6tXie1FoOB7xlMfCsN19pSvRAeYUHOdFWOlhpQ/LlbfTqYwwmEDFV0h8GGuqmCmKmT+pxcUV/Nt2gQ==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.3.tgz", + "integrity": "sha512-9Gyu2F7+bg4Vv+pjbovuYDhHX+mqdqITykfzdM9UyKqKHlsE5aAjRhR+oOEfXW5vBeu8tarzlJFIZva4ZjAdrQ==", "dev": true, + "license": "MIT", "dependencies": { "@types/bonjour": "^3.5.13", "@types/connect-history-api-fallback": "^1.5.4", - "@types/express": "^4.17.21", + "@types/express": "^4.17.25", + "@types/express-serve-static-core": "^4.17.21", "@types/serve-index": "^1.9.4", "@types/serve-static": "^1.15.5", "@types/sockjs": "^0.3.36", @@ -25040,18 +24603,17 @@ "bonjour-service": "^1.2.1", "chokidar": "^3.6.0", "colorette": "^2.0.10", - "compression": "^1.7.4", + "compression": "^1.8.1", "connect-history-api-fallback": "^2.0.0", - "express": "^4.19.2", + "express": "^4.22.1", "graceful-fs": "^4.2.6", - "html-entities": "^2.4.0", - "http-proxy-middleware": "^2.0.3", + "http-proxy-middleware": "^2.0.9", "ipaddr.js": "^2.1.0", "launch-editor": "^2.6.1", "open": "^10.0.3", "p-retry": "^6.2.0", "schema-utils": "^4.2.0", - "selfsigned": "^2.4.1", + "selfsigned": "^5.5.0", "serve-index": "^1.9.1", "sockjs": "^0.3.24", "spdy": "^4.0.2", @@ -25080,6 +24642,19 @@ } } }, + "node_modules/webpack-dev-server/node_modules/@types/express-serve-static-core": { + "version": "4.19.8", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", + "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "node_modules/webpack-dev-server/node_modules/define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", From d959828908bcf48dfeafb6df1b15b238fa829a33 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 23 Jan 2026 13:48:24 +0500 Subject: [PATCH 015/640] Implement pagination for `/api/project/list` and `/api/users/list` (#3489) * Implement pagination for /api/projects/list * Test /api/projects/list pagination * Update ProjectsAPIClient.list() * Return projects total_count * Fix APIClient.list() * Test APIClient.list() * Implement pagination for /api/users/list * Add name_pattern for projects * Add TestProjectsAPIClientList * Add server-side validation for project name * Allow _ in name_pattern * Add name_pattern for users * Add @overload for ProjectsAPIClient.list() --- src/dstack/_internal/core/models/projects.py | 12 +- src/dstack/_internal/core/models/users.py | 12 +- .../_internal/server/routers/projects.py | 16 +- src/dstack/_internal/server/routers/users.py | 32 +++- src/dstack/_internal/server/schemas/fleets.py | 6 +- .../_internal/server/schemas/projects.py | 40 +++- src/dstack/_internal/server/schemas/users.py | 48 ++++- .../_internal/server/services/projects.py | 122 ++++++++---- src/dstack/_internal/server/services/users.py | 86 ++++++++- src/dstack/api/server/_projects.py | 76 +++++++- src/dstack/api/server/_users.py | 50 ++++- .../_internal/server/routers/test_projects.py | 180 +++++++++++++++++- .../_internal/server/routers/test_users.py | 160 +++++++++++++++- src/tests/api/common.py | 29 +++ src/tests/api/test_projects.py | 62 ++++++ src/tests/api/test_users.py | 54 ++++++ 16 files changed, 903 insertions(+), 82 deletions(-) create mode 100644 src/tests/api/common.py create mode 100644 src/tests/api/test_projects.py create mode 100644 src/tests/api/test_users.py diff --git a/src/dstack/_internal/core/models/projects.py b/src/dstack/_internal/core/models/projects.py index 9748ece1ae..63adf91962 100644 --- a/src/dstack/_internal/core/models/projects.py +++ b/src/dstack/_internal/core/models/projects.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional +from typing import List, Optional, Union from pydantic import UUID4 @@ -28,6 +28,16 @@ class Project(CoreModel): is_public: bool = False +class ProjectsInfoList(CoreModel): + total_count: Optional[int] = None + projects: List[Project] + + +# For backward compatibility with 0.20 clients, endpoints return `List[Project]` if `total_count` is None. +# TODO: Replace with ProjectsInfoList in 0.21. +ProjectsInfoListOrProjectsList = Union[List[Project], ProjectsInfoList] + + class ProjectHookConfig(CoreModel): """ This class can be inherited to extend the project creation configuration passed to the hooks. diff --git a/src/dstack/_internal/core/models/users.py b/src/dstack/_internal/core/models/users.py index 99fb8823e3..8e70e092d6 100644 --- a/src/dstack/_internal/core/models/users.py +++ b/src/dstack/_internal/core/models/users.py @@ -1,6 +1,6 @@ import enum from datetime import datetime -from typing import Optional +from typing import List, Optional, Union from pydantic import UUID4 @@ -42,6 +42,16 @@ class UserWithCreds(User): ssh_private_key: Optional[str] = None +class UsersInfoList(CoreModel): + total_count: Optional[int] = None + users: List[User] + + +# For backward compatibility with 0.20 clients, endpoints return `List[User]` if `total_count` is None. +# TODO: Replace with UsersInfoList in 0.21. +UsersInfoListOrUsersList = Union[List[User], UsersInfoList] + + class UserHookConfig(CoreModel): """ This class can be inherited to extend the user creation configuration passed to the hooks. diff --git a/src/dstack/_internal/server/routers/projects.py b/src/dstack/_internal/server/routers/projects.py index b07b7b1c62..aaf08809b7 100644 --- a/src/dstack/_internal/server/routers/projects.py +++ b/src/dstack/_internal/server/routers/projects.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.models.projects import Project +from dstack._internal.core.models.projects import Project, ProjectsInfoListOrProjectsList from dstack._internal.server.db import get_session from dstack._internal.server.models import ProjectModel, UserModel from dstack._internal.server.schemas.projects import ( @@ -36,14 +36,14 @@ ) -@router.post("/list", response_model=List[Project]) +@router.post("/list", response_model=ProjectsInfoListOrProjectsList) async def list_projects( body: Optional[ListProjectsRequest] = None, session: AsyncSession = Depends(get_session), user: UserModel = Depends(Authenticated()), ): """ - Returns projects visible to the user, sorted by ascending `created_at`. + Returns projects visible to the user. Returns all accessible projects (member projects for regular users, all non-deleted projects for global admins, plus public projects if `include_not_joined` is `True`). @@ -55,7 +55,15 @@ async def list_projects( body = ListProjectsRequest() return CustomORJSONResponse( await projects.list_user_accessible_projects( - session=session, user=user, include_not_joined=body.include_not_joined + session=session, + user=user, + include_not_joined=body.include_not_joined, + return_total_count=body.return_total_count, + name_pattern=body.name_pattern, + prev_created_at=body.prev_created_at, + prev_id=body.prev_id, + limit=body.limit, + ascending=body.ascending, ) ) diff --git a/src/dstack/_internal/server/routers/users.py b/src/dstack/_internal/server/routers/users.py index 6030416f50..6cd72f00a1 100644 --- a/src/dstack/_internal/server/routers/users.py +++ b/src/dstack/_internal/server/routers/users.py @@ -1,16 +1,17 @@ -from typing import List +from typing import Optional from fastapi import APIRouter, Depends from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.errors import ResourceNotExistsError -from dstack._internal.core.models.users import User, UserWithCreds +from dstack._internal.core.models.users import User, UsersInfoListOrUsersList, UserWithCreds from dstack._internal.server.db import get_session from dstack._internal.server.models import UserModel from dstack._internal.server.schemas.users import ( CreateUserRequest, DeleteUsersRequest, GetUserRequest, + ListUsersRequest, RefreshTokenRequest, UpdateUserRequest, ) @@ -28,12 +29,35 @@ ) -@router.post("/list", response_model=List[User]) +@router.post("/list", response_model=UsersInfoListOrUsersList) async def list_users( + body: Optional[ListUsersRequest] = None, session: AsyncSession = Depends(get_session), user: UserModel = Depends(Authenticated()), ): - return CustomORJSONResponse(await users.list_users_for_user(session=session, user=user)) + """ + Returns users visible to the user, sorted by descending `created_at`. + + Admins see all non-deleted users. Non-admins only see themselves. + + The results are paginated. To get the next page, pass `created_at` and `id` of + the last user from the previous page as `prev_created_at` and `prev_id`. + """ + if body is None: + # For backward compatibility + body = ListUsersRequest() + return CustomORJSONResponse( + await users.list_users_for_user( + session=session, + user=user, + return_total_count=body.return_total_count, + name_pattern=body.name_pattern, + prev_created_at=body.prev_created_at, + prev_id=body.prev_id, + limit=body.limit, + ascending=body.ascending, + ) + ) @router.post("/get_my_user", response_model=UserWithCreds) diff --git a/src/dstack/_internal/server/schemas/fleets.py b/src/dstack/_internal/server/schemas/fleets.py index ae66818ab9..3df43d12ce 100644 --- a/src/dstack/_internal/server/schemas/fleets.py +++ b/src/dstack/_internal/server/schemas/fleets.py @@ -9,10 +9,10 @@ class ListFleetsRequest(CoreModel): - project_name: Optional[str] + project_name: Optional[str] = None only_active: bool = False - prev_created_at: Optional[datetime] - prev_id: Optional[UUID] + prev_created_at: Optional[datetime] = None + prev_id: Optional[UUID] = None limit: int = Field(100, ge=0, le=100) ascending: bool = False diff --git a/src/dstack/_internal/server/schemas/projects.py b/src/dstack/_internal/server/schemas/projects.py index ec05c1fb47..5f0133ab72 100644 --- a/src/dstack/_internal/server/schemas/projects.py +++ b/src/dstack/_internal/server/schemas/projects.py @@ -1,4 +1,6 @@ -from typing import Annotated, List +from datetime import datetime +from typing import Annotated, List, Optional +from uuid import UUID from pydantic import Field @@ -8,8 +10,42 @@ class ListProjectsRequest(CoreModel): include_not_joined: Annotated[ - bool, Field(description="Include public projects where user is not a member") + bool, Field(description="Include public projects where user is not a member.") ] = True + return_total_count: Annotated[ + bool, Field(description="Return `total_count` with the total number of projects.") + ] = False + name_pattern: Annotated[ + Optional[str], + Field( + description="Include only projects with the name containing `name_pattern`.", + regex="^[a-zA-Z0-9-_]*$", + ), + ] = None + prev_created_at: Annotated[ + Optional[datetime], + Field( + description="Paginate projects by specifying `created_at` of the last (first) project in previous batch for descending (ascending)." + ), + ] = None + prev_id: Annotated[ + Optional[UUID], + Field( + description=( + "Paginate projects by specifying `id` of the last (first) project in previous batch for descending (ascending)." + " Must be used together with `prev_created_at`." + ) + ), + ] = None + limit: Annotated[ + int, Field(ge=0, le=2000, description="Limit number of projects returned.") + ] = 2000 + ascending: Annotated[ + bool, + Field( + description="Return projects sorted by `created_at` in ascending order. Defaults to descending." + ), + ] = False class CreateProjectRequest(CoreModel): diff --git a/src/dstack/_internal/server/schemas/users.py b/src/dstack/_internal/server/schemas/users.py index 6579d96572..574d5b093e 100644 --- a/src/dstack/_internal/server/schemas/users.py +++ b/src/dstack/_internal/server/schemas/users.py @@ -1,9 +1,55 @@ -from typing import List, Optional +from datetime import datetime +from typing import Annotated, List, Optional +from uuid import UUID + +from pydantic import Field from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.users import GlobalRole +class ListUsersRequest(CoreModel): + return_total_count: Annotated[ + bool, Field(description="Return `total_count` with the total number of users.") + ] = False + name_pattern: Annotated[ + Optional[str], + Field( + description="Include only users with the name containing `name_pattern`.", + regex="^[a-zA-Z0-9-_]*$", + ), + ] = None + prev_created_at: Annotated[ + Optional[datetime], + Field( + description=( + "Paginate users by specifying `created_at` of the last (first) user in previous " + "batch for descending (ascending)." + ) + ), + ] = None + prev_id: Annotated[ + Optional[UUID], + Field( + description=( + "Paginate users by specifying `id` of the last (first) user in previous batch " + "for descending (ascending). Must be used together with `prev_created_at`." + ) + ), + ] = None + limit: Annotated[int, Field(ge=0, le=2000, description="Limit number of users returned.")] = ( + 2000 + ) + ascending: Annotated[ + bool, + Field( + description=( + "Return users sorted by `created_at` in ascending order. Defaults to descending." + ) + ), + ] = False + + class GetUserRequest(CoreModel): username: str diff --git a/src/dstack/_internal/server/services/projects.py b/src/dstack/_internal/server/services/projects.py index 937247f5a1..2383594690 100644 --- a/src/dstack/_internal/server/services/projects.py +++ b/src/dstack/_internal/server/services/projects.py @@ -1,8 +1,10 @@ +import re import secrets import uuid +from datetime import datetime from typing import Awaitable, Callable, List, Optional, Tuple -from sqlalchemy import delete, select, update +from sqlalchemy import and_, delete, literal_column, or_, select, update from sqlalchemy import func as safunc from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import QueryableAttribute, joinedload, load_only @@ -19,6 +21,8 @@ MemberPermissions, Project, ProjectHookConfig, + ProjectsInfoList, + ProjectsInfoListOrProjectsList, ) from dstack._internal.core.models.runs import RunStatus from dstack._internal.core.models.users import GlobalRole, ProjectRole @@ -62,57 +66,87 @@ async def get_or_create_default_project( return default_project, True -async def list_user_projects( - session: AsyncSession, - user: UserModel, -) -> List[Project]: - """ - Returns projects where the user is a member or all projects for global admins. - """ - projects = await list_user_project_models( - session=session, - user=user, - ) - projects = sorted(projects, key=lambda p: p.created_at) - return [ - project_model_to_project(p, include_backends=False, include_members=False) - for p in projects - ] - - async def list_user_accessible_projects( session: AsyncSession, user: UserModel, include_not_joined: bool, -) -> List[Project]: + return_total_count: bool, + name_pattern: Optional[str], + prev_created_at: Optional[datetime], + prev_id: Optional[uuid.UUID], + limit: int, + ascending: bool, +) -> ProjectsInfoListOrProjectsList: """ Returns all projects accessible to the user: + - All projects for global admins - Projects where user is a member (public or private) - if `include_not_joined`: Public projects where user is NOT a member """ - if user.global_role == GlobalRole.ADMIN: - projects = await list_project_models(session=session) - else: - projects = await list_member_project_models(session=session, user=user) + filters = [ProjectModel.deleted == False] + if name_pattern: + name_pattern = name_pattern.replace("_", "/_") + filters.append(ProjectModel.name.ilike(f"%{name_pattern}%", escape="/")) + stmt = select(ProjectModel).where(*filters) + if user.global_role != GlobalRole.ADMIN: + stmt = stmt.outerjoin( + MemberModel, + onclause=and_( + MemberModel.project_id == ProjectModel.id, + MemberModel.user_id == user.id, + ), + ) if include_not_joined: - public_projects = await list_public_non_member_project_models( - session=session, user=user + stmt = stmt.where( + or_( + ProjectModel.is_public == True, + MemberModel.user_id.is_not(None), + ) ) - projects += public_projects - - projects = sorted(projects, key=lambda p: p.created_at) - return [ - project_model_to_project(p, include_backends=False, include_members=False) - for p in projects - ] - - -async def list_projects(session: AsyncSession) -> List[Project]: - projects = await list_project_models(session=session) - return [ + else: + stmt = stmt.where(MemberModel.user_id.is_not(None)) + pagination_filters = [] + if prev_created_at is not None: + if ascending: + if prev_id is None: + pagination_filters.append(ProjectModel.created_at > prev_created_at) + else: + pagination_filters.append( + or_( + ProjectModel.created_at > prev_created_at, + and_( + ProjectModel.created_at == prev_created_at, ProjectModel.id < prev_id + ), + ) + ) + else: + if prev_id is None: + pagination_filters.append(ProjectModel.created_at < prev_created_at) + else: + pagination_filters.append( + or_( + ProjectModel.created_at < prev_created_at, + and_( + ProjectModel.created_at == prev_created_at, ProjectModel.id > prev_id + ), + ) + ) + order_by = (ProjectModel.created_at.desc(), ProjectModel.id) + if ascending: + order_by = (ProjectModel.created_at.asc(), ProjectModel.id.desc()) + total_count = None + if return_total_count: + res = await session.execute(stmt.with_only_columns(safunc.count(literal_column("1")))) + total_count = res.scalar_one() + res = await session.execute(stmt.where(*pagination_filters).order_by(*order_by).limit(limit)) + project_models = res.unique().scalars().all() + projects = [ project_model_to_project(p, include_backends=False, include_members=False) - for p in projects + for p in project_models ] + if total_count is None: + return projects + return ProjectsInfoList(total_count=total_count, projects=projects) async def get_project_by_name( @@ -543,6 +577,7 @@ async def get_project_model_by_id_or_error( async def create_project_model( session: AsyncSession, owner: UserModel, project_name: str, is_public: bool = False ) -> ProjectModel: + validate_project_name(project_name) private_bytes, public_bytes = await run_async( generate_rsa_key_pair_bytes, f"{project_name}@dstack" ) @@ -649,6 +684,15 @@ def get_member_permissions(member_model: MemberModel) -> MemberPermissions: ) +def validate_project_name(project_name: str): + if not is_valid_project_name(project_name): + raise ServerClientError("Project name should match regex '^[a-zA-Z0-9-_]{1,50}$'") + + +def is_valid_project_name(project_name: str) -> bool: + return re.match("^[a-zA-Z0-9-_]{1,50}$", project_name) is not None + + _CREATE_PROJECT_HOOKS = [] diff --git a/src/dstack/_internal/server/services/users.py b/src/dstack/_internal/server/services/users.py index 3f8f6afa7b..73ceebe0ef 100644 --- a/src/dstack/_internal/server/services/users.py +++ b/src/dstack/_internal/server/services/users.py @@ -5,9 +5,10 @@ import uuid from collections.abc import AsyncGenerator from contextlib import asynccontextmanager +from datetime import datetime from typing import Awaitable, Callable, List, Optional, Tuple -from sqlalchemy import delete, select +from sqlalchemy import and_, delete, literal_column, or_, select from sqlalchemy import func as safunc from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import load_only @@ -21,6 +22,8 @@ User, UserHookConfig, UserPermissions, + UsersInfoList, + UsersInfoListOrUsersList, UserTokenCreds, UserWithCreds, ) @@ -55,23 +58,90 @@ async def get_or_create_admin_user(session: AsyncSession) -> Tuple[UserModel, bo async def list_users_for_user( session: AsyncSession, user: UserModel, -) -> List[User]: + return_total_count: bool, + name_pattern: Optional[str], + prev_created_at: Optional[datetime], + prev_id: Optional[uuid.UUID], + limit: int, + ascending: bool, +) -> UsersInfoListOrUsersList: if user.global_role == GlobalRole.ADMIN: - return await list_all_users(session=session) - return [user_model_to_user(user)] + return await list_all_users( + session=session, + include_deleted=False, + return_total_count=return_total_count, + name_pattern=name_pattern, + prev_created_at=prev_created_at, + prev_id=prev_id, + limit=limit, + ascending=ascending, + ) + users = [] + if not user.deleted and (name_pattern is None or name_pattern.lower() in user.name.lower()): + users.append(user_model_to_user(user)) + if return_total_count: + return UsersInfoList(total_count=len(users), users=users) + return users async def list_all_users( session: AsyncSession, include_deleted: bool = False, -) -> List[User]: + return_total_count: bool = False, + name_pattern: Optional[str] = None, + prev_created_at: Optional[datetime] = None, + prev_id: Optional[uuid.UUID] = None, + limit: int = 2000, + ascending: bool = False, +) -> UsersInfoListOrUsersList: filters = [] if not include_deleted: filters.append(UserModel.deleted == False) - res = await session.execute(select(UserModel).where(*filters)) + if name_pattern: + name_pattern = name_pattern.replace("_", "/_") + filters.append(UserModel.name.ilike(f"%{name_pattern}%", escape="/")) + stmt = select(UserModel).where(*filters) + pagination_filters = [] + if prev_created_at is not None: + if ascending: + if prev_id is None: + pagination_filters.append(UserModel.created_at > prev_created_at) + else: + pagination_filters.append( + or_( + UserModel.created_at > prev_created_at, + and_( + UserModel.created_at == prev_created_at, + UserModel.id < prev_id, + ), + ) + ) + else: + if prev_id is None: + pagination_filters.append(UserModel.created_at < prev_created_at) + else: + pagination_filters.append( + or_( + UserModel.created_at < prev_created_at, + and_( + UserModel.created_at == prev_created_at, + UserModel.id > prev_id, + ), + ) + ) + order_by = (UserModel.created_at.desc(), UserModel.id) + if ascending: + order_by = (UserModel.created_at.asc(), UserModel.id.desc()) + total_count = None + if return_total_count: + res = await session.execute(stmt.with_only_columns(safunc.count(literal_column("1")))) + total_count = res.scalar_one() + res = await session.execute(stmt.where(*pagination_filters).order_by(*order_by).limit(limit)) user_models = res.scalars().all() - user_models = sorted(user_models, key=lambda u: u.created_at) - return [user_model_to_user(u) for u in user_models] + users = [user_model_to_user(u) for u in user_models] + if total_count is None: + return users + return UsersInfoList(total_count=total_count, users=users) async def get_user_with_creds_by_name( diff --git a/src/dstack/api/server/_projects.py b/src/dstack/api/server/_projects.py index 31bdc3b2de..96a1f511f7 100644 --- a/src/dstack/api/server/_projects.py +++ b/src/dstack/api/server/_projects.py @@ -1,14 +1,20 @@ -from typing import List +import json +from datetime import datetime +from typing import Any, List, Literal, Optional, Union, overload +from uuid import UUID from pydantic import parse_obj_as -from dstack._internal.core.models.projects import Project +from dstack._internal.core.models.projects import ( + Project, + ProjectsInfoList, + ProjectsInfoListOrProjectsList, +) from dstack._internal.core.models.users import ProjectRole from dstack._internal.server.schemas.projects import ( AddProjectMemberRequest, CreateProjectRequest, DeleteProjectsRequest, - ListProjectsRequest, MemberSetting, RemoveProjectMemberRequest, SetProjectMembersRequest, @@ -17,10 +23,66 @@ class ProjectsAPIClient(APIClientGroup): - def list(self, include_not_joined: bool = True) -> List[Project]: - body = ListProjectsRequest(include_not_joined=include_not_joined) - resp = self._request("/api/projects/list", body=body.json()) - return parse_obj_as(List[Project.__response__], resp.json()) + @overload + def list( + self, + include_not_joined: bool = True, + *, + return_total_count: Literal[True], + name_pattern: Optional[str] = None, + prev_created_at: Optional[datetime] = None, + prev_id: Optional[UUID] = None, + limit: Optional[int] = None, + ascending: Optional[bool] = None, + ) -> ProjectsInfoList: + pass + + @overload + def list( + self, + include_not_joined: bool = True, + *, + return_total_count: Union[Literal[False], None] = None, + name_pattern: Optional[str] = None, + prev_created_at: Optional[datetime] = None, + prev_id: Optional[UUID] = None, + limit: Optional[int] = None, + ascending: Optional[bool] = None, + ) -> List[Project]: + pass + + def list( + self, + include_not_joined: bool = True, + *, + return_total_count: Optional[bool] = None, + name_pattern: Optional[str] = None, + prev_created_at: Optional[datetime] = None, + prev_id: Optional[UUID] = None, + limit: Optional[int] = None, + ascending: Optional[bool] = None, + ) -> ProjectsInfoListOrProjectsList: + # Passing only non-None fields for backward compatibility with 0.20 servers. + body: dict[str, Any] = { + "include_not_joined": include_not_joined, + } + if return_total_count is not None: + body["return_total_count"] = return_total_count + if name_pattern is not None: + body["name_pattern"] = name_pattern + if prev_created_at is not None: + body["prev_created_at"] = prev_created_at.isoformat() + if prev_id is not None: + body["prev_id"] = str(prev_id) + if limit is not None: + body["limit"] = limit + if ascending is not None: + body["ascending"] = ascending + resp = self._request("/api/projects/list", body=json.dumps(body)) + resp_json = resp.json() + if isinstance(resp_json, list): + return parse_obj_as(List[Project.__response__], resp_json) + return parse_obj_as(ProjectsInfoList, resp_json) def create(self, project_name: str, is_public: bool = False) -> Project: body = CreateProjectRequest(project_name=project_name, is_public=is_public) diff --git a/src/dstack/api/server/_users.py b/src/dstack/api/server/_users.py index 6082636c4b..885eae54a2 100644 --- a/src/dstack/api/server/_users.py +++ b/src/dstack/api/server/_users.py @@ -1,8 +1,18 @@ -from typing import List +import json +from datetime import datetime +from typing import Any, List, Optional +from uuid import UUID from pydantic import parse_obj_as +from pydantic.json import pydantic_encoder -from dstack._internal.core.models.users import GlobalRole, User, UserWithCreds +from dstack._internal.core.models.users import ( + GlobalRole, + User, + UsersInfoList, + UsersInfoListOrUsersList, + UserWithCreds, +) from dstack._internal.server.schemas.users import ( CreateUserRequest, GetUserRequest, @@ -13,9 +23,39 @@ class UsersAPIClient(APIClientGroup): - def list(self) -> List[User]: - resp = self._request("/api/users/list") - return parse_obj_as(List[User.__response__], resp.json()) + def list( + self, + return_total_count: Optional[bool] = None, + name_pattern: Optional[str] = None, + prev_created_at: Optional[datetime] = None, + prev_id: Optional[UUID] = None, + limit: Optional[int] = None, + ascending: Optional[bool] = None, + ) -> UsersInfoListOrUsersList: + # Passing only non-None fields for backward compatibility with 0.20 servers. + body: dict[str, Any] = {} + if return_total_count is not None: + body["return_total_count"] = return_total_count + if name_pattern is not None: + body["name_pattern"] = name_pattern + if prev_created_at is not None: + body["prev_created_at"] = prev_created_at + if prev_id is not None: + body["prev_id"] = prev_id + if limit is not None: + body["limit"] = limit + if ascending is not None: + body["ascending"] = ascending + if body: + resp = self._request( + "/api/users/list", body=json.dumps(body, default=pydantic_encoder) + ) + else: + resp = self._request("/api/users/list") + resp_json = resp.json() + if isinstance(resp_json, list): + return parse_obj_as(List[User.__response__], resp_json) + return parse_obj_as(UsersInfoList, resp_json) def get_my_user(self) -> UserWithCreds: resp = self._request("/api/users/get_my_user") diff --git a/src/tests/_internal/server/routers/test_projects.py b/src/tests/_internal/server/routers/test_projects.py index 5c9ef42ffb..2da65c7d5c 100644 --- a/src/tests/_internal/server/routers/test_projects.py +++ b/src/tests/_internal/server/routers/test_projects.py @@ -15,7 +15,6 @@ from dstack._internal.server.services.permissions import DefaultPermissions from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( - create_backend, create_fleet, create_project, create_repo, @@ -65,10 +64,6 @@ async def test_returns_projects(self, test_db, session: AsyncSession, client: As await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) - await create_backend( - session=session, - project_id=project.id, - ) response = await client.post("/api/projects/list", headers=get_auth_headers(user.token)) assert response.status_code in [200] assert response.json() == [ @@ -216,6 +211,181 @@ async def test_member_sees_both_public_and_private_projects( assert "public_project" in project_names assert "private_project" in project_names + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_paginated_projects( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user( + session=session, + created_at=datetime(2023, 1, 2, 3, 0, tzinfo=timezone.utc), + global_role=GlobalRole.ADMIN, + ) + project1 = await create_project( + session=session, + name="project1", + owner=user, + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + project2 = await create_project( + session=session, + name="project2", + owner=user, + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + ) + project3 = await create_project( + session=session, + name="project3", + owner=user, + created_at=datetime(2023, 1, 2, 3, 6, tzinfo=timezone.utc), + ) + response = await client.post( + "/api/projects/list", + headers=get_auth_headers(user.token), + json={"limit": 1}, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "project_id": str(project3.id), + "project_name": project3.name, + "owner": { + "id": str(user.id), + "username": user.name, + "created_at": "2023-01-02T03:00:00+00:00", + "global_role": user.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + }, + "created_at": "2023-01-02T03:06:00+00:00", + "backends": [], + "members": [], + "is_public": False, + } + ] + response = await client.post( + "/api/projects/list", + headers=get_auth_headers(user.token), + json={ + "prev_created_at": "2023-01-02T03:06:00+00:00", + "prev_id": str(project3.id), + "limit": 1, + }, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "project_id": str(project2.id), + "project_name": project2.name, + "owner": { + "id": str(user.id), + "username": user.name, + "created_at": "2023-01-02T03:00:00+00:00", + "global_role": user.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + }, + "created_at": "2023-01-02T03:05:00+00:00", + "backends": [], + "members": [], + "is_public": False, + } + ] + response = await client.post( + "/api/projects/list", + headers=get_auth_headers(user.token), + json={ + "prev_created_at": "2023-01-02T03:05:00+00:00", + "prev_id": str(project2.id), + "limit": 1, + }, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "project_id": str(project1.id), + "project_name": project1.name, + "owner": { + "id": str(user.id), + "username": user.name, + "created_at": "2023-01-02T03:00:00+00:00", + "global_role": user.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + }, + "created_at": "2023-01-02T03:04:00+00:00", + "backends": [], + "members": [], + "is_public": False, + } + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_total_count(self, test_db, session: AsyncSession, client: AsyncClient): + user = await create_user( + session=session, + created_at=datetime(2023, 1, 2, 3, 0, tzinfo=timezone.utc), + global_role=GlobalRole.ADMIN, + ) + await create_project( + session=session, + name="project1", + owner=user, + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + project3 = await create_project( + session=session, + name="project3", + owner=user, + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + ) + response = await client.post( + "/api/projects/list", + headers=get_auth_headers(user.token), + json={"limit": 1, "return_total_count": True}, + ) + assert response.status_code == 200 + assert response.json() == { + "total_count": 2, + "projects": [ + { + "project_id": str(project3.id), + "project_name": project3.name, + "owner": { + "id": str(user.id), + "username": user.name, + "created_at": "2023-01-02T03:00:00+00:00", + "global_role": user.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + }, + "created_at": "2023-01-02T03:05:00+00:00", + "backends": [], + "members": [], + "is_public": False, + } + ], + } + + +class TestListOnlyNoFleets: @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_only_no_fleets_returns_projects_without_active_fleets( diff --git a/src/tests/_internal/server/routers/test_users.py b/src/tests/_internal/server/routers/test_users.py index 6c5b373a63..5042e75d6b 100644 --- a/src/tests/_internal/server/routers/test_users.py +++ b/src/tests/_internal/server/routers/test_users.py @@ -39,7 +39,7 @@ async def test_admins_see_all_non_deleted_users( admin = await create_user( session=session, name="admin", - created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), global_role=GlobalRole.ADMIN, ) other_user = await create_user( @@ -61,7 +61,7 @@ async def test_admins_see_all_non_deleted_users( { "id": str(admin.id), "username": admin.name, - "created_at": "2023-01-02T03:04:00+00:00", + "created_at": "2023-01-02T03:05:00+00:00", "global_role": admin.global_role, "email": None, "active": True, @@ -84,6 +84,162 @@ async def test_admins_see_all_non_deleted_users( }, ] + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_total_count(self, test_db, session: AsyncSession, client: AsyncClient): + admin = await create_user( + session=session, + name="admin", + created_at=datetime(2023, 1, 2, 3, 6, tzinfo=timezone.utc), + global_role=GlobalRole.ADMIN, + ) + await create_user( + session=session, + name="user_one", + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + ) + await create_user( + session=session, + name="deleted_user", + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + deleted=True, + ) + response = await client.post( + "/api/users/list", + headers=get_auth_headers(admin.token), + json={"limit": 1, "return_total_count": True}, + ) + assert response.status_code == 200 + assert response.json() == { + "total_count": 2, + "users": [ + { + "id": str(admin.id), + "username": admin.name, + "created_at": "2023-01-02T03:06:00+00:00", + "global_role": admin.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + } + ], + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_paginates_results(self, test_db, session: AsyncSession, client: AsyncClient): + admin = await create_user( + session=session, + name="admin", + created_at=datetime(2023, 1, 2, 3, 6, tzinfo=timezone.utc), + global_role=GlobalRole.ADMIN, + ) + user_one = await create_user( + session=session, + name="user_one", + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + ) + await create_user( + session=session, + name="user_two", + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + ) + response = await client.post( + "/api/users/list", + headers=get_auth_headers(admin.token), + json={"limit": 1}, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "id": str(admin.id), + "username": admin.name, + "created_at": "2023-01-02T03:06:00+00:00", + "global_role": admin.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + } + ] + response = await client.post( + "/api/users/list", + headers=get_auth_headers(admin.token), + json={ + "prev_created_at": "2023-01-02T03:06:00+00:00", + "prev_id": str(admin.id), + "limit": 1, + }, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "id": str(user_one.id), + "username": user_one.name, + "created_at": "2023-01-02T03:05:00+00:00", + "global_role": user_one.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + } + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_filters_by_name_pattern( + self, test_db, session: AsyncSession, client: AsyncClient + ): + admin = await create_user( + session=session, + name="admin", + created_at=datetime(2023, 1, 2, 3, 6, tzinfo=timezone.utc), + global_role=GlobalRole.ADMIN, + ) + matching_user = await create_user( + session=session, + name="alpha_user", + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + ) + await create_user( + session=session, + name="bravo", + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + global_role=GlobalRole.USER, + ) + response = await client.post( + "/api/users/list", + headers=get_auth_headers(admin.token), + json={"name_pattern": "alpha"}, + ) + assert response.status_code == 200 + assert response.json() == [ + { + "id": str(matching_user.id), + "username": matching_user.name, + "created_at": "2023-01-02T03:05:00+00:00", + "global_role": matching_user.global_role, + "email": None, + "active": True, + "permissions": { + "can_create_projects": True, + }, + "ssh_public_key": None, + } + ] + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_non_admins_see_only_themselves( diff --git a/src/tests/api/common.py b/src/tests/api/common.py new file mode 100644 index 0000000000..c453b6afee --- /dev/null +++ b/src/tests/api/common.py @@ -0,0 +1,29 @@ +import json +from dataclasses import dataclass, field +from typing import Any, Optional + +import requests + + +@dataclass +class RequestRecorder: + payload: Any + last_path: Optional[str] = None + last_body: Optional[str] = None + last_kwargs: dict[str, Any] = field(default_factory=dict) + + def __call__( + self, + path: str, + body: Optional[str] = None, + raise_for_status: bool = True, + method: str = "POST", + **kwargs, + ) -> requests.Response: + self.last_path = path + self.last_body = body + self.last_kwargs = kwargs + resp = requests.Response() + resp.status_code = 200 + resp._content = json.dumps(self.payload).encode("utf-8") + return resp diff --git a/src/tests/api/test_projects.py b/src/tests/api/test_projects.py new file mode 100644 index 0000000000..38b93bae5a --- /dev/null +++ b/src/tests/api/test_projects.py @@ -0,0 +1,62 @@ +import json +import logging +from datetime import datetime, timezone +from uuid import UUID + +from dstack.api.server._projects import ProjectsAPIClient +from tests.api.common import RequestRecorder + +PROJECT_PAYLOAD = { + "project_id": "1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e", + "project_name": "p", + "owner": { + "id": "2b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e", + "username": "u", + "created_at": "2023-01-02T03:04:00+00:00", + "global_role": "user", + "email": None, + "active": True, + "permissions": {"can_create_projects": True}, + "ssh_public_key": None, + }, + "created_at": "2023-01-02T03:04:00+00:00", + "backends": [], + "members": [], + "is_public": False, +} + + +class TestProjectsAPIClientList: + def test_projects_list_serializes_pagination_and_parses_info_list(self): + request = RequestRecorder(payload={"total_count": 1, "projects": [PROJECT_PAYLOAD]}) + client = ProjectsAPIClient(_request=request, _logger=logging.getLogger("test")) + dt = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc) + pid = UUID("3b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e") + + result = client.list( + return_total_count=True, + prev_created_at=dt, + name_pattern="p", + prev_id=pid, + limit=1, + ascending=True, + ) + + payload = json.loads(request.last_body) + assert request.last_path == "/api/projects/list" + assert payload["include_not_joined"] is True + assert payload["return_total_count"] is True + assert payload["name_pattern"] == "p" + assert payload["prev_created_at"] == dt.isoformat() + assert payload["prev_id"] == str(pid) + assert payload["limit"] == 1 + assert payload["ascending"] is True + assert result.total_count == 1 + assert result.projects[0].project_name == "p" + + def test_projects_list_parses_list_response(self): + request = RequestRecorder(payload=[PROJECT_PAYLOAD]) + client = ProjectsAPIClient(_request=request, _logger=logging.getLogger("test")) + result = client.list() + assert isinstance(result, list) + assert result[0].project_name == PROJECT_PAYLOAD["project_name"] diff --git a/src/tests/api/test_users.py b/src/tests/api/test_users.py new file mode 100644 index 0000000000..c01703b811 --- /dev/null +++ b/src/tests/api/test_users.py @@ -0,0 +1,54 @@ +import json +import logging +from datetime import datetime, timezone +from uuid import UUID + +from dstack.api.server._users import UsersAPIClient +from tests.api.common import RequestRecorder + +USER_PAYLOAD = { + "id": "11111111-1111-4111-8111-111111111111", + "username": "user", + "created_at": "2023-01-02T03:04:00+00:00", + "global_role": "user", + "email": None, + "active": True, + "permissions": {"can_create_projects": True}, + "ssh_public_key": None, +} + + +class TestUsersAPIClientList: + def test_serializes_pagination_and_parses_info_list(self): + recorder = RequestRecorder({"total_count": 1, "users": [USER_PAYLOAD]}) + client = UsersAPIClient(_request=recorder, _logger=logging.getLogger("test")) + dt = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc) + uid = UUID("22222222-2222-4222-8222-222222222222") + + result = client.list( + return_total_count=True, + name_pattern="user", + prev_created_at=dt, + prev_id=uid, + limit=1, + ascending=True, + ) + + payload = json.loads(recorder.last_body) + assert recorder.last_path == "/api/users/list" + assert payload["return_total_count"] is True + assert payload["name_pattern"] == "user" + assert payload["prev_created_at"] == dt.isoformat() + assert payload["prev_id"] == str(uid) + assert payload["limit"] == 1 + assert payload["ascending"] is True + assert result.total_count == 1 + assert result.users[0].username == "user" + + def test_parses_list_response(self): + recorder = RequestRecorder([USER_PAYLOAD]) + client = UsersAPIClient(_request=recorder, _logger=logging.getLogger("test")) + result = client.list() + + assert isinstance(result, list) + assert result[0].username == "user" From 1c3b7f8078de3955841be1611bb4d3b691b0614b Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 23 Jan 2026 14:02:11 +0500 Subject: [PATCH 016/640] Update dstack server CLI logo (#3438) * Update dstack server CLI logo * Remove extra padding --- src/dstack/_internal/server/app.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index b41152c149..dbea6f777b 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -419,16 +419,11 @@ def _sentry_traces_sampler(sampling_context: SamplingContext) -> float: def _print_dstack_logo(): console.print( - """[purple]╱╱╭╮╱╱╭╮╱╱╱╱╱╱╭╮ -╱╱┃┃╱╭╯╰╮╱╱╱╱╱┃┃ -╭━╯┣━┻╮╭╋━━┳━━┫┃╭╮ -┃╭╮┃━━┫┃┃╭╮┃╭━┫╰╯╯ -┃╰╯┣━━┃╰┫╭╮┃╰━┫╭╮╮ -╰━━┻━━┻━┻╯╰┻━━┻╯╰╯ -╭━━┳━━┳━┳╮╭┳━━┳━╮ -┃━━┫┃━┫╭┫╰╯┃┃━┫╭╯ -┣━━┃┃━┫┃╰╮╭┫┃━┫┃ -╰━━┻━━┻╯╱╰╯╰━━┻╯ + r"""[purple] _ _ _ + __| |___| |_ __ _ ___| | __ ___ ___ _ ____ _____ _ __ + / _` / __| __/ _` |/ __| |/ / / __|/ _ \ '__\ \ / / _ \ '__| +| (_| \__ \ || (_| | (__| < \__ \ __/ | \ V / __/ | + \__,_|___/\__\__,_|\___|_|\_\ |___/\___|_| \_/ \___|_| [/]""" ) From 330eb1f0ba9b1b05ab3953303d6e0aeb8540d437 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 23 Jan 2026 16:33:25 +0500 Subject: [PATCH 017/640] Move pytest.ini options to pyproject.toml (#3491) * Move pytest.ini options to pyproject.toml * Fix test_logs * Fix decorator order * Fix TestRESTPlugin --- pyproject.toml | 24 +++++++- pytest.ini | 12 ---- src/tests/_internal/server/conftest.py | 2 +- .../_internal/server/services/test_logs.py | 2 +- src/tests/plugins/test_rest_plugin.py | 57 +++++++++---------- 5 files changed, 53 insertions(+), 44 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 2fe97f2cbb..8e4e7bc070 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,11 +96,33 @@ ignore = [ "src/dstack/_internal/server/migrations/versions", ] +[tool.pytest.ini_options] +testpaths = ["src/tests"] +addopts = [ + "--disable-socket", + "--allow-hosts=127.0.0.1,localhost", + # unix socket for Docker/testcontainers + "--allow-unix-socket", +] +markers = [ + "shim_version", + "dockerized", +] +env = [ + "DSTACK_CLI_RICH_FORCE_TERMINAL=0", +] +filterwarnings = [ + # testcontainers modules use deprecated decorators – nothing we can do: + # https://github.com/testcontainers/testcontainers-python/issues/874 + "ignore:^The @wait_container_is_ready decorator:DeprecationWarning" +] + [dependency-groups] dev = [ "pre-commit>=4.2.0", - "pytest~=7.2", + "pytest~=8.0", "pytest-asyncio>=0.23.8", + "pytest-mock>=3.14.0", "pytest-httpbin>=2.1.0", "pytest-socket>=0.7.0", "pytest-env>=1.1.0", diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 30c0e62811..0000000000 --- a/pytest.ini +++ /dev/null @@ -1,12 +0,0 @@ -[pytest] -testpaths = src/tests -addopts = - --disable-socket - --allow-hosts=127.0.0.1,localhost - ; unix socket for Docker/testcontainers - --allow-unix-socket -markers = - shim_version - dockerized -env = - DSTACK_CLI_RICH_FORCE_TERMINAL=0 diff --git a/src/tests/_internal/server/conftest.py b/src/tests/_internal/server/conftest.py index 623723f07c..9bb508c5d6 100644 --- a/src/tests/_internal/server/conftest.py +++ b/src/tests/_internal/server/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture -def client(event_loop): +def client(): transport = httpx.ASGITransport(app=app) return httpx.AsyncClient(transport=transport, base_url="http://test") diff --git a/src/tests/_internal/server/services/test_logs.py b/src/tests/_internal/server/services/test_logs.py index 892ed2d77b..06bfca7dea 100644 --- a/src/tests/_internal/server/services/test_logs.py +++ b/src/tests/_internal/server/services/test_logs.py @@ -798,8 +798,8 @@ async def test_poll_logs_descending_malformed_lines( class TestCloudWatchLogStorage: FAKE_NOW = datetime(2023, 10, 6, 10, 1, 54, tzinfo=timezone.utc) - @freeze_time(FAKE_NOW) @pytest_asyncio.fixture + @freeze_time(FAKE_NOW) async def project(self, test_db, session: AsyncSession) -> ProjectModel: project = await create_project(session=session, name="test-proj") return project diff --git a/src/tests/plugins/test_rest_plugin.py b/src/tests/plugins/test_rest_plugin.py index 0b725ff95b..7d9e35a51d 100644 --- a/src/tests/plugins/test_rest_plugin.py +++ b/src/tests/plugins/test_rest_plugin.py @@ -1,7 +1,6 @@ import json import os from contextlib import nullcontext as does_not_raise -from unittest import mock from unittest.mock import Mock import pytest @@ -101,14 +100,14 @@ async def test_on_run_apply_plugin_service_uri_not_set(self): CustomApplyPolicy() @pytest.mark.asyncio - @mock.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( "spec", ["run_spec", "fleet_spec", "volume_spec", "gateway_spec"], indirect=True ) async def test_on_apply_plugin_service_returns_mutated_spec( - self, test_db, user, project, spec + self, mocker, test_db, user, project, spec ): + mocker.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) policy = CustomApplyPolicy() mock_response = Mock() response_dict = {"spec": spec.dict(), "error": None} @@ -120,55 +119,54 @@ async def test_on_apply_plugin_service_returns_mutated_spec( mock_response.text = json.dumps(response_dict) mock_response.raise_for_status = Mock() - with mock.patch("requests.post", return_value=mock_response): - result = policy.on_apply(user=user.name, project=project.name, spec=spec) - assert result == type(spec)(**response_dict["spec"]) + mocker.patch("requests.post", return_value=mock_response) + result = policy.on_apply(user=user.name, project=project.name, spec=spec) + assert result == type(spec)(**response_dict["spec"]) @pytest.mark.asyncio - @mock.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( "spec", ["run_spec", "fleet_spec", "volume_spec", "gateway_spec"], indirect=True ) - async def test_on_apply_plugin_service_call_fails(self, test_db, user, project, spec): + async def test_on_apply_plugin_service_call_fails(self, mocker, test_db, user, project, spec): + mocker.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) policy = CustomApplyPolicy() - with mock.patch("requests.post", side_effect=requests.RequestException("fail")): - with pytest.raises(ServerClientError): - policy.on_apply(user=user.name, project=project.name, spec=spec) + mocker.patch("requests.post", side_effect=requests.RequestException("fail")) + with pytest.raises(ServerClientError): + policy.on_apply(user=user.name, project=project.name, spec=spec) @pytest.mark.asyncio - @mock.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( "spec", ["run_spec", "fleet_spec", "volume_spec", "gateway_spec"], indirect=True ) - async def test_on_apply_plugin_service_connection_fails(self, test_db, user, project, spec): + async def test_on_apply_plugin_service_connection_fails( + self, mocker, test_db, user, project, spec + ): + mocker.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) policy = CustomApplyPolicy() - with mock.patch( - "requests.post", side_effect=requests.ConnectionError("Failed to connect") - ): - with pytest.raises(ServerClientError): - policy.on_apply(user=user.name, project=project.name, spec=spec) + mocker.patch("requests.post", side_effect=requests.ConnectionError("Failed to connect")) + with pytest.raises(ServerClientError): + policy.on_apply(user=user.name, project=project.name, spec=spec) @pytest.mark.asyncio - @mock.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( "spec", ["run_spec", "fleet_spec", "volume_spec", "gateway_spec"], indirect=True ) async def test_on_apply_plugin_service_returns_invalid_spec( - self, test_db, user, project, spec + self, mocker, test_db, user, project, spec ): + mocker.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) policy = CustomApplyPolicy() mock_response = Mock() mock_response.text = json.dumps({"invalid-key": "abc"}) mock_response.raise_for_status = Mock() - with mock.patch("requests.post", return_value=mock_response): - with pytest.raises(ServerClientError): - policy.on_apply(user.name, project=project.name, spec=spec) + mocker.patch("requests.post", return_value=mock_response) + with pytest.raises(ServerClientError): + policy.on_apply(user.name, project=project.name, spec=spec) @pytest.mark.asyncio - @mock.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize( "spec", ["run_spec", "fleet_spec", "volume_spec", "gateway_spec"], indirect=True @@ -194,14 +192,15 @@ async def test_on_apply_plugin_service_returns_invalid_spec( ], ) async def test_on_apply_plugin_service_error_handling( - self, test_db, user, project, spec, error, expectation + self, mocker, test_db, user, project, spec, error, expectation ): + mocker.patch.dict(os.environ, {PLUGIN_SERVICE_URI_ENV_VAR_NAME: "http://mock"}) policy = CustomApplyPolicy() mock_response = Mock() response_dict = {"spec": spec.dict(), "error": error} mock_response.text = json.dumps(response_dict) mock_response.raise_for_status = Mock() - with mock.patch("requests.post", return_value=mock_response): - with expectation: - result = policy.on_apply(user=user.name, project=project.name, spec=spec) - assert result == type(spec)(**response_dict["spec"]) + mocker.patch("requests.post", return_value=mock_response) + with expectation: + result = policy.on_apply(user=user.name, project=project.name, spec=spec) + assert result == type(spec)(**response_dict["spec"]) From 802c450ad4a34f7352d8be29afa6847c5b84be60 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:52:18 +0100 Subject: [PATCH 018/640] [UX] Make `dstack project` and `dstack project set-default` interactive for default project selection (#3488) --- pyproject.toml | 2 +- src/dstack/_internal/cli/commands/project.py | 118 +++++++++++++++++-- 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e4e7bc070..37588341e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "rich", "rich-argparse", "tqdm", - "simple-term-menu", + "questionary>=2.0.1", "pydantic>=1.10.10,<2.0.0", "pydantic-duality>=1.2.4", "websocket-client", diff --git a/src/dstack/_internal/cli/commands/project.py b/src/dstack/_internal/cli/commands/project.py index 0f6e5b4db5..db4a7a5eb9 100644 --- a/src/dstack/_internal/cli/commands/project.py +++ b/src/dstack/_internal/cli/commands/project.py @@ -1,19 +1,77 @@ import argparse -from typing import Any, Union +import sys +from typing import Any, Optional, Union from requests import HTTPError from rich.table import Table +try: + import questionary + + is_project_menu_supported = sys.stdin.isatty() +except (ImportError, NotImplementedError, AttributeError): + is_project_menu_supported = False + import dstack.api.server from dstack._internal.cli.commands import BaseCommand from dstack._internal.cli.utils.common import add_row_from_dict, confirm_ask, console from dstack._internal.core.errors import ClientError, CLIError +from dstack._internal.core.models.config import ProjectConfig from dstack._internal.core.services.configs import ConfigManager from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) +def select_default_project( + project_configs: list[ProjectConfig], default_project: Optional[ProjectConfig] +) -> Optional[ProjectConfig]: + """Show an interactive menu to select a default project. + + This method only prompts for selection and does not update the configuration. + Use `ConfigManager.configure_project()` and `ConfigManager.save()` to persist + the selected project as default. + + Args: + project_configs: Non-empty list of available project configurations. + default_project: Currently default project, if any. + + Returns: + Selected project configuration, or None if cancelled. + + Raises: + CLIError: If `is_project_menu_supported` is False or `project_configs` is empty. + """ + if not is_project_menu_supported: + raise CLIError("Interactive menu is not supported on this platform") + + if len(project_configs) == 0: + raise CLIError("No projects configured") + + menu_entries = [] + default_index = None + for i, project_config in enumerate(project_configs): + is_default = project_config.name == default_project.name if default_project else False + entry = f"{project_config.name} ({project_config.url})" + if is_default: + default_index = i + menu_entries.append((entry, i)) + + choices = [questionary.Choice(title=entry, value=index) for entry, index in menu_entries] # pyright: ignore[reportPossiblyUnboundVariable] + default_value = default_index + selected_index = questionary.select( # pyright: ignore[reportPossiblyUnboundVariable] + message="Select the default project:", + choices=choices, + default=default_value, # pyright: ignore[reportArgumentType] + qmark="", + instruction="(↑↓ Enter)", + ).ask() + + if selected_index is not None and isinstance(selected_index, int): + return project_configs[selected_index] + return None + + class ProjectCommand(BaseCommand): NAME = "project" DESCRIPTION = "Manage projects configs" @@ -67,14 +125,17 @@ def _register(self): # Set default subcommand set_default_parser = subparsers.add_parser("set-default", help="Set default project") set_default_parser.add_argument( - "name", type=str, help="The name of the project to set as default" + "name", + type=str, + nargs="?" if is_project_menu_supported else None, + help="The name of the project to set as default", ) set_default_parser.set_defaults(subfunc=self._set_default) def _command(self, args: argparse.Namespace): super()._command(args) if not hasattr(args, "subfunc"): - args.subfunc = self._list + args.subfunc = self._project args.subfunc(args) def _add(self, args: argparse.Namespace): @@ -156,14 +217,47 @@ def _list(self, args: argparse.Namespace): console.print(table) + def _project(self, args: argparse.Namespace): + if is_project_menu_supported and not getattr(args, "verbose", False): + config_manager = ConfigManager() + project_configs = config_manager.list_project_configs() + default_project = config_manager.get_project_config() + selected_project = select_default_project(project_configs, default_project) + if selected_project is not None: + config_manager.configure_project( + name=selected_project.name, + url=selected_project.url, + token=selected_project.token, + default=True, + ) + config_manager.save() + console.print("[grey58]OK[/]") + else: + self._list(args) + def _set_default(self, args: argparse.Namespace): - config_manager = ConfigManager() - project_config = config_manager.get_project_config(args.name) - if project_config is None: - raise CLIError(f"Project '{args.name}' not found") + if args.name: + config_manager = ConfigManager() + project_config = config_manager.get_project_config(args.name) + if project_config is None: + raise CLIError(f"Project '{args.name}' not found") - config_manager.configure_project( - name=args.name, url=project_config.url, token=project_config.token, default=True - ) - config_manager.save() - console.print("[grey58]OK[/]") + config_manager.configure_project( + name=args.name, url=project_config.url, token=project_config.token, default=True + ) + config_manager.save() + console.print("[grey58]OK[/]") + else: + config_manager = ConfigManager() + project_configs = config_manager.list_project_configs() + default_project = config_manager.get_project_config() + selected_project = select_default_project(project_configs, default_project) + if selected_project is not None: + config_manager.configure_project( + name=selected_project.name, + url=selected_project.url, + token=selected_project.token, + default=True, + ) + config_manager.save() + console.print("[grey58]OK[/]") From bd2d485f73bee893560f832e3459b39bd7e6d3a2 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Fri, 23 Jan 2026 22:36:02 +0545 Subject: [PATCH 019/640] Add replica groups in dstack-service (#3408) * Add replica groups in dstack-service add_replica_groups_model Replica Groups AutoScaling Rolling deployment and UI Replica Groups implementation clean up * Resolve Merge Conflict & Rename replica_groups to replicas * Resolve pyright type check * Rename replicas to count and make replica names optional * Resolve review comments on probes and rate limits * Resolve tests * Transform to ReplicaGroup in the replica_groups property * Resolve review comments * Resolve test_runs * Resolved major comments * Remove create_group_run_spec and use Job Configurator instead * Resolve Minor Issues * Resolve Minor Issues - Additional fixes * Resolve conflict with master branch * Resolve Major Comments * Resolve some minor comments * Resolve some minor comments --------- Co-authored-by: Bihan Rana --- src/dstack/_internal/cli/utils/run.py | 66 ++++- .../_internal/core/models/configurations.py | 236 ++++++++++++++++-- src/dstack/_internal/core/models/runs.py | 2 + .../server/background/tasks/process_runs.py | 207 +++++++++++++-- ...a7d_add_runmodel_desired_replica_counts.py | 26 ++ src/dstack/_internal/server/models.py | 2 +- .../server/services/jobs/__init__.py | 23 +- .../services/jobs/configurators/base.py | 12 +- .../server/services/jobs/configurators/dev.py | 6 +- .../services/jobs/configurators/service.py | 3 + .../server/services/runs/__init__.py | 85 +++++-- .../_internal/server/services/runs/plan.py | 148 +++++++---- .../server/services/runs/replicas.py | 225 ++++++++++++++--- .../_internal/server/services/runs/spec.py | 13 +- .../server/services/services/__init__.py | 28 ++- .../server/services/services/autoscalers.py | 29 +-- .../_internal/server/routers/test_runs.py | 2 + 17 files changed, 929 insertions(+), 184 deletions(-) create mode 100644 src/dstack/_internal/server/migrations/versions/706e0acc3a7d_add_runmodel_desired_replica_counts.py diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index dec354e984..588f74bc69 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -285,16 +285,38 @@ def _format_job_name( show_deployment_num: bool, show_replica: bool, show_job: bool, + group_index: Optional[int] = None, + last_shown_group_index: Optional[int] = None, ) -> str: name_parts = [] + prefix = "" if show_replica: - name_parts.append(f"replica={job.job_spec.replica_num}") + # Show group information if replica groups are used + if group_index is not None: + # Show group=X replica=Y when group changes, or just replica=Y when same group + if group_index != last_shown_group_index: + # First job in group: use 3 spaces indent + prefix = " " + name_parts.append(f"group={group_index} replica={job.job_spec.replica_num}") + else: + # Subsequent job in same group: align "replica=" with first job's "replica=" + # Calculate padding: width of " group={last_shown_group_index} " + padding_width = 3 + len(f"group={last_shown_group_index}") + 1 + prefix = " " * padding_width + name_parts.append(f"replica={job.job_spec.replica_num}") + else: + # Legacy behavior: no replica groups + prefix = " " + name_parts.append(f"replica={job.job_spec.replica_num}") + else: + prefix = " " + if show_job: name_parts.append(f"job={job.job_spec.job_num}") name_suffix = ( f" deployment={latest_job_submission.deployment_num}" if show_deployment_num else "" ) - name_value = " " + (" ".join(name_parts) if name_parts else "") + name_value = prefix + (" ".join(name_parts) if name_parts else "") name_value += name_suffix return name_value @@ -363,6 +385,17 @@ def get_runs_table( ) merge_job_rows = len(run.jobs) == 1 and not show_deployment_num + group_name_to_index: Dict[str, int] = {} + if run.run_spec.configuration.type == "service" and hasattr( + run.run_spec.configuration, "replica_groups" + ): + replica_groups = run.run_spec.configuration.replica_groups + if replica_groups: + for idx, group in enumerate(replica_groups): + assert group.name is not None, "Group name is always set" + group_name = group.name + group_name_to_index[group_name] = idx + run_row: Dict[Union[str, int], Any] = { "NAME": _format_run_name(run, show_deployment_num), "SUBMITTED": format_date(run.submitted_at), @@ -376,13 +409,35 @@ def get_runs_table( if not merge_job_rows: add_row_from_dict(table, run_row) - for job in run.jobs: + # Sort jobs by group index first, then by replica_num within each group + def get_job_sort_key(job: Job) -> tuple: + group_index = None + if group_name_to_index: + group_index = group_name_to_index.get(job.job_spec.replica_group) + # Use a large number for jobs without groups to put them at the end + return (group_index if group_index is not None else 999999, job.job_spec.replica_num) + + sorted_jobs = sorted(run.jobs, key=get_job_sort_key) + + last_shown_group_index: Optional[int] = None + for job in sorted_jobs: latest_job_submission = job.job_submissions[-1] status_formatted = _format_job_submission_status(latest_job_submission, verbose) + # Get group index for this job + group_index: Optional[int] = None + if group_name_to_index: + group_index = group_name_to_index.get(job.job_spec.replica_group) + job_row: Dict[Union[str, int], Any] = { "NAME": _format_job_name( - job, latest_job_submission, show_deployment_num, show_replica, show_job + job, + latest_job_submission, + show_deployment_num, + show_replica, + show_job, + group_index=group_index, + last_shown_group_index=last_shown_group_index, ), "STATUS": status_formatted, "PROBES": _format_job_probes( @@ -394,6 +449,9 @@ def get_runs_table( "GPU": "-", "PRICE": "-", } + # Update last shown group index for next iteration + if group_index is not None: + last_shown_group_index = group_index jpd = latest_job_submission.job_provisioning_data if jpd is not None: shared_offer: Optional[InstanceOfferWithAvailability] = None diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 4558aebb11..6bd9f0827e 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -31,6 +31,7 @@ from dstack._internal.core.models.services import AnyModel, OpenAIChatModel from dstack._internal.core.models.unix import UnixUser from dstack._internal.core.models.volumes import MountPoint, VolumeConfiguration, parse_mount_point +from dstack._internal.core.services import is_valid_dstack_resource_name from dstack._internal.utils.common import has_duplicates, list_enum_values_for_annotation from dstack._internal.utils.json_schema import add_extra_schema_types from dstack._internal.utils.json_utils import ( @@ -54,6 +55,7 @@ DEFAULT_PROBE_READY_AFTER = 1 DEFAULT_PROBE_METHOD = "get" MAX_PROBE_URL_LEN = 2048 +DEFAULT_REPLICA_GROUP_NAME = "default" class RunConfigurationType(str, Enum): @@ -612,6 +614,11 @@ class ConfigurationWithCommandsParams(CoreModel): @root_validator def check_image_or_commands_present(cls, values): + # If replicas is list, skip validation - commands come from replica groups + replicas = values.get("replicas") + if isinstance(replicas, list): + return values + if not values.get("commands") and not values.get("image"): raise ValueError("Either `commands` or `image` must be set") return values @@ -734,6 +741,68 @@ def schema_extra(schema: Dict[str, Any]): ) +def _validate_replica_range(v: Range[int]) -> Range[int]: + """Validate a Range[int] used for replica counts.""" + if v.max is None: + raise ValueError("The maximum number of replicas is required") + if v.min is None: + v.min = 0 + if v.min < 0: + raise ValueError("The minimum number of replicas must be greater than or equal to 0") + return v + + +class ReplicaGroup(CoreModel): + name: Annotated[ + Optional[str], + Field( + description="The name of the replica group. If not provided, defaults to 'replica-group-0', 'replica-group-1', etc. based on position." + ), + ] + count: Annotated[ + Range[int], + Field( + description="The number of replicas. Can be a number (e.g. `2`) or a range (`0..4` or `1..8`). " + "If it's a range, the `scaling` property is required" + ), + ] + scaling: Annotated[ + Optional[ScalingSpec], + Field(description="The auto-scaling rules. Required if `count` is set to a range"), + ] = None + + resources: Annotated[ + ResourcesSpec, + Field(description="The resources requirements for replicas in this group"), + ] = ResourcesSpec() + + commands: Annotated[ + CommandsList, + Field(description="The shell commands to run for replicas in this group"), + ] = [] + + @validator("name") + def validate_name(cls, v: Optional[str]) -> Optional[str]: + if v is not None: + if not is_valid_dstack_resource_name(v): + raise ValueError("Resource name should match regex '^[a-z][a-z0-9-]{1,40}$'") + return v + + @validator("count") + def convert_count(cls, v: Range[int]) -> Range[int]: + return _validate_replica_range(v) + + @root_validator() + def validate_scaling(cls, values): + scaling = values.get("scaling") + count = values.get("count") + if count and count.min != count.max and not scaling: + raise ValueError("When you set `count` to a range, ensure to specify `scaling`.") + if count and count.min == count.max and scaling: + raise ValueError("To use `scaling`, `count` must be set to a range.") + return values + + class ServiceConfigurationParams(CoreModel): port: Annotated[ # NOTE: it's a PortMapping for historical reasons. Only `port.container_port` is used. @@ -775,13 +844,7 @@ class ServiceConfigurationParams(CoreModel): SERVICE_HTTPS_DEFAULT ) auth: Annotated[bool, Field(description="Enable the authorization")] = True - replicas: Annotated[ - Range[int], - Field( - description="The number of replicas. Can be a number (e.g. `2`) or a range (`0..4` or `1..8`). " - "If it's a range, the `scaling` property is required" - ), - ] = Range[int](min=1, max=1) + scaling: Annotated[ Optional[ScalingSpec], Field(description="The auto-scaling rules. Required if `replicas` is set to a range"), @@ -792,6 +855,20 @@ class ServiceConfigurationParams(CoreModel): Field(description="List of probes used to determine job health"), ] = [] + replicas: Annotated[ + Optional[Union[List[ReplicaGroup], Range[int]]], + Field( + description=( + "The number of replicas or a list of replica groups. " + "Can be an integer (e.g., `2`), a range (e.g., `0..4`), or a list of replica groups. " + "Each replica group defines replicas with shared configuration " + "(commands, resources, scaling). " + "When `replicas` is a list of replica groups, top-level `scaling`, `commands`, " + "and `resources` are not allowed and must be specified in each replica group instead. " + ) + ), + ] = None + @validator("port") def convert_port(cls, v) -> PortMapping: if isinstance(v, int): @@ -806,26 +883,6 @@ def convert_model(cls, v: Optional[Union[AnyModel, str]]) -> Optional[AnyModel]: return OpenAIChatModel(type="chat", name=v, format="openai") return v - @validator("replicas") - def convert_replicas(cls, v: Range[int]) -> Range[int]: - if v.max is None: - raise ValueError("The maximum number of replicas is required") - if v.min is None: - v.min = 0 - if v.min < 0: - raise ValueError("The minimum number of replicas must be greater than or equal to 0") - return v - - @root_validator() - def validate_scaling(cls, values): - scaling = values.get("scaling") - replicas = values.get("replicas") - if replicas and replicas.min != replicas.max and not scaling: - raise ValueError("When you set `replicas` to a range, ensure to specify `scaling`.") - if replicas and replicas.min == replicas.max and scaling: - raise ValueError("To use `scaling`, `replicas` must be set to a range.") - return values - @validator("rate_limits") def validate_rate_limits(cls, v: list[RateLimit]) -> list[RateLimit]: counts = Counter(limit.prefix for limit in v) @@ -847,6 +904,103 @@ def validate_probes(cls, v: list[ProbeConfig]) -> list[ProbeConfig]: raise ValueError("Probes must be unique") return v + @validator("replicas") + def validate_replicas( + cls, v: Optional[Union[Range[int], List[ReplicaGroup]]] + ) -> Optional[Union[Range[int], List[ReplicaGroup]]]: + if v is None: + return v + if isinstance(v, Range): + return _validate_replica_range(v) + + if isinstance(v, list): + if not v: + raise ValueError("`replicas` cannot be an empty list") + + # Assign default names to groups without names + for index, group in enumerate(v): + if group.name is None: + group.name = f"replica-group-{index}" + + # Check for duplicate names + names = [group.name for group in v] + if len(names) != len(set(names)): + duplicates = [name for name in set(names) if names.count(name) > 1] + raise ValueError( + f"Duplicate replica group names found: {duplicates}. " + "Each replica group must have a unique name." + ) + return v + + @root_validator() + def validate_scaling(cls, values): + scaling = values.get("scaling") + replicas = values.get("replicas") + + if isinstance(replicas, Range): + if replicas and replicas.min != replicas.max and not scaling: + raise ValueError( + "When you set `replicas` to a range, ensure to specify `scaling`." + ) + if replicas and replicas.min == replicas.max and scaling: + raise ValueError("To use `scaling`, `replicas` must be set to a range.") + return values + + @root_validator() + def validate_top_level_properties_with_replica_groups(cls, values): + """ + When replicas is a list of ReplicaGroup, forbid top-level scaling, commands, and resources + """ + replicas = values.get("replicas") + + if not isinstance(replicas, list): + return values + + scaling = values.get("scaling") + if scaling is not None: + raise ValueError( + "Top-level `scaling` is not allowed when `replicas` is a list. " + "Specify `scaling` in each replica group instead." + ) + + commands = values.get("commands", []) + if commands: + raise ValueError( + "Top-level `commands` is not allowed when `replicas` is a list. " + "Specify `commands` in each replica group instead." + ) + + resources = values.get("resources") + + default_resources = ResourcesSpec() + if resources and resources.dict() != default_resources.dict(): + raise ValueError( + "Top-level `resources` is not allowed when `replicas` is a list. " + "Specify `resources` in each replica group instead." + ) + + return values + + @root_validator() + def validate_replica_groups_have_commands_or_image(cls, values): + """ + When replicas is a list, ensure each ReplicaGroup has commands OR service has image. + """ + replicas = values.get("replicas") + image = values.get("image") + + if not isinstance(replicas, list): + return values + + for group in replicas: + if not group.commands and not image: + raise ValueError( + f"Replica group '{group.name}' has no commands. " + "Either set `commands` in the replica group or set `image` at the service level." + ) + + return values + class ServiceConfigurationConfig( ProfileParamsConfig, @@ -869,6 +1023,34 @@ class ServiceConfiguration( ): type: Literal["service"] = "service" + @property + def replica_groups(self) -> List[ReplicaGroup]: + if self.replicas is None: + return [ + ReplicaGroup( + name=DEFAULT_REPLICA_GROUP_NAME, + count=Range[int](min=1, max=1), + commands=self.commands, + resources=self.resources, + scaling=self.scaling, + ) + ] + if isinstance(self.replicas, list): + return self.replicas + if isinstance(self.replicas, Range): + return [ + ReplicaGroup( + name=DEFAULT_REPLICA_GROUP_NAME, + count=self.replicas, + commands=self.commands, + resources=self.resources, + scaling=self.scaling, + ) + ] + raise ValueError( + f"Invalid replicas type: {type(self.replicas)}. Expected None, Range[int], or List[ReplicaGroup]" + ) + AnyRunConfiguration = Union[DevEnvironmentConfiguration, TaskConfiguration, ServiceConfiguration] diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index a966bc34a0..27c6f430c1 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -17,6 +17,7 @@ ) from dstack._internal.core.models.configurations import ( DEFAULT_PROBE_METHOD, + DEFAULT_REPLICA_GROUP_NAME, LEGACY_REPO_DIR, AnyRunConfiguration, HTTPHeaderSpec, @@ -253,6 +254,7 @@ class JobSpec(CoreModel): job_num: int job_name: str jobs_per_replica: int = 1 # default value for backward compatibility + replica_group: str = DEFAULT_REPLICA_GROUP_NAME app_specs: Optional[List[AppSpec]] user: Optional[UnixUser] = None # default value for backward compatibility commands: List[str] diff --git a/src/dstack/_internal/server/background/tasks/process_runs.py b/src/dstack/_internal/server/background/tasks/process_runs.py index ad42e7ed40..ee907e519b 100644 --- a/src/dstack/_internal/server/background/tasks/process_runs.py +++ b/src/dstack/_internal/server/background/tasks/process_runs.py @@ -1,5 +1,6 @@ import asyncio import datetime +import json from typing import List, Optional, Set, Tuple from sqlalchemy import and_, func, or_, select @@ -8,6 +9,7 @@ import dstack._internal.server.services.services.autoscalers as autoscalers from dstack._internal.core.errors import ServerError +from dstack._internal.core.models.configurations import ReplicaGroup from dstack._internal.core.models.profiles import RetryEvent, StopCriteria from dstack._internal.core.models.runs import ( Job, @@ -45,9 +47,14 @@ switch_run_status, ) from dstack._internal.server.services.runs.replicas import ( + build_replica_lists, + has_out_of_date_replicas, is_replica_registered, + job_belongs_to_group, retry_run_replica_jobs, + scale_down_replicas, scale_run_replicas, + scale_run_replicas_per_group, ) from dstack._internal.server.services.secrets import get_project_secrets_mapping from dstack._internal.server.services.services import update_service_desired_replica_count @@ -234,9 +241,10 @@ async def _process_pending_run(session: AsyncSession, run_model: RunModel): logger.debug("%s: retrying run is not yet ready for resubmission", fmt(run_model)) return - run_model.desired_replica_count = 1 if run.run_spec.configuration.type == "service": - run_model.desired_replica_count = run.run_spec.configuration.replicas.min or 0 + run_model.desired_replica_count = sum( + group.count.min or 0 for group in run.run_spec.configuration.replica_groups + ) await update_service_desired_replica_count( session, run_model, @@ -245,12 +253,18 @@ async def _process_pending_run(session: AsyncSession, run_model: RunModel): last_scaled_at=None, ) - if run_model.desired_replica_count == 0: - # stay zero scaled - return + if run_model.desired_replica_count == 0: + # stay zero scaled + return + + replicas: List[ReplicaGroup] = run.run_spec.configuration.replica_groups + + await scale_run_replicas_per_group(session, run_model, replicas) + else: + run_model.desired_replica_count = 1 + await scale_run_replicas(session, run_model, replicas_diff=run_model.desired_replica_count) - await scale_run_replicas(session, run_model, replicas_diff=run_model.desired_replica_count) - switch_run_status(session, run_model, RunStatus.SUBMITTED) + switch_run_status(session=session, run_model=run_model, new_status=RunStatus.SUBMITTED) def _retrying_run_ready_for_resubmission(run_model: RunModel, run: Run) -> bool: @@ -486,9 +500,60 @@ async def _handle_run_replicas( # FIXME: should only include scaling events, not retries and deployments last_scaled_at=max((r.timestamp for r in replicas_info), default=None), ) + replicas: List[ReplicaGroup] = run_spec.configuration.replica_groups + assert replicas, "replica groups should always return at least one group" + + await scale_run_replicas_per_group(session, run_model, replicas) + + # Handle per-group rolling deployment + await _update_jobs_to_new_deployment_in_place( + session=session, + run_model=run_model, + run_spec=run_spec, + replicas=replicas, + ) + # Process per-group rolling deployment + for group in replicas: + await _handle_rolling_deployment_for_group( + session=session, run_model=run_model, group=group, run_spec=run_spec + ) + # Terminate replicas from groups that were removed from the configuration + existing_group_names = set() + for job in run_model.jobs: + if job.status.is_finished(): + continue + try: + job_spec = JobSpec.__response__.parse_raw(job.job_spec_data) + existing_group_names.add(job_spec.replica_group) + except Exception: + continue + new_group_names = {group.name for group in replicas} + removed_group_names = existing_group_names - new_group_names + for removed_group_name in removed_group_names: + # Build replica lists for this removed group + active_replicas, inactive_replicas = build_replica_lists( + run_model=run_model, + group_filter=removed_group_name, + ) + + total_replicas = len(active_replicas) + len(inactive_replicas) + if total_replicas > 0: + logger.info( + "%s: terminating %d replica(s) from removed group '%s'", + fmt(run_model), + total_replicas, + removed_group_name, + ) + # Terminate all active replicas in the removed group + if active_replicas: + scale_down_replicas(session, active_replicas, len(active_replicas)) + # Terminate all inactive replicas in the removed group + if inactive_replicas: + scale_down_replicas(session, inactive_replicas, len(inactive_replicas)) + return max_replica_count = run_model.desired_replica_count - if _has_out_of_date_replicas(run_model): + if has_out_of_date_replicas(run_model): # allow extra replicas when deployment is in progress max_replica_count += ROLLING_DEPLOYMENT_MAX_SURGE @@ -506,7 +571,7 @@ async def _handle_run_replicas( run_model=run_model, run_spec=run_spec, ) - if _has_out_of_date_replicas(run_model): + if has_out_of_date_replicas(run_model): assert run_spec.configuration.type == "service", ( "Rolling deployment is only supported for services" ) @@ -551,7 +616,10 @@ async def _handle_run_replicas( async def _update_jobs_to_new_deployment_in_place( - session: AsyncSession, run_model: RunModel, run_spec: RunSpec + session: AsyncSession, + run_model: RunModel, + run_spec: RunSpec, + replicas: Optional[List] = None, ) -> None: """ Bump deployment_num for jobs that do not require redeployment. @@ -560,16 +628,26 @@ async def _update_jobs_to_new_deployment_in_place( session=session, project=run_model.project, ) + for replica_num, job_models in group_jobs_by_replica_latest(run_model.jobs): if all(j.status.is_finished() for j in job_models): continue if all(j.deployment_num == run_model.deployment_num for j in job_models): continue + + # Determine which group this replica belongs to + replica_group_name = None + + if replicas: + job_spec = JobSpec.__response__.parse_raw(job_models[0].job_spec_data) + replica_group_name = job_spec.replica_group + # FIXME: Handle getting image configuration errors or skip it. new_job_specs = await get_job_specs_from_run_spec( run_spec=run_spec, secrets=secrets, replica_num=replica_num, + replica_group_name=replica_group_name, ) assert len(new_job_specs) == len(job_models), ( "Changing the number of jobs within a replica is not yet supported" @@ -585,15 +663,6 @@ async def _update_jobs_to_new_deployment_in_place( job_model.deployment_num = run_model.deployment_num -def _has_out_of_date_replicas(run: RunModel) -> bool: - for job in run.jobs: - if job.deployment_num < run.deployment_num and not ( - job.status.is_finished() or job.termination_reason == JobTerminationReason.SCALED_DOWN - ): - return True - return False - - async def _should_retry_job( session: AsyncSession, run: Run, @@ -671,3 +740,103 @@ def _should_stop_on_master_done(run: Run) -> bool: if is_master_job(job) and job.job_submissions[-1].status == JobStatus.DONE: return True return False + + +async def _handle_rolling_deployment_for_group( + session: AsyncSession, run_model: RunModel, group: ReplicaGroup, run_spec: RunSpec +) -> None: + """ + Handle rolling deployment for a single replica group. + """ + from dstack._internal.server.services.runs.replicas import ( + build_replica_lists, + scale_run_replicas_for_group, + ) + + desired_replica_counts = ( + json.loads(run_model.desired_replica_counts) if run_model.desired_replica_counts else {} + ) + + group_desired = desired_replica_counts.get(group.name, group.count.min or 0) + + # Check if group has out-of-date replicas + if not has_out_of_date_replicas(run_model, group_filter=group.name): + return # Group is up-to-date + + # Calculate max replicas (allow surge during deployment) + group_max_replica_count = group_desired + ROLLING_DEPLOYMENT_MAX_SURGE + + # Count non-terminated replicas for this group only + + non_terminated_replica_count = len( + { + j.replica_num + for j in run_model.jobs + if not j.status.is_finished() + and group.name is not None + and job_belongs_to_group(job=j, group_name=group.name) + } + ) + + # Start new up-to-date replicas if needed + if non_terminated_replica_count < group_max_replica_count: + active_replicas, inactive_replicas = build_replica_lists( + run_model=run_model, + group_filter=group.name, + ) + + await scale_run_replicas_for_group( + session=session, + run_model=run_model, + group=group, + replicas_diff=group_max_replica_count - non_terminated_replica_count, + run_spec=run_spec, + active_replicas=active_replicas, + inactive_replicas=inactive_replicas, + ) + + # Stop out-of-date replicas that are not registered + replicas_to_stop_count = 0 + for _, jobs in group_jobs_by_replica_latest(run_model.jobs): + assert group.name is not None, "Group name is always set" + if not job_belongs_to_group(jobs[0], group.name): + continue + # Check if replica is out-of-date and not registered + if ( + any(j.deployment_num < run_model.deployment_num for j in jobs) + and any( + j.status not in [JobStatus.TERMINATING] + JobStatus.finished_statuses() + for j in jobs + ) + and not is_replica_registered(jobs) + ): + replicas_to_stop_count += 1 + + # Stop excessive registered out-of-date replicas + non_terminating_registered_replicas_count = 0 + for _, jobs in group_jobs_by_replica_latest(run_model.jobs): + assert group.name is not None, "Group name is always set" + if not job_belongs_to_group(jobs[0], group.name): + continue + + if is_replica_registered(jobs) and all(j.status != JobStatus.TERMINATING for j in jobs): + non_terminating_registered_replicas_count += 1 + + replicas_to_stop_count += max(0, non_terminating_registered_replicas_count - group_desired) + + if replicas_to_stop_count > 0: + # Build lists again to get current state + active_replicas, inactive_replicas = build_replica_lists( + run_model=run_model, + group_filter=group.name, + ) + + await scale_run_replicas_for_group( + session=session, + run_model=run_model, + group=group, + replicas_diff=-replicas_to_stop_count, + run_spec=run_spec, + active_replicas=active_replicas, + inactive_replicas=inactive_replicas, + ) diff --git a/src/dstack/_internal/server/migrations/versions/706e0acc3a7d_add_runmodel_desired_replica_counts.py b/src/dstack/_internal/server/migrations/versions/706e0acc3a7d_add_runmodel_desired_replica_counts.py new file mode 100644 index 0000000000..e993df7bec --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/706e0acc3a7d_add_runmodel_desired_replica_counts.py @@ -0,0 +1,26 @@ +"""add runmodel desired_replica_counts + +Revision ID: 706e0acc3a7d +Revises: 903c91e24634 +Create Date: 2025-12-18 10:54:13.508297 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "706e0acc3a7d" +down_revision = "903c91e24634" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("runs", schema=None) as batch_op: + batch_op.add_column(sa.Column("desired_replica_counts", sa.Text(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("runs", schema=None) as batch_op: + batch_op.drop_column("desired_replica_counts") diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 6a8aa41eb4..982ade215f 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -405,7 +405,7 @@ class RunModel(BaseModel): priority: Mapped[int] = mapped_column(Integer, default=0) deployment_num: Mapped[int] = mapped_column(Integer) desired_replica_count: Mapped[int] = mapped_column(Integer) - + desired_replica_counts: Mapped[Optional[str]] = mapped_column(Text, nullable=True) jobs: Mapped[List["JobModel"]] = relationship( back_populates="run", lazy="selectin", order_by="[JobModel.replica_num, JobModel.job_num]" ) diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index 18d410c133..2ddadbfb1e 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -99,7 +99,10 @@ def switch_job_status( async def get_jobs_from_run_spec( - run_spec: RunSpec, secrets: Dict[str, str], replica_num: int + run_spec: RunSpec, + secrets: Dict[str, str], + replica_num: int, + replica_group_name: Optional[str] = None, ) -> List[Job]: return [ Job(job_spec=s, job_submissions=[]) @@ -107,14 +110,20 @@ async def get_jobs_from_run_spec( run_spec=run_spec, secrets=secrets, replica_num=replica_num, + replica_group_name=replica_group_name, ) ] async def get_job_specs_from_run_spec( - run_spec: RunSpec, secrets: Dict[str, str], replica_num: int + run_spec: RunSpec, + secrets: Dict[str, str], + replica_num: int, + replica_group_name: Optional[str] = None, ) -> List[JobSpec]: - job_configurator = _get_job_configurator(run_spec=run_spec, secrets=secrets) + job_configurator = _get_job_configurator( + run_spec=run_spec, secrets=secrets, replica_group_name=replica_group_name + ) job_specs = await job_configurator.get_job_specs(replica_num=replica_num) return job_specs @@ -242,10 +251,14 @@ def is_master_job(job: Job) -> bool: return job.job_spec.job_num == 0 -def _get_job_configurator(run_spec: RunSpec, secrets: Dict[str, str]) -> JobConfigurator: +def _get_job_configurator( + run_spec: RunSpec, secrets: Dict[str, str], replica_group_name: Optional[str] = None +) -> JobConfigurator: configuration_type = RunConfigurationType(run_spec.configuration.type) configurator_class = _configuration_type_to_configurator_class_map[configuration_type] - return configurator_class(run_spec=run_spec, secrets=secrets) + return configurator_class( + run_spec=run_spec, secrets=secrets, replica_group_name=replica_group_name + ) _job_configurator_classes = [ diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index 72821f1dd3..df6738a774 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -16,6 +16,7 @@ DEFAULT_PROBE_READY_AFTER, DEFAULT_PROBE_TIMEOUT, DEFAULT_PROBE_URL, + DEFAULT_REPLICA_GROUP_NAME, LEGACY_REPO_DIR, PortMapping, ProbeConfig, @@ -90,9 +91,11 @@ def __init__( self, run_spec: RunSpec, secrets: Optional[Dict[str, str]] = None, + replica_group_name: Optional[str] = None, ): self.run_spec = run_spec self.secrets = secrets or {} + self.replica_group_name = replica_group_name async def get_job_specs(self, replica_num: int) -> List[JobSpec]: job_spec = await self._get_job_spec(replica_num=replica_num, job_num=0, jobs_per_replica=1) @@ -150,6 +153,7 @@ async def _get_job_spec( job_num=job_num, job_name=f"{self.run_spec.run_name}-{job_num}-{replica_num}", jobs_per_replica=jobs_per_replica, + replica_group=self.replica_group_name or DEFAULT_REPLICA_GROUP_NAME, app_specs=self._app_specs(), commands=await self._commands(), env=self._env(), @@ -308,9 +312,15 @@ def _registry_auth(self) -> Optional[RegistryAuth]: return self.run_spec.configuration.registry_auth def _requirements(self, jobs_per_replica: int) -> Requirements: + resources = self.run_spec.configuration.resources + if self.run_spec.configuration.type == "service": + for group in self.run_spec.configuration.replica_groups: + if group.name == self.replica_group_name: + resources = group.resources + break spot_policy = self._spot_policy() return Requirements( - resources=self.run_spec.configuration.resources, + resources=resources, max_price=self.run_spec.merged_profile.max_price, spot=None if spot_policy == SpotPolicy.AUTO else (spot_policy == SpotPolicy.SPOT), reservation=self.run_spec.merged_profile.reservation, diff --git a/src/dstack/_internal/server/services/jobs/configurators/dev.py b/src/dstack/_internal/server/services/jobs/configurators/dev.py index da683a60cc..42bdc4376e 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/dev.py +++ b/src/dstack/_internal/server/services/jobs/configurators/dev.py @@ -18,7 +18,9 @@ class DevEnvironmentJobConfigurator(JobConfigurator): TYPE: RunConfigurationType = RunConfigurationType.DEV_ENVIRONMENT - def __init__(self, run_spec: RunSpec, secrets: Dict[str, str]): + def __init__( + self, run_spec: RunSpec, secrets: Dict[str, str], replica_group_name: Optional[str] = None + ): assert run_spec.configuration.type == "dev-environment" if run_spec.configuration.ide == "vscode": @@ -34,7 +36,7 @@ def __init__(self, run_spec: RunSpec, secrets: Dict[str, str]): version=run_spec.configuration.version, extensions=["ms-python.python", "ms-toolsai.jupyter"], ) - super().__init__(run_spec=run_spec, secrets=secrets) + super().__init__(run_spec=run_spec, secrets=secrets, replica_group_name=replica_group_name) def _shell_commands(self) -> List[str]: assert self.run_spec.configuration.type == "dev-environment" diff --git a/src/dstack/_internal/server/services/jobs/configurators/service.py b/src/dstack/_internal/server/services/jobs/configurators/service.py index be15c4b23c..6b5aa8c2d3 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/service.py +++ b/src/dstack/_internal/server/services/jobs/configurators/service.py @@ -10,6 +10,9 @@ class ServiceJobConfigurator(JobConfigurator): def _shell_commands(self) -> List[str]: assert self.run_spec.configuration.type == "service" + for group in self.run_spec.configuration.replica_groups: + if group.name == self.replica_group_name: + return group.commands return self.run_spec.configuration.commands def _default_single_branch(self) -> bool: diff --git a/src/dstack/_internal/server/services/runs/__init__.py b/src/dstack/_internal/server/services/runs/__init__.py index 5773403cff..f99934dc3f 100644 --- a/src/dstack/_internal/server/services/runs/__init__.py +++ b/src/dstack/_internal/server/services/runs/__init__.py @@ -490,8 +490,6 @@ async def submit_run( if run_spec.merged_profile.schedule is not None: initial_status = RunStatus.PENDING initial_replicas = 0 - elif run_spec.configuration.type == "service": - initial_replicas = run_spec.configuration.replicas.min or 0 run_model = RunModel( id=uuid.uuid4(), @@ -519,32 +517,67 @@ async def submit_run( if run_spec.configuration.type == "service": await services.register_service(session, run_model, run_spec) + service_config = run_spec.configuration - for replica_num in range(initial_replicas): - jobs = await get_jobs_from_run_spec( - run_spec=run_spec, - secrets=secrets, - replica_num=replica_num, - ) - for job in jobs: - job_model = create_job_model_for_new_submission( - run_model=run_model, - job=job, - status=JobStatus.SUBMITTED, - ) - session.add(job_model) - events.emit( - session, - f"Job created on run submission. Status: {job_model.status.upper()}", - # Set `SystemActor` for consistency with all other places where jobs can be - # created (retry, scaling, rolling deployments, etc). Think of the run as being - # created by the user, while the job is created by the system to satisfy the - # run spec. - actor=events.SystemActor(), - targets=[ - events.Target.from_model(job_model), - ], + global_replica_num = 0 # Global counter across all groups for unique replica_num + + for replica_group in service_config.replica_groups: + if run_spec.merged_profile.schedule is not None: + group_initial_replicas = 0 + else: + group_initial_replicas = replica_group.count.min or 0 + + # Each replica in this group gets the same group-specific configuration + for group_replica_num in range(group_initial_replicas): + jobs = await get_jobs_from_run_spec( + run_spec=run_spec, + secrets=secrets, + replica_num=global_replica_num, + replica_group_name=replica_group.name, + ) + + for job in jobs: + job_model = create_job_model_for_new_submission( + run_model=run_model, + job=job, + status=JobStatus.SUBMITTED, + ) + session.add(job_model) + events.emit( + session, + f"Job created on run submission. Status: {job_model.status.upper()}", + actor=events.SystemActor(), + targets=[ + events.Target.from_model(job_model), + ], + ) + global_replica_num += 1 + else: + for replica_num in range(initial_replicas): + jobs = await get_jobs_from_run_spec( + run_spec=run_spec, + secrets=secrets, + replica_num=replica_num, ) + for job in jobs: + job_model = create_job_model_for_new_submission( + run_model=run_model, + job=job, + status=JobStatus.SUBMITTED, + ) + session.add(job_model) + events.emit( + session, + f"Job created on run submission. Status: {job_model.status.upper()}", + # Set `SystemActor` for consistency with all other places where jobs can be + # created (retry, scaling, rolling deployments, etc). Think of the run as being + # created by the user, while the job is created by the system to satisfy the + # run spec. + actor=events.SystemActor(), + targets=[ + events.Target.from_model(job_model), + ], + ) await session.commit() await session.refresh(run_model) diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index dd1ad1b284..5e3b6e5a02 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -78,57 +78,117 @@ async def get_job_plans( run_spec.run_name = "dry-run" secrets = await get_project_secrets_mapping(session=session, project=project) - jobs = await get_jobs_from_run_spec( - run_spec=run_spec, - secrets=secrets, - replica_num=0, - ) - volumes = await get_job_configured_volumes( - session=session, - project=project, - run_spec=run_spec, - job_num=0, - ) - candidate_fleet_models = await _select_candidate_fleet_models( - session=session, - project=project, - run_model=None, - run_spec=run_spec, - ) - fleet_model, instance_offers, backend_offers = await find_optimal_fleet_with_offers( - project=project, - fleet_models=candidate_fleet_models, - run_model=None, - run_spec=run_spec, - job=jobs[0], - master_job_provisioning_data=None, - volumes=volumes, - exclude_not_available=False, - ) - if _should_force_non_fleet_offers(run_spec) or ( - FeatureFlags.AUTOCREATED_FLEETS_ENABLED and profile.fleets is None and fleet_model is None - ): - # Keep the old behavior returning all offers irrespective of fleets. - # Needed for supporting offers with autocreated fleets flow (and for `dstack offer`). - instance_offers, backend_offers = await _get_non_fleet_offers( + + job_plans = [] + + if run_spec.configuration.type == "service": + volumes = await get_job_configured_volumes( session=session, project=project, - profile=profile, + run_spec=run_spec, + job_num=0, + ) + candidate_fleet_models = await _select_candidate_fleet_models( + session=session, + project=project, + run_model=None, + run_spec=run_spec, + ) + for replica_group in run_spec.configuration.replica_groups: + jobs = await get_jobs_from_run_spec( + run_spec=run_spec, + secrets=secrets, + replica_num=0, + replica_group_name=replica_group.name, + ) + fleet_model, instance_offers, backend_offers = await find_optimal_fleet_with_offers( + project=project, + fleet_models=candidate_fleet_models, + run_model=None, + run_spec=run_spec, + job=jobs[0], + master_job_provisioning_data=None, + volumes=volumes, + exclude_not_available=False, + ) + if _should_force_non_fleet_offers(run_spec) or ( + FeatureFlags.AUTOCREATED_FLEETS_ENABLED + and profile.fleets is None + and fleet_model is None + ): + # Keep the old behavior returning all offers irrespective of fleets. + # Needed for supporting offers with autocreated fleets flow (and for `dstack offer`). + instance_offers, backend_offers = await _get_non_fleet_offers( + session=session, + project=project, + profile=profile, + run_spec=run_spec, + job=jobs[0], + volumes=volumes, + ) + + for job in jobs: + job_plan = _get_job_plan( + instance_offers=instance_offers, + backend_offers=backend_offers, + profile=profile, + job=job, + max_offers=max_offers, + ) + job_plans.append(job_plan) + else: + jobs = await get_jobs_from_run_spec( + run_spec=run_spec, + secrets=secrets, + replica_num=0, + ) + volumes = await get_job_configured_volumes( + session=session, + project=project, + run_spec=run_spec, + job_num=0, + ) + candidate_fleet_models = await _select_candidate_fleet_models( + session=session, + project=project, + run_model=None, + run_spec=run_spec, + ) + fleet_model, instance_offers, backend_offers = await find_optimal_fleet_with_offers( + project=project, + fleet_models=candidate_fleet_models, + run_model=None, run_spec=run_spec, job=jobs[0], + master_job_provisioning_data=None, volumes=volumes, + exclude_not_available=False, ) + if _should_force_non_fleet_offers(run_spec) or ( + FeatureFlags.AUTOCREATED_FLEETS_ENABLED + and profile.fleets is None + and fleet_model is None + ): + # Keep the old behavior returning all offers irrespective of fleets. + # Needed for supporting offers with autocreated fleets flow (and for `dstack offer`). + instance_offers, backend_offers = await _get_non_fleet_offers( + session=session, + project=project, + profile=profile, + run_spec=run_spec, + job=jobs[0], + volumes=volumes, + ) - job_plans = [] - for job in jobs: - job_plan = _get_job_plan( - instance_offers=instance_offers, - backend_offers=backend_offers, - profile=profile, - job=job, - max_offers=max_offers, - ) - job_plans.append(job_plan) + for job in jobs: + job_plan = _get_job_plan( + instance_offers=instance_offers, + backend_offers=backend_offers, + profile=profile, + job=job, + max_offers=max_offers, + ) + job_plans.append(job_plan) run_spec.run_name = run_name return job_plans diff --git a/src/dstack/_internal/server/services/runs/replicas.py b/src/dstack/_internal/server/services/runs/replicas.py index e994e77ddc..4f6c7ee19d 100644 --- a/src/dstack/_internal/server/services/runs/replicas.py +++ b/src/dstack/_internal/server/services/runs/replicas.py @@ -1,8 +1,10 @@ -from typing import List +import json +from typing import List, Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession -from dstack._internal.core.models.runs import JobStatus, JobTerminationReason, RunSpec +from dstack._internal.core.models.configurations import ReplicaGroup +from dstack._internal.core.models.runs import JobSpec, JobStatus, JobTerminationReason, RunSpec from dstack._internal.server.models import JobModel, RunModel from dstack._internal.server.services import events from dstack._internal.server.services.jobs import ( @@ -11,7 +13,10 @@ switch_job_status, ) from dstack._internal.server.services.logging import fmt -from dstack._internal.server.services.runs import create_job_model_for_new_submission, logger +from dstack._internal.server.services.runs import ( + create_job_model_for_new_submission, + logger, +) from dstack._internal.server.services.secrets import get_project_secrets_mapping @@ -23,10 +28,17 @@ async def retry_run_replica_jobs( session=session, project=run_model.project, ) + + # Determine replica group from existing job + run_spec = RunSpec.__response__.parse_raw(run_model.run_spec) + job_spec = JobSpec.__response__.parse_raw(latest_jobs[0].job_spec_data) + replica_group_name = job_spec.replica_group + new_jobs = await get_jobs_from_run_spec( - run_spec=RunSpec.__response__.parse_raw(run_model.run_spec), + run_spec=run_spec, secrets=secrets, replica_num=latest_jobs[0].replica_num, + replica_group_name=replica_group_name, ) assert len(new_jobs) == len(latest_jobs), ( "Changing the number of jobs within a replica is not yet supported" @@ -64,7 +76,6 @@ def is_replica_registered(jobs: list[JobModel]) -> bool: async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replicas_diff: int): if replicas_diff == 0: - # nothing to do return logger.info( @@ -74,14 +85,42 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica abs(replicas_diff), ) + active_replicas, inactive_replicas = build_replica_lists(run_model) + run_spec = RunSpec.__response__.parse_raw(run_model.run_spec) + + if replicas_diff < 0: + scale_down_replicas(session, active_replicas, abs(replicas_diff)) + else: + await _scale_up_replicas( + session, + run_model, + inactive_replicas, + replicas_diff, + run_spec, + group_name=None, + ) + + +def build_replica_lists( + run_model: RunModel, + group_filter: Optional[str] = None, +) -> Tuple[ + List[Tuple[int, bool, int, List[JobModel]]], List[Tuple[int, bool, int, List[JobModel]]] +]: # lists of (importance, is_out_of_date, replica_num, jobs) active_replicas: list[tuple[int, bool, int, list[JobModel]]] = [] inactive_replicas: list[tuple[int, bool, int, list[JobModel]]] = [] for replica_num, replica_jobs in group_jobs_by_replica_latest(run_model.jobs): + # Filter by group if specified + if group_filter is not None: + if not job_belongs_to_group(replica_jobs[0], group_filter): + continue + statuses = set(job.status for job in replica_jobs) deployment_num = replica_jobs[0].deployment_num # same for all jobs is_out_of_date = deployment_num < run_model.deployment_num + if {JobStatus.TERMINATING, *JobStatus.finished_statuses()} & statuses: # if there are any terminating or finished jobs, the replica is inactive inactive_replicas.append((0, is_out_of_date, replica_num, replica_jobs)) @@ -98,42 +137,68 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica # all jobs are running and ready, the replica is active and has the importance of 3 active_replicas.append((3, is_out_of_date, replica_num, replica_jobs)) - # sort by is_out_of_date (up-to-date first), importance (desc), and replica_num (asc) + # Sort by is_out_of_date (up-to-date first), importance (desc), and replica_num (asc) active_replicas.sort(key=lambda r: (r[1], -r[0], r[2])) - run_spec = RunSpec.__response__.parse_raw(run_model.run_spec) - if replicas_diff < 0: - for _, _, _, replica_jobs in reversed(active_replicas[-abs(replicas_diff) :]): - # scale down the less important replicas first - for job in replica_jobs: - if job.status.is_finished() or job.status == JobStatus.TERMINATING: - continue - job.termination_reason = JobTerminationReason.SCALED_DOWN - switch_job_status(session, job, JobStatus.TERMINATING, events.SystemActor()) - # background task will process the job later - else: - scheduled_replicas = 0 + return active_replicas, inactive_replicas + + +def scale_down_replicas( + session: AsyncSession, + active_replicas: List[Tuple[int, bool, int, List[JobModel]]], + count: int, +) -> None: + """Scale down by terminating the least important replicas""" + if count <= 0: + return + + for _, _, _, replica_jobs in reversed(active_replicas[-count:]): + for job in replica_jobs: + if job.status.is_finished() or job.status == JobStatus.TERMINATING: + continue + job.termination_reason = JobTerminationReason.SCALED_DOWN + switch_job_status(session, job, JobStatus.TERMINATING, events.SystemActor()) + # background task will process the job later + + +async def _scale_up_replicas( + session: AsyncSession, + run_model: RunModel, + inactive_replicas: List[Tuple[int, bool, int, List[JobModel]]], + replicas_diff: int, + run_spec: RunSpec, + group_name: Optional[str] = None, +) -> None: + """Scale up by retrying inactive replicas and creating new ones""" + if replicas_diff <= 0: + return + + scheduled_replicas = 0 - # rerun inactive replicas - for _, _, _, replica_jobs in inactive_replicas: - if scheduled_replicas == replicas_diff: - break - await retry_run_replica_jobs(session, run_model, replica_jobs, only_failed=False) - scheduled_replicas += 1 + # Retry inactive replicas first + for _, _, _, replica_jobs in inactive_replicas: + if scheduled_replicas == replicas_diff: + break + await retry_run_replica_jobs(session, run_model, replica_jobs, only_failed=False) + scheduled_replicas += 1 + # Create new replicas + if scheduled_replicas < replicas_diff: secrets = await get_project_secrets_mapping( session=session, project=run_model.project, ) - for replica_num in range( - len(active_replicas) + scheduled_replicas, len(active_replicas) + replicas_diff - ): - # FIXME: Handle getting image configuration errors or skip it. + max_replica_num = max((job.replica_num for job in run_model.jobs), default=-1) + + new_replicas_needed = replicas_diff - scheduled_replicas + for i in range(new_replicas_needed): + new_replica_num = max_replica_num + 1 + i jobs = await get_jobs_from_run_spec( run_spec=run_spec, secrets=secrets, - replica_num=replica_num, + replica_num=new_replica_num, + replica_group_name=group_name, ) for job in jobs: job_model = create_job_model_for_new_submission( @@ -148,3 +213,105 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica actor=events.SystemActor(), targets=[events.Target.from_model(job_model)], ) + # Append to run_model.jobs so that when processing later replica groups in the same + # transaction, run_model.jobs includes jobs from previously processed groups. + run_model.jobs.append(job_model) + + +async def scale_run_replicas_per_group( + session: AsyncSession, + run_model: RunModel, + replicas: List[ReplicaGroup], +) -> None: + """Scale each replica group independently""" + if not replicas: + return + + desired_replica_counts = ( + json.loads(run_model.desired_replica_counts) if run_model.desired_replica_counts else {} + ) + + for group in replicas: + assert group.name is not None, "Group name is always set" + group_desired = desired_replica_counts.get(group.name, group.count.min or 0) + + # Build replica lists filtered by this group + active_replicas, inactive_replicas = build_replica_lists( + run_model=run_model, group_filter=group.name + ) + + # Count active replicas + active_group_count = len(active_replicas) + group_diff = group_desired - active_group_count + + if group_diff != 0: + # Check if rolling deployment is in progress for THIS GROUP + + group_has_out_of_date = has_out_of_date_replicas(run_model, group_filter=group.name) + + # During rolling deployment, don't scale down old replicas + # Let rolling deployment handle stopping old replicas + if group_diff < 0 and group_has_out_of_date: + # Skip scaling down during rolling deployment + continue + await scale_run_replicas_for_group( + session=session, + run_model=run_model, + group=group, + replicas_diff=group_diff, + run_spec=RunSpec.__response__.parse_raw(run_model.run_spec), + active_replicas=active_replicas, + inactive_replicas=inactive_replicas, + ) + + +async def scale_run_replicas_for_group( + session: AsyncSession, + run_model: RunModel, + group: ReplicaGroup, + replicas_diff: int, + run_spec: RunSpec, + active_replicas: List[Tuple[int, bool, int, List[JobModel]]], + inactive_replicas: List[Tuple[int, bool, int, List[JobModel]]], +) -> None: + """Scale a specific replica group up or down""" + if replicas_diff == 0: + return + + logger.info( + "%s: scaling %s %s replica(s) for group '%s'", + fmt(run_model), + "UP" if replicas_diff > 0 else "DOWN", + abs(replicas_diff), + group.name, + ) + + if replicas_diff < 0: + scale_down_replicas(session, active_replicas, abs(replicas_diff)) + else: + await _scale_up_replicas( + session=session, + run_model=run_model, + inactive_replicas=inactive_replicas, + replicas_diff=replicas_diff, + run_spec=run_spec, + group_name=group.name, + ) + + +def job_belongs_to_group(job: JobModel, group_name: str) -> bool: + job_spec = JobSpec.__response__.parse_raw(job.job_spec_data) + return job_spec.replica_group == group_name + + +def has_out_of_date_replicas(run: RunModel, group_filter: Optional[str] = None) -> bool: + for job in run.jobs: + # Filter jobs by group if specified + if group_filter is not None: + if not job_belongs_to_group(job, group_filter): + continue + if job.deployment_num < run.deployment_num and not ( + job.status.is_finished() or job.termination_reason == JobTerminationReason.SCALED_DOWN + ): + return True + return False diff --git a/src/dstack/_internal/server/services/runs/spec.py b/src/dstack/_internal/server/services/runs/spec.py index 73b6d9fc7a..db81eb724a 100644 --- a/src/dstack/_internal/server/services/runs/spec.py +++ b/src/dstack/_internal/server/services/runs/spec.py @@ -88,7 +88,9 @@ def validate_run_spec_and_set_defaults( f"Maximum utilization_policy.time_window is {settings.SERVER_METRICS_RUNNING_TTL_SECONDS}s" ) if isinstance(run_spec.configuration, ServiceConfiguration): - if run_spec.merged_profile.schedule and run_spec.configuration.replicas.min == 0: + if run_spec.merged_profile.schedule and all( + group.count.min == 0 for group in run_spec.configuration.replica_groups + ): raise ServerClientError( "Scheduled services with autoscaling to zero are not supported" ) @@ -149,11 +151,10 @@ def get_nodes_required_num(run_spec: RunSpec) -> int: nodes_required_num = 1 if run_spec.configuration.type == "task": nodes_required_num = run_spec.configuration.nodes - elif ( - run_spec.configuration.type == "service" - and run_spec.configuration.replicas.min is not None - ): - nodes_required_num = run_spec.configuration.replicas.min + elif run_spec.configuration.type == "service": + nodes_required_num = sum( + group.count.min or 0 for group in run_spec.configuration.replica_groups + ) return nodes_required_num diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 39e8e98c6a..1f7dcde79d 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -2,6 +2,7 @@ Application logic related to `type: service` runs. """ +import json import uuid from datetime import datetime from typing import Optional @@ -145,7 +146,11 @@ def _register_service_in_server(run_model: RunModel, run_spec: RunSpec) -> Servi "The `https` configuration property is not applicable when running services without a gateway." " Please configure a gateway or remove the `https` property from the service configuration" ) - if run_spec.configuration.replicas.min != run_spec.configuration.replicas.max: + # Check if any group has autoscaling (min != max) + has_autoscaling = any( + group.count.min != group.count.max for group in run_spec.configuration.replica_groups + ) + if has_autoscaling: raise ServerClientError( "Auto-scaling is not supported when running services without a gateway." " Please configure a gateway or set `replicas` to a fixed value in the service configuration" @@ -303,13 +308,24 @@ async def update_service_desired_replica_count( configuration: ServiceConfiguration, last_scaled_at: Optional[datetime], ) -> None: - scaler = get_service_scaler(configuration) stats = None if run_model.gateway_id is not None: conn = await get_or_add_gateway_connection(session, run_model.gateway_id) stats = await conn.get_stats(run_model.project.name, run_model.run_name) - run_model.desired_replica_count = scaler.get_desired_count( - current_desired_count=run_model.desired_replica_count, - stats=stats, - last_scaled_at=last_scaled_at, + replica_groups = configuration.replica_groups + desired_replica_counts = {} + total = 0 + prev_counts = ( + json.loads(run_model.desired_replica_counts) if run_model.desired_replica_counts else {} ) + for group in replica_groups: + scaler = get_service_scaler(group.count, group.scaling) + group_desired = scaler.get_desired_count( + current_desired_count=prev_counts.get(group.name, group.count.min or 0), + stats=stats, + last_scaled_at=last_scaled_at, + ) + desired_replica_counts[group.name] = group_desired + total += group_desired + run_model.desired_replica_counts = json.dumps(desired_replica_counts) + run_model.desired_replica_count = total diff --git a/src/dstack/_internal/server/services/services/autoscalers.py b/src/dstack/_internal/server/services/services/autoscalers.py index cd6d06e588..641d2cee4d 100644 --- a/src/dstack/_internal/server/services/services/autoscalers.py +++ b/src/dstack/_internal/server/services/services/autoscalers.py @@ -6,7 +6,8 @@ from pydantic import BaseModel import dstack._internal.utils.common as common_utils -from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.configurations import ScalingSpec +from dstack._internal.core.models.resources import Range from dstack._internal.proxy.gateway.schemas.stats import PerWindowStats @@ -119,21 +120,21 @@ def get_desired_count( return new_desired_count -def get_service_scaler(conf: ServiceConfiguration) -> BaseServiceScaler: - assert conf.replicas.min is not None - assert conf.replicas.max is not None - if conf.scaling is None: +def get_service_scaler(count: Range[int], scaling: Optional[ScalingSpec]) -> BaseServiceScaler: + assert count.min is not None + assert count.max is not None + if scaling is None: return ManualScaler( - min_replicas=conf.replicas.min, - max_replicas=conf.replicas.max, + min_replicas=count.min, + max_replicas=count.max, ) - if conf.scaling.metric == "rps": + if scaling.metric == "rps": return RPSAutoscaler( # replicas count validated by configuration model - min_replicas=conf.replicas.min, - max_replicas=conf.replicas.max, - target=conf.scaling.target, - scale_up_delay=conf.scaling.scale_up_delay, - scale_down_delay=conf.scaling.scale_down_delay, + min_replicas=count.min, + max_replicas=count.max, + target=scaling.target, + scale_up_delay=scaling.scale_up_delay, + scale_down_delay=scaling.scale_down_delay, ) - raise ValueError(f"No scaler found for scaling parameters {conf.scaling}") + raise ValueError(f"No scaler found for scaling parameters {scaling}") diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 627fa8a167..9485ed1abb 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -257,6 +257,7 @@ def get_dev_env_run_plan_dict( "replica_num": 0, "job_num": 0, "jobs_per_replica": 1, + "replica_group": "default", "single_branch": False, "max_duration": None, "stop_duration": 300, @@ -493,6 +494,7 @@ def get_dev_env_run_dict( "replica_num": 0, "job_num": 0, "jobs_per_replica": 1, + "replica_group": "default", "single_branch": False, "max_duration": None, "stop_duration": 300, From be788642ce9bf7ab651f56ba75fb4a137e2dff13 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:03:44 +0000 Subject: [PATCH 020/640] [chore]: Add `list_events` utility for unit tests (#3493) --- src/dstack/_internal/server/testing/common.py | 7 +++++++ .../server/background/tasks/test_process_events.py | 10 +++------- src/tests/_internal/server/services/test_instances.py | 10 ++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index eb325ad156..640a3932dd 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -7,6 +7,7 @@ from uuid import UUID import gpuhunt +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.backends.base.compute import ( @@ -90,6 +91,7 @@ BackendModel, ComputeGroupModel, DecryptedString, + EventModel, FileArchiveModel, FleetModel, GatewayComputeModel, @@ -1111,6 +1113,11 @@ async def create_secret( return secret_model +async def list_events(session: AsyncSession) -> list[EventModel]: + res = await session.execute(select(EventModel).order_by(EventModel.recorded_at, EventModel.id)) + return list(res.scalars().all()) + + def get_private_key_string() -> str: return """ -----BEGIN RSA PRIVATE KEY----- diff --git a/src/tests/_internal/server/background/tasks/test_process_events.py b/src/tests/_internal/server/background/tasks/test_process_events.py index 899f2946e8..21043e0bae 100644 --- a/src/tests/_internal/server/background/tasks/test_process_events.py +++ b/src/tests/_internal/server/background/tasks/test_process_events.py @@ -3,14 +3,12 @@ import pytest from freezegun import freeze_time -from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.server import settings from dstack._internal.server.background.tasks.process_events import delete_events -from dstack._internal.server.models import EventModel from dstack._internal.server.services import events -from dstack._internal.server.testing.common import create_user +from dstack._internal.server.testing.common import create_user, list_events @pytest.mark.asyncio @@ -27,8 +25,7 @@ async def test_deletes_old_events(test_db, session: AsyncSession) -> None: ) await session.commit() - res = await session.execute(select(EventModel)) - all_events = res.scalars().all() + all_events = await list_events(session) assert len(all_events) == 10 with ( @@ -37,8 +34,7 @@ async def test_deletes_old_events(test_db, session: AsyncSession) -> None: ): await delete_events() - res = await session.execute(select(EventModel).order_by(EventModel.recorded_at)) - remaining_events = res.scalars().all() + remaining_events = await list_events(session) assert len(remaining_events) == 5 assert [e.message for e in remaining_events] == [ "Event 5", diff --git a/src/tests/_internal/server/services/test_instances.py b/src/tests/_internal/server/services/test_instances.py index 9e4cb02e3a..ca6432d61e 100644 --- a/src/tests/_internal/server/services/test_instances.py +++ b/src/tests/_internal/server/services/test_instances.py @@ -1,7 +1,6 @@ import uuid import pytest -from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession import dstack._internal.server.services.instances as instances_services @@ -15,13 +14,14 @@ Resources, ) from dstack._internal.core.models.profiles import Profile -from dstack._internal.server.models import EventModel, InstanceModel +from dstack._internal.server.models import InstanceModel from dstack._internal.server.testing.common import ( create_instance, create_project, create_user, get_volume, get_volume_configuration, + list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -41,8 +41,7 @@ async def test_includes_termination_reason_in_event_messages_only_once( instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATING) instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATED) - res = await session.execute(select(EventModel)) - events = res.scalars().all() + events = await list_events(session) assert len(events) == 2 assert {e.message for e in events} == { "Instance status changed PENDING -> TERMINATING. Termination reason: ERROR (Some err)", @@ -63,8 +62,7 @@ async def test_includes_termination_reason_in_event_message_when_switching_direc instance.termination_reason_message = "Some err" instances_services.switch_instance_status(session, instance, InstanceStatus.TERMINATED) - res = await session.execute(select(EventModel)) - events = res.scalars().all() + events = await list_events(session) assert len(events) == 1 assert events[0].message == ( "Instance status changed PENDING -> TERMINATED. Termination reason: ERROR (Some err)" From 5205c8916b10f345946ca8f09f1dba8d11446b65 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:15:21 +0000 Subject: [PATCH 021/640] [Docs]: Fix k8s backend config example (#3495) --- docs/docs/concepts/backends.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/concepts/backends.md b/docs/docs/concepts/backends.md index 574342318f..9a1c90ec5c 100644 --- a/docs/docs/concepts/backends.md +++ b/docs/docs/concepts/backends.md @@ -1015,13 +1015,13 @@ To use the `kubernetes` backend with `dstack`, you need to configure it with the ```yaml projects: - name: main - backends: - - type: kubernetes - kubeconfig: - filename: ~/.kube/config - proxy_jump: - hostname: 204.12.171.137 - port: 32000 + backends: + - type: kubernetes + kubeconfig: + filename: ~/.kube/config + proxy_jump: + hostname: 204.12.171.137 + port: 32000 ``` From 883b45550d10dffb6a98c3df51d203537bef4436 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 26 Jan 2026 12:15:02 +0500 Subject: [PATCH 022/640] Move ruff.toml to pyproject.toml (#3496) --- pyproject.toml | 14 ++++++++++++++ ruff.toml | 13 ------------- 2 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 ruff.toml diff --git a/pyproject.toml b/pyproject.toml index 37588341e5..748b5211a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,20 @@ ignore-case = true [tool.uv.sources] dstack-plugin-server = { path = "examples/plugins/example_plugin_server", editable = true } +[tool.ruff] +target-version = "py39" +line-length = 99 + +[tool.ruff.lint] +select = ["E", "F", "I", "Q", "W", "PGH", "FLY", "S113"] +ignore = [ + "E501", + "E712", +] + +[tool.ruff.lint.isort] +known-first-party = ["dstack"] + [tool.pyright] typeCheckingMode = "standard" include = [ diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index b61fb7d00b..0000000000 --- a/ruff.toml +++ /dev/null @@ -1,13 +0,0 @@ -target-version = "py39" -line-length = 99 - -[lint] -select = ['E', 'F', 'I' ,'Q', 'W', 'PGH', 'FLY', 'S113'] -ignore = [ - 'E501', - 'E712', -] - -[lint.isort] -known-first-party = ["dstack"] -known-third-party = ["mkdocs_gen_files", "datacrunch"] From ca2172cd5f556c6c1941ed3fd4a47002cf79713c Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Mon, 26 Jan 2026 08:07:51 +0000 Subject: [PATCH 023/640] Events: instance/job reachability and health (#3482) Add new events: - Job became reachable/unreachable - Instance became reachable/unreachable - Instance health changed Additionally, do not set `unreachable=True` for instances in statuses other than `idle` and `busy` to avoid unnecessary and potentially misleading events during instance provisioning. --- .../background/tasks/process_instances.py | 31 +++++++++++++++++-- .../background/tasks/process_running_jobs.py | 29 ++++++++++++++--- .../tasks/test_process_instances.py | 10 ++++++ .../tasks/test_process_running_jobs.py | 4 +++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/server/background/tasks/process_instances.py b/src/dstack/_internal/server/background/tasks/process_instances.py index c2bc27ee85..3cb53322a7 100644 --- a/src/dstack/_internal/server/background/tasks/process_instances.py +++ b/src/dstack/_internal/server/background/tasks/process_instances.py @@ -43,6 +43,7 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.fleets import InstanceGroupPlacement from dstack._internal.core.models.instances import ( + HealthStatus, InstanceAvailability, InstanceOfferWithAvailability, InstanceRuntime, @@ -75,6 +76,7 @@ InstanceHealthResponse, ) from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services import events from dstack._internal.server.services.fleets import ( fleet_model_to_fleet, get_create_instance_offers, @@ -759,8 +761,8 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non ) session.add(health_check_model) - instance.health = health_status - instance.unreachable = not instance_check.reachable + _set_health(session, instance, health_status) + _set_unreachable(session, instance, unreachable=not instance_check.reachable) if instance_check.reachable: instance.termination_deadline = None @@ -1093,6 +1095,31 @@ async def _terminate(session: AsyncSession, instance: InstanceModel) -> None: switch_instance_status(session, instance, InstanceStatus.TERMINATED) +def _set_health(session: AsyncSession, instance: InstanceModel, health: HealthStatus) -> None: + if instance.health != health: + events.emit( + session, + f"Instance health changed {instance.health.upper()} -> {health.upper()}", + actor=events.SystemActor(), + targets=[events.Target.from_model(instance)], + ) + instance.health = health + + +def _set_unreachable(session: AsyncSession, instance: InstanceModel, unreachable: bool) -> None: + if ( + instance.status.is_available() # avoid misleading event during provisioning + and instance.unreachable != unreachable + ): + events.emit( + session, + "Instance became unreachable" if unreachable else "Instance became reachable", + actor=events.SystemActor(), + targets=[events.Target.from_model(instance)], + ) + instance.unreachable = unreachable + + def _next_termination_retry_at(instance: InstanceModel) -> datetime.datetime: assert instance.last_termination_retry_at is not None return instance.last_termination_retry_at + TERMINATION_RETRY_TIMEOUT diff --git a/src/dstack/_internal/server/background/tasks/process_running_jobs.py b/src/dstack/_internal/server/background/tasks/process_running_jobs.py index f5ca6c61ae..9de0fffcc0 100644 --- a/src/dstack/_internal/server/background/tasks/process_running_jobs.py +++ b/src/dstack/_internal/server/background/tasks/process_running_jobs.py @@ -51,9 +51,9 @@ UserModel, ) from dstack._internal.server.schemas.runner import GPUDevice, TaskStatus +from dstack._internal.server.services import events, services from dstack._internal.server.services import files as files_services from dstack._internal.server.services import logs as logs_services -from dstack._internal.server.services import services from dstack._internal.server.services.instances import get_instance_ssh_private_keys from dstack._internal.server.services.jobs import ( find_job, @@ -355,7 +355,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel): ) if success: - job_model.disconnected_at = None + _reset_disconnected_at(session, job_model) else: if job_model.termination_reason: logger.warning( @@ -368,8 +368,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel): # job will be terminated and instance will be emptied by process_terminating_jobs else: # No job_model.termination_reason set means ssh connection failed - if job_model.disconnected_at is None: - job_model.disconnected_at = common_utils.get_current_datetime() + _set_disconnected_at_now(session, job_model) if _should_terminate_job_due_to_disconnect(job_model): # TODO: Replace with JobTerminationReason.INSTANCE_UNREACHABLE for on-demand. job_model.termination_reason = JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY @@ -933,6 +932,28 @@ def _should_terminate_due_to_low_gpu_util(min_util: int, gpus_util: Iterable[Ite return False +def _set_disconnected_at_now(session: AsyncSession, job_model: JobModel) -> None: + if job_model.disconnected_at is None: + job_model.disconnected_at = common_utils.get_current_datetime() + events.emit( + session, + "Job became unreachable", + actor=events.SystemActor(), + targets=[events.Target.from_model(job_model)], + ) + + +def _reset_disconnected_at(session: AsyncSession, job_model: JobModel) -> None: + if job_model.disconnected_at is not None: + job_model.disconnected_at = None + events.emit( + session, + "Job became reachable", + actor=events.SystemActor(), + targets=[events.Target.from_model(job_model)], + ) + + def _get_cluster_info( jobs: List[Job], replica_num: int, diff --git a/src/tests/_internal/server/background/tasks/test_process_instances.py b/src/tests/_internal/server/background/tasks/test_process_instances.py index 38bffc4421..8691f3e7e6 100644 --- a/src/tests/_internal/server/background/tasks/test_process_instances.py +++ b/src/tests/_internal/server/background/tasks/test_process_instances.py @@ -77,6 +77,7 @@ get_job_provisioning_data, get_placement_group_provisioning_data, get_remote_connection_info, + list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -324,10 +325,13 @@ async def test_check_shim_process_ureachable_state( healthcheck.assert_called() await session.refresh(instance) + events = await list_events(session) assert instance is not None assert instance.status == InstanceStatus.IDLE assert not instance.unreachable + assert len(events) == 1 + assert events[0].message == "Instance became reachable" @pytest.mark.asyncio @pytest.mark.parametrize("health_status", [HealthStatus.HEALTHY, HealthStatus.FAILURE]) @@ -351,12 +355,15 @@ async def test_check_shim_switch_to_unreachable_state( await process_instances() await session.refresh(instance) + events = await list_events(session) assert instance is not None assert instance.status == InstanceStatus.IDLE assert instance.unreachable # Should keep the previous status assert instance.health == health_status + assert len(events) == 1 + assert events[0].message == "Instance became unreachable" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -384,11 +391,14 @@ async def test_check_shim_check_instance_health(self, test_db, session: AsyncSes await process_instances() await session.refresh(instance) + events = await list_events(session) assert instance is not None assert instance.status == InstanceStatus.IDLE assert not instance.unreachable assert instance.health == HealthStatus.WARNING + assert len(events) == 1 + assert events[0].message == "Instance health changed HEALTHY -> WARNING" res = await session.execute(select(InstanceHealthCheckModel)) health_check = res.scalars().one() diff --git a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py index e3cc011be9..601fbe1ee7 100644 --- a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py +++ b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py @@ -68,6 +68,7 @@ get_job_runtime_data, get_run_spec, get_volume_configuration, + list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -515,9 +516,12 @@ async def test_pulling_shim_failed(self, test_db, session: AsyncSession): await process_running_jobs() assert SSHTunnelMock.call_count == 3 await session.refresh(job) + events = await list_events(session) assert job is not None assert job.disconnected_at is not None assert job.status == JobStatus.PULLING + assert len(events) == 1 + assert events[0].message == "Job became unreachable" with ( patch("dstack._internal.server.services.runner.ssh.SSHTunnel") as SSHTunnelMock, patch("dstack._internal.server.services.runner.ssh.time.sleep"), From 90b0579212d5a9bf90946c2c5a8c3fe448585ca6 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Mon, 26 Jan 2026 08:08:07 +0000 Subject: [PATCH 024/640] Volume events (#3494) - Support volume events in the API, CLI, and UI - Add the following events: - Volume created - Volume status changed - Volume deleted - Volume deleted due to exceeding `auto_cleanup_duration` --- .../List/hooks/useColumnDefinitions.tsx | 13 +++++ .../src/pages/Events/List/hooks/useFilters.ts | 9 ++++ frontend/src/types/event.d.ts | 5 +- src/dstack/_internal/cli/commands/event.py | 12 +++++ src/dstack/_internal/cli/services/events.py | 1 + src/dstack/_internal/core/models/events.py | 1 + .../background/tasks/process_idle_volumes.py | 9 +++- .../background/tasks/process_volumes.py | 9 ++-- src/dstack/_internal/server/models.py | 1 + src/dstack/_internal/server/routers/events.py | 1 + .../_internal/server/routers/volumes.py | 6 ++- src/dstack/_internal/server/schemas/events.py | 11 +++++ .../_internal/server/services/events.py | 17 +++++++ .../_internal/server/services/volumes.py | 48 ++++++++++++++----- src/dstack/api/server/_events.py | 4 ++ .../tasks/test_process_idle_volumes.py | 4 ++ .../tasks/test_process_submitted_volumes.py | 4 ++ .../_internal/server/routers/test_volumes.py | 9 ++++ 18 files changed, 142 insertions(+), 22 deletions(-) diff --git a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx index 88e067814e..ad337cf5f1 100644 --- a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx +++ b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx @@ -112,6 +112,19 @@ export const useColumnsDefinitions = () => { ); + case 'volume': + return ( +
+ Volume{' '} + {target.project_name && ( + + {target.project_name} + + )} + /{target.name} +
+ ); + default: return '---'; } diff --git a/frontend/src/pages/Events/List/hooks/useFilters.ts b/frontend/src/pages/Events/List/hooks/useFilters.ts index 56aa1f67df..a3d510718f 100644 --- a/frontend/src/pages/Events/List/hooks/useFilters.ts +++ b/frontend/src/pages/Events/List/hooks/useFilters.ts @@ -17,6 +17,7 @@ type RequestParamsKeys = keyof Pick< | 'target_instances' | 'target_runs' | 'target_jobs' + | 'target_volumes' | 'within_projects' | 'within_fleets' | 'within_runs' @@ -31,6 +32,7 @@ const filterKeys: Record = { TARGET_INSTANCES: 'target_instances', TARGET_RUNS: 'target_runs', TARGET_JOBS: 'target_jobs', + TARGET_VOLUMES: 'target_volumes', WITHIN_PROJECTS: 'within_projects', WITHIN_FLEETS: 'within_fleets', WITHIN_RUNS: 'within_runs', @@ -47,6 +49,7 @@ const multipleChoiseKeys: RequestParamsKeys[] = [ 'target_instances', 'target_runs', 'target_jobs', + 'target_volumes', 'within_projects', 'within_fleets', 'within_runs', @@ -61,6 +64,7 @@ const targetTypes = [ { label: 'Instance', value: 'instance' }, { label: 'Run', value: 'run' }, { label: 'Job', value: 'job' }, + { label: 'Volume', value: 'volume' }, ]; export const useFilters = () => { @@ -153,6 +157,11 @@ export const useFilters = () => { operators: ['='], propertyLabel: 'Target jobs', }, + { + key: filterKeys.TARGET_VOLUMES, + operators: ['='], + propertyLabel: 'Target volumes', + }, { key: filterKeys.WITHIN_PROJECTS, diff --git a/frontend/src/types/event.d.ts b/frontend/src/types/event.d.ts index 4ef6bfcb89..3aadfa1f31 100644 --- a/frontend/src/types/event.d.ts +++ b/frontend/src/types/event.d.ts @@ -1,4 +1,4 @@ -declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job'; +declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job' | 'volume'; declare type TEventListRequestParams = Omit & { prev_recorded_at?: string; @@ -8,6 +8,7 @@ declare type TEventListRequestParams = Omit EventListFilters: filters.target_runs = [ api.client.runs.get(api.project, name).id for name in args.target_runs ] + elif args.target_volumes: + filters.target_volumes = [ + api.client.volumes.get(project_name=api.project, name=name).id + for name in args.target_volumes + ] if args.within_fleets: filters.within_fleets = [ diff --git a/src/dstack/_internal/cli/services/events.py b/src/dstack/_internal/cli/services/events.py index 4f3f620b76..c2903065c9 100644 --- a/src/dstack/_internal/cli/services/events.py +++ b/src/dstack/_internal/cli/services/events.py @@ -16,6 +16,7 @@ class EventListFilters: target_fleets: Optional[list[uuid.UUID]] = None target_runs: Optional[list[uuid.UUID]] = None + target_volumes: Optional[list[uuid.UUID]] = None within_projects: Optional[list[uuid.UUID]] = None within_fleets: Optional[list[uuid.UUID]] = None within_runs: Optional[list[uuid.UUID]] = None diff --git a/src/dstack/_internal/core/models/events.py b/src/dstack/_internal/core/models/events.py index fc7f51601a..6dae2dc178 100644 --- a/src/dstack/_internal/core/models/events.py +++ b/src/dstack/_internal/core/models/events.py @@ -16,6 +16,7 @@ class EventTargetType(str, Enum): INSTANCE = "instance" RUN = "run" JOB = "job" + VOLUME = "volume" class EventTarget(CoreModel): diff --git a/src/dstack/_internal/server/background/tasks/process_idle_volumes.py b/src/dstack/_internal/server/background/tasks/process_idle_volumes.py index 2557012c2b..cd5b66bc70 100644 --- a/src/dstack/_internal/server/background/tasks/process_idle_volumes.py +++ b/src/dstack/_internal/server/background/tasks/process_idle_volumes.py @@ -12,6 +12,7 @@ from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ProjectModel, UserModel, VolumeModel from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services import events from dstack._internal.server.services.locking import get_locker from dstack._internal.server.services.volumes import ( get_volume_configuration, @@ -100,8 +101,12 @@ async def _delete_idle_volumes(session: AsyncSession, volumes: List[VolumeModel] volume_model.deleted = True volume_model.deleted_at = get_current_datetime() - - logger.info("Deleted idle volume %s", volume_model.name) + events.emit( + session=session, + message="Volume deleted due to exceeding auto_cleanup_duration", + actor=events.SystemActor(), + targets=[events.Target.from_model(volume_model)], + ) await session.commit() diff --git a/src/dstack/_internal/server/background/tasks/process_volumes.py b/src/dstack/_internal/server/background/tasks/process_volumes.py index 534af8d48f..66124619a4 100644 --- a/src/dstack/_internal/server/background/tasks/process_volumes.py +++ b/src/dstack/_internal/server/background/tasks/process_volumes.py @@ -16,6 +16,7 @@ from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import volumes as volumes_services from dstack._internal.server.services.locking import get_locker +from dstack._internal.server.services.volumes import switch_volume_status from dstack._internal.server.utils import sentry_utils from dstack._internal.utils.common import get_current_datetime, run_async from dstack._internal.utils.logging import get_logger @@ -79,8 +80,8 @@ async def _process_submitted_volume(session: AsyncSession, volume_model: VolumeM volume.name, volume.configuration.backend.value, ) - volume_model.status = VolumeStatus.FAILED volume_model.status_message = "Backend not available" + switch_volume_status(session, volume_model, VolumeStatus.FAILED) volume_model.last_processed_at = get_current_datetime() await session.commit() return @@ -102,18 +103,18 @@ async def _process_submitted_volume(session: AsyncSession, volume_model: VolumeM ) except BackendError as e: logger.info("Failed to create volume %s: %s", volume_model.name, repr(e)) - volume_model.status = VolumeStatus.FAILED status_message = f"Backend error: {repr(e)}" if len(e.args) > 0: status_message = str(e.args[0]) volume_model.status_message = status_message + switch_volume_status(session, volume_model, VolumeStatus.FAILED) volume_model.last_processed_at = get_current_datetime() await session.commit() return except Exception as e: logger.exception("Got exception when creating volume %s", volume_model.name) - volume_model.status = VolumeStatus.FAILED volume_model.status_message = f"Unexpected error: {repr(e)}" + switch_volume_status(session, volume_model, VolumeStatus.FAILED) volume_model.last_processed_at = get_current_datetime() await session.commit() return @@ -123,6 +124,6 @@ async def _process_submitted_volume(session: AsyncSession, volume_model: VolumeM # Provisioned volumes marked as active since they become available almost immediately in AWS # TODO: Consider checking volume state volume_model.volume_provisioning_data = vpd.json() - volume_model.status = VolumeStatus.ACTIVE + switch_volume_status(session, volume_model, VolumeStatus.ACTIVE) volume_model.last_processed_at = get_current_datetime() await session.commit() diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 982ade215f..7e9db282d1 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -745,6 +745,7 @@ class VolumeModel(BaseModel): deleted: Mapped[bool] = mapped_column(Boolean, default=False) deleted_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + # NOTE: `status` must be changed only via `switch_volume_status()` status: Mapped[VolumeStatus] = mapped_column(EnumAsString(VolumeStatus, 100), index=True) status_message: Mapped[Optional[str]] = mapped_column(Text) diff --git a/src/dstack/_internal/server/routers/events.py b/src/dstack/_internal/server/routers/events.py index 3895767d6f..be75cccbb4 100644 --- a/src/dstack/_internal/server/routers/events.py +++ b/src/dstack/_internal/server/routers/events.py @@ -44,6 +44,7 @@ async def list_events( target_instances=body.target_instances, target_runs=body.target_runs, target_jobs=body.target_jobs, + target_volumes=body.target_volumes, within_projects=body.within_projects, within_fleets=body.within_fleets, within_runs=body.within_runs, diff --git a/src/dstack/_internal/server/routers/volumes.py b/src/dstack/_internal/server/routers/volumes.py index 2ac5034707..ead5465c48 100644 --- a/src/dstack/_internal/server/routers/volumes.py +++ b/src/dstack/_internal/server/routers/volumes.py @@ -116,5 +116,7 @@ async def delete_volumes( """ Deletes one or more volumes. """ - _, project = user_project - await volumes_services.delete_volumes(session=session, project=project, names=body.names) + user, project = user_project + await volumes_services.delete_volumes( + session=session, project=project, names=body.names, user=user + ) diff --git a/src/dstack/_internal/server/schemas/events.py b/src/dstack/_internal/server/schemas/events.py index 537a731970..66ea2e3404 100644 --- a/src/dstack/_internal/server/schemas/events.py +++ b/src/dstack/_internal/server/schemas/events.py @@ -80,6 +80,17 @@ class ListEventsRequest(CoreModel): max_items=MAX_FILTER_ITEMS, ), ] = None + target_volumes: Annotated[ + Optional[list[uuid.UUID]], + Field( + description=( + "List of volume IDs." + " The response will only include events that target the specified volumes" + ), + min_items=MIN_FILTER_ITEMS, + max_items=MAX_FILTER_ITEMS, + ), + ] = None within_projects: Annotated[ Optional[list[uuid.UUID]], Field( diff --git a/src/dstack/_internal/server/services/events.py b/src/dstack/_internal/server/services/events.py index c9818ef9ee..80d81734b5 100644 --- a/src/dstack/_internal/server/services/events.py +++ b/src/dstack/_internal/server/services/events.py @@ -20,6 +20,7 @@ ProjectModel, RunModel, UserModel, + VolumeModel, ) from dstack._internal.server.services.logging import fmt_entity from dstack._internal.utils.common import get_current_datetime @@ -91,6 +92,7 @@ def from_model( ProjectModel, RunModel, UserModel, + VolumeModel, ], ) -> "Target": if isinstance(model, FleetModel): @@ -135,6 +137,13 @@ def from_model( id=model.id, name=model.name, ) + if isinstance(model, VolumeModel): + return Target( + type=EventTargetType.VOLUME, + project_id=model.project_id or model.project.id, + id=model.id, + name=model.name, + ) raise ValueError(f"Unsupported model type: {type(model)}") def fmt(self) -> str: @@ -212,6 +221,7 @@ async def list_events( target_instances: Optional[list[uuid.UUID]], target_runs: Optional[list[uuid.UUID]], target_jobs: Optional[list[uuid.UUID]], + target_volumes: Optional[list[uuid.UUID]], within_projects: Optional[list[uuid.UUID]], within_fleets: Optional[list[uuid.UUID]], within_runs: Optional[list[uuid.UUID]], @@ -281,6 +291,13 @@ async def list_events( EventTargetModel.entity_id.in_(target_jobs), ) ) + if target_volumes is not None: + target_filters.append( + and_( + EventTargetModel.entity_type == EventTargetType.VOLUME, + EventTargetModel.entity_id.in_(target_volumes), + ) + ) if within_projects is not None: target_filters.append(EventTargetModel.entity_project_id.in_(within_projects)) if within_fleets is not None: diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index eb8f4bab64..49a3d79594 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from typing import List, Optional -from sqlalchemy import and_, func, or_, select, update +from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload, selectinload @@ -33,6 +33,7 @@ VolumeModel, ) from dstack._internal.server.services import backends as backends_services +from dstack._internal.server.services import events from dstack._internal.server.services.instances import get_instance_provisioning_data from dstack._internal.server.services.locking import ( get_locker, @@ -46,6 +47,24 @@ logger = get_logger(__name__) +def switch_volume_status( + session: AsyncSession, + volume_model: VolumeModel, + new_status: VolumeStatus, + actor: events.AnyActor = events.SystemActor(), +): + old_status = volume_model.status + if old_status == new_status: + return + + volume_model.status = new_status + + msg = f"Volume status changed {old_status.upper()} -> {new_status.upper()}" + if volume_model.status_message is not None: + msg += f" ({volume_model.status_message})" + events.emit(session, msg, actor=actor, targets=[events.Target.from_model(volume_model)]) + + async def list_volumes( session: AsyncSession, user: UserModel, @@ -245,11 +264,19 @@ async def create_volume( attachments=[], ) session.add(volume_model) + events.emit( + session, + message=f"Volume created. Status: {volume_model.status.upper()}", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(volume_model)], + ) await session.commit() return volume_model_to_volume(volume_model) -async def delete_volumes(session: AsyncSession, project: ProjectModel, names: List[str]): +async def delete_volumes( + session: AsyncSession, project: ProjectModel, names: List[str], user: UserModel +): res = await session.execute( select(VolumeModel).where( VolumeModel.project_id == project.id, @@ -287,17 +314,14 @@ async def delete_volumes(session: AsyncSession, project: ProjectModel, names: Li await _delete_volume(session=session, project=project, volume_model=volume_model) except Exception: logger.exception("Error when deleting volume %s", volume_model.name) - await session.execute( - update(VolumeModel) - .where( - VolumeModel.project_id == project.id, - VolumeModel.id.in_(volumes_ids), - ) - .values( - deleted=True, - deleted_at=common.get_current_datetime(), + volume_model.deleted = True + volume_model.deleted_at = common.get_current_datetime() + events.emit( + session, + message="Volume deleted", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(volume_model)], ) - ) await session.commit() diff --git a/src/dstack/api/server/_events.py b/src/dstack/api/server/_events.py index 2f5d6639a3..d9bf828394 100644 --- a/src/dstack/api/server/_events.py +++ b/src/dstack/api/server/_events.py @@ -27,6 +27,9 @@ def list( prev_id: Optional[UUID] = None, limit: int = LIST_EVENTS_DEFAULT_LIMIT, ascending: bool = False, + *, + # NOTE: New parameters go here. Avoid positional parameters, they can break compatibility. + target_volumes: Optional[list[UUID]] = None, ) -> list[Event]: if prev_recorded_at is not None: # Time zones other than UTC are misinterpreted by the server: @@ -39,6 +42,7 @@ def list( target_instances=target_instances, target_runs=target_runs, target_jobs=target_jobs, + target_volumes=target_volumes, within_projects=within_projects, within_fleets=within_fleets, within_runs=within_runs, diff --git a/src/tests/_internal/server/background/tasks/test_process_idle_volumes.py b/src/tests/_internal/server/background/tasks/test_process_idle_volumes.py index 13f557df2f..9d73afbb78 100644 --- a/src/tests/_internal/server/background/tasks/test_process_idle_volumes.py +++ b/src/tests/_internal/server/background/tasks/test_process_idle_volumes.py @@ -20,6 +20,7 @@ create_volume, get_volume_configuration, get_volume_provisioning_data, + list_events, ) from dstack._internal.utils.common import get_current_datetime @@ -73,10 +74,13 @@ async def test_deletes_idle_volumes(self, test_db, session: AsyncSession): await session.refresh(volume1) await session.refresh(volume2) + events = await list_events(session) assert volume1.deleted assert volume1.deleted_at is not None assert not volume2.deleted assert volume2.deleted_at is None + assert len(events) == 1 + assert events[0].message == "Volume deleted due to exceeding auto_cleanup_duration" @pytest.mark.asyncio diff --git a/src/tests/_internal/server/background/tasks/test_process_submitted_volumes.py b/src/tests/_internal/server/background/tasks/test_process_submitted_volumes.py index 0b2f0b1948..dfeef1e42e 100644 --- a/src/tests/_internal/server/background/tasks/test_process_submitted_volumes.py +++ b/src/tests/_internal/server/background/tasks/test_process_submitted_volumes.py @@ -11,6 +11,7 @@ create_project, create_user, create_volume, + list_events, ) @@ -51,3 +52,6 @@ async def test_provisiones_volumes(self, test_db, session: AsyncSession): aws_mock.compute.return_value.create_volume.assert_called_once() await session.refresh(volume) assert volume.status == VolumeStatus.ACTIVE + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Volume status changed SUBMITTED -> ACTIVE" diff --git a/src/tests/_internal/server/routers/test_volumes.py b/src/tests/_internal/server/routers/test_volumes.py index 1949747f62..f7719f9374 100644 --- a/src/tests/_internal/server/routers/test_volumes.py +++ b/src/tests/_internal/server/routers/test_volumes.py @@ -22,6 +22,7 @@ get_auth_headers, get_volume_configuration, get_volume_provisioning_data, + list_events, ) @@ -357,6 +358,9 @@ async def test_creates_volume(self, test_db, session: AsyncSession, client: Asyn } res = await session.execute(select(VolumeModel)) assert res.scalar_one() + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Volume created. Status: SUBMITTED" class TestDeleteVolumes: @@ -397,6 +401,9 @@ async def test_deletes_volumes(self, test_db, session: AsyncSession, client: Asy assert response.status_code == 200 await session.refresh(volume) assert volume.deleted + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Volume deleted" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -428,3 +435,5 @@ async def test_returns_400_when_volumes_in_use( assert response.status_code == 400 await session.refresh(volume) assert not volume.deleted + events = await list_events(session) + assert len(events) == 0 From 198534107ed07a50e05074395178a656305a76a2 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 26 Jan 2026 14:30:36 +0500 Subject: [PATCH 025/640] Set JobTerminationReason.INSTANCE_UNREACHABLE for unreachable on-demand instances (#3497) --- .../server/background/tasks/process_running_jobs.py | 8 ++++++-- .../server/background/tasks/test_process_running_jobs.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/server/background/tasks/process_running_jobs.py b/src/dstack/_internal/server/background/tasks/process_running_jobs.py index 9de0fffcc0..bcb35a0898 100644 --- a/src/dstack/_internal/server/background/tasks/process_running_jobs.py +++ b/src/dstack/_internal/server/background/tasks/process_running_jobs.py @@ -370,8 +370,12 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel): # No job_model.termination_reason set means ssh connection failed _set_disconnected_at_now(session, job_model) if _should_terminate_job_due_to_disconnect(job_model): - # TODO: Replace with JobTerminationReason.INSTANCE_UNREACHABLE for on-demand. - job_model.termination_reason = JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY + if job_provisioning_data.instance_type.resources.spot: + job_model.termination_reason = ( + JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY + ) + else: + job_model.termination_reason = JobTerminationReason.INSTANCE_UNREACHABLE job_model.termination_reason_message = "Instance is unreachable" switch_job_status(session, job_model, JobStatus.TERMINATING) else: diff --git a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py index 601fbe1ee7..9e318866c5 100644 --- a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py +++ b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py @@ -532,7 +532,7 @@ async def test_pulling_shim_failed(self, test_db, session: AsyncSession): assert SSHTunnelMock.call_count == 3 await session.refresh(job) assert job.status == JobStatus.TERMINATING - assert job.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY + assert job.termination_reason == JobTerminationReason.INSTANCE_UNREACHABLE assert job.remove_at is None @pytest.mark.asyncio From b4b69fb4d74d946ddc342fe0e9469466cd2c8844 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Tue, 27 Jan 2026 08:22:24 +0000 Subject: [PATCH 026/640] Support gateway events in API, CLI, and UI (#3499) --- .../List/hooks/useColumnDefinitions.tsx | 13 +++++++++++++ .../src/pages/Events/List/hooks/useFilters.ts | 9 +++++++++ frontend/src/types/event.d.ts | 5 +++-- src/dstack/_internal/cli/commands/event.py | 19 +++++++++++++++++++ src/dstack/_internal/cli/services/events.py | 1 + src/dstack/_internal/core/models/events.py | 1 + src/dstack/_internal/core/models/gateways.py | 4 ++++ src/dstack/_internal/server/routers/events.py | 1 + src/dstack/_internal/server/schemas/events.py | 11 +++++++++++ .../_internal/server/services/events.py | 17 +++++++++++++++++ .../server/services/gateways/__init__.py | 1 + src/dstack/api/server/_events.py | 2 ++ .../_internal/server/routers/test_gateways.py | 8 ++++++++ 13 files changed, 90 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx index ad337cf5f1..d6e5b846ea 100644 --- a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx +++ b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx @@ -125,6 +125,19 @@ export const useColumnsDefinitions = () => { ); + case 'gateway': + return ( +
+ Gateway{' '} + {target.project_name && ( + + {target.project_name} + + )} + /{target.name} +
+ ); + default: return '---'; } diff --git a/frontend/src/pages/Events/List/hooks/useFilters.ts b/frontend/src/pages/Events/List/hooks/useFilters.ts index a3d510718f..d463770b30 100644 --- a/frontend/src/pages/Events/List/hooks/useFilters.ts +++ b/frontend/src/pages/Events/List/hooks/useFilters.ts @@ -18,6 +18,7 @@ type RequestParamsKeys = keyof Pick< | 'target_runs' | 'target_jobs' | 'target_volumes' + | 'target_gateways' | 'within_projects' | 'within_fleets' | 'within_runs' @@ -33,6 +34,7 @@ const filterKeys: Record = { TARGET_RUNS: 'target_runs', TARGET_JOBS: 'target_jobs', TARGET_VOLUMES: 'target_volumes', + TARGET_GATEWAYS: 'target_gateways', WITHIN_PROJECTS: 'within_projects', WITHIN_FLEETS: 'within_fleets', WITHIN_RUNS: 'within_runs', @@ -50,6 +52,7 @@ const multipleChoiseKeys: RequestParamsKeys[] = [ 'target_runs', 'target_jobs', 'target_volumes', + 'target_gateways', 'within_projects', 'within_fleets', 'within_runs', @@ -65,6 +68,7 @@ const targetTypes = [ { label: 'Run', value: 'run' }, { label: 'Job', value: 'job' }, { label: 'Volume', value: 'volume' }, + { label: 'Gateway', value: 'gateway' }, ]; export const useFilters = () => { @@ -162,6 +166,11 @@ export const useFilters = () => { operators: ['='], propertyLabel: 'Target volumes', }, + { + key: filterKeys.TARGET_GATEWAYS, + operators: ['='], + propertyLabel: 'Target gateways', + }, { key: filterKeys.WITHIN_PROJECTS, diff --git a/frontend/src/types/event.d.ts b/frontend/src/types/event.d.ts index 3aadfa1f31..618ea6673f 100644 --- a/frontend/src/types/event.d.ts +++ b/frontend/src/types/event.d.ts @@ -1,4 +1,4 @@ -declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job' | 'volume'; +declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job' | 'volume' | 'gateway'; declare type TEventListRequestParams = Omit & { prev_recorded_at?: string; @@ -9,6 +9,7 @@ declare type TEventListRequestParams = Omit EventListFilters: api.client.volumes.get(project_name=api.project, name=name).id for name in args.target_volumes ] + elif args.target_gateways: + filters.target_gateways = [] + for name in args.target_gateways: + id = api.client.gateways.get(api.project, name).id + if id is None: + # TODO(0.21): Remove this check once `Gateway.id` is required. + raise CLIError( + "Cannot determine gateway ID, most likely due to an outdated dstack server." + " Update the server to 0.20.7 or higher or remove --target-gateway." + ) + filters.target_gateways.append(id) if args.within_fleets: filters.within_fleets = [ diff --git a/src/dstack/_internal/cli/services/events.py b/src/dstack/_internal/cli/services/events.py index c2903065c9..11f764bd15 100644 --- a/src/dstack/_internal/cli/services/events.py +++ b/src/dstack/_internal/cli/services/events.py @@ -17,6 +17,7 @@ class EventListFilters: target_fleets: Optional[list[uuid.UUID]] = None target_runs: Optional[list[uuid.UUID]] = None target_volumes: Optional[list[uuid.UUID]] = None + target_gateways: Optional[list[uuid.UUID]] = None within_projects: Optional[list[uuid.UUID]] = None within_fleets: Optional[list[uuid.UUID]] = None within_runs: Optional[list[uuid.UUID]] = None diff --git a/src/dstack/_internal/core/models/events.py b/src/dstack/_internal/core/models/events.py index 6dae2dc178..289c4fc674 100644 --- a/src/dstack/_internal/core/models/events.py +++ b/src/dstack/_internal/core/models/events.py @@ -17,6 +17,7 @@ class EventTargetType(str, Enum): RUN = "run" JOB = "job" VOLUME = "volume" + GATEWAY = "gateway" class EventTarget(CoreModel): diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py index 2dfeb5b181..b342c0a73b 100644 --- a/src/dstack/_internal/core/models/gateways.py +++ b/src/dstack/_internal/core/models/gateways.py @@ -1,4 +1,5 @@ import datetime +import uuid from enum import Enum from typing import Dict, Optional, Union @@ -93,6 +94,9 @@ class GatewaySpec(CoreModel): class Gateway(CoreModel): + # ID is only optional on the client side for compatibility with pre-0.20.7 servers. + # TODO(0.21): Make required. + id: Optional[uuid.UUID] = None name: str configuration: GatewayConfiguration created_at: datetime.datetime diff --git a/src/dstack/_internal/server/routers/events.py b/src/dstack/_internal/server/routers/events.py index be75cccbb4..4250eb4d7a 100644 --- a/src/dstack/_internal/server/routers/events.py +++ b/src/dstack/_internal/server/routers/events.py @@ -45,6 +45,7 @@ async def list_events( target_runs=body.target_runs, target_jobs=body.target_jobs, target_volumes=body.target_volumes, + target_gateways=body.target_gateways, within_projects=body.within_projects, within_fleets=body.within_fleets, within_runs=body.within_runs, diff --git a/src/dstack/_internal/server/schemas/events.py b/src/dstack/_internal/server/schemas/events.py index 66ea2e3404..30f7fe3244 100644 --- a/src/dstack/_internal/server/schemas/events.py +++ b/src/dstack/_internal/server/schemas/events.py @@ -91,6 +91,17 @@ class ListEventsRequest(CoreModel): max_items=MAX_FILTER_ITEMS, ), ] = None + target_gateways: Annotated[ + Optional[list[uuid.UUID]], + Field( + description=( + "List of gateway IDs." + " The response will only include events that target the specified gateways" + ), + min_items=MIN_FILTER_ITEMS, + max_items=MAX_FILTER_ITEMS, + ), + ] = None within_projects: Annotated[ Optional[list[uuid.UUID]], Field( diff --git a/src/dstack/_internal/server/services/events.py b/src/dstack/_internal/server/services/events.py index 80d81734b5..c6d35a4577 100644 --- a/src/dstack/_internal/server/services/events.py +++ b/src/dstack/_internal/server/services/events.py @@ -14,6 +14,7 @@ EventModel, EventTargetModel, FleetModel, + GatewayModel, InstanceModel, JobModel, MemberModel, @@ -87,6 +88,7 @@ def __post_init__(self): def from_model( model: Union[ FleetModel, + GatewayModel, InstanceModel, JobModel, ProjectModel, @@ -102,6 +104,13 @@ def from_model( id=model.id, name=model.name, ) + if isinstance(model, GatewayModel): + return Target( + type=EventTargetType.GATEWAY, + project_id=model.project_id or model.project.id, + id=model.id, + name=model.name, + ) if isinstance(model, InstanceModel): return Target( type=EventTargetType.INSTANCE, @@ -222,6 +231,7 @@ async def list_events( target_runs: Optional[list[uuid.UUID]], target_jobs: Optional[list[uuid.UUID]], target_volumes: Optional[list[uuid.UUID]], + target_gateways: Optional[list[uuid.UUID]], within_projects: Optional[list[uuid.UUID]], within_fleets: Optional[list[uuid.UUID]], within_runs: Optional[list[uuid.UUID]], @@ -298,6 +308,13 @@ async def list_events( EventTargetModel.entity_id.in_(target_volumes), ) ) + if target_gateways is not None: + target_filters.append( + and_( + EventTargetModel.entity_type == EventTargetType.GATEWAY, + EventTargetModel.entity_id.in_(target_gateways), + ) + ) if within_projects is not None: target_filters.append(EventTargetModel.entity_project_id.in_(within_projects)) if within_fleets is not None: diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 4ab80a8331..cf41b53973 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -558,6 +558,7 @@ def gateway_model_to_gateway(gateway_model: GatewayModel) -> Gateway: configuration = get_gateway_configuration(gateway_model) configuration.default = gateway_model.project.default_gateway_id == gateway_model.id return Gateway( + id=gateway_model.id, name=gateway_model.name, ip_address=ip_address, instance_id=instance_id, diff --git a/src/dstack/api/server/_events.py b/src/dstack/api/server/_events.py index d9bf828394..d403fb2427 100644 --- a/src/dstack/api/server/_events.py +++ b/src/dstack/api/server/_events.py @@ -30,6 +30,7 @@ def list( *, # NOTE: New parameters go here. Avoid positional parameters, they can break compatibility. target_volumes: Optional[list[UUID]] = None, + target_gateways: Optional[list[UUID]] = None, ) -> list[Event]: if prev_recorded_at is not None: # Time zones other than UTC are misinterpreted by the server: @@ -43,6 +44,7 @@ def list( target_runs=target_runs, target_jobs=target_jobs, target_volumes=target_volumes, + target_gateways=target_gateways, within_projects=within_projects, within_fleets=within_fleets, within_runs=within_runs, diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index b909c7d729..70f6b22b7e 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -17,6 +17,7 @@ create_user, get_auth_headers, ) +from dstack._internal.server.testing.matchers import SomeUUID4Str class TestListAndGetGateways: @@ -54,6 +55,7 @@ async def test_list(self, test_db, session: AsyncSession, client: AsyncClient): assert response.status_code == 200 assert response.json() == [ { + "id": SomeUUID4Str(), "backend": backend.type.value, "created_at": response.json()[0]["created_at"], "default": False, @@ -107,6 +109,7 @@ async def test_get(self, test_db, session: AsyncSession, client: AsyncClient): ) assert response.status_code == 200 assert response.json() == { + "id": SomeUUID4Str(), "backend": backend.type.value, "created_at": response.json()["created_at"], "default": False, @@ -189,6 +192,7 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn ) assert response.status_code == 200 assert response.json() == { + "id": SomeUUID4Str(), "name": "test", "backend": "aws", "region": "us", @@ -243,6 +247,7 @@ async def test_create_gateway_without_name( g.assert_called_once() assert response.status_code == 200 assert response.json() == { + "id": SomeUUID4Str(), "name": "random-name", "backend": "aws", "region": "us", @@ -347,6 +352,7 @@ async def test_set_default_gateway(self, test_db, session: AsyncSession, client: ) assert response.status_code == 200 assert response.json() == { + "id": SomeUUID4Str(), "backend": backend.type.value, "created_at": response.json()["created_at"], "default": True, @@ -471,6 +477,7 @@ def get_backend(project, backend_type): assert response.status_code == 200 assert response.json() == [ { + "id": str(gateway_gcp.id), "backend": backend_gcp.type.value, "created_at": response.json()[0]["created_at"], "default": False, @@ -542,6 +549,7 @@ async def test_set_wildcard_domain(self, test_db, session: AsyncSession, client: ) assert response.status_code == 200 assert response.json() == { + "id": SomeUUID4Str(), "backend": backend.type.value, "created_at": response.json()["created_at"], "status": "submitted", From 5efca70ef47c10ac19b5c7ebac6fa5dfdd13ba38 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Tue, 27 Jan 2026 14:08:26 +0545 Subject: [PATCH 027/640] Use numeric replica-group names (#3502) Co-authored-by: Bihan Rana --- src/dstack/_internal/core/models/configurations.py | 12 ++++++------ src/dstack/_internal/core/services/__init__.py | 4 ++++ src/tests/_internal/server/routers/test_runs.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 6bd9f0827e..3b2c7812b9 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -31,7 +31,7 @@ from dstack._internal.core.models.services import AnyModel, OpenAIChatModel from dstack._internal.core.models.unix import UnixUser from dstack._internal.core.models.volumes import MountPoint, VolumeConfiguration, parse_mount_point -from dstack._internal.core.services import is_valid_dstack_resource_name +from dstack._internal.core.services import is_valid_replica_group_name from dstack._internal.utils.common import has_duplicates, list_enum_values_for_annotation from dstack._internal.utils.json_schema import add_extra_schema_types from dstack._internal.utils.json_utils import ( @@ -55,7 +55,7 @@ DEFAULT_PROBE_READY_AFTER = 1 DEFAULT_PROBE_METHOD = "get" MAX_PROBE_URL_LEN = 2048 -DEFAULT_REPLICA_GROUP_NAME = "default" +DEFAULT_REPLICA_GROUP_NAME = "0" class RunConfigurationType(str, Enum): @@ -756,7 +756,7 @@ class ReplicaGroup(CoreModel): name: Annotated[ Optional[str], Field( - description="The name of the replica group. If not provided, defaults to 'replica-group-0', 'replica-group-1', etc. based on position." + description="The name of the replica group. If not provided, defaults to '0', '1', etc. based on position." ), ] count: Annotated[ @@ -784,8 +784,8 @@ class ReplicaGroup(CoreModel): @validator("name") def validate_name(cls, v: Optional[str]) -> Optional[str]: if v is not None: - if not is_valid_dstack_resource_name(v): - raise ValueError("Resource name should match regex '^[a-z][a-z0-9-]{1,40}$'") + if not is_valid_replica_group_name(v): + raise ValueError("Resource name should match regex '^[a-z0-9][a-z0-9-]{0,39}$'") return v @validator("count") @@ -920,7 +920,7 @@ def validate_replicas( # Assign default names to groups without names for index, group in enumerate(v): if group.name is None: - group.name = f"replica-group-{index}" + group.name = str(index) # Check for duplicate names names = [group.name for group in v] diff --git a/src/dstack/_internal/core/services/__init__.py b/src/dstack/_internal/core/services/__init__.py index 89ec6d8520..6d698e1228 100644 --- a/src/dstack/_internal/core/services/__init__.py +++ b/src/dstack/_internal/core/services/__init__.py @@ -10,3 +10,7 @@ def validate_dstack_resource_name(resource_name: str): def is_valid_dstack_resource_name(resource_name: str) -> bool: return re.match("^[a-z][a-z0-9-]{1,40}$", resource_name) is not None + + +def is_valid_replica_group_name(name: str) -> bool: + return re.match("^[a-z0-9][a-z0-9-]{0,39}$", name) is not None diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 9485ed1abb..70ab54bd16 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -257,7 +257,7 @@ def get_dev_env_run_plan_dict( "replica_num": 0, "job_num": 0, "jobs_per_replica": 1, - "replica_group": "default", + "replica_group": "0", "single_branch": False, "max_duration": None, "stop_duration": 300, @@ -494,7 +494,7 @@ def get_dev_env_run_dict( "replica_num": 0, "job_num": 0, "jobs_per_replica": 1, - "replica_group": "default", + "replica_group": "0", "single_branch": False, "max_duration": None, "stop_duration": 300, From b4c6f17839ed3e46d41e4af7470da4f6d32d60a1 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Tue, 27 Jan 2026 08:44:37 +0000 Subject: [PATCH 028/640] Add gateway lifecycle events (#3500) - Gateway created - Gateway status changed - Gateway deleted - Gateway set as default - Gateway unset as default - Gateway wildcard domain changed --- .../background/tasks/process_gateways.py | 24 +-- .../_internal/server/routers/gateways.py | 15 +- .../server/services/gateways/__init__.py | 141 ++++++++++++++---- src/dstack/_internal/server/testing/common.py | 15 +- .../background/tasks/test_process_gateways.py | 16 ++ .../_internal/server/routers/test_gateways.py | 57 ++++++- 6 files changed, 211 insertions(+), 57 deletions(-) diff --git a/src/dstack/_internal/server/background/tasks/process_gateways.py b/src/dstack/_internal/server/background/tasks/process_gateways.py index a54cb9e319..2566a4f4d8 100644 --- a/src/dstack/_internal/server/background/tasks/process_gateways.py +++ b/src/dstack/_internal/server/background/tasks/process_gateways.py @@ -14,6 +14,7 @@ GatewayConnection, create_gateway_compute, gateway_connections_pool, + switch_gateway_status, ) from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker from dstack._internal.server.services.logging import fmt @@ -60,14 +61,6 @@ async def process_gateways(): logger.error( "%s: unexpected gateway status %r", fmt(gateway_model), initial_status.upper() ) - if gateway_model.status != initial_status: - logger.info( - "%s: gateway status has changed %s -> %s%s", - fmt(gateway_model), - initial_status.upper(), - gateway_model.status.upper(), - f": {gateway_model.status_message}" if gateway_model.status_message else "", - ) gateway_model.last_processed_at = get_current_datetime() await session.commit() finally: @@ -128,8 +121,8 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew project=gateway_model.project, backend_type=configuration.backend ) except BackendNotAvailable: - gateway_model.status = GatewayStatus.FAILED gateway_model.status_message = "Backend not available" + switch_gateway_status(session, gateway_model, GatewayStatus.FAILED) return try: @@ -140,18 +133,17 @@ async def _process_submitted_gateway(session: AsyncSession, gateway_model: Gatew backend_id=backend_model.id, ) session.add(gateway_model) - gateway_model.status = GatewayStatus.PROVISIONING + switch_gateway_status(session, gateway_model, GatewayStatus.PROVISIONING) except BackendError as e: - logger.info("%s: failed to create gateway compute: %r", fmt(gateway_model), e) - gateway_model.status = GatewayStatus.FAILED status_message = f"Backend error: {repr(e)}" if len(e.args) > 0: status_message = str(e.args[0]) gateway_model.status_message = status_message + switch_gateway_status(session, gateway_model, GatewayStatus.FAILED) except Exception as e: logger.exception("%s: got exception when creating gateway compute", fmt(gateway_model)) - gateway_model.status = GatewayStatus.FAILED gateway_model.status_message = f"Unexpected error: {repr(e)}" + switch_gateway_status(session, gateway_model, GatewayStatus.FAILED) async def _process_provisioning_gateway( @@ -179,18 +171,18 @@ async def _process_provisioning_gateway( gateway_model.gateway_compute ) if connection is None: - gateway_model.status = GatewayStatus.FAILED gateway_model.status_message = "Failed to connect to gateway" + switch_gateway_status(session, gateway_model, GatewayStatus.FAILED) gateway_model.gateway_compute.deleted = True return try: await gateways_services.configure_gateway(connection) except Exception: logger.exception("%s: failed to configure gateway", fmt(gateway_model)) - gateway_model.status = GatewayStatus.FAILED gateway_model.status_message = "Failed to configure gateway" + switch_gateway_status(session, gateway_model, GatewayStatus.FAILED) await gateway_connections_pool.remove(gateway_model.gateway_compute.ip_address) gateway_model.gateway_compute.active = False return - gateway_model.status = GatewayStatus.RUNNING + switch_gateway_status(session, gateway_model, GatewayStatus.RUNNING) diff --git a/src/dstack/_internal/server/routers/gateways.py b/src/dstack/_internal/server/routers/gateways.py index fb03a3d69c..0f89e5db45 100644 --- a/src/dstack/_internal/server/routers/gateways.py +++ b/src/dstack/_internal/server/routers/gateways.py @@ -72,11 +72,12 @@ async def delete_gateways( session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): - _, project = user_project + user, project = user_project await gateways.delete_gateways( session=session, project=project, gateways_names=body.names, + user=user, ) @@ -86,8 +87,8 @@ async def set_default_gateway( session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): - _, project = user_project - await gateways.set_default_gateway(session=session, project=project, name=body.name) + user, project = user_project + await gateways.set_default_gateway(session=session, project=project, name=body.name, user=user) @router.post("/set_wildcard_domain", response_model=models.Gateway) @@ -96,9 +97,13 @@ async def set_gateway_wildcard_domain( session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): - _, project = user_project + user, project = user_project return CustomORJSONResponse( await gateways.set_gateway_wildcard_domain( - session=session, project=project, name=body.name, wildcard_domain=body.wildcard_domain + session=session, + project=project, + name=body.name, + wildcard_domain=body.wildcard_domain, + user=user, ) ) diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index cf41b53973..bff20466a8 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -1,6 +1,8 @@ import asyncio import datetime import uuid +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from datetime import timedelta from functools import partial from typing import List, Optional, Sequence @@ -45,6 +47,7 @@ ProjectModel, UserModel, ) +from dstack._internal.server.services import events from dstack._internal.server.services.backends import ( check_backend_type_available, get_project_backend_by_type_or_error, @@ -66,6 +69,24 @@ logger = get_logger(__name__) +def switch_gateway_status( + session: AsyncSession, + gateway_model: GatewayModel, + new_status: GatewayStatus, + actor: events.AnyActor = events.SystemActor(), +): + old_status = gateway_model.status + if old_status == new_status: + return + + gateway_model.status = new_status + + msg = f"Gateway status changed {old_status.upper()} -> {new_status.upper()}" + if gateway_model.status_message is not None: + msg += f" ({gateway_model.status_message})" + events.emit(session, msg, actor=actor, targets=[events.Target.from_model(gateway_model)]) + + GATEWAY_CONNECT_ATTEMPTS = 30 GATEWAY_CONNECT_DELAY = 10 GATEWAY_CONFIGURE_ATTEMPTS = 50 @@ -163,6 +184,7 @@ async def create_gateway( configuration.name = await generate_gateway_name(session=session, project=project) gateway = GatewayModel( + id=uuid.uuid4(), name=configuration.name, region=configuration.region, project_id=project.id, @@ -173,11 +195,19 @@ async def create_gateway( last_processed_at=get_current_datetime(), ) session.add(gateway) + events.emit( + session, + f"Gateway created. Status: {gateway.status.upper()}", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(gateway)], + ) await session.commit() default_gateway = await get_project_default_gateway_model(session=session, project=project) if default_gateway is None or configuration.default: - await set_default_gateway(session=session, project=project, name=configuration.name) + await set_default_gateway( + session=session, project=project, name=configuration.name, user=user + ) return gateway_model_to_gateway(gateway) @@ -214,6 +244,7 @@ async def delete_gateways( session: AsyncSession, project: ProjectModel, gateways_names: List[str], + user: UserModel, ): res = await session.execute( select(GatewayModel).where( @@ -273,46 +304,51 @@ async def delete_gateways( gateway_model.gateway_compute.deleted = True session.add(gateway_model.gateway_compute) await session.delete(gateway_model) + events.emit( + session, + "Gateway deleted", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(gateway_model)], + ) await session.commit() async def set_gateway_wildcard_domain( - session: AsyncSession, project: ProjectModel, name: str, wildcard_domain: Optional[str] + session: AsyncSession, + project: ProjectModel, + name: str, + wildcard_domain: Optional[str], + user: UserModel, ) -> Gateway: - gateway = await get_project_gateway_model_by_name( - session=session, - project=project, - name=name, - ) - if gateway is None: - raise ResourceNotExistsError() - if gateway.backend.type == BackendType.DSTACK: - raise ServerClientError("Custom domains for dstack Sky gateway are not supported") - await session.execute( - update(GatewayModel) - .where( - GatewayModel.project_id == project.id, - GatewayModel.name == name, - ) - .values( - wildcard_domain=wildcard_domain, - ) - ) - await session.commit() - gateway = await get_project_gateway_model_by_name( - session=session, - project=project, - name=name, - ) - if gateway is None: - raise ResourceNotExistsError() + async with get_project_gateway_model_by_name_for_update( + session=session, project=project, name=name + ) as gateway: + if gateway is None: + raise ResourceNotExistsError() + if gateway.backend.type == BackendType.DSTACK: + raise ServerClientError("Custom domains for dstack Sky gateway are not supported") + old_domain = gateway.wildcard_domain + if old_domain != wildcard_domain: + gateway.wildcard_domain = wildcard_domain + events.emit( + session, + f"Gateway wildcard domain changed {old_domain!r} -> {gateway.wildcard_domain!r}", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(gateway)], + ) + await session.commit() return gateway_model_to_gateway(gateway) -async def set_default_gateway(session: AsyncSession, project: ProjectModel, name: str): +async def set_default_gateway( + session: AsyncSession, project: ProjectModel, name: str, user: Optional[UserModel] +): gateway = await get_project_gateway_model_by_name(session=session, project=project, name=name) if gateway is None: raise ResourceNotExistsError() + if project.default_gateway_id == gateway.id: + return + previous_gateway = await get_project_default_gateway_model(session, project) await session.execute( update(ProjectModel) .where( @@ -322,6 +358,19 @@ async def set_default_gateway(session: AsyncSession, project: ProjectModel, name default_gateway_id=gateway.id, ) ) + if previous_gateway is not None: + events.emit( + session, + "Gateway unset as default", + actor=events.UserActor.from_user(user) if user is not None else events.SystemActor(), + targets=[events.Target.from_model(previous_gateway)], + ) + events.emit( + session, + "Gateway set as default", + actor=events.UserActor.from_user(user) if user is not None else events.SystemActor(), + targets=[events.Target.from_model(gateway)], + ) await session.commit() @@ -343,6 +392,38 @@ async def get_project_gateway_model_by_name( return res.scalar() +@asynccontextmanager +async def get_project_gateway_model_by_name_for_update( + session: AsyncSession, project: ProjectModel, name: str +) -> AsyncGenerator[Optional[GatewayModel], None]: + """ + Fetch the gateway from the database and lock it for update. + + **NOTE**: commit changes to the database before exiting from this context manager, + so that in-memory locks are only released after commit. + """ + + filters = [ + GatewayModel.project_id == project.id, + GatewayModel.name == name, + ] + res = await session.execute(select(GatewayModel.id).where(*filters)) + gateway_id = res.scalar_one_or_none() + if gateway_id is None: + yield None + else: + async with get_locker(get_db().dialect_name).lock_ctx( + GatewayModel.__tablename__, [gateway_id] + ): + # Refetch after lock + res = await session.execute( + select(GatewayModel) + .where(GatewayModel.id.in_([gateway_id]), *filters) + .with_for_update(key_share=True, of=GatewayModel) + ) + yield res.scalar_one_or_none() + + async def get_project_default_gateway_model( session: AsyncSession, project: ProjectModel ) -> Optional[GatewayModel]: diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 640a3932dd..cca5212576 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -7,8 +7,9 @@ from uuid import UUID import gpuhunt -from sqlalchemy import select +from sqlalchemy import delete, select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload from dstack._internal.core.backends.base.compute import ( Compute, @@ -1114,8 +1115,16 @@ async def create_secret( async def list_events(session: AsyncSession) -> list[EventModel]: - res = await session.execute(select(EventModel).order_by(EventModel.recorded_at, EventModel.id)) - return list(res.scalars().all()) + res = await session.execute( + select(EventModel) + .order_by(EventModel.recorded_at, EventModel.id) + .options(joinedload(EventModel.targets)) + ) + return list(res.scalars().unique().all()) + + +async def clear_events(session: AsyncSession) -> None: + await session.execute(delete(EventModel)) def get_private_key_string() -> str: diff --git a/src/tests/_internal/server/background/tasks/test_process_gateways.py b/src/tests/_internal/server/background/tasks/test_process_gateways.py index 3460f18cb9..b280b8948d 100644 --- a/src/tests/_internal/server/background/tasks/test_process_gateways.py +++ b/src/tests/_internal/server/background/tasks/test_process_gateways.py @@ -13,6 +13,7 @@ create_gateway, create_gateway_compute, create_project, + list_events, ) @@ -46,6 +47,9 @@ async def test_submitted_to_provisioning(self, test_db, session: AsyncSession): assert gateway.status == GatewayStatus.PROVISIONING assert gateway.gateway_compute is not None assert gateway.gateway_compute.ip_address == "2.2.2.2" + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Gateway status changed SUBMITTED -> PROVISIONING" async def test_marks_gateway_as_failed_if_gateway_creation_errors( self, test_db, session: AsyncSession @@ -71,6 +75,9 @@ async def test_marks_gateway_as_failed_if_gateway_creation_errors( await session.refresh(gateway) assert gateway.status == GatewayStatus.FAILED assert gateway.status_message == "Some error" + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Gateway status changed SUBMITTED -> FAILED (Some error)" @pytest.mark.asyncio @@ -96,6 +103,9 @@ async def test_provisioning_to_running(self, test_db, session: AsyncSession): pool_add.assert_called_once() await session.refresh(gateway) assert gateway.status == GatewayStatus.RUNNING + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Gateway status changed PROVISIONING -> RUNNING" async def test_marks_gateway_as_failed_if_fails_to_connect( self, test_db, session: AsyncSession @@ -119,3 +129,9 @@ async def test_marks_gateway_as_failed_if_fails_to_connect( await session.refresh(gateway) assert gateway.status == GatewayStatus.FAILED assert gateway.status_message == "Failed to connect to gateway" + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message + == "Gateway status changed PROVISIONING -> FAILED (Failed to connect to gateway)" + ) diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index 70f6b22b7e..f80537a1b1 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -10,12 +10,14 @@ from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( ComputeMockSpec, + clear_events, create_backend, create_gateway, create_gateway_compute, create_project, create_user, get_auth_headers, + list_events, ) from dstack._internal.server.testing.matchers import SomeUUID4Str @@ -218,6 +220,8 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn "tags": None, }, } + events = await list_events(session) + assert events[0].message == "Gateway created. Status: SUBMITTED" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -273,6 +277,8 @@ async def test_create_gateway_without_name( "tags": None, }, } + events = await list_events(session) + assert events[0].message == "Gateway created. Status: SUBMITTED" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -337,6 +343,7 @@ async def test_set_default_gateway(self, test_db, session: AsyncSession, client: project_id=project.id, backend_id=backend.id, gateway_compute_id=gateway_compute.id, + name="first_gateway", ) response = await client.post( f"/api/project/{project.name}/gateways/set_default", @@ -378,6 +385,40 @@ async def test_set_default_gateway(self, test_db, session: AsyncSession, client: "tags": None, }, } + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Gateway set as default" + + second_gateway_compute = await create_gateway_compute( + session=session, + backend_id=backend.id, + ) + second_gateway = await create_gateway( + session=session, + project_id=project.id, + backend_id=backend.id, + gateway_compute_id=second_gateway_compute.id, + name="second_gateway", + ) + await clear_events(session) + response = await client.post( + f"/api/project/{project.name}/gateways/set_default", + json={"name": second_gateway.name}, + headers=get_auth_headers(user.token), + ) + assert response.status_code == 200 + events = await list_events(session) + assert len(events) == 2 + actual_events = [(e.targets[0].entity_name, e.message) for e in events] + expected_events = [ + ("first_gateway", "Gateway unset as default"), + ("second_gateway", "Gateway set as default"), + ] + assert ( + actual_events == expected_events + # in case events are emitted exactly at the same time + or actual_events == expected_events[::-1] + ) @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -504,6 +545,10 @@ def get_backend(project, backend_type): }, } ] + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Gateway deleted" + assert events[0].targets[0].entity_name == "gateway-aws" class TestUpdateGateway: @@ -541,10 +586,11 @@ async def test_set_wildcard_domain(self, test_db, session: AsyncSession, client: project_id=project.id, backend_id=backend.id, gateway_compute_id=gateway_compute.id, + wildcard_domain="old.example", ) response = await client.post( f"/api/project/{project.name}/gateways/set_wildcard_domain", - json={"name": gateway.name, "wildcard_domain": "test.com"}, + json={"name": gateway.name, "wildcard_domain": "new.example"}, headers=get_auth_headers(user.token), ) assert response.status_code == 200 @@ -560,7 +606,7 @@ async def test_set_wildcard_domain(self, test_db, session: AsyncSession, client: "hostname": gateway_compute.ip_address, "name": gateway.name, "region": gateway.region, - "wildcard_domain": "test.com", + "wildcard_domain": "new.example", "configuration": { "type": "gateway", "name": gateway.name, @@ -568,13 +614,18 @@ async def test_set_wildcard_domain(self, test_db, session: AsyncSession, client: "region": gateway.region, "instance_type": None, "router": None, - "domain": "test.com", + "domain": "new.example", "default": False, "public_ip": True, "certificate": {"type": "lets-encrypt"}, "tags": None, }, } + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Gateway wildcard domain changed 'old.example' -> 'new.example'" + ) @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) From dde4a07055f77470048a61a195281d5583a18872 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Tue, 27 Jan 2026 13:56:37 +0100 Subject: [PATCH 029/640] Docs minor improvements (#3501) * [Docs] Improved documentation structure (WIP) - [x] Introduced `More` under `Concepts` - [x] Moved `Metrics` to `Concepts` - [x] Improved `Installation` (removed SSH fleets - only keep it in `Backends`; moved `Configure` after `Set up the server`) - [x] Mention `server restart is required after updating server/config.yml` in `Backends` - [x] Improved `Distributed tasks` (structure; links to `Fleets` and `Examples`) * [Docs] Documentation improvements - [x] Improved `Fleets` documentation - [x] Minor improvements of the `Tasks` page under `Concepts` - [x] Minor improvements on the home page * [Docs] Minor updates to `README.md`, `Overview`, `Fleets`, `Quickstart`, and examples --- README.md | 13 +- docs/blog/posts/gpu-health-checks.md | 4 +- docs/blog/posts/metrics-ui.md | 2 +- docs/blog/posts/prometheus.md | 4 +- docs/docs/concepts/backends.md | 21 +- docs/docs/concepts/fleets.md | 511 ++++++++---------- docs/docs/{guides => concepts}/metrics.md | 0 docs/docs/concepts/tasks.md | 23 +- docs/docs/guides/protips.md | 2 +- docs/docs/guides/troubleshooting.md | 2 +- docs/docs/index.md | 8 +- docs/docs/installation/index.md | 31 +- docs/docs/quickstart.md | 10 +- docs/overrides/home.html | 43 +- examples/clusters/nccl-rccl-tests/README.md | 2 +- .../distributed-training/axolotl/README.md | 2 +- .../distributed-training/ray-ragen/README.md | 2 +- examples/distributed-training/trl/README.md | 2 +- mkdocs.yml | 25 +- 19 files changed, 325 insertions(+), 382 deletions(-) rename docs/docs/{guides => concepts}/metrics.md (100%) diff --git a/README.md b/README.md index 71d8d1a8b7..bbba8e136a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It streamlines development, training, and inference, and is compatible with any hardware, open-source tools, and frameworks. -#### Hardware +#### Accelerators `dstack` supports `NVIDIA`, `AMD`, `Google TPU`, `Intel Gaudi`, and `Tenstorrent` accelerators out of the box. @@ -46,7 +46,7 @@ It streamlines development, training, and inference, and is compatible with any ##### Configure backends -To orchestrate compute across cloud providers or existing Kubernetes clusters, you need to configure backends. +To orchestrate compute across GPU clouds or Kubernetes clusters, you need to configure backends. Backends can be set up in `~/.dstack/server/config.yml` or through the [project settings page](https://dstack.ai/docs/concepts/projects#backends) in the UI. @@ -123,12 +123,11 @@ Configuration is updated at ~/.dstack/config.yml `dstack` supports the following configurations: -* [Dev environments](https://dstack.ai/docs/dev-environments) — for interactive development using a desktop IDE -* [Tasks](https://dstack.ai/docs/tasks) — for scheduling jobs (incl. distributed jobs) or running web apps -* [Services](https://dstack.ai/docs/services) — for deployment of models and web apps (with auto-scaling and authorization) -* [Fleets](https://dstack.ai/docs/fleets) — for managing cloud and on-prem clusters +* [Fleets](https://dstack.ai/docs/concepts/fleets) — for managing cloud and on-prem clusters +* [Dev environments](https://dstack.ai/docs/concepts/dev-environments) — for interactive development using a desktop IDE +* [Tasks](https://dstack.ai/docs/concepts/tasks) — for scheduling jobs (incl. distributed jobs) or running web apps +* [Services](https://dstack.ai/docs/concepts/services) — for deployment of models and web apps (with auto-scaling and authorization) * [Volumes](https://dstack.ai/docs/concepts/volumes) — for managing persisted volumes -* [Gateways](https://dstack.ai/docs/concepts/gateways) — for configuring the ingress traffic and public endpoints Configuration can be defined as YAML files within your repo. diff --git a/docs/blog/posts/gpu-health-checks.md b/docs/blog/posts/gpu-health-checks.md index c10557e753..b864e77855 100644 --- a/docs/blog/posts/gpu-health-checks.md +++ b/docs/blog/posts/gpu-health-checks.md @@ -12,7 +12,7 @@ categories: In large-scale training, a single bad GPU can derail progress. Sometimes the failure is obvious — jobs crash outright. Other times it’s subtle: correctable memory errors, intermittent instability, or thermal throttling that quietly drags down throughput. In big experiments, these issues can go unnoticed for hours or days, wasting compute and delaying results. -`dstack` already supports GPU telemetry monitoring through NVIDIA DCGM [metrics](../../docs/guides/metrics.md), covering utilization, memory, and temperature. This release extends that capability with passive hardware health checks powered by DCGM [background health checks](https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/feature-overview.html#background-health-checks). With these, `dstack` continuously evaluates fleet GPUs for hardware reliability and displays their status before scheduling workloads. +`dstack` already supports GPU telemetry monitoring through NVIDIA DCGM [metrics](../../docs/concepts/metrics.md), covering utilization, memory, and temperature. This release extends that capability with passive hardware health checks powered by DCGM [background health checks](https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/feature-overview.html#background-health-checks). With these, `dstack` continuously evaluates fleet GPUs for hardware reliability and displays their status before scheduling workloads. @@ -69,5 +69,5 @@ If you have experience with GPU reliability or ideas for automated recovery, joi !!! info "What's next?" 1. Check [Quickstart](../../docs/quickstart.md) 2. Explore the [clusters](../../docs/guides/clusters.md) guide - 3. Learn more about [metrics](../../docs/guides/metrics.md) + 3. Learn more about [metrics](../../docs/concepts/metrics.md) 4. Join [Discord](https://discord.gg/u8SmfwPpMd) diff --git a/docs/blog/posts/metrics-ui.md b/docs/blog/posts/metrics-ui.md index db21cf019a..877ae9fca8 100644 --- a/docs/blog/posts/metrics-ui.md +++ b/docs/blog/posts/metrics-ui.md @@ -53,6 +53,6 @@ For persistent storage and long-term access to metrics, we still recommend setti metrics from `dstack`. !!! info "What's next?" - 1. See [Metrics](../../docs/guides/metrics.md) + 1. See [Metrics](../../docs/concepts/metrics.md) 2. Check [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md) 3. Join [Discord](https://discord.gg/u8SmfwPpMd) diff --git a/docs/blog/posts/prometheus.md b/docs/blog/posts/prometheus.md index 8a4d579c04..08aecb4cf5 100644 --- a/docs/blog/posts/prometheus.md +++ b/docs/blog/posts/prometheus.md @@ -45,7 +45,7 @@ Overall, `dstack` collects three groups of metrics: | **Runs** | Run metrics include run counters for each user in each project. | | **Jobs** | A run consists of one or more jobs, each mapped to a container. Job metrics offer insights into execution time, cost, GPU model, NVIDIA DCGM telemetry, and more. | -For a full list of available metrics and labels, check out [Metrics](../../docs/guides/metrics.md). +For a full list of available metrics and labels, check out [Metrics](../../docs/concepts/metrics.md). ??? info "NVIDIA" NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends, @@ -59,7 +59,7 @@ For a full list of available metrics and labels, check out [Metrics](../../docs/ only accessible through the UI and the [`dstack metrics`](dstack-metrics.md) CLI. !!! info "What's next?" - 1. See [Metrics](../../docs/guides/metrics.md) + 1. See [Metrics](../../docs/concepts/metrics.md) 1. Check [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md) diff --git a/docs/docs/concepts/backends.md b/docs/docs/concepts/backends.md index 9a1c90ec5c..572d4e0411 100644 --- a/docs/docs/concepts/backends.md +++ b/docs/docs/concepts/backends.md @@ -1,21 +1,22 @@ # Backends -Backends allow `dstack` to provision fleets across cloud providers or Kubernetes clusters. +Backends allow `dstack` to provision fleets across GPU clouds or Kubernetes clusters. `dstack` supports two types of backends: * [VM-based](#vm-based) – use `dstack`'s native integration with cloud providers to provision VMs, manage clusters, and orchestrate container-based runs. * [Container-based](#container-based) – use either `dstack`'s native integration with cloud providers or Kubernetes to orchestrate container-based runs; provisioning in this case is delegated to the cloud provider or Kubernetes. -??? info "SSH fleets" +!!! info "SSH fleets" When using `dstack` with on-prem servers, backend configuration isn’t required. Simply create [SSH fleets](../concepts/fleets.md#ssh-fleets) once the server is up. Backends can be configured via `~/.dstack/server/config.yml` or through the [project settings page](../concepts/projects.md#backends) in the UI. See the examples of backend configuration below. +> If you update `~/.dstack/server/config.yml`, you have to restart the server. + ## VM-based -VM-based backends allow `dstack` users to manage clusters and orchestrate container-based runs across a wide range of cloud providers. -Under the hood, `dstack` uses native integrations with these providers to provision clusters on demand. +VM-based backends allow `dstack` users to manage clusters and orchestrate container-based runs across a wide range of cloud providers. Under the hood, `dstack` uses native integrations with these providers to provision clusters on demand. Compared to [container-based](#container-based) backends, this approach offers finer-grained, simpler control over cluster provisioning and eliminates the dependency on a Kubernetes layer. @@ -1036,9 +1037,13 @@ projects: No additional setup is required — `dstack` configures and manages the proxy automatically. -??? info "NVIDIA GPU Operator" - For `dstack` to correctly detect GPUs in your Kubernetes cluster, the cluster must have the - [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/index.html) pre-installed. +??? info "Required operators" + === "NVIDIA" + For `dstack` to correctly detect GPUs in your Kubernetes cluster, the cluster must have the + [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/index.html) pre-installed. + === "AMD" + For `dstack` to correctly detect GPUs in your Kubernetes cluster, the cluster must have the + [AMD GPU Operator](https://github.com/ROCm/gpu-operator) pre-installed. 100% +
- FLEET INSTANCE BACKEND GPU PRICE STATUS CREATED - my-fleet - - - - - - -``` + ```shell + $ dstack apply -f fleet.dstack.yml + + # BACKEND REGION RESOURCES SPOT PRICE + 1 gcp us-west4 2xCPU, 8GB, 100GB (disk) yes $0.010052 + 2 azure westeurope 2xCPU, 8GB, 100GB (disk) yes $0.0132 + 3 gcp europe-central2 2xCPU, 8GB, 100GB (disk) yes $0.013248 -
+ Create the fleet? [y/n]: y -If `nodes` is a range that starts above `0`, `dstack` pre-creates the initial number of instances up front, while any additional ones are created on demand. + FLEET INSTANCE BACKEND GPU PRICE STATUS CREATED + my-fleet 0 gcp (europe-west-1) L4:24GB (spot) $0.1624 idle 3 mins ago + 1 gcp (europe-west-1) L4:24GB (spot) $0.1624 idle 3 mins ago + ``` -> Setting the `nodes` range to start above `0` is supported only for [VM-based backends](backends.md#vm-based). + -??? info "Target number of nodes" + If the `nodes` range starts with `0`, `dstack apply` creates only a template. Instances are provisioned only when you submit runs. - If `nodes` is defined as a range, you can start with more than the minimum number of instances by using the `target` parameter when creating the fleet. +=== "SSH fleets" + If you have a group of on-prem servers accessible via SSH, you can create an SSH fleet as follows:
- + ```yaml type: fleet - name: my-fleet + + # Uncomment if instances are interconnected + #placement: cluster - nodes: - min: 0 - max: 2 - - # Provision 2 instances initially - target: 2 - - # Deprovision instances above the minimum if they remain idle - idle_duration: 1h + ssh_config: + user: ubuntu + identity_file: ~/.ssh/id_rsa + hosts: + - 3.255.177.51 + - 3.255.177.52 ``` - +
-By default, when you submit a [dev environment](dev-environments.md), [task](tasks.md), or [service](services.md), `dstack` tries all available fleets. However, you can explicitly specify the [`fleets`](../reference/dstack.yml/dev-environment.md#fleets) in your run configuration -or via [`--fleet`](../reference/cli/dstack/apply.md#fleet) with `dstack apply`. + Pass the fleet configuration to `dstack apply`: -### Configuration options +
-#### Placement { #backend-placement } + ```shell + $ dstack apply -f fleet.dstack.yml + + Provisioning... + ---> 100% -To ensure instances are interconnected (e.g., for -[distributed tasks](tasks.md#distributed-tasks)), set `placement` to `cluster`. -This ensures all instances are provisioned with optimal inter-node connectivity. + FLEET INSTANCE BACKEND GPU PRICE STATUS CREATED + my-fleet 0 ssh (remote) L4:24GB $0 idle 3 mins ago + 1 ssh (remote) L4:24GB $0 idle 3 mins ago + ``` -??? info "AWS" - When you create a fleet with AWS, [Elastic Fabric Adapter networking](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html) is automatically configured if it’s supported for the corresponding instance type. - Note, EFA requires the `public_ips` to be set to `false` in the `aws` backend configuration. - Otherwise, instances are only connected by the default VPC subnet. +
- Refer to the [AWS](../../examples/clusters/aws/index.md) example for more details. + `dstack apply` automatically connects to on-prem servers, installs the required dependencies, and adds them to the created fleet. -??? info "GCP" - When you create a fleet with GCP, `dstack` automatically configures [GPUDirect-TCPXO and GPUDirect-TCPX](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx-autopilot) networking for the A3 Mega and A3 High instance types, as well as RoCE networking for the A4 instance type. + ??? info "Host requirements" + 1. Hosts must be pre-installed with Docker. - !!! info "Backend configuration" - You may need to configure `extra_vpcs` and `roce_vpcs` in the `gcp` backend configuration. - Refer to the [GCP](../../examples/clusters/gcp/index.md) examples for more details. + === "NVIDIA" + 2. Hosts with NVIDIA GPUs must also be pre-installed with CUDA 12.1 and + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). -??? info "Nebius" - When you create a fleet with Nebius, [InfiniBand networking](https://docs.nebius.com/compute/clusters/gpu) is automatically configured if it’s supported for the corresponding instance type. - Otherwise, instances are only connected by the default VPC subnet. + === "AMD" + 2. Hosts with AMD GPUs must also be pre-installed with AMDGPU-DKMS kernel driver (e.g. via + [native package manager](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/native-install/index.html) + or [AMDGPU installer](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/amdgpu-install.html).) - An InfiniBand fabric for the cluster is selected automatically. If you prefer to use some specific fabrics, configure them in the - [backend settings](../reference/server/config.yml.md#nebius). + === "Intel Gaudi" + 2. Hosts with Intel Gaudi accelerators must be pre-installed with [Gaudi software and drivers](https://docs.habana.ai/en/latest/Installation_Guide/Driver_Installation.html#driver-installation). + This must include the drivers, `hl-smi`, and Habana Container Runtime. -The `cluster` placement is supported for `aws`, `azure`, `gcp`, `nebius`, `oci`, and `vultr` -backends. + === "Tenstorrent" + 2. Hosts with Tenstorrent accelerators must be pre-installed with [Tenstorrent software](https://docs.tenstorrent.com/getting-started/README.html#software-installation). + This must include the drivers, `tt-smi`, and HugePages. -> For more details on optimal inter-node connectivity, read the [Clusters](../guides/clusters.md) guide. + 3. The user specified must have passwordless `sudo` access. - + 4. The SSH server must be running and configured with `AllowTcpForwarding yes` in `/etc/ssh/sshd_config`. -#### Resources + 5. The firewall must allow SSH and should forbid any other connections from external networks. For `placement: cluster` fleets, it should also allow any communication between fleet nodes. -When you specify a resource value like `cpu` or `memory`, -you can either use an exact value (e.g. `24GB`) or a -range (e.g. `24GB..`, or `24GB..80GB`, or `..80GB`). +> Once the fleet is created, you can run [dev environments](dev-environments.md), [tasks](tasks.md), and [services](services.md). -
+## Configuration options -```yaml -type: fleet -# The name is optional, if not specified, generated randomly -name: my-fleet +Backend fleets support [many options](../reference/dstack.yml/fleet.md); see some major configuration examples below. -nodes: 2 +### Cluster placement -resources: - # 200GB or more RAM - memory: 200GB.. - # 4 GPUs from 40GB to 80GB - gpu: 40GB..80GB:4 - # Disk size - disk: 500GB -``` +Both [backend fleets](#backend-fleet) and [SSH fleets](#ssh-fleet) allow the `placement` property to be set to `cluster`. -
- -The `gpu` property allows specifying not only memory size but also GPU vendor, names -and their quantity. Examples: `nvidia` (one NVIDIA GPU), `A100` (one A100), `A10G,A100` (either A10G or A100), -`A100:80GB` (one A100 of 80GB), `A100:2` (two A100), `24GB..40GB:2` (two GPUs between 24GB and 40GB), -`A100:40GB:2` (two A100 GPUs of 40GB). +This property ensures that instances are interconnected. This is required for running [distributed tasks](tasks.md#distributed-tasks). -??? info "Google Cloud TPU" - To use TPUs, specify its architecture via the `gpu` property. +=== "Backend fleets" + Backend fleets allow to provision interconnected clusters across supported backends. +
+ ```yaml type: fleet - # The name is optional, if not specified, generated randomly name: my-fleet nodes: 2 - + placement: cluster + resources: - gpu: v2-8 + gpu: H100:8 ``` + +
- Currently, only 8 TPU cores can be specified, supporting single TPU device workloads. Multi-TPU support is coming soon. - -> If you’re unsure which offers (hardware configurations) are available from the configured backends, use the -> [`dstack offer`](../reference/cli/dstack/offer.md#list-gpu-offers) command to list them. - -#### Blocks { #backend-blocks } - -For backend fleets, `blocks` function the same way as in SSH fleets. -See the [`Blocks`](#ssh-blocks) section under SSH fleets for details on the blocks concept. - -
- -```yaml -type: fleet - -name: my-fleet + #### Backends -resources: - gpu: NVIDIA:80GB:8 + Fast interconnect is supported on the `aws`, `gcp`, `nebius`, `kubernetes`, and `runpod` backends. Some backends may require additional configuration. -# Split into 4 blocks, each with 2 GPUs -blocks: 4 -``` + === "AWS" + On AWS, `dstack` requires `public_ips` to be set to `false` in the backend configuration. + Refer to the [AWS](../../examples/clusters/aws/index.md) example for more details. -
+ === "GCP" + On GCP, you may need to configure `extra_vpcs` and `roce_vpcs` in the `gcp` backend configuration. + Refer to the [GCP](../../examples/clusters/gcp/index.md) examples for more details. -#### Idle duration + === "Nebius" + On [Nebius](https://docs.nebius.com/compute/clusters/gpu), `dstack` automatically configures InfiniBand networking if it is supported by the selected instance type. -By default, fleet instances stay `idle` for 3 days and can be reused within that time. -If an instance is not reused within this period, it is automatically terminated. + === "Kubernetes" + If the Kubernetes cluster has interconnect configured, `dstack` can use it without additional setup. + See the [Lambda](../../examples/clusters/lambda/index.md#kubernetes) or [Crusoe](../../examples/clusters/crusoe/index.md#kubernetes) examples. + + === "Runpod" + On [Runpod](https://docs.runpod.io/instant-clusters), `dstack` automatically configures InfiniBand networking if it is supported by the selected instance type. + + > See the [Clusters](../../examples.md#clusters) examples. -To change the default idle duration, set -[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) in the fleet configuration (e.g., `0s`, `1m`, or `off` for -unlimited). +=== "SSH fleets" + If the hosts in the SSH fleet have interconnect configured, you only need to set `placement` to `cluster`. -
- +
+ ```yaml type: fleet - # The name is optional, if not specified, generated randomly name: my-fleet - - nodes: 2 - # Terminate instances idle for more than 1 hour - idle_duration: 1h - - resources: - gpu: 24GB - ``` - -
+ placement: cluster -#### Spot policy + ssh_config: + user: ubuntu + identity_file: ~/.ssh/id_rsa + hosts: + - 3.255.177.51 + - 3.255.177.52 + ``` + +
-By default, `dstack` uses on-demand instances. However, you can change that -via the [`spot_policy`](../reference/dstack.yml/fleet.md#spot_policy) property. It accepts `spot`, `on-demand`, and `auto`. + !!! info "Network" + By default, `dstack` automatically detects the network shared by the hosts. However, it's possible to configure it explicitly via the [`network`](../reference/dstack.yml/fleet.md#network) property. -#### Retry policy + -By default, if `dstack` fails to provision an instance or an instance is interrupted, no retry is attempted. +### Nodes -If you'd like `dstack` to do it, configure the -[retry](../reference/dstack.yml/fleet.md#retry) property accordingly: +The `nodes` property is supported only by backend fleets and specifies how many nodes `dstack` must or can provision. -
+
```yaml type: fleet -# The name is optional, if not specified, generated randomly name: my-fleet -nodes: 1 +# Allow to provision of up to 2 instances +nodes: 0..2 -resources: - gpu: 24GB +# Uncomment to ensure instances are inter-connected +#placement: cluster + +# Deprovision instances above the minimum if they remain idle +idle_duration: 1h -retry: - # Retry on specific events - on_events: [no-capacity, interruption] - # Retry for up to 1 hour - duration: 1h +resources: + # Allow to provision up to 8 GPUs + gpu: 0..8 ```
-!!! info "Reference" - Backend fleets support many more configuration options, - incl. [`backends`](../reference/dstack.yml/fleet.md#backends), - [`regions`](../reference/dstack.yml/fleet.md#regions), - [`max_price`](../reference/dstack.yml/fleet.md#max_price), and - among [others](../reference/dstack.yml/fleet.md). +#### Pre-provisioning -## SSH fleets +If the `nodes` range starts with `0`, `dstack apply` creates only a template, and instances are provisioned when you submit runs. -If you have a group of on-prem servers accessible via SSH, you can create an SSH fleet. +To provision instances up front, set the `nodes` range to start above `0`. This pre-creates the initial number of instances; additional instances (if any) are provisioned on demand. -### Apply a configuration -Define a fleet configuration as a YAML file in your project directory. The file must have a -`.dstack.yml` extension (e.g. `.dstack.yml` or `fleet.dstack.yml`). +
+ + ```yaml + type: fleet + name: my-fleet + + nodes: 2..10 -
+ # Uncomment to ensure instances are inter-connected + #placement: cluster + resources: + gpu: H100:8 + ``` + +
+ +Pre-provisioning is supported only for [VM-based backends](backends.md#vm-based). + +??? info "Target number" + To pre-provision more than the minimum number of instances, set the `target` parameter. + +
+ ```yaml type: fleet - # The name is optional, if not specified, generated randomly name: my-fleet - # Uncomment if instances are interconnected - #placement: cluster + nodes: + min: 2 + max: 10 + target: 6 - # SSH credentials for the on-prem servers - ssh_config: - user: ubuntu - identity_file: ~/.ssh/id_rsa - hosts: - - 3.255.177.51 - - 3.255.177.52 + # Deprovision instances above the minimum if they remain idle + idle_duration: 1h ``` - -
-??? info "Requirements" - 1. Hosts must be pre-installed with Docker. +
- === "NVIDIA" - 2. Hosts with NVIDIA GPUs must also be pre-installed with CUDA 12.1 and - [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + `dstack apply` pre-provisions up to `target` and scales back to `min` after `idle_duration`. - === "AMD" - 2. Hosts with AMD GPUs must also be pre-installed with AMDGPU-DKMS kernel driver (e.g. via - [native package manager](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/native-install/index.html) - or [AMDGPU installer](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/amdgpu-install.html).) +### Resources - === "Intel Gaudi" - 2. Hosts with Intel Gaudi accelerators must be pre-installed with [Gaudi software and drivers](https://docs.habana.ai/en/latest/Installation_Guide/Driver_Installation.html#driver-installation). - This must include the drivers, `hl-smi`, and Habana Container Runtime. +Backend fleets allow you to specify the resource requirements for the instances to be provisioned. The `resources` property syntax is the same as for [run configurations](dev-environments.md#resources). - === "Tenstorrent" - 2. Hosts with Tenstorrent accelerators must be pre-installed with [Tenstorrent software](https://docs.tenstorrent.com/getting-started/README.html#software-installation). - This must include the drivers, `tt-smi`, and HugePages. +> Not directly related, but in addition to `resources`, you can specify [`spot_policy`](../reference/dstack.yml/fleet.md#instance_types), [`instance_types`](../reference/dstack.yml/fleet.md#instance_types), [`max_price`](../reference/dstack.yml/fleet.md#max_price), [`region`](../reference/dstack.yml/fleet.md#max_price), and other [options](../reference/dstack.yml/fleet.md#). - 3. The user specified must have passwordless `sudo` access. + - 4. The SSH server must be running and configured with `AllowTcpForwarding yes` in `/etc/ssh/sshd_config`. +### Backends - 5. The firewall must allow SSH and should forbid any other connections from external networks. For `placement: cluster` fleets, it should also allow any communication between fleet nodes. +### Idle duration -To create or update the fleet, pass the fleet configuration to [`dstack apply`](../reference/cli/dstack/apply.md): +By default, instances of a backend fleet stay `idle` for 3 days and can be reused within that time. +If an instance is not reused within this period, it is automatically terminated. -
+To change the default idle duration, set +[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) in the fleet configuration (e.g., `0s`, `1m`, or `off` for +unlimited). -```shell -$ dstack apply -f examples/misc/fleets/.dstack.yml +
+ +```yaml +type: fleet +name: my-fleet -Provisioning... ----> 100% +nodes: 2 + +# Terminate instances idle for more than 1 hour +idle_duration: 1h - FLEET INSTANCE GPU PRICE STATUS CREATED - my-fleet 0 L4:24GB (spot) $0 idle 3 mins ago - 1 L4:24GB (spot) $0 idle 3 mins ago +resources: + gpu: 24GB ```
-When you apply, `dstack` connects to the specified hosts using the provided SSH credentials, -installs the dependencies, and configures these hosts as a fleet. - -Once the status of instances changes to `idle`, they can be used by dev environments, tasks, and services. - -### Configuration options +### Blocks -#### Placement { #ssh-placement } +By default, a job uses the entire instance—e.g., all 8 GPUs. To allow multiple jobs on the same instance, set the `blocks` property to divide the instance. Each job can then use one or more blocks, up to the full instance. -If the hosts are interconnected (i.e. share the same network), set `placement` to `cluster`. -This is required if you'd like to use the fleet for [distributed tasks](tasks.md#distributed-tasks). +=== "Backend fleets" +
-??? info "Network" - By default, `dstack` automatically detects the network shared by the hosts. - However, it's possible to configure it explicitly via - the [`network`](../reference/dstack.yml/fleet.md#network) property. + ```yaml + type: fleet + name: my-fleet - [//]: # (TODO: Provide an example and more detail) + nodes: 0..2 -> For more details on optimal inter-node connectivity, read the [Clusters](../guides/clusters.md) guide. + resources: + gpu: H100:8 -#### Blocks { #ssh-blocks } + # Split into 4 blocks, each with 2 GPUs + blocks: 4 + ``` -By default, a job uses the entire instance—e.g., all 8 GPUs. To allow multiple jobs on the same instance, set the `blocks` property to divide the instance. Each job can then use one or more blocks, up to the full instance. +
-
+=== "SSH fleets" +
```yaml type: fleet @@ -386,7 +356,7 @@ By default, a job uses the entire instance—e.g., all 8 GPUs. To allow multiple blocks: 1 ``` -
+
All resources (GPU, CPU, memory) are split evenly across blocks, while disk is shared. @@ -396,37 +366,16 @@ Set `blocks` to `auto` to match the number of blocks to the number of GPUs. !!! info "Distributed tasks" Distributed tasks require exclusive access to all host resources and therefore must use all blocks on each node. - -#### Environment variables - -If needed, you can specify environment variables that will be used by `dstack-shim` and passed to containers. - -[//]: # (TODO: Explain what dstack-shim is) - -For example, these variables can be used to configure a proxy: - -```yaml -type: fleet -name: my-fleet -env: - - HTTP_PROXY=http://proxy.example.com:80 - - HTTPS_PROXY=http://proxy.example.com:80 - - NO_PROXY=localhost,127.0.0.1 +### SSH config -ssh_config: - user: ubuntu - identity_file: ~/.ssh/id_rsa - hosts: - - 3.255.177.51 - - 3.255.177.52 -``` + #### Proxy jump -If fleet hosts are behind a head node (aka "login node"), configure [`proxy_jump`](../reference/dstack.yml/fleet.md#proxy_jump): +If hosts are behind a head node (aka "login node"), configure [`proxy_jump`](../reference/dstack.yml/fleet.md#proxy_jump): -
+
```yaml type: fleet @@ -446,8 +395,7 @@ If fleet hosts are behind a head node (aka "login node"), configure [`proxy_jump
-To be able to attach to runs, both explicitly with `dstack attach` and implicitly with `dstack apply`, you must either -add a front node key (`~/.ssh/head_node_key`) to an SSH agent or configure a key path in `~/.ssh/config`: +To be able to attach to runs, both explicitly with `dstack attach` and implicitly with `dstack apply`, you must either add a front node key (`~/.ssh/head_node_key`) to an SSH agent or configure a key path in `~/.ssh/config`:
@@ -458,22 +406,33 @@ add a front node key (`~/.ssh/head_node_key`) to an SSH agent or configure a key
-where `Host` must match `ssh_config.proxy_jump.hostname` or `ssh_config.hosts[n].proxy_jump.hostname` if you configure head nodes -on a per-worker basis. +where `Host` must match `ssh_config.proxy_jump.hostname` or `ssh_config.hosts[n].proxy_jump.hostname` if you configure head nodes on a per-worker basis. -!!! info "Reference" - For all SSH fleet configuration options, refer to the [reference](../reference/dstack.yml/fleet.md). +### Environment variables -#### Troubleshooting +If needed, you can specify environment variables that will be automatically passed to any jobs running on this fleet. + +For example, these variables can be used to configure a proxy: + +```yaml +type: fleet +name: my-fleet -!!! info "Resources" - Once the fleet is created, double-check that the GPU, memory, and disk are detected correctly. +env: + - HTTP_PROXY=http://proxy.example.com:80 + - HTTPS_PROXY=http://proxy.example.com:80 + - NO_PROXY=localhost,127.0.0.1 -If the status does not change to `idle` after a few minutes or the resources are not displayed correctly, ensure that -all host requirements are satisfied. +ssh_config: + user: ubuntu + identity_file: ~/.ssh/id_rsa + hosts: + - 3.255.177.51 + - 3.255.177.52 +``` -If the requirements are met but the fleet still fails to be created correctly, check the logs at -`/root/.dstack/shim.log` on the hosts for error details. +!!! info "Reference" + The fleet configuration file supports many more options. See the [reference](../reference/dstack.yml/fleet.md). ## Manage fleets @@ -513,4 +472,6 @@ To terminate and delete specific instances from a fleet, pass `-i INSTANCE_NUM`. !!! info "What's next?" 1. Check [dev environments](dev-environments.md), [tasks](tasks.md), and [services](services.md) - 2. Read the [Clusters](../guides/clusters.md) guide + 2. Read about [Backends](backends.md) guide + 3. Explore the [`.dstack.yml` reference](../reference/dstack.yml/fleet.md) + 4. See the [Clusters](../../examples.md#clusters) example diff --git a/docs/docs/guides/metrics.md b/docs/docs/concepts/metrics.md similarity index 100% rename from docs/docs/guides/metrics.md rename to docs/docs/concepts/metrics.md diff --git a/docs/docs/concepts/tasks.md b/docs/docs/concepts/tasks.md index ac94415d4d..6f3f2fabb7 100644 --- a/docs/docs/concepts/tasks.md +++ b/docs/docs/concepts/tasks.md @@ -135,18 +135,18 @@ resources:
-Nodes can communicate using their private IP addresses. -Use `DSTACK_MASTER_NODE_IP`, `DSTACK_NODES_IPS`, `DSTACK_NODE_RANK`, and other -[System environment variables](#system-environment-variables) for inter-node communication. - -`dstack` is easy to use with `accelerate`, `torchrun`, Ray, Spark, and any other distributed frameworks. +!!! info "Cluster placement" + To submit a distributed task, you must create at least one fleet with a [cluster placement](fleets.md#cluster-placement). + +Jobs on each node communicate using their private IP addresses. Use `DSTACK_MASTER_NODE_IP`, `DSTACK_NODES_IPS`, `DSTACK_NODE_RANK`, and other [system environment variables](#system-environment-variables) for inter-node communication. -!!! info "MPI" - If want to use MPI, you can set `startup_order` to `workers-first` and `stop_criteria` to `master-done`, and use `DSTACK_MPI_HOSTFILE`. - See the [NCCL/RCCL tests](../../examples/clusters/nccl-rccl-tests/index.md) examples. + -> For detailed examples, see [distributed training](../../examples.md#distributed-training) examples. +`dstack` is easy to use with `accelerate`, `torchrun`, Ray, Spark, and any other distributed frameworks. + +> For detailed examples, see the [distributed training](../../examples.md#distributed-training) + and [clusters](../../examples.md#clusters) examples. ??? info "Network interface" Distributed frameworks usually detect the correct network interface automatically, @@ -172,11 +172,6 @@ Use `DSTACK_MASTER_NODE_IP`, `DSTACK_NODES_IPS`, `DSTACK_NODE_RANK`, and other For convenience, `~/.ssh/config` is preconfigured with these options, so a simple `ssh ` is enough. For a list of nodes IPs check the `DSTACK_NODES_IPS` environment variable. -!!! info "Cluster fleets" - To run distributed tasks, you need to create a fleet with [`placement: cluster`](fleets.md#cloud-placement). - -> See the [Clusters](../guides/clusters.md) guide for more details on how to use `dstack` on clusters. - ### Resources When you specify a resource value like `cpu` or `memory`, diff --git a/docs/docs/guides/protips.md b/docs/docs/guides/protips.md index 167b8f1b4b..dfb7abf0b6 100644 --- a/docs/docs/guides/protips.md +++ b/docs/docs/guides/protips.md @@ -482,7 +482,7 @@ The `offer` command allows you to filter and group offers with various [advanced ## Metrics -`dstack` tracks essential metrics accessible via the CLI and UI. To access advanced metrics like DCGM, configure the server to export metrics to Prometheus. See [Metrics](metrics.md) for details. +`dstack` tracks essential metrics accessible via the CLI and UI. To access advanced metrics like DCGM, configure the server to export metrics to Prometheus. See [Metrics](../concepts/metrics.md) for details. ## Service quotas diff --git a/docs/docs/guides/troubleshooting.md b/docs/docs/guides/troubleshooting.md index 5d17b894d0..2b4356fb5c 100644 --- a/docs/docs/guides/troubleshooting.md +++ b/docs/docs/guides/troubleshooting.md @@ -119,7 +119,7 @@ one of these features, `dstack` will only select offers from the backends that s [Instance volumes](../concepts/volumes.md#instance-volumes), and [Privileged containers](../reference/dstack.yml/dev-environment.md#privileged) are supported by all backends except `runpod`, `vastai`, and `kubernetes`. -- [Clusters](../concepts/fleets.md#cloud-placement) +- [Clusters](../concepts/fleets.md#cluster-placement) and [distributed tasks](../concepts/tasks.md#distributed-tasks) are only supported by the `aws`, `azure`, `gcp`, `nebius`, `oci`, and `vultr` backends, as well as SSH fleets. diff --git a/docs/docs/index.md b/docs/docs/index.md index b0228fb2c9..121a379150 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -4,9 +4,8 @@ It streamlines development, training, and inference, and is compatible with any hardware, open-source tools, and frameworks. -#### Hardware - -`dstack` supports `NVIDIA`, `AMD`, `TPU`, `Intel Gaudi`, and `Tenstorrent` accelerators out of the box. +!!! info "Accelerators" + `dstack` supports `NVIDIA`, `AMD`, `TPU`, `Intel Gaudi`, and `Tenstorrent` accelerators out of the box. ## How does it work? @@ -20,12 +19,11 @@ It streamlines development, training, and inference, and is compatible with any `dstack` supports the following configurations: +* [Fleets](concepts/fleets.md) — for managing cloud and on-prem clusters * [Dev environments](concepts/dev-environments.md) — for interactive development using a desktop IDE * [Tasks](concepts/tasks.md) — for scheduling jobs, incl. distributed ones (or running web apps) * [Services](concepts/services.md) — for deploying models (or web apps) -* [Fleets](concepts/fleets.md) — for managing cloud and on-prem clusters * [Volumes](concepts/volumes.md) — for managing network volumes (to persist data) -* [Gateways](concepts/gateways.md) — for publishing services with a custom domain and HTTPS Configuration can be defined as YAML files within your repo. diff --git a/docs/docs/installation/index.md b/docs/docs/installation/index.md index aad8741b66..e179a8e663 100644 --- a/docs/docs/installation/index.md +++ b/docs/docs/installation/index.md @@ -6,15 +6,6 @@ ## Set up the server -### Configure backends - -To orchestrate compute across cloud providers or Kubernetes clusters, you need to configure [backends](../concepts/backends.md). - -??? info "SSH fleets" - When using `dstack` with on-prem servers, backend configuration isn’t required. Simply create [SSH fleets](../concepts/fleets.md#ssh-fleets) once the server is up. - -### Start the server - The server can run on your laptop or any environment with access to the cloud and on-prem clusters you plan to use. === "uv" @@ -72,16 +63,17 @@ The server can run on your laptop or any environment with access to the cloud an
-To verify that backends are properly configured, use the [`dstack offer`](../reference/cli/dstack/offer.md#list-gpu-offers) command to list available GPU offers. +For more details on server deployment options, see the [Server deployment](../guides/server-deployment.md) guide. -!!! info "Server deployment" - For more details on server deployment options, see the [Server deployment](../guides/server-deployment.md) guide. +### Configure backends + +> To orchestrate compute across GPU clouds or Kubernetes clusters, you need to configure [backends](../concepts/backends.md). ## Set up the CLI Once the server is up, you can access it via the `dstack` CLI. -> The CLI can be set up via `pip` or `uv` on Linux, macOS, and Windows. It requires Git and OpenSSH. +> The CLI can be used on Linux, macOS, and Windows. It requires Git and OpenSSH. === "uv" @@ -105,13 +97,15 @@ Once the server is up, you can access it via the `dstack` CLI. ??? info "Windows" To use the CLI on Windows, ensure you've installed Git and OpenSSH via - [Git for Windows:material-arrow-top-right-thin:{ .external }](https://git-scm.com/download/win). + [Git for Windows](https://git-scm.com/download/win). When installing it, ensure you've checked `Git from the command line and also from 3-rd party software` (or `Use Git and optional Unix tools from the Command Prompt`), and `Use bundled OpenSSH`. +### Configure the default project + To point the CLI to the `dstack` server, configure it with the server address, user token, and project name: @@ -130,6 +124,10 @@ Configuration is updated at ~/.dstack/config.yml This configuration is stored in `~/.dstack/config.yml`. +### Check offers + +To verify that both the server and CLI are properly configured, use the [`dstack offer`](../reference/cli/dstack/offer.md#list-gpu-offers) command to list available GPU offers. If you don't see valid offers, ensure you've set up [backends](../concepts/backends.md). + ??? info "Shell autocompletion" `dstack` supports shell autocompletion for `bash` and `zsh`. @@ -195,11 +193,10 @@ This configuration is stored in `~/.dstack/config.yml`. > If you get an error similar to `2: command not found: compdef`, then add the following line to the beginning of your `~/.zshrc` file: > `autoload -Uz compinit && compinit`. - !!! info "What's next?" - 1. Follow [Quickstart](../quickstart.md) - 2. See [Backends](../concepts/backends.md) + 1. See [Backends](../concepts/backends.md) + 2. Follow [Quickstart](../quickstart.md) 3. Check the [server deployment](../guides/server-deployment.md) guide 4. Browse [examples](../../examples.md) 5. Join the community via [Discord](https://discord.gg/u8SmfwPpMd) diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md index 759ec1b573..2a8d3f0610 100644 --- a/docs/docs/quickstart.md +++ b/docs/docs/quickstart.md @@ -1,11 +1,11 @@ # Quickstart -??? info "Prerequsites" +!!! info "Prerequsites" Before using `dstack`, ensure you've [installed](installation/index.md) the server and the CLI. ## Create a fleet -Before you can submit your first run, you have to create a [fleet](concepts/fleets.md). +> Before submitting runs, you must create a [fleet](concepts/fleets.md). === "Backend fleet" If you're using cloud providers or Kubernetes clusters and have configured the corresponding [backends](concepts/backends.md), create a fleet as follows: @@ -49,11 +49,9 @@ Before you can submit your first run, you have to create a [fleet](concepts/flee
- If `nodes` is a range that starts above `0`, `dstack` pre-creates the initial number of instances up front, while any additional ones are created on demand. - - > Setting the `nodes` range to start above `0` is supported only for [VM-based backends](concepts/backends.md#vm-based). + If the `nodes` range starts with `0`, `dstack apply` creates only a template. Instances are provisioned only when you submit runs. - If the fleet needs to be a cluster, the [placement](concepts/fleets.md#backend-placement) property must be set to `cluster`. + If the fleet needs to be a cluster, the [placement](concepts/fleets.md#cluster-placement) property must be set to `cluster`. === "SSH fleet" If you have a group of on-prem servers accessible via SSH, you can create an SSH fleet as follows: diff --git a/docs/overrides/home.html b/docs/overrides/home.html index ced53fb1e8..7cebed7b6a 100644 --- a/docs/overrides/home.html +++ b/docs/overrides/home.html @@ -190,15 +190,6 @@

Native integration with GPU clouds

fill-rule="nonzero" fill="currentColor" class="fill-main"> - - - Kubernetes - - - -

@@ -217,17 +208,17 @@

Easy to use with on-prem clusters

- - Kubernetes + + SSH fleets - - - SSH fleets + + + Kubernetes Single-node & distributed tasks Tasks - - - Clusters - -

@@ -320,7 +305,7 @@

Single-node & distributed tasks

-

Scalable service endpoints

+

Scalable model inference

With dstack, you can easily deploy any model as a secure, @@ -334,6 +319,12 @@

Scalable service endpoints

Services + + + Gateways + +

@@ -596,10 +587,12 @@

Get started in minutes

- + + Installation + +

diff --git a/examples/clusters/nccl-rccl-tests/README.md b/examples/clusters/nccl-rccl-tests/README.md index 7248a4f422..0a2a138b1a 100644 --- a/examples/clusters/nccl-rccl-tests/README.md +++ b/examples/clusters/nccl-rccl-tests/README.md @@ -3,7 +3,7 @@ This example shows how to run [NCCL](https://github.com/NVIDIA/nccl-tests) or [RCCL](https://github.com/ROCm/rccl-tests) tests on a cluster using [distributed tasks](https://dstack.ai/docs/concepts/tasks#distributed-tasks). !!! info "Prerequisites" - Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#backend-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). + Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#cluster-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). ## Running as a task diff --git a/examples/distributed-training/axolotl/README.md b/examples/distributed-training/axolotl/README.md index 9ddd77a363..1454732ad5 100644 --- a/examples/distributed-training/axolotl/README.md +++ b/examples/distributed-training/axolotl/README.md @@ -3,7 +3,7 @@ This example walks you through how to run distributed fine-tune using [Axolotl](https://github.com/axolotl-ai-cloud/axolotl) and [distributed tasks](https://dstack.ai/docs/concepts/tasks#distributed-tasks). !!! info "Prerequisites" - Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#backend-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). + Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#cluster-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). ## Define a configuration diff --git a/examples/distributed-training/ray-ragen/README.md b/examples/distributed-training/ray-ragen/README.md index e79f27f788..32ce0173fd 100644 --- a/examples/distributed-training/ray-ragen/README.md +++ b/examples/distributed-training/ray-ragen/README.md @@ -6,7 +6,7 @@ to fine-tune an agent on multiple nodes. Under the hood `RAGEN` uses [verl](https://github.com/volcengine/verl) for Reinforcement Learning and [Ray](https://docs.ray.io/en/latest/) for distributed training. !!! info "Prerequisites" - Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#backend-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). + Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#cluster-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). ## Run a Ray cluster diff --git a/examples/distributed-training/trl/README.md b/examples/distributed-training/trl/README.md index c6231e5170..9df482da52 100644 --- a/examples/distributed-training/trl/README.md +++ b/examples/distributed-training/trl/README.md @@ -3,7 +3,7 @@ This example walks you through how to run distributed fine-tune using [TRL](https://github.com/huggingface/trl), [Accelerate](https://github.com/huggingface/accelerate) and [Deepspeed](https://github.com/deepspeedai/DeepSpeed). !!! info "Prerequisites" - Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#backend-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). + Before running a distributed task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](https://dstack.ai/docs/concepts/fleets#cluster-placement) or an [SSH fleet](https://dstack.ai/docs/concepts/fleets#ssh-placement)). ## Define a configuration diff --git a/mkdocs.yml b/mkdocs.yml index 07eed5f3b7..5dcdd69999 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,15 +74,13 @@ plugins: - docs/concepts/tasks.md: How to run tasks - for training or fine-tuning, including distributed tasks - docs/concepts/services.md: How to deploy services - for model inference or web apps - docs/concepts/volumes.md: How to manage volumes - for persistent storage or caching + - docs/concepts/gateways.md: How to manage gateways - enabling auto-scaling, rate limits, and custom domains - docs/concepts/secrets.md: How to manage secrets - for API keys or other sensitive data - docs/concepts/projects.md: How to manage projects - for managing separate teams - - docs/concepts/gateways.md: How to manage gateways - enabling auto-scaling, rate limits, and custom domains + - docs/concepts/metrics.md: How to manage gateways - enabling auto-scaling, rate limits, and custom domains Guides: - - docs/guides/clusters.md: How to work with clusters - for distributed tasks - - docs/guides/kubernetes.md: How to work with Kubernetes - docs/guides/server-deployment.md: Detailed guide on how to deploy the dstack server - docs/guides/troubleshooting.md: Common issues and how to troubleshoot them - - docs/guides/metrics.md: How to monitor metrics - docs/guides/protips.md: Pro tips - tips and tricks to use dstack more efficiently Examples: - examples/single-node-training/trl/index.md: TRL @@ -149,8 +147,8 @@ plugins: "blog/data-centers-and-private-clouds.md": "blog/posts/gpu-blocks-and-proxy-jump.md" "blog/distributed-training-with-aws-efa.md": "examples/clusters/aws/index.md" "blog/dstack-stats.md": "blog/posts/dstack-metrics.md" - "docs/concepts/metrics.md": "docs/guides/metrics.md" - "docs/guides/monitoring.md": "docs/guides/metrics.md" + "docs/guides/metrics.md": "docs/concepts/metrics.md" + "docs/guides/monitoring.md": "docs/concepts/metrics.md" "blog/nvidia-and-amd-on-vultr.md.md": "blog/posts/nvidia-and-amd-on-vultr.md" "examples/misc/nccl-tests/index.md": "examples/clusters/nccl-rccl-tests/index.md" "examples/misc/a3high-clusters/index.md": "examples/clusters/gcp/index.md" @@ -268,19 +266,18 @@ nav: - Tasks: docs/concepts/tasks.md - Services: docs/concepts/services.md - Volumes: docs/concepts/volumes.md - - Secrets: docs/concepts/secrets.md - - Projects: docs/concepts/projects.md - - Gateways: docs/concepts/gateways.md + - More: + - Gateways: docs/concepts/gateways.md + - Secrets: docs/concepts/secrets.md + - Projects: docs/concepts/projects.md + - Metrics: docs/concepts/metrics.md - Guides: - - Clusters: docs/guides/clusters.md - - Kubernetes: docs/guides/kubernetes.md - Server deployment: docs/guides/server-deployment.md - Troubleshooting: docs/guides/troubleshooting.md - - Metrics: docs/guides/metrics.md - Protips: docs/guides/protips.md - Upgrade: docs/guides/upgrade.md - - Migration: - - Slurm: docs/guides/migration/slurm.md + - Migration: + - Slurm: docs/guides/migration/slurm.md - Reference: - .dstack.yml: - dev-environment: docs/reference/dstack.yml/dev-environment.md From b93476e187450ecbf0ac729a5c195d278580e1b3 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:36:58 +0000 Subject: [PATCH 030/640] Support secret events in API, CLI, and UI (#3504) --- .../Events/List/hooks/useColumnDefinitions.tsx | 13 +++++++++++++ .../src/pages/Events/List/hooks/useFilters.ts | 9 +++++++++ frontend/src/types/event.d.ts | 5 +++-- src/dstack/_internal/cli/commands/event.py | 11 +++++++++++ src/dstack/_internal/cli/services/events.py | 1 + src/dstack/_internal/core/models/events.py | 1 + src/dstack/_internal/server/routers/events.py | 1 + src/dstack/_internal/server/schemas/events.py | 11 +++++++++++ src/dstack/_internal/server/services/events.py | 17 +++++++++++++++++ src/dstack/api/server/_events.py | 2 ++ 10 files changed, 69 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx index d6e5b846ea..bc9a07aa0c 100644 --- a/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx +++ b/frontend/src/pages/Events/List/hooks/useColumnDefinitions.tsx @@ -138,6 +138,19 @@ export const useColumnsDefinitions = () => { ); + case 'secret': + return ( +
+ Secret{' '} + {target.project_name && ( + + {target.project_name} + + )} + /{target.name} +
+ ); + default: return '---'; } diff --git a/frontend/src/pages/Events/List/hooks/useFilters.ts b/frontend/src/pages/Events/List/hooks/useFilters.ts index d463770b30..e6a97c71e8 100644 --- a/frontend/src/pages/Events/List/hooks/useFilters.ts +++ b/frontend/src/pages/Events/List/hooks/useFilters.ts @@ -19,6 +19,7 @@ type RequestParamsKeys = keyof Pick< | 'target_jobs' | 'target_volumes' | 'target_gateways' + | 'target_secrets' | 'within_projects' | 'within_fleets' | 'within_runs' @@ -35,6 +36,7 @@ const filterKeys: Record = { TARGET_JOBS: 'target_jobs', TARGET_VOLUMES: 'target_volumes', TARGET_GATEWAYS: 'target_gateways', + TARGET_SECRETS: 'target_secrets', WITHIN_PROJECTS: 'within_projects', WITHIN_FLEETS: 'within_fleets', WITHIN_RUNS: 'within_runs', @@ -53,6 +55,7 @@ const multipleChoiseKeys: RequestParamsKeys[] = [ 'target_jobs', 'target_volumes', 'target_gateways', + 'target_secrets', 'within_projects', 'within_fleets', 'within_runs', @@ -69,6 +72,7 @@ const targetTypes = [ { label: 'Job', value: 'job' }, { label: 'Volume', value: 'volume' }, { label: 'Gateway', value: 'gateway' }, + { label: 'Secret', value: 'secret' }, ]; export const useFilters = () => { @@ -171,6 +175,11 @@ export const useFilters = () => { operators: ['='], propertyLabel: 'Target gateways', }, + { + key: filterKeys.TARGET_SECRETS, + operators: ['='], + propertyLabel: 'Target secrets', + }, { key: filterKeys.WITHIN_PROJECTS, diff --git a/frontend/src/types/event.d.ts b/frontend/src/types/event.d.ts index 618ea6673f..dd0147fe15 100644 --- a/frontend/src/types/event.d.ts +++ b/frontend/src/types/event.d.ts @@ -1,4 +1,4 @@ -declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job' | 'volume' | 'gateway'; +declare type TEventTargetType = 'project' | 'user' | 'fleet' | 'instance' | 'run' | 'job' | 'volume' | 'gateway' | 'secret'; declare type TEventListRequestParams = Omit & { prev_recorded_at?: string; @@ -10,6 +10,7 @@ declare type TEventListRequestParams = Omit EventListFilters: " Update the server to 0.20.7 or higher or remove --target-gateway." ) filters.target_gateways.append(id) + elif args.target_secrets: + filters.target_secrets = [ + api.client.secrets.get(api.project, name=name).id for name in args.target_secrets + ] if args.within_fleets: filters.within_fleets = [ diff --git a/src/dstack/_internal/cli/services/events.py b/src/dstack/_internal/cli/services/events.py index 11f764bd15..0f0eb0f4b4 100644 --- a/src/dstack/_internal/cli/services/events.py +++ b/src/dstack/_internal/cli/services/events.py @@ -18,6 +18,7 @@ class EventListFilters: target_runs: Optional[list[uuid.UUID]] = None target_volumes: Optional[list[uuid.UUID]] = None target_gateways: Optional[list[uuid.UUID]] = None + target_secrets: Optional[list[uuid.UUID]] = None within_projects: Optional[list[uuid.UUID]] = None within_fleets: Optional[list[uuid.UUID]] = None within_runs: Optional[list[uuid.UUID]] = None diff --git a/src/dstack/_internal/core/models/events.py b/src/dstack/_internal/core/models/events.py index 289c4fc674..f2efb80d0e 100644 --- a/src/dstack/_internal/core/models/events.py +++ b/src/dstack/_internal/core/models/events.py @@ -18,6 +18,7 @@ class EventTargetType(str, Enum): JOB = "job" VOLUME = "volume" GATEWAY = "gateway" + SECRET = "secret" class EventTarget(CoreModel): diff --git a/src/dstack/_internal/server/routers/events.py b/src/dstack/_internal/server/routers/events.py index 4250eb4d7a..036a8b2be8 100644 --- a/src/dstack/_internal/server/routers/events.py +++ b/src/dstack/_internal/server/routers/events.py @@ -46,6 +46,7 @@ async def list_events( target_jobs=body.target_jobs, target_volumes=body.target_volumes, target_gateways=body.target_gateways, + target_secrets=body.target_secrets, within_projects=body.within_projects, within_fleets=body.within_fleets, within_runs=body.within_runs, diff --git a/src/dstack/_internal/server/schemas/events.py b/src/dstack/_internal/server/schemas/events.py index 30f7fe3244..3899b1f398 100644 --- a/src/dstack/_internal/server/schemas/events.py +++ b/src/dstack/_internal/server/schemas/events.py @@ -102,6 +102,17 @@ class ListEventsRequest(CoreModel): max_items=MAX_FILTER_ITEMS, ), ] = None + target_secrets: Annotated[ + Optional[list[uuid.UUID]], + Field( + description=( + "List of secret IDs." + " The response will only include events that target the specified secrets" + ), + min_items=MIN_FILTER_ITEMS, + max_items=MAX_FILTER_ITEMS, + ), + ] = None within_projects: Annotated[ Optional[list[uuid.UUID]], Field( diff --git a/src/dstack/_internal/server/services/events.py b/src/dstack/_internal/server/services/events.py index c6d35a4577..d46b43e201 100644 --- a/src/dstack/_internal/server/services/events.py +++ b/src/dstack/_internal/server/services/events.py @@ -20,6 +20,7 @@ MemberModel, ProjectModel, RunModel, + SecretModel, UserModel, VolumeModel, ) @@ -93,6 +94,7 @@ def from_model( JobModel, ProjectModel, RunModel, + SecretModel, UserModel, VolumeModel, ], @@ -139,6 +141,13 @@ def from_model( id=model.id, name=model.run_name, ) + if isinstance(model, SecretModel): + return Target( + type=EventTargetType.SECRET, + project_id=model.project_id or model.project.id, + id=model.id, + name=model.name, + ) if isinstance(model, UserModel): return Target( type=EventTargetType.USER, @@ -232,6 +241,7 @@ async def list_events( target_jobs: Optional[list[uuid.UUID]], target_volumes: Optional[list[uuid.UUID]], target_gateways: Optional[list[uuid.UUID]], + target_secrets: Optional[list[uuid.UUID]], within_projects: Optional[list[uuid.UUID]], within_fleets: Optional[list[uuid.UUID]], within_runs: Optional[list[uuid.UUID]], @@ -315,6 +325,13 @@ async def list_events( EventTargetModel.entity_id.in_(target_gateways), ) ) + if target_secrets is not None: + target_filters.append( + and_( + EventTargetModel.entity_type == EventTargetType.SECRET, + EventTargetModel.entity_id.in_(target_secrets), + ) + ) if within_projects is not None: target_filters.append(EventTargetModel.entity_project_id.in_(within_projects)) if within_fleets is not None: diff --git a/src/dstack/api/server/_events.py b/src/dstack/api/server/_events.py index d403fb2427..5aff9c9d21 100644 --- a/src/dstack/api/server/_events.py +++ b/src/dstack/api/server/_events.py @@ -31,6 +31,7 @@ def list( # NOTE: New parameters go here. Avoid positional parameters, they can break compatibility. target_volumes: Optional[list[UUID]] = None, target_gateways: Optional[list[UUID]] = None, + target_secrets: Optional[list[UUID]] = None, ) -> list[Event]: if prev_recorded_at is not None: # Time zones other than UTC are misinterpreted by the server: @@ -45,6 +46,7 @@ def list( target_jobs=target_jobs, target_volumes=target_volumes, target_gateways=target_gateways, + target_secrets=target_secrets, within_projects=within_projects, within_fleets=within_fleets, within_runs=within_runs, From ca56b3b9f9d43879d866242548e4f5e9706e6581 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Tue, 27 Jan 2026 21:21:00 +0100 Subject: [PATCH 031/640] [Docs] Events #3397 (#3506) * [Docs] Events #3397 * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> * Update docs/docs/concepts/events.md Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> --------- Co-authored-by: jvstme <36324149+jvstme@users.noreply.github.com> --- docs/docs/concepts/events.md | 70 +++++++++++++++++++++++++ docs/docs/reference/cli/dstack/event.md | 5 +- mkdocs.yml | 4 +- 3 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 docs/docs/concepts/events.md diff --git a/docs/docs/concepts/events.md b/docs/docs/concepts/events.md new file mode 100644 index 0000000000..fb6e0938a2 --- /dev/null +++ b/docs/docs/concepts/events.md @@ -0,0 +1,70 @@ +# Events + +Events provide a chronological record of notable state changes and operations affecting `dstack` resources. They are designed for auditing, debugging, and understanding the lifecycle of runs, jobs, fleets, and other resources. + +Each event includes the following fields: + +| Field | Description | +| --------- | ----------------------------------------------------------- | +| Timestamp | When the event occurred | +| Actor | The user or system that initiated the change, if applicable | +| Targets | The resources affected by the event | +| Message | A description of the change or additional event details | + +Events can be queried by targeting a specific resource or within a group of related resources. For example, you can query events targeting a particular job, or query events within a run, including the run itself and all of its jobs. + +Events are accessible through the UI, CLI, and API. + +## UI + +The UI allows you to query events either globally on the dedicated `Events` page or within a specific group on the page of a run, job, fleet, and other resources. + +### Global page + +The global page shows events from all projects that the user has access to and allows filtering by many fields. + +![](https://dstack.ai/static-assets/static-assets/images/dstack-ui-events-global.png){ width=800 } + +This page allows you to query events targeting a specific resource or within a particular group. + +### Resource page + +The resource page shows events within that specific group. For example, if you open a run and switch to the `Events` tab, you will see all events about that run and its jobs. + +![](https://dstack.ai/static-assets/static-assets/images/dstack-ui-events-run.png){ width=800 } + +## CLI + +To query events via the CLI, use the `dstack event` command. This command provides several arguments that allow filtering by target and within scopes. + +Here is an example of querying all events within a particular run: + +
+ +```shell +$ dstack event --within-run cursor + +[2026-01-21 13:09:37] [👤admin] [run cursor] Run submitted. Status: SUBMITTED +[2026-01-21 13:09:37] [job cursor-0-0] Job created on run submission. Status: SUBMITTED +[2026-01-21 13:09:57] [job cursor-0-0] Job status changed SUBMITTED -> PROVISIONING +[2026-01-21 13:09:58] [job cursor-0-0, instance some-fleet-0] Instance created for job. Instance status: PROVISIONING +[2026-01-21 13:09:59] [run cursor] Run status changed SUBMITTED -> PROVISIONING +[2026-01-21 13:11:22] [job cursor-0-0] Job status changed PROVISIONING -> PULLING +[2026-01-21 13:11:49] [job cursor-0-0] Job status changed PULLING -> RUNNING +[2026-01-21 13:11:51] [run cursor] Run status changed PROVISIONING -> RUNNING +[2026-01-21 13:18:41] [👤admin] [run cursor] Run status changed RUNNING -> TERMINATING. Termination reason: STOPPED_BY_USER +[2026-01-21 13:18:48] [job cursor-0-0] Job status changed RUNNING -> TERMINATING. Termination reason: TERMINATED_BY_USER +[2026-01-21 13:19:05] [instance some-fleet-0, job cursor-0-0] Job unassigned from instance. Instance blocks: 0/1 busy +[2026-01-21 13:19:05] [job cursor-0-0] Job status changed TERMINATING -> TERMINATED +[2026-01-21 13:19:07] [run cursor] Run status changed TERMINATING -> TERMINATED +``` + +
+ +To see all supported arguments, check the [reference](../reference/cli/dstack/event.md). + +If you invoke the command without arguments, it will include all events targeting resources in the project. + +## TTL + +By default, `dstack` stores each event for 30 days and then deletes it. This can be overridden by server administrators using the `DSTACK_SERVER_EVENTS_TTL_SECONDS` environment variable. diff --git a/docs/docs/reference/cli/dstack/event.md b/docs/docs/reference/cli/dstack/event.md index 9854609208..8f90e456c9 100644 --- a/docs/docs/reference/cli/dstack/event.md +++ b/docs/docs/reference/cli/dstack/event.md @@ -1,9 +1,8 @@ # dstack event -The `dstack event` command (an alias for `dstack event list`) allows you to view events. - - +The `dstack event` command, an alias for `dstack event list`, allows you to view events. +For more details, see [Events](../../../concepts/events.md). ## Usage diff --git a/mkdocs.yml b/mkdocs.yml index 5dcdd69999..c7afe30256 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,7 +77,8 @@ plugins: - docs/concepts/gateways.md: How to manage gateways - enabling auto-scaling, rate limits, and custom domains - docs/concepts/secrets.md: How to manage secrets - for API keys or other sensitive data - docs/concepts/projects.md: How to manage projects - for managing separate teams - - docs/concepts/metrics.md: How to manage gateways - enabling auto-scaling, rate limits, and custom domains + - docs/concepts/metrics.md: How to access metrics - for monitoring of runs/jobs/fleets, hardware usage (e.g. GPU util), etc + - docs/concepts/events.md: How to access events - aka audit log - for tracking state changes, resource creation/deletion, etc Guides: - docs/guides/server-deployment.md: Detailed guide on how to deploy the dstack server - docs/guides/troubleshooting.md: Common issues and how to troubleshoot them @@ -271,6 +272,7 @@ nav: - Secrets: docs/concepts/secrets.md - Projects: docs/concepts/projects.md - Metrics: docs/concepts/metrics.md + - Events: docs/concepts/events.md - Guides: - Server deployment: docs/guides/server-deployment.md - Troubleshooting: docs/guides/troubleshooting.md From a1f0d58d1adae1da4a7cc6048251ef13c034a765 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Tue, 27 Jan 2026 21:21:32 +0100 Subject: [PATCH 032/640] [UX] Extend `dstack login` with interactive selection of `url` and default project (#3492) * [UX] Extend `dstack login` with interactive selection of `url` and default project * Updated tests * Addressing issues from the PR review (removed overly-agressibe catching of import and other exceptions when detecting if interactive menu is available). Plus, added handling KeyboardException in `dstack login`. --- src/dstack/_internal/cli/commands/login.py | 165 ++++++- src/dstack/_internal/cli/commands/project.py | 14 +- src/dstack/_internal/cli/utils/common.py | 6 +- .../_internal/cli/commands/test_login.py | 414 ++++++++++++++++-- 4 files changed, 520 insertions(+), 79 deletions(-) diff --git a/src/dstack/_internal/cli/commands/login.py b/src/dstack/_internal/cli/commands/login.py index 0919494e63..8431a76c61 100644 --- a/src/dstack/_internal/cli/commands/login.py +++ b/src/dstack/_internal/cli/commands/login.py @@ -1,18 +1,34 @@ import argparse import queue +import sys import threading import urllib.parse import webbrowser from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Optional +from typing import Any, Optional + +import questionary +from rich.prompt import Prompt as RichPrompt +from rich.text import Text from dstack._internal.cli.commands import BaseCommand +from dstack._internal.cli.commands.project import select_default_project from dstack._internal.cli.utils.common import console, resolve_url from dstack._internal.core.errors import ClientError, CLIError from dstack._internal.core.models.users import UserWithCreds +from dstack._internal.utils.logging import get_logger from dstack.api._public.runs import ConfigManager from dstack.api.server import APIClient +logger = get_logger(__name__) + +is_project_menu_supported = sys.stdin.isatty() + + +class UrlPrompt(RichPrompt): + def render_default(self, default: Any) -> Text: + return Text(f"({default})", style="bold orange1") + class LoginCommand(BaseCommand): NAME = "login" @@ -23,7 +39,7 @@ def _register(self): self._parser.add_argument( "--url", help="The server URL, e.g. https://sky.dstack.ai", - required=True, + required=not is_project_menu_supported, ) self._parser.add_argument( "-p", @@ -33,10 +49,25 @@ def _register(self): " Selected automatically if the server supports only one provider." ), ) + self._parser.add_argument( + "-y", + "--yes", + help="Don't ask for confirmation (e.g. set first project as default)", + action="store_true", + ) + self._parser.add_argument( + "-n", + "--no", + help="Don't ask for confirmation (e.g. do not change default project)", + action="store_true", + ) def _command(self, args: argparse.Namespace): super()._command(args) - base_url = _normalize_url_or_error(args.url) + url = args.url + if url is None: + url = self._prompt_url() + base_url = _normalize_url_or_error(url) api_client = APIClient(base_url=base_url) provider = self._select_provider_or_error(api_client=api_client, provider=args.provider) server = _LoginServer(api_client=api_client, provider=provider) @@ -56,9 +87,9 @@ def _command(self, args: argparse.Namespace): server.shutdown() if user is None: raise CLIError("CLI authentication failed") - console.print(f"Logged in as [code]{user.username}[/].") + console.print(f"Logged in as [code]{user.username}[/]") api_client = APIClient(base_url=base_url, token=user.creds.token) - self._configure_projects(api_client=api_client, user=user) + self._configure_projects(api_client=api_client, user=user, args=args) def _select_provider_or_error(self, api_client: APIClient, provider: Optional[str]) -> str: providers = api_client.auth.list_providers() @@ -67,6 +98,8 @@ def _select_provider_or_error(self, api_client: APIClient, provider: Optional[st raise CLIError("No SSO providers configured on the server.") if provider is None: if len(available_providers) > 1: + if is_project_menu_supported: + return self._prompt_provider(available_providers) raise CLIError( "Specify -p/--provider to choose SSO provider" f" Available providers: {', '.join(available_providers)}" @@ -79,7 +112,37 @@ def _select_provider_or_error(self, api_client: APIClient, provider: Optional[st ) return provider - def _configure_projects(self, api_client: APIClient, user: UserWithCreds): + def _prompt_url(self) -> str: + try: + url = UrlPrompt.ask( + "Enter the server URL", + default="https://sky.dstack.ai", + console=console, + ) + except KeyboardInterrupt: + console.print("\nCancelled by user") + raise SystemExit(1) + if url is None: + raise CLIError("URL is required") + return url + + def _prompt_provider(self, available_providers: list[str]) -> str: + choices = [ + questionary.Choice(title=provider, value=provider) for provider in available_providers + ] + selected_provider = questionary.select( + message="Select SSO provider:", + choices=choices, + qmark="", + instruction="(↑↓ Enter)", + ).ask() + if selected_provider is None: + raise SystemExit(1) + return selected_provider + + def _configure_projects( + self, api_client: APIClient, user: UserWithCreds, args: argparse.Namespace + ): projects = api_client.projects.list(include_not_joined=False) if len(projects) == 0: console.print( @@ -89,30 +152,88 @@ def _configure_projects(self, api_client: APIClient, user: UserWithCreds): return config_manager = ConfigManager() default_project = config_manager.get_project_config() - new_default_project = None - for i, project in enumerate(projects): - set_as_default = ( - default_project is None - and i == 0 - or default_project is not None - and default_project.name == project.project_name - ) - if set_as_default: - new_default_project = project + for project in projects: config_manager.configure_project( name=project.project_name, url=api_client.base_url, token=user.creds.token, - default=set_as_default, + default=False, ) config_manager.save() + project_names = ", ".join(f"[code]{p.project_name}[/]" for p in projects) console.print( - f"Configured projects: {', '.join(f'[code]{p.project_name}[/]' for p in projects)}." + f"Added {project_names} project{'' if len(projects) == 1 else 's'} at {config_manager.config_filepath}" ) - if new_default_project: - console.print( - f"Set project [code]{new_default_project.project_name}[/] as default project." - ) + + project_configs = config_manager.list_project_configs() + + if args.no: + return + + if args.yes: + if len(projects) > 0: + first_project_from_server = projects[0] + first_project_config = next( + ( + pc + for pc in project_configs + if pc.name == first_project_from_server.project_name + ), + None, + ) + if first_project_config is not None: + config_manager.configure_project( + name=first_project_config.name, + url=first_project_config.url, + token=first_project_config.token, + default=True, + ) + config_manager.save() + console.print( + f"Set [code]{first_project_config.name}[/] project as default at {config_manager.config_filepath}" + ) + return + + if len(project_configs) == 1 or not is_project_menu_supported: + selected_project = None + if len(project_configs) == 1: + selected_project = project_configs[0] + else: + for i, project in enumerate(projects): + set_as_default = ( + default_project is None + and i == 0 + or default_project is not None + and default_project.name == project.project_name + ) + if set_as_default: + selected_project = next( + (pc for pc in project_configs if pc.name == project.project_name), + None, + ) + break + if selected_project is not None: + config_manager.configure_project( + name=selected_project.name, + url=selected_project.url, + token=selected_project.token, + default=True, + ) + config_manager.save() + console.print( + f"Set [code]{selected_project.name}[/] project as default at {config_manager.config_filepath}" + ) + else: + console.print() + selected_project = select_default_project(project_configs, default_project) + if selected_project is not None: + config_manager.configure_project( + name=selected_project.name, + url=selected_project.url, + token=selected_project.token, + default=True, + ) + config_manager.save() class _BadRequestError(Exception): diff --git a/src/dstack/_internal/cli/commands/project.py b/src/dstack/_internal/cli/commands/project.py index db4a7a5eb9..5edd4e64ec 100644 --- a/src/dstack/_internal/cli/commands/project.py +++ b/src/dstack/_internal/cli/commands/project.py @@ -2,16 +2,10 @@ import sys from typing import Any, Optional, Union +import questionary from requests import HTTPError from rich.table import Table -try: - import questionary - - is_project_menu_supported = sys.stdin.isatty() -except (ImportError, NotImplementedError, AttributeError): - is_project_menu_supported = False - import dstack.api.server from dstack._internal.cli.commands import BaseCommand from dstack._internal.cli.utils.common import add_row_from_dict, confirm_ask, console @@ -22,6 +16,8 @@ logger = get_logger(__name__) +is_project_menu_supported = sys.stdin.isatty() + def select_default_project( project_configs: list[ProjectConfig], default_project: Optional[ProjectConfig] @@ -57,9 +53,9 @@ def select_default_project( default_index = i menu_entries.append((entry, i)) - choices = [questionary.Choice(title=entry, value=index) for entry, index in menu_entries] # pyright: ignore[reportPossiblyUnboundVariable] + choices = [questionary.Choice(title=entry, value=index) for entry, index in menu_entries] default_value = default_index - selected_index = questionary.select( # pyright: ignore[reportPossiblyUnboundVariable] + selected_index = questionary.select( message="Select the default project:", choices=choices, default=default_value, # pyright: ignore[reportArgumentType] diff --git a/src/dstack/_internal/cli/utils/common.py b/src/dstack/_internal/cli/utils/common.py index d53b84567b..afbfb2d8cb 100644 --- a/src/dstack/_internal/cli/utils/common.py +++ b/src/dstack/_internal/cli/utils/common.py @@ -99,7 +99,11 @@ def configure_logging(): def confirm_ask(prompt, **kwargs) -> bool: kwargs["console"] = console - return Confirm.ask(prompt=prompt, **kwargs) + try: + return Confirm.ask(prompt=prompt, **kwargs) + except KeyboardInterrupt: + console.print("\nCancelled by user") + raise SystemExit(1) def add_row_from_dict(table: Table, data: Dict[Union[str, int], Any], **kwargs): diff --git a/src/tests/_internal/cli/commands/test_login.py b/src/tests/_internal/cli/commands/test_login.py index bd7eba584f..4e0c27ae34 100644 --- a/src/tests/_internal/cli/commands/test_login.py +++ b/src/tests/_internal/cli/commands/test_login.py @@ -8,34 +8,72 @@ class TestLogin: + @staticmethod + def _setup_auth_mocks(api_client_mock, login_server_mock, user_token="token"): + """Set up common authentication mocks.""" + api_client_mock.return_value.auth.list_providers.return_value = [ + SimpleNamespace(name="github", enabled=True) + ] + api_client_mock.return_value.auth.authorize.return_value = SimpleNamespace( + authorization_url="http://auth_url" + ) + user = SimpleNamespace(username="me", creds=SimpleNamespace(token=user_token)) + login_server_mock.return_value.get_logged_in_user.return_value = user + return user + + @staticmethod + def _setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path: Path, project_configs: list[SimpleNamespace] + ): + """Set up ConfigManager mock with state tracking via side effects.""" + config_manager_mock.return_value.config_filepath = tmp_path / "config.yml" + config_manager_mock.return_value.list_project_configs.return_value = project_configs + + def configure_project_side_effect(name, url, token, default): + for pc in project_configs: + if pc.name == name: + pc.url = url + pc.token = token + if default: + for p in project_configs: + p.default = False + pc.default = default or pc.default + return + + def get_project_config_side_effect(name=None): + if name is None: + for pc in project_configs: + if pc.default: + return pc + return None + for pc in project_configs: + if pc.name == name: + return pc + return None + + config_manager_mock.return_value.configure_project.side_effect = ( + configure_project_side_effect + ) + config_manager_mock.return_value.get_project_config.side_effect = ( + get_project_config_side_effect + ) + def test_login_no_projects(self, capsys: CaptureFixture, tmp_path: Path): with ( patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, - patch("dstack._internal.cli.commands.login.APIClient") as APIClientMock, - patch("dstack._internal.cli.commands.login._LoginServer") as LoginServerMock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, patch( "dstack._internal.cli.commands.login._normalize_url_or_error" - ) as _normalize_url_or_error_mock, + ) as normalize_url_mock, ): webbrowser_mock.open.return_value = True - _normalize_url_or_error_mock.return_value = "http://127.0.0.1:31313" - APIClientMock.return_value.auth.list_providers.return_value = [ - SimpleNamespace(name="github", enabled=True) - ] - APIClientMock.return_value.auth.authorize.return_value = SimpleNamespace( - authorization_url="http://auth_url" - ) - APIClientMock.return_value.projects.list.return_value = [] - user = SimpleNamespace(username="me", creds=SimpleNamespace(token="token")) - LoginServerMock.return_value.get_logged_in_user.return_value = user + normalize_url_mock.return_value = "http://127.0.0.1:31313" + self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [] + exit_code = run_dstack_cli( - [ - "login", - "--url", - "http://127.0.0.1:31313", - "--provider", - "github", - ], + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github"], home_dir=tmp_path, ) @@ -43,53 +81,331 @@ def test_login_no_projects(self, capsys: CaptureFixture, tmp_path: Path): assert capsys.readouterr().out.replace("\n", "") == ( "Your browser has been opened to log in with Github:" "http://auth_url" - "Logged in as me." + "Logged in as me" "No projects configured. Create your own project via the UI or contact a project manager to add you to the project." ) def test_login_configures_projects(self, capsys: CaptureFixture, tmp_path: Path): with ( patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, - patch("dstack._internal.cli.commands.login.APIClient") as APIClientMock, - patch("dstack._internal.cli.commands.login.ConfigManager") as ConfigManagerMock, - patch("dstack._internal.cli.commands.login._LoginServer") as LoginServerMock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login.ConfigManager") as config_manager_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, patch( "dstack._internal.cli.commands.login._normalize_url_or_error" - ) as _normalize_url_or_error_mock, + ) as normalize_url_mock, ): - _normalize_url_or_error_mock.return_value = "http://127.0.0.1:31313" webbrowser_mock.open.return_value = True - APIClientMock.return_value.auth.list_providers.return_value = [ - SimpleNamespace(name="github", enabled=True) + normalize_url_mock.return_value = "http://127.0.0.1:31313" + user = self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [ + SimpleNamespace(project_name="project1"), + SimpleNamespace(project_name="project2"), ] - APIClientMock.return_value.auth.authorize.return_value = SimpleNamespace( - authorization_url="http://auth_url" + api_client_mock.return_value.base_url = "http://127.0.0.1:31313" + + project_configs = [ + SimpleNamespace( + name="project1", url="http://127.0.0.1:31313", token="token", default=False + ), + SimpleNamespace( + name="project2", url="http://127.0.0.1:31313", token="token", default=False + ), + ] + config_manager_mock.return_value.get_project_config.return_value = None + self._setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path, project_configs + ) + + exit_code = run_dstack_cli( + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github"], + home_dir=tmp_path, + ) + + config_manager_mock.return_value.configure_project.assert_has_calls( + [ + call( + name="project1", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + call( + name="project2", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + call( + name="project1", url="http://127.0.0.1:31313", token="token", default=True + ), + ] ) - APIClientMock.return_value.projects.list.return_value = [ + config_manager_mock.return_value.save.assert_called() + final_default = config_manager_mock.return_value.get_project_config() + assert final_default is not None + assert final_default.name == "project1" + + assert exit_code == 0 + assert capsys.readouterr().out.replace("\n", "") == ( + "Your browser has been opened to log in with Github:" + "http://auth_url" + "Logged in as me" + f"Added project1, project2 projects at {tmp_path / 'config.yml'}" + f"Set project1 project as default at {tmp_path / 'config.yml'}" + ) + + def test_login_configures_projects_yes_sets_first_project_default( + self, capsys: CaptureFixture, tmp_path: Path + ): + with ( + patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login.ConfigManager") as config_manager_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, + patch( + "dstack._internal.cli.commands.login._normalize_url_or_error" + ) as normalize_url_mock, + ): + webbrowser_mock.open.return_value = True + normalize_url_mock.return_value = "http://127.0.0.1:31313" + user = self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [ + SimpleNamespace(project_name="project1"), + SimpleNamespace(project_name="project2"), + ] + api_client_mock.return_value.base_url = "http://127.0.0.1:31313" + + project_configs = [ + SimpleNamespace( + name="project1", url="http://127.0.0.1:31313", token="token", default=False + ), + SimpleNamespace( + name="project2", url="http://127.0.0.1:31313", token="token", default=True + ), + ] + self._setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path, project_configs + ) + + exit_code = run_dstack_cli( + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github", "--yes"], + home_dir=tmp_path, + ) + + config_manager_mock.return_value.configure_project.assert_has_calls( + [ + call( + name="project1", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + call( + name="project2", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + call( + name="project1", url="http://127.0.0.1:31313", token="token", default=True + ), + ] + ) + final_default = config_manager_mock.return_value.get_project_config() + assert final_default is not None + assert final_default.name == "project1" + + assert exit_code == 0 + assert capsys.readouterr().out.replace("\n", "") == ( + "Your browser has been opened to log in with Github:" + "http://auth_url" + "Logged in as me" + f"Added project1, project2 projects at {tmp_path / 'config.yml'}" + f"Set project1 project as default at {tmp_path / 'config.yml'}" + ) + + def test_login_configures_projects_no_does_not_change_default( + self, capsys: CaptureFixture, tmp_path: Path + ): + with ( + patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login.ConfigManager") as config_manager_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, + patch( + "dstack._internal.cli.commands.login._normalize_url_or_error" + ) as normalize_url_mock, + ): + webbrowser_mock.open.return_value = True + normalize_url_mock.return_value = "http://127.0.0.1:31313" + user = self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [ SimpleNamespace(project_name="project1"), SimpleNamespace(project_name="project2"), ] - APIClientMock.return_value.base_url = "http://127.0.0.1:31313" - ConfigManagerMock.return_value.get_project_config.return_value = None - user = SimpleNamespace(username="me", creds=SimpleNamespace(token="token")) - LoginServerMock.return_value.get_logged_in_user.return_value = user + api_client_mock.return_value.base_url = "http://127.0.0.1:31313" + + project_configs = [ + SimpleNamespace( + name="project1", url="http://127.0.0.1:31313", token="token", default=False + ), + SimpleNamespace( + name="project2", url="http://127.0.0.1:31313", token="token", default=True + ), + ] + self._setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path, project_configs + ) + exit_code = run_dstack_cli( + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github", "--no"], + home_dir=tmp_path, + ) + + config_manager_mock.return_value.configure_project.assert_has_calls( [ - "login", - "--url", - "http://127.0.0.1:31313", - "--provider", - "github", - ], + call( + name="project1", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + call( + name="project2", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, + ), + ] + ) + assert ( + call(name="project1", url="http://127.0.0.1:31313", token="token", default=True) + not in config_manager_mock.return_value.configure_project.mock_calls + ) + final_default = config_manager_mock.return_value.get_project_config() + assert final_default is not None + assert final_default.name == "project2" + + assert exit_code == 0 + assert capsys.readouterr().out.replace("\n", "") == ( + "Your browser has been opened to log in with Github:" + "http://auth_url" + "Logged in as me" + f"Added project1, project2 projects at {tmp_path / 'config.yml'}" + ) + + def test_login_single_project_auto_default(self, capsys: CaptureFixture, tmp_path: Path): + with ( + patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login.ConfigManager") as config_manager_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, + patch( + "dstack._internal.cli.commands.login._normalize_url_or_error" + ) as normalize_url_mock, + ): + webbrowser_mock.open.return_value = True + normalize_url_mock.return_value = "http://127.0.0.1:31313" + user = self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [ + SimpleNamespace(project_name="project1"), + ] + api_client_mock.return_value.base_url = "http://127.0.0.1:31313" + + project_configs = [ + SimpleNamespace( + name="project1", url="http://127.0.0.1:31313", token="token", default=False + ), + ] + config_manager_mock.return_value.get_project_config.return_value = None + self._setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path, project_configs + ) + + exit_code = run_dstack_cli( + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github"], home_dir=tmp_path, ) - ConfigManagerMock.return_value.configure_project.assert_has_calls( + + config_manager_mock.return_value.configure_project.assert_has_calls( [ call( name="project1", url="http://127.0.0.1:31313", token=user.creds.token, - default=True, + default=False, + ), + call( + name="project1", url="http://127.0.0.1:31313", token="token", default=True + ), + ] + ) + final_default = config_manager_mock.return_value.get_project_config() + assert final_default is not None + assert final_default.name == "project1" + + assert exit_code == 0 + assert capsys.readouterr().out.replace("\n", "") == ( + "Your browser has been opened to log in with Github:" + "http://auth_url" + "Logged in as me" + f"Added project1 project at {tmp_path / 'config.yml'}" + f"Set project1 project as default at {tmp_path / 'config.yml'}" + ) + + def test_login_interactive_prompts_for_default_project( + self, capsys: CaptureFixture, tmp_path: Path + ): + with ( + patch("dstack._internal.cli.commands.login.webbrowser") as webbrowser_mock, + patch("dstack._internal.cli.commands.login.APIClient") as api_client_mock, + patch("dstack._internal.cli.commands.login.ConfigManager") as config_manager_mock, + patch("dstack._internal.cli.commands.login._LoginServer") as login_server_mock, + patch( + "dstack._internal.cli.commands.login._normalize_url_or_error" + ) as normalize_url_mock, + patch( + "dstack._internal.cli.commands.login.select_default_project" + ) as select_default_project_mock, + patch("dstack._internal.cli.commands.login.is_project_menu_supported", True), + ): + webbrowser_mock.open.return_value = True + normalize_url_mock.return_value = "http://127.0.0.1:31313" + user = self._setup_auth_mocks(api_client_mock, login_server_mock) + api_client_mock.return_value.projects.list.return_value = [ + SimpleNamespace(project_name="project1"), + SimpleNamespace(project_name="project2"), + ] + api_client_mock.return_value.base_url = "http://127.0.0.1:31313" + + project_configs = [ + SimpleNamespace( + name="project1", url="http://127.0.0.1:31313", token="token", default=False + ), + SimpleNamespace( + name="project2", url="http://127.0.0.1:31313", token="token", default=False + ), + ] + config_manager_mock.return_value.get_project_config.return_value = None + self._setup_config_manager_with_state_tracking( + config_manager_mock, tmp_path, project_configs + ) + select_default_project_mock.return_value = project_configs[1] + + exit_code = run_dstack_cli( + ["login", "--url", "http://127.0.0.1:31313", "--provider", "github"], + home_dir=tmp_path, + ) + + select_default_project_mock.assert_called_once() + config_manager_mock.return_value.configure_project.assert_has_calls( + [ + call( + name="project1", + url="http://127.0.0.1:31313", + token=user.creds.token, + default=False, ), call( name="project2", @@ -97,15 +413,19 @@ def test_login_configures_projects(self, capsys: CaptureFixture, tmp_path: Path) token=user.creds.token, default=False, ), + call( + name="project2", url="http://127.0.0.1:31313", token="token", default=True + ), ] ) - ConfigManagerMock.return_value.save.assert_called() + final_default = config_manager_mock.return_value.get_project_config() + assert final_default is not None + assert final_default.name == "project2" assert exit_code == 0 assert capsys.readouterr().out.replace("\n", "") == ( "Your browser has been opened to log in with Github:" "http://auth_url" - "Logged in as me." - "Configured projects: project1, project2." - "Set project project1 as default project." + "Logged in as me" + f"Added project1, project2 projects at {tmp_path / 'config.yml'}" ) From bef08a593979e2b68ac3e6125061c5f85611dcfb Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:30:58 +0000 Subject: [PATCH 033/640] Add secret lifecycle events (#3505) - Secret created - Secret updated - Secret deleted --- .../_internal/server/routers/secrets.py | 6 +- .../_internal/server/services/secrets.py | 127 ++++++++++++------ .../_internal/server/routers/test_secrets.py | 32 +++++ 3 files changed, 120 insertions(+), 45 deletions(-) diff --git a/src/dstack/_internal/server/routers/secrets.py b/src/dstack/_internal/server/routers/secrets.py index c19f15bccc..30cbdc60c7 100644 --- a/src/dstack/_internal/server/routers/secrets.py +++ b/src/dstack/_internal/server/routers/secrets.py @@ -59,13 +59,14 @@ async def create_or_update_secret( session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): - _, project = user_project + user, project = user_project return CustomORJSONResponse( await secrets_services.create_or_update_secret( session=session, project=project, name=body.name, value=body.value, + user=user, ) ) @@ -76,9 +77,10 @@ async def delete_secrets( session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()), ): - _, project = user_project + user, project = user_project await secrets_services.delete_secrets( session=session, project=project, names=body.secrets_names, + user=user, ) diff --git a/src/dstack/_internal/server/services/secrets.py b/src/dstack/_internal/server/services/secrets.py index 1cea57f655..ed12576256 100644 --- a/src/dstack/_internal/server/services/secrets.py +++ b/src/dstack/_internal/server/services/secrets.py @@ -1,8 +1,11 @@ import re +import uuid +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from typing import Dict, List, Optional import sqlalchemy.exc -from sqlalchemy import delete, select, update +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.errors import ( @@ -11,11 +14,10 @@ ServerClientError, ) from dstack._internal.core.models.secrets import Secret -from dstack._internal.server.models import DecryptedString, ProjectModel, SecretModel -from dstack._internal.utils.logging import get_logger - -logger = get_logger(__name__) - +from dstack._internal.server.db import get_db +from dstack._internal.server.models import DecryptedString, ProjectModel, SecretModel, UserModel +from dstack._internal.server.services import events +from dstack._internal.server.services.locking import get_locker _SECRET_NAME_REGEX = "^[A-Za-z0-9-_]{1,200}$" _SECRET_VALUE_MAX_LENGTH = 5000 @@ -57,6 +59,7 @@ async def create_or_update_secret( project: ProjectModel, name: str, value: str, + user: UserModel, ) -> Secret: _validate_secret(name=name, value=value) try: @@ -65,6 +68,7 @@ async def create_or_update_secret( project=project, name=name, value=value, + user=user, ) except ResourceExistsError: secret_model = await update_secret( @@ -72,6 +76,7 @@ async def create_or_update_secret( project=project, name=name, value=value, + user=user, ) return secret_model_to_secret(secret_model, include_value=True) @@ -80,26 +85,24 @@ async def delete_secrets( session: AsyncSession, project: ProjectModel, names: List[str], + user: UserModel, ): - existing_secrets_query = await session.execute( - select(SecretModel).where( - SecretModel.project_id == project.id, - SecretModel.name.in_(names), - ) - ) - existing_names = [s.name for s in existing_secrets_query.scalars().all()] - missing_names = set(names) - set(existing_names) - if missing_names: - raise ResourceNotExistsError(f"Secrets not found: {', '.join(missing_names)}") - - await session.execute( - delete(SecretModel).where( - SecretModel.project_id == project.id, - SecretModel.name.in_(names), - ) - ) - await session.commit() - logger.info("Deleted secrets %s in project %s", names, project.name) + async with get_project_secret_models_by_name_for_update( + session=session, project=project, names=names + ) as secret_models: + existing_names = [s.name for s in secret_models] + missing_names = set(names) - set(existing_names) + if missing_names: + raise ResourceNotExistsError(f"Secrets not found: {', '.join(missing_names)}") + for secret_model in secret_models: + await session.delete(secret_model) + events.emit( + session, + "Secret deleted", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(secret_model)], + ) + await session.commit() def secret_model_to_secret(secret_model: SecretModel, include_value: bool = False) -> Secret: @@ -142,13 +145,47 @@ async def get_project_secret_model_by_name( return res.scalar_one_or_none() +@asynccontextmanager +async def get_project_secret_models_by_name_for_update( + session: AsyncSession, project: ProjectModel, names: list[str] +) -> AsyncGenerator[list[SecretModel], None]: + """ + Fetch secrets from the database and lock them for update. + + **NOTE**: commit changes to the database before exiting from this context manager, + so that in-memory locks are only released after commit. + """ + filters = [ + SecretModel.project_id == project.id, + SecretModel.name.in_(names), + ] + res = await session.execute(select(SecretModel.id).where(*filters)) + secret_ids = res.scalars().all() + if not secret_ids: + yield [] + else: + async with get_locker(get_db().dialect_name).lock_ctx( + SecretModel.__tablename__, sorted(secret_ids) + ): + # Refetch after lock + res = await session.execute( + select(SecretModel) + .where(SecretModel.id.in_(secret_ids), *filters) + .with_for_update(key_share=True) + .order_by(SecretModel.id) # take locks in order + ) + yield list(res.scalars().all()) + + async def create_secret( session: AsyncSession, project: ProjectModel, name: str, value: str, + user: UserModel, ) -> SecretModel: secret_model = SecretModel( + id=uuid.uuid4(), project_id=project.id, name=name, value=DecryptedString(plaintext=value), @@ -156,6 +193,12 @@ async def create_secret( try: async with session.begin_nested(): session.add(secret_model) + events.emit( + session, + "Secret created", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(secret_model)], + ) except sqlalchemy.exc.IntegrityError: raise ResourceExistsError() await session.commit() @@ -167,25 +210,23 @@ async def update_secret( project: ProjectModel, name: str, value: str, + user: UserModel, ) -> SecretModel: - await session.execute( - update(SecretModel) - .where( - SecretModel.project_id == project.id, - SecretModel.name == name, - ) - .values( - value=DecryptedString(plaintext=value), - ) - ) - await session.commit() - secret_model = await get_project_secret_model_by_name( - session=session, - project=project, - name=name, - ) - if secret_model is None: - raise ResourceNotExistsError() + async with get_project_secret_models_by_name_for_update( + session=session, project=project, names=[name] + ) as secret_models: + if not secret_models: + raise ResourceNotExistsError() + secret_model = secret_models[0] + if secret_model.value.get_plaintext_or_error() != value: + secret_model.value = DecryptedString(plaintext=value) + events.emit( + session, + "Secret updated", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(secret_model)], + ) + await session.commit() return secret_model diff --git a/src/tests/_internal/server/routers/test_secrets.py b/src/tests/_internal/server/routers/test_secrets.py index a2279c770c..302591b881 100644 --- a/src/tests/_internal/server/routers/test_secrets.py +++ b/src/tests/_internal/server/routers/test_secrets.py @@ -11,6 +11,7 @@ create_secret, create_user, get_auth_headers, + list_events, ) @@ -145,6 +146,9 @@ async def test_creates_secret(self, test_db, session: AsyncSession, client: Asyn res = await session.execute(select(SecretModel)) secret_model = res.scalar() assert secret_model is not None + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Secret created" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -165,6 +169,29 @@ async def test_updates_secret(self, test_db, session: AsyncSession, client: Asyn assert response.status_code == 200 await session.refresh(secret) assert secret.value.get_plaintext_or_error() == "new_value" + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Secret updated" + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_no_event_if_value_unchanged( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session=session, global_role=GlobalRole.USER) + project = await create_project(session=session, owner=user) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + await create_secret(session=session, project=project, name="secret1", value="value") + response = await client.post( + f"/api/project/{project.name}/secrets/create_or_update", + headers=get_auth_headers(user.token), + json={"name": "secret1", "value": "value"}, + ) + assert response.status_code == 200 + events = await list_events(session) + assert len(events) == 0 @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -253,6 +280,11 @@ async def test_deletes_secrets(self, test_db, session: AsyncSession, client: Asy assert len(secrets) == 1 assert secrets[0].name == "secret2" + # Verify event was emitted + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Secret deleted" + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_delete_nonexistent_secret_raises_error( From b7f637b1885e120b292a300f499c79c6f49541d3 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:33:15 +0000 Subject: [PATCH 034/640] Fix apply plan compatibility with old servers (#3507) --- src/dstack/_internal/core/compatibility/runs.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/compatibility/runs.py b/src/dstack/_internal/core/compatibility/runs.py index 5bec348d67..f1b7dc5222 100644 --- a/src/dstack/_internal/core/compatibility/runs.py +++ b/src/dstack/_internal/core/compatibility/runs.py @@ -1,7 +1,12 @@ from typing import Optional from dstack._internal.core.models.common import IncludeExcludeDictType, IncludeExcludeSetType -from dstack._internal.core.models.runs import ApplyRunPlanInput, JobSpec, RunSpec +from dstack._internal.core.models.runs import ( + DEFAULT_REPLICA_GROUP_NAME, + ApplyRunPlanInput, + JobSpec, + RunSpec, +) from dstack._internal.server.schemas.runs import GetRunPlanRequest, ListRunsRequest @@ -23,7 +28,13 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD current_resource = plan.current_resource if current_resource is not None: current_resource_excludes: IncludeExcludeDictType = {} + apply_plan_excludes["current_resource"] = current_resource_excludes current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec) + current_resource_excludes["jobs"] = { + "__all__": { + "job_spec": get_job_spec_excludes([job.job_spec for job in current_resource.jobs]), + } + } return {"plan": apply_plan_excludes} @@ -70,4 +81,6 @@ def get_job_spec_excludes(job_specs: list[JobSpec]) -> IncludeExcludeDictType: clients backward-compatibility with older servers. """ spec_excludes: IncludeExcludeDictType = {} + if all(s.replica_group == DEFAULT_REPLICA_GROUP_NAME for s in job_specs): + spec_excludes["replica_group"] = True return spec_excludes From 6da1554063294252261db0e6b2e42af550fb2d0e Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:26:10 +0100 Subject: [PATCH 035/640] [UI] Minor tweaks (#3508) --- .../AppLayout/TutorialPanel/constants.tsx | 10 +++++----- .../src/layouts/AppLayout/TutorialPanel/hooks.ts | 16 ++++++++-------- frontend/src/layouts/AppLayout/index.tsx | 2 +- frontend/src/locale/en.json | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/frontend/src/layouts/AppLayout/TutorialPanel/constants.tsx b/frontend/src/layouts/AppLayout/TutorialPanel/constants.tsx index 7d4bb378b2..4ee909c0cf 100644 --- a/frontend/src/layouts/AppLayout/TutorialPanel/constants.tsx +++ b/frontend/src/layouts/AppLayout/TutorialPanel/constants.tsx @@ -8,7 +8,7 @@ import { Box } from 'components'; export const tutorialPanelI18nStrings: TutorialPanelProps.I18nStrings = { labelsTaskStatus: { pending: 'Pending', 'in-progress': 'In progress', success: 'Success' }, loadingText: 'Loading', - tutorialListTitle: 'Take a tour', + tutorialListTitle: '', tutorialListDescription: 'Follow the tutorials below to get up to speed with dstack Sky.', tutorialListDownloadLinkText: 'Download PDF version', tutorialCompletedText: 'Completed', @@ -50,7 +50,7 @@ export enum HotspotIds { export const BILLING_TUTORIAL: TutorialPanelProps.Tutorial = { completed: false, - title: 'Set up billing', + title: 'Billing', description: ( <> @@ -80,7 +80,7 @@ export const BILLING_TUTORIAL: TutorialPanelProps.Tutorial = { export const CONFIGURE_CLI_TUTORIAL: TutorialPanelProps.Tutorial = { completed: false, - title: 'Set up the CLI', + title: 'CLI', prerequisitesAlert: 'Please, create a project before set up the CLI', description: ( <> @@ -111,7 +111,7 @@ export const CONFIGURE_CLI_TUTORIAL: TutorialPanelProps.Tutorial = { export const CREATE_FIRST_PROJECT: TutorialPanelProps.Tutorial = { completed: false, - title: 'Create a project', + title: 'Project', description: ( <> @@ -136,7 +136,7 @@ export const CREATE_FIRST_PROJECT: TutorialPanelProps.Tutorial = { export const JOIN_DISCORD_TUTORIAL: TutorialPanelProps.Tutorial = { completed: false, - title: 'Community', + title: 'Discord', description: ( <> diff --git a/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts b/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts index d3a465fcb5..74d166ba8d 100644 --- a/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts +++ b/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts @@ -167,14 +167,6 @@ export const useTutorials = () => { prerequisitesNeeded: !createProjectCompleted, }, - { - ...BILLING_TUTORIAL, - id: 4, - completed: billingCompleted, - startCallback: startBillingTutorial, - finishCallback: finishBillingTutorial, - }, - { ...QUICKSTART_TUTORIAL, id: 5, @@ -190,6 +182,14 @@ export const useTutorials = () => { completed: discordCompleted, startCallback: startDiscordTutorial, }, + + { + ...BILLING_TUTORIAL, + id: 4, + completed: billingCompleted, + startCallback: startBillingTutorial, + finishCallback: finishBillingTutorial, + }, ]; }, [ billingUrl, diff --git a/frontend/src/layouts/AppLayout/index.tsx b/frontend/src/layouts/AppLayout/index.tsx index d417779d41..3dd4db470b 100644 --- a/frontend/src/layouts/AppLayout/index.tsx +++ b/frontend/src/layouts/AppLayout/index.tsx @@ -183,7 +183,7 @@ const AppLayout: React.FC<{ children: React.ReactNode }> = ({ children }) => { }, process.env.UI_VERSION === 'sky' && { type: 'button', - iconName: 'suggestions', + iconName: 'support', title: t('common.tutorial_other'), onClick: toggleTutorialPanel, }, diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index c821abf317..342ce0cdd7 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -46,7 +46,7 @@ "continue": "Continue", "select_visible_columns": "Select visible columns", "tutorial": "Tutorials", - "tutorial_other": "Tour", + "tutorial_other": "Take a tour", "docs": "Docs", "discord": "Discord", "danger_zone": "Danger Zone", From 7ba2f3c17155f0082a4a3706361d23d6781744da Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:30:59 +0000 Subject: [PATCH 036/640] Fix `dstack event` compat. with older servers (#3509) Fixes this error: ``` $ dstack event Server validation error: {'detail': [{'loc': ['body', 'target_gateways'], 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}, {'loc': ['body', 'target_secrets'], 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}, {'loc': ['body', 'target_volumes'], 'msg': 'extra fields not permitted', 'type': 'value_error.extra'}]} ``` --- src/dstack/_internal/core/compatibility/events.py | 13 +++++++++++++ src/dstack/api/server/_events.py | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/dstack/_internal/core/compatibility/events.py diff --git a/src/dstack/_internal/core/compatibility/events.py b/src/dstack/_internal/core/compatibility/events.py new file mode 100644 index 0000000000..b28db11587 --- /dev/null +++ b/src/dstack/_internal/core/compatibility/events.py @@ -0,0 +1,13 @@ +from dstack._internal.core.models.common import IncludeExcludeDictType +from dstack._internal.server.schemas.events import ListEventsRequest + + +def get_list_events_excludes(request: ListEventsRequest) -> IncludeExcludeDictType: + list_gpus_excludes: IncludeExcludeDictType = {} + if request.target_volumes is None: + list_gpus_excludes["target_volumes"] = True + if request.target_gateways is None: + list_gpus_excludes["target_gateways"] = True + if request.target_secrets is None: + list_gpus_excludes["target_secrets"] = True + return list_gpus_excludes diff --git a/src/dstack/api/server/_events.py b/src/dstack/api/server/_events.py index 5aff9c9d21..22cd8893cd 100644 --- a/src/dstack/api/server/_events.py +++ b/src/dstack/api/server/_events.py @@ -4,6 +4,7 @@ from pydantic import parse_obj_as +from dstack._internal.core.compatibility.events import get_list_events_excludes from dstack._internal.core.models.events import Event, EventTargetType from dstack._internal.server.schemas.events import LIST_EVENTS_DEFAULT_LIMIT, ListEventsRequest from dstack.api.server._group import APIClientGroup @@ -57,5 +58,7 @@ def list( limit=limit, ascending=ascending, ) - resp = self._request("/api/events/list", body=req.json()) + resp = self._request( + "/api/events/list", body=req.json(exclude=get_list_events_excludes(req)) + ) return parse_obj_as(list[Event.__response__], resp.json()) From 763092d92074d22888ae7ab91192ddeb8085c249 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:31:12 +0000 Subject: [PATCH 037/640] Fix scaling during update to replica groups (#3510) This fix prevents the number of replicas from dropping to `replicas.min` for all existing services during a server update to 0.20.7. --- .../server/services/services/__init__.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 1f7dcde79d..45bb1fe0fe 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -19,7 +19,11 @@ ServerClientError, SSHError, ) -from dstack._internal.core.models.configurations import SERVICE_HTTPS_DEFAULT, ServiceConfiguration +from dstack._internal.core.models.configurations import ( + DEFAULT_REPLICA_GROUP_NAME, + SERVICE_HTTPS_DEFAULT, + ServiceConfiguration, +) from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.core.models.runs import JobSpec, Run, RunSpec, ServiceModelSpec, ServiceSpec @@ -318,8 +322,18 @@ async def update_service_desired_replica_count( prev_counts = ( json.loads(run_model.desired_replica_counts) if run_model.desired_replica_counts else {} ) + if ( + prev_counts == {} + and len(replica_groups) == 1 + and replica_groups[0].name == DEFAULT_REPLICA_GROUP_NAME + ): + # Special case to avoid dropping the replica count to group.count.min + # when a 0.20.7+ server first processes a service created by a pre-0.20.7 server. + # TODO: remove once most users upgrade to 0.20.7+. + prev_counts = {DEFAULT_REPLICA_GROUP_NAME: run_model.desired_replica_count} for group in replica_groups: scaler = get_service_scaler(group.count, group.scaling) + assert group.name is not None, "Group name is always set" group_desired = scaler.get_desired_count( current_desired_count=prev_counts.get(group.name, group.count.min or 0), stats=stats, From bb0278886dd4cb662b4c60fdc52ab4879779e09d Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Wed, 28 Jan 2026 22:39:07 +0545 Subject: [PATCH 038/640] [Docs] Replica groups (#3511) * Add Replica Groups Docs * Minor edits --------- Co-authored-by: Bihan Rana Co-authored-by: peterschmidt85 --- docs/docs/concepts/services.md | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/docs/concepts/services.md b/docs/docs/concepts/services.md index 745f78e3f0..0f6bf07bb8 100644 --- a/docs/docs/concepts/services.md +++ b/docs/docs/concepts/services.md @@ -164,6 +164,57 @@ Setting the minimum number of replicas to `0` allows the service to scale down t > The `scaling` property requires creating a [gateway](gateways.md). +??? info "Replica groups" + A service can include multiple replica groups. Each group can define its own `commands`, `resources` requirements, and `scaling` rules. + +
+ + ```yaml + type: service + name: llama-8b-service + + image: lmsysorg/sglang:latest + env: + - MODEL_ID=deepseek-ai/DeepSeek-R1-Distill-Llama-8B + + replicas: + - count: 1..2 + scaling: + metric: rps + target: 10 + commands: + - | + python -m sglang.launch_server \ + --model-path $MODEL_ID \ + --port 8000 \ + --trust-remote-code + resources: + gpu: 48GB + + - count: 1..4 + scaling: + metric: rps + target: 5 + commands: + - | + python -m sglang.launch_server \ + --model-path $MODEL_ID \ + --port 8000 \ + --trust-remote-code + resources: + gpu: 24GB + + port: 8000 + model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B + ``` + +
+ + > Properties such as `regions`, `port`, `image`, `env` and some other cannot be configured per replica group. This support is coming soon. + +??? info "Disaggregated serving" + Native support for disaggregated prefill and decode, allowing both worker types to run within a single service, is coming soon. + ### Model If the service is running a chat model with an OpenAI-compatible interface, From 0d58216d0ea088df7281130a4d45ed9996972040 Mon Sep 17 00:00:00 2001 From: Oleg Vavilov Date: Wed, 28 Jan 2026 22:51:40 +0300 Subject: [PATCH 039/640] Showed counters for Project and User List --- frontend/src/pages/Events/List/hooks/useFilters.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/pages/Events/List/hooks/useFilters.ts b/frontend/src/pages/Events/List/hooks/useFilters.ts index e6a97c71e8..d9457f9064 100644 --- a/frontend/src/pages/Events/List/hooks/useFilters.ts +++ b/frontend/src/pages/Events/List/hooks/useFilters.ts @@ -148,22 +148,22 @@ export const useFilters = () => { { key: filterKeys.TARGET_FLEETS, operators: ['='], - propertyLabel: 'Target fleets', + propertyLabel: 'Target fleet IDs', }, { key: filterKeys.TARGET_INSTANCES, operators: ['='], - propertyLabel: 'Target instances', + propertyLabel: 'Target instance IDs', }, { key: filterKeys.TARGET_RUNS, operators: ['='], - propertyLabel: 'Target runs', + propertyLabel: 'Target run IDs', }, { key: filterKeys.TARGET_JOBS, operators: ['='], - propertyLabel: 'Target jobs', + propertyLabel: 'Target job IDs', }, { key: filterKeys.TARGET_VOLUMES, @@ -191,13 +191,13 @@ export const useFilters = () => { { key: filterKeys.WITHIN_FLEETS, operators: ['='], - propertyLabel: 'Within fleets', + propertyLabel: 'Within fleet IDs', }, { key: filterKeys.WITHIN_RUNS, operators: ['='], - propertyLabel: 'Within runs', + propertyLabel: 'Within run IDs', }, { From 916b94e3388b74614443cbc2043508e53eddbe25 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:42:00 +0100 Subject: [PATCH 040/640] [Docs] Added `Spot policy` (#3512) --- docs/docs/concepts/fleets.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/docs/concepts/fleets.md b/docs/docs/concepts/fleets.md index 4def218456..dad06cb80a 100644 --- a/docs/docs/concepts/fleets.md +++ b/docs/docs/concepts/fleets.md @@ -281,12 +281,38 @@ Pre-provisioning is supported only for [VM-based backends](backends.md#vm-based) Backend fleets allow you to specify the resource requirements for the instances to be provisioned. The `resources` property syntax is the same as for [run configurations](dev-environments.md#resources). -> Not directly related, but in addition to `resources`, you can specify [`spot_policy`](../reference/dstack.yml/fleet.md#instance_types), [`instance_types`](../reference/dstack.yml/fleet.md#instance_types), [`max_price`](../reference/dstack.yml/fleet.md#max_price), [`region`](../reference/dstack.yml/fleet.md#max_price), and other [options](../reference/dstack.yml/fleet.md#). +### Spot policy - +Backend fleets allow you to specify a `spot policy`. By default, it is set to `on-demand`. If you want to use spot instances, you must set it to `auto` if you plan to use both on-demand and spot instances, or to `spot` if only spot instances are allowed. + +
+ +```yaml +type: fleet +name: my-fleet + +nodes: 0..2 + +# Uncomment to ensure instances are inter-connected +#placement: cluster + +# Allows both on-demand and spot +spot_policy: auto + +idle_duration: 1h + +resources: + gpu: 0..8 +``` + +
+ +Note that run configurations must specify their own `spot policy` which is also set to `on-demand` by default. ### Backends +Backend fleets allow you to set `backends` to specify which backends are allowed to be used. + ### Idle duration By default, instances of a backend fleet stay `idle` for 3 days and can be reused within that time. @@ -432,7 +458,7 @@ ssh_config: ``` !!! info "Reference" - The fleet configuration file supports many more options. See the [reference](../reference/dstack.yml/fleet.md). + The fleet configuration file supports additional options, including [`instance_types`](../reference/dstack.yml/fleet.md#instance_types), [`max_price`](../reference/dstack.yml/fleet.md#max_price), [`regions`](../reference/dstack.yml/fleet.md#max_price), among others. For the complete list, see the [reference](../reference/dstack.yml/fleet.md). ## Manage fleets From 9c81898b5c7cac18e4f27608682146a5d31df425 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 29 Jan 2026 12:30:46 +0300 Subject: [PATCH 041/640] Switch UI to pagination-based projects and users API (#3503) Switch UI to pagination-based projects and users API #3490 --- .../App/Login/LoginByGithubCallback/index.tsx | 2 +- .../useCheckingForFleetsInProjectsOfMember.ts | 13 +- frontend/src/hooks/useInfiniteScroll.ts | 61 +++++++-- frontend/src/hooks/useProjectFilter.ts | 10 +- .../layouts/AppLayout/TutorialPanel/hooks.ts | 12 +- .../src/pages/Events/List/hooks/useFilters.ts | 18 +-- frontend/src/pages/Models/List/hooks.tsx | 4 +- frontend/src/pages/Project/List/index.tsx | 98 +++++++------- .../Members/UsersAutosuggest/index.tsx | 4 +- frontend/src/pages/Project/Members/index.tsx | 4 +- .../hooks/useGetRunSpecFromYaml.ts | 1 + .../pages/Runs/CreateDevEnvironment/index.tsx | 1 + .../src/pages/Runs/List/hooks/useFilters.ts | 4 +- .../src/pages/User/Details/Projects/index.tsx | 9 +- frontend/src/pages/User/List/hooks.tsx | 39 ++++++ frontend/src/pages/User/List/index.tsx | 126 +++++++----------- frontend/src/services/project.ts | 17 ++- frontend/src/services/user.ts | 17 ++- frontend/src/types/project.d.ts | 10 +- frontend/src/types/user.d.ts | 12 ++ 20 files changed, 270 insertions(+), 192 deletions(-) create mode 100644 frontend/src/pages/User/List/hooks.tsx diff --git a/frontend/src/App/Login/LoginByGithubCallback/index.tsx b/frontend/src/App/Login/LoginByGithubCallback/index.tsx index af88aa72f1..45be311b6b 100644 --- a/frontend/src/App/Login/LoginByGithubCallback/index.tsx +++ b/frontend/src/App/Login/LoginByGithubCallback/index.tsx @@ -41,7 +41,7 @@ export const LoginByGithubCallback: React.FC = () => { .then(async ({ creds: { token } }) => { dispatch(setAuthData({ token })); if (process.env.UI_VERSION === 'sky') { - const result = await getProjects().unwrap(); + const result = await getProjects({}).unwrap(); if (result?.length === 0) { navigate(ROUTES.PROJECT.ADD); return; diff --git a/frontend/src/hooks/useCheckingForFleetsInProjectsOfMember.ts b/frontend/src/hooks/useCheckingForFleetsInProjectsOfMember.ts index d91b78a3b1..b330358b46 100644 --- a/frontend/src/hooks/useCheckingForFleetsInProjectsOfMember.ts +++ b/frontend/src/hooks/useCheckingForFleetsInProjectsOfMember.ts @@ -5,9 +5,12 @@ import { useGetOnlyNoFleetsProjectsQuery, useGetProjectsQuery } from 'services/p type Args = { projectNames?: IProject['project_name'][] }; export const useCheckingForFleetsInProjects = ({ projectNames }: Args) => { - const { data: projectsData } = useGetProjectsQuery(undefined, { - skip: !!projectNames?.length, - }); + const { data: projectsData } = useGetProjectsQuery( + {}, + { + skip: !!projectNames?.length, + }, + ); const { data: noFleetsProjectsData } = useGetOnlyNoFleetsProjectsQuery(); @@ -16,8 +19,8 @@ export const useCheckingForFleetsInProjects = ({ projectNames }: Args) => { return projectNames; } - if (projectsData) { - return projectsData.map((project) => project.project_name); + if (projectsData?.data) { + return projectsData.data.map((project) => project.project_name); } return []; diff --git a/frontend/src/hooks/useInfiniteScroll.ts b/frontend/src/hooks/useInfiniteScroll.ts index 727586ab00..b11115d63f 100644 --- a/frontend/src/hooks/useInfiniteScroll.ts +++ b/frontend/src/hooks/useInfiniteScroll.ts @@ -9,10 +9,14 @@ const SCROLL_POSITION_GAP = 400; type InfinityListArgs = Partial>; type ListResponse = DataItem[]; +type ResponseWithDataProp = { data: ListResponse; total_count: number }; + +type LazyQueryResponse = ResponseWithDataProp | ListResponse; type UseInfinityParams = { - useLazyQuery: UseLazyQuery, any>>; + useLazyQuery: UseLazyQuery, any>>; args: { limit?: number } & Args; + getResponseItems?: (listItem: DataItem) => Partial; getPaginationParams: (listItem: DataItem) => Partial; skip?: boolean; // options?: UseQueryStateOptions, Record>; @@ -26,22 +30,31 @@ export const useInfiniteScroll = ({ skip, }: UseInfinityParams) => { const [data, setData] = useState>([]); + const [totalCount, setTotalCount] = useState(); const scrollElement = useRef(document.documentElement); const isLoadingRef = useRef(false); + const isDisabledMoreRef = useRef(false); const lastRequestParams = useRef(undefined); - const [disabledMore, setDisabledMore] = useState(false); const { limit, ...argsProp } = args; const lastArgsProps = useRef>(null); const [getItems, { isLoading, isFetching }] = useLazyQuery({ ...args } as Args); const getDataRequest = (params: Args) => { - lastRequestParams.current = params; + if (isEqual(params, lastRequestParams.current)) { + return Promise.reject(); + } - return getItems({ + const request = getItems({ limit, ...params, } as Args).unwrap(); + + request.then(() => { + lastRequestParams.current = { ...params }; + }); + + return request; }; const getEmptyList = () => { @@ -49,9 +62,18 @@ export const useInfiniteScroll = ({ setData([]); - getDataRequest(argsProp as Args).then((result) => { - setDisabledMore(false); - setData(result as ListResponse); + getDataRequest(argsProp as Args).then((result: LazyQueryResponse) => { + // setDisabledMore(false); + isDisabledMoreRef.current = false; + + if ('data' in result) { + setData(result.data as ListResponse); + setTotalCount(result.total_count); + } else { + setData(result as ListResponse); + setTotalCount(); + } + isLoadingRef.current = false; }); }; @@ -64,7 +86,7 @@ export const useInfiniteScroll = ({ }, [argsProp, lastArgsProps, skip]); const getMore = async () => { - if (isLoadingRef.current || disabledMore || skip) { + if (isLoadingRef.current || isDisabledMoreRef.current || skip) { return; } @@ -76,10 +98,20 @@ export const useInfiniteScroll = ({ ...getPaginationParams(data[data.length - 1]), } as Args); - if (result.length > 0) { - setData((prev) => [...prev, ...result]); + let listResponse: ListResponse; + + if ('data' in result) { + listResponse = result.data; + setTotalCount(result.total_count); + } else { + listResponse = result; + setTotalCount(); + } + + if (listResponse.length > 0) { + setData((prev) => [...prev, ...listResponse]); } else { - setDisabledMore(true); + isDisabledMoreRef.current = true; } } catch (e) { console.log(e); @@ -87,7 +119,7 @@ export const useInfiniteScroll = ({ setTimeout(() => { isLoadingRef.current = false; - }, 10); + }, 50); }; useLayoutEffect(() => { @@ -101,7 +133,7 @@ export const useInfiniteScroll = ({ }, [data]); const onScroll = useCallback(() => { - if (disabledMore || isLoadingRef.current) { + if (isDisabledMoreRef.current || isLoadingRef.current) { return; } @@ -112,7 +144,7 @@ export const useInfiniteScroll = ({ if (scrollPositionFromBottom < SCROLL_POSITION_GAP) { getMore().catch(console.log); } - }, [disabledMore, getMore]); + }, [getMore]); useEffect(() => { document.addEventListener('scroll', onScroll); @@ -126,6 +158,7 @@ export const useInfiniteScroll = ({ return { data, + totalCount, isLoading: isLoading || (data.length === 0 && isFetching), isLoadingMore, refreshList: getEmptyList, diff --git a/frontend/src/hooks/useProjectFilter.ts b/frontend/src/hooks/useProjectFilter.ts index b4573b1f8e..58e573d31d 100644 --- a/frontend/src/hooks/useProjectFilter.ts +++ b/frontend/src/hooks/useProjectFilter.ts @@ -16,20 +16,20 @@ export const useProjectFilter = ({ localStorePrefix }: Args) => { null, ); - const { data: projectsData } = useGetProjectsQuery(); + const { data: projectsData } = useGetProjectsQuery({}); const projectOptions = useMemo(() => { - if (!projectsData?.length) return []; + if (!projectsData?.data?.length) return []; - return projectsData.map((project) => ({ label: project.project_name, value: project.project_name })); + return projectsData.data.map((project) => ({ label: project.project_name, value: project.project_name })); }, [projectsData]); useEffect(() => { - if (!projectsData || !selectedProject) { + if (!projectsData?.data || !selectedProject) { return; } - const hasSelectedProject = projectsData.some(({ project_name }) => selectedProject?.value === project_name); + const hasSelectedProject = projectsData.data.some(({ project_name }) => selectedProject?.value === project_name); if (!hasSelectedProject) { setSelectedProject(null); diff --git a/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts b/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts index 74d166ba8d..305a711a05 100644 --- a/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts +++ b/frontend/src/layouts/AppLayout/TutorialPanel/hooks.ts @@ -44,7 +44,7 @@ export const useTutorials = () => { } = useAppSelector(selectTutorialPanel); const { data: userBillingData } = useGetUserBillingInfoQuery({ username: useName ?? '' }, { skip: !useName }); - const { data: projectData } = useGetProjectsQuery(); + const { data: projectData } = useGetProjectsQuery({}); const { data: runsData } = useGetRunsQuery({ limit: 1, }); @@ -54,14 +54,14 @@ export const useTutorials = () => { useEffect(() => { if ( userBillingData && - projectData && + projectData?.data && runsData && !completeIsChecked.current && location.pathname !== ROUTES.PROJECT.ADD ) { const billingCompleted = userBillingData.balance > 0; const configureCLICompleted = runsData.length > 0; - const createProjectCompleted = projectData.length > 0; + const createProjectCompleted = projectData.data.length > 0; let tempHideStartUp = hideStartUp; @@ -88,7 +88,7 @@ export const useTutorials = () => { }, [userBillingData, runsData, projectData, location.pathname]); useEffect(() => { - if (projectData && projectData.length > 0 && !createProjectCompleted) { + if (projectData?.data && projectData.data.length > 0 && !createProjectCompleted) { dispatch( updateTutorialPanelState({ createProjectCompleted: true, @@ -114,8 +114,8 @@ export const useTutorials = () => { }, []); const startConfigCliTutorial = useCallback(() => { - if (projectData?.length) { - navigate(ROUTES.PROJECT.DETAILS.SETTINGS.FORMAT(projectData[0].project_name)); + if (projectData?.data?.length) { + navigate(ROUTES.PROJECT.DETAILS.SETTINGS.FORMAT(projectData.data[0].project_name)); } }, [projectData]); diff --git a/frontend/src/pages/Events/List/hooks/useFilters.ts b/frontend/src/pages/Events/List/hooks/useFilters.ts index d9457f9064..535582195b 100644 --- a/frontend/src/pages/Events/List/hooks/useFilters.ts +++ b/frontend/src/pages/Events/List/hooks/useFilters.ts @@ -77,8 +77,8 @@ const targetTypes = [ export const useFilters = () => { const [searchParams, setSearchParams] = useSearchParams(); - const { data: projectsData } = useGetProjectsQuery(); - const { data: usersData } = useGetUserListQuery(); + const { data: projectsData } = useGetProjectsQuery({}); + const { data: usersData } = useGetUserListQuery({}); const [propertyFilterQuery, setPropertyFilterQuery] = useState(() => requestParamsToTokens({ searchParams, filterKeys }), @@ -92,7 +92,7 @@ export const useFilters = () => { const filteringOptions = useMemo(() => { const options: PropertyFilterProps.FilteringOption[] = []; - projectsData?.forEach(({ project_name }) => { + projectsData?.data?.forEach(({ project_name }) => { options.push({ propertyKey: filterKeys.TARGET_PROJECTS, value: project_name, @@ -104,7 +104,7 @@ export const useFilters = () => { }); }); - usersData?.forEach(({ username }) => { + usersData?.data?.forEach(({ username }) => { options.push({ propertyKey: filterKeys.TARGET_USERS, value: username, @@ -247,14 +247,16 @@ export const useFilters = () => { ...(params[filterKeys.TARGET_PROJECTS] && Array.isArray(params[filterKeys.TARGET_PROJECTS]) ? { [filterKeys.TARGET_PROJECTS]: params[filterKeys.TARGET_PROJECTS]?.map( - (name: string) => projectsData?.find(({ project_name }) => project_name === name)?.['project_id'], + (name: string) => + projectsData?.data?.find(({ project_name }) => project_name === name)?.['project_id'], ), } : {}), ...(params[filterKeys.WITHIN_PROJECTS] && Array.isArray(params[filterKeys.WITHIN_PROJECTS]) ? { [filterKeys.WITHIN_PROJECTS]: params[filterKeys.WITHIN_PROJECTS]?.map( - (name: string) => projectsData?.find(({ project_name }) => project_name === name)?.['project_id'], + (name: string) => + projectsData?.data?.find(({ project_name }) => project_name === name)?.['project_id'], ), } : {}), @@ -262,7 +264,7 @@ export const useFilters = () => { ...(params[filterKeys.TARGET_USERS] && Array.isArray(params[filterKeys.TARGET_USERS]) ? { [filterKeys.TARGET_USERS]: params[filterKeys.TARGET_USERS]?.map( - (name: string) => usersData?.find(({ username }) => username === name)?.['id'], + (name: string) => usersData?.data?.find(({ username }) => username === name)?.['id'], ), } : {}), @@ -270,7 +272,7 @@ export const useFilters = () => { ...(params[filterKeys.ACTORS] && Array.isArray(params[filterKeys.ACTORS]) ? { [filterKeys.ACTORS]: params[filterKeys.ACTORS]?.map( - (name: string) => usersData?.find(({ username }) => username === name)?.['id'], + (name: string) => usersData?.data?.find(({ username }) => username === name)?.['id'], ), } : {}), diff --git a/frontend/src/pages/Models/List/hooks.tsx b/frontend/src/pages/Models/List/hooks.tsx index 912875b394..461bf28a3b 100644 --- a/frontend/src/pages/Models/List/hooks.tsx +++ b/frontend/src/pages/Models/List/hooks.tsx @@ -129,7 +129,7 @@ const filterKeys: Record = { export const useFilters = (localStorePrefix = 'models-list-page') => { const [searchParams, setSearchParams] = useSearchParams(); const { projectOptions } = useProjectFilter({ localStorePrefix }); - const { data: usersData } = useGetUserListQuery(); + const { data: usersData } = useGetUserListQuery({}); const [propertyFilterQuery, setPropertyFilterQuery] = useState(() => requestParamsToTokens({ searchParams, filterKeys }), @@ -151,7 +151,7 @@ export const useFilters = (localStorePrefix = 'models-list-page') => { }); }); - usersData?.forEach(({ username }) => { + usersData?.data?.forEach(({ username }) => { options.push({ propertyKey: filterKeys.USER_NAME, value: username, diff --git a/frontend/src/pages/Project/List/index.tsx b/frontend/src/pages/Project/List/index.tsx index af4afcf5dc..eb896df91d 100644 --- a/frontend/src/pages/Project/List/index.tsx +++ b/frontend/src/pages/Project/List/index.tsx @@ -1,44 +1,36 @@ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useNavigate } from 'react-router-dom'; -import { get as _get } from 'lodash'; - -import { - Button, - ButtonWithConfirmation, - Header, - ListEmptyMessage, - Pagination, - SpaceBetween, - Table, - TextFilter, -} from 'components'; - -import { useBreadcrumbs, useCollection } from 'hooks'; + +import { Button, ButtonWithConfirmation, Header, ListEmptyMessage, Loader, SpaceBetween, Table, TextFilter } from 'components'; + +import { DEFAULT_TABLE_PAGE_SIZE } from 'consts'; +import { useBreadcrumbs, useCollection, useInfiniteScroll } from 'hooks'; import { ROUTES } from 'routes'; -import { useGetProjectsQuery } from 'services/project'; +import { useLazyGetProjectsQuery } from 'services/project'; import { useCheckAvailableProjectPermission } from '../hooks/useCheckAvailableProjectPermission'; import { useDeleteProject } from '../hooks/useDeleteProject'; import { useColumnsDefinitions } from './hooks'; -const SEARCHABLE_COLUMNS = ['project_name', 'owner.username']; - export const ProjectList: React.FC = () => { const { t } = useTranslation(); - const { isLoading, isFetching, data, refetch } = useGetProjectsQuery(); const { isAvailableDeletingPermission, isAvailableProjectManaging } = useCheckAvailableProjectPermission(); const { deleteProject, deleteProjects, isDeleting } = useDeleteProject(); + const [filteringText, setFilteringText] = useState(''); + const [namePattern, setNamePattern] = useState(''); const navigate = useNavigate(); - const sortedData = useMemo(() => { - if (!data) return []; + const { data, isLoading, refreshList, isLoadingMore, totalCount } = useInfiniteScroll({ + useLazyQuery: useLazyGetProjectsQuery, + args: { name_pattern: namePattern, limit: DEFAULT_TABLE_PAGE_SIZE }, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - return [...data].sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); - }, [data]); + getPaginationParams: (lastProject) => ({ + prev_created_at: lastProject.created_at, + prev_id: lastProject.project_id, + }), + }); useBreadcrumbs([ { @@ -51,7 +43,24 @@ export const ProjectList: React.FC = () => { navigate(ROUTES.PROJECT.ADD); }; + const onClearFilter = () => { + setNamePattern(''); + setFilteringText(''); + }; + const renderEmptyMessage = (): React.ReactNode => { + if (isLoading) { + return null; + } + + if (filteringText) { + return ( + + + + ); + } + return ( {isAvailableProjectManaging && } @@ -59,28 +68,10 @@ export const ProjectList: React.FC = () => { ); }; - const renderNoMatchMessage = (onClearFilter: () => void): React.ReactNode => { - return ( - - - - ); - }; - - const { items, actions, filteredItemsCount, collectionProps, filterProps, paginationProps } = useCollection(sortedData, { + const { items, collectionProps } = useCollection(data, { filtering: { empty: renderEmptyMessage(), - noMatch: renderNoMatchMessage(() => actions.setFiltering('')), - - filteringFunction: (projectItem: IProject, filteringText) => { - const filteringTextLowerCase = filteringText.toLowerCase(); - - return SEARCHABLE_COLUMNS.map((key) => _get(projectItem, key)).some( - (value) => typeof value === 'string' && value.trim().toLowerCase().indexOf(filteringTextLowerCase) > -1, - ); - }, }, - pagination: { pageSize: 20 }, selection: {}, }); @@ -104,9 +95,9 @@ export const ProjectList: React.FC = () => { }); const renderCounter = () => { - if (!data?.length) return ''; + if (typeof totalCount !== 'number') return ''; - return `(${data.length})`; + return `(${totalCount})`; }; return ( @@ -116,7 +107,7 @@ export const ProjectList: React.FC = () => { variant="full-page" columnDefinitions={columns} items={items} - loading={isLoading || isFetching} + loading={isLoading} loadingText={t('common.loading')} selectionType={isAvailableProjectManaging ? 'multi' : undefined} stickyHeader={true} @@ -141,9 +132,9 @@ export const ProjectList: React.FC = () => { + + ); + } + return ( @@ -91,22 +76,10 @@ export const UserList: React.FC = () => { ); }; - const renderNoMatchMessage = (onClearFilter: () => void): React.ReactNode => { - return ( - - - - ); - }; - - const { items, actions, filteredItemsCount, collectionProps, filterProps, paginationProps } = useCollection(sortedData, { + const { items, actions, collectionProps } = useCollection(data, { filtering: { empty: renderEmptyMessage(), - noMatch: renderNoMatchMessage(() => actions.setFiltering('')), - filteringFunction: (user, filteringText) => - includeSubString(user.username, filteringText) || includeSubString(user.email ?? '', filteringText), }, - pagination: { pageSize: 20 }, selection: {}, }); @@ -145,13 +118,9 @@ export const UserList: React.FC = () => { }, [collectionProps.selectedItems]); const renderCounter = () => { - const { selectedItems } = collectionProps; - - if (!data?.length) return ''; - - if (selectedItems?.length) return `(${selectedItems?.length}/${data?.length ?? 0})`; + if (typeof totalCount !== 'number') return ''; - return `(${data.length})`; + return `(${totalCount})`; }; return ( @@ -160,9 +129,11 @@ export const UserList: React.FC = () => { {...collectionProps} variant="full-page" isItemDisabled={getIsTableItemDisabled} - columnDefinitions={COLUMN_DEFINITIONS} + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + columnDefinitions={columns} items={items} - loading={isLoading || isFetching} + loading={isLoading} loadingText={t('common.loading')} selectionType="multi" stickyHeader={true} @@ -186,9 +157,9 @@ export const UserList: React.FC = () => { + )} + + + } + title="On-demand GPUs for AI workloads" + imageFirst + > + Access affordable on-demand and spot GPUs with pay-as-you-go pricing. +
+
+ You can also bring compute from your own cloud accounts or on-prem infrastructure + and use it alongside Sky GPUs. + + + + dstack Sky provides first-class primitives for compute management, training, + inference, and observability across heterogeneous AI compute. Use one interface + to efficiently utilize cloud GPUs, manage your own clusters, or run your own + AI token factory at scale. +
+
+ Sky is built on dstack and uses the same CLI and YAML configurations as a + self-hosted dstack server. + + + + + + + + + + + ); +} diff --git a/website/src/pages/Sky/pricing.ts b/website/src/pages/Sky/pricing.ts new file mode 100644 index 0000000000..744a1e469e --- /dev/null +++ b/website/src/pages/Sky/pricing.ts @@ -0,0 +1,13 @@ +// Indicative USD per GPU-hour from Sky API quotes, 2026-09-14; availability varies. +export const gpuPrices = [ + { name: 'B300', memory: '288GB', onDemand: '$7.58–17.80', spot: '$3.79–6.49' }, + { name: 'B200', memory: '192GB', onDemand: '$6.23–17.91', spot: '$2.41–5.44' }, + { name: 'H200', memory: '141GB', onDemand: '$4.20–9.89', spot: '$2.10–9.89' }, + { name: 'H100', memory: '80GB', onDemand: '$2.89–12.74', spot: '$1.09–8.94' }, + // Includes AMD Developer Cloud and Hot Aisle backend quotes. + { name: 'MI300X', memory: '192GB', onDemand: '$1.99–2.99', spot: null }, + { name: 'RTX PRO 6000', memory: '96GB', onDemand: '$0.97–7.20', spot: '$0.50–3.17' }, + { name: 'A100', memory: '80GB', onDemand: '$1.59–5.58', spot: '$0.87–4.29' }, + { name: 'A100', memory: '40GB', onDemand: '$1.99–3.75', spot: '$0.34–3.57' }, + { name: 'L40S', memory: '48GB', onDemand: '$1.09–4.71', spot: '$0.54–4.71' }, +]; diff --git a/website/src/router.tsx b/website/src/router.tsx index 6b63521522..fb46c14ddd 100644 --- a/website/src/router.tsx +++ b/website/src/router.tsx @@ -1,19 +1,17 @@ import { Navigate, createBrowserRouter } from 'react-router-dom'; import { App } from './App'; import { HomePage } from './pages/Home'; -import { OldPage } from './pages/Old'; +import { SkyPage } from './pages/Sky/SkyPage'; import { ROUTES } from './routes'; -// Single data router. In production this app owns only `/` (the landing) — docs and blog -// are served by MkDocs on the same origin. `/old` is kept as a template for future product -// pages: reachable in dev, and harmless in production (MkDocs serves unknown paths). Stray -// paths redirect home. +// The landing and product pages share one layout. Docs and blog are served by MkDocs +// on the same origin; stray paths redirect home. export const router = createBrowserRouter([ { element: , children: [ { index: true, element: }, - { path: ROUTES.OLD, element: }, + { path: ROUTES.SKY, element: }, { path: '*', element: }, ], }, diff --git a/website/src/routes.ts b/website/src/routes.ts index dd9c5fac47..4d759478cc 100644 --- a/website/src/routes.ts +++ b/website/src/routes.ts @@ -1,6 +1,6 @@ // Central route table + cross-links to the MkDocs-served parts of the site. // -// This app (the landing) owns only `/`. Docs and blog are served by MkDocs on the +// This app owns `/` and `/products/sky`. Docs and blog are served by MkDocs on the // SAME origin in production (`/docs`, `/blog`). For standalone landing development you // can point those links at the live site by setting VITE_DOCS_BASE, e.g. // VITE_DOCS_BASE=https://dstack.ai npm run dev @@ -8,10 +8,7 @@ const SITE_BASE = (import.meta.env.VITE_DOCS_BASE ?? '').replace(/\/+$/, ''); export const ROUTES = { HOME: '/', - // Kept as a template/reference for building future product pages. Reachable in dev - // (`npm run dev` at /old); not part of the integrated production deploy (where this app - // only owns `/` and MkDocs serves everything else). - OLD: '/old', + SKY: '/products/sky', } as const; export type Route = (typeof ROUTES)[keyof typeof ROUTES]; diff --git a/website/src/styles.css b/website/src/styles.css index cb83a43ec1..13d8fa9197 100644 --- a/website/src/styles.css +++ b/website/src/styles.css @@ -32,15 +32,12 @@ --nav-height: 4.875rem; --frame: 80rem; --page-gutter: clamp(1.5rem, 5.5vw, 5rem); - --section-gap: clamp(2.5rem, 4vw, 4rem); --card-gap: 1.25rem; --radius: 0; - --media-column: minmax(18rem, 40rem); --doc-article-width: 49.5rem; --doc-rail-width: 17.5rem; --doc-right-gap: 2.5rem; --doc-gap: var(--doc-right-gap); - --doc-main-width: calc(var(--doc-article-width) + var(--doc-right-gap) + var(--doc-rail-width)); } :root[data-theme='dark'] { @@ -220,16 +217,6 @@ p { display: none; } -/* On the Old page the nav drops the brand gradient accent for a plain 1px hairline in the - text color, matching the Cloudscape reference header. */ -.site-nav--old { - border-bottom: 1px solid var(--cs-text); -} - -.site-nav--old::after { - display: none; -} - .site-nav__inner { position: relative; z-index: 1; @@ -278,10 +265,6 @@ p { min-width: 0; } -.site-desktop-trigger { - flex: 0 0 auto; -} - .site-mobile-trigger, .site-mobile-spacer { display: none; @@ -376,12 +359,14 @@ p { pointer-events: none; } +.home-hero h1, .home-hero h2, .home-hero p, .home-hero__actions { pointer-events: auto; } +.home-hero h1, .home-hero h2 { margin-top: 0; max-width: 720px; @@ -439,6 +424,23 @@ p { z-index: 1; } +.home-main--sky::before { + position: absolute; + top: calc(-1 * var(--nav-height)); + left: 0; + right: 0; + height: 36rem; + z-index: -1; + content: ''; + pointer-events: none; + background: linear-gradient( + 180deg, + rgba(65, 157, 255, 0.5) 0%, + rgba(176, 65, 255, 0.06) 35%, + rgba(176, 65, 255, 0) 100% + ); +} + .home-stack { padding: 0 0 96px; } @@ -450,12 +452,6 @@ p { var(--doc-rail-width); } -.home-with-rail .image-text-row, -.home-with-rail .image-text-row:not(.image-first) { - grid-template-columns: minmax(16rem, 1fr) minmax(0, 1fr); - gap: var(--doc-gap); -} - .media-card h3, .doc-media-card h3 { margin-bottom: 8px; @@ -463,47 +459,6 @@ p { line-height: 1.2; } -.image-text-row { - display: grid; - grid-template-columns: var(--media-column) minmax(0, 1fr); - gap: var(--section-gap); - align-items: center; - margin-top: 40px; -} - -.image-text-row:not(.image-first) { - grid-template-columns: minmax(0, 1fr) var(--media-column); -} - -.image-text-row.image-first .landing-image { - order: -1; -} - -.docs-shell--old-page .image-text-row, -.docs-shell--old-page .image-text-row:not(.image-first) { - grid-template-columns: minmax(14rem, 24rem) minmax(0, 1fr); - gap: var(--doc-gap); -} - -.docs-shell--old-page .image-text-row:not(.image-first) { - grid-template-columns: minmax(0, 1fr) minmax(14rem, 24rem); -} - -.landing-image { - width: 100%; - aspect-ratio: 876 / 528; - height: auto; - object-fit: cover; - border-radius: var(--radius); -} - -.landing-copy { - max-width: 604px; -} - -.landing-copy h2, -.overview-section h2, -.start-building h2, .docs-section > h2 { margin-bottom: 12px; font-size: 28px; @@ -538,65 +493,6 @@ p { font-size: 16px; } -.landing-copy p + p { - margin-top: 18px; -} - -.overview-section { - margin-top: 56px; -} - -.stats-grid { - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: var(--card-gap); - margin-top: 20px; -} - -.stat-card { - display: grid; - grid-template-columns: 72px minmax(0, 1fr); - gap: var(--card-gap); - align-items: center; - min-height: 160px; - padding: 24px 40px; - border-radius: 0; - background: var(--cs-panel); -} - -.stat-card span { - color: #3867ff; - font-size: 44px; - line-height: 1; - font-weight: 400; -} - -.stat-card p { - max-width: 14ch; - font-size: 24px; - line-height: 1.15; -} - -.core-features { - margin-top: 36px; -} - -.core-features > h3 { - margin-bottom: 4px; - font-size: 20px; -} - -.core-features > p { - max-width: 1230px; -} - -.feature-card-grid { - display: grid; - grid-template-columns: repeat(4, 1fr); - gap: 20px; - margin-top: 32px; -} - .media-card { border: 1px solid var(--cs-border); /* 1px / 12px */ border-radius: 12px; @@ -625,49 +521,6 @@ p { font-size: var(--font-small); } -.start-building { - margin-top: 64px; - padding-top: 48px; - border-top: 0.5px solid var(--cs-border); -} - -.start-grid { - display: grid; - grid-template-columns: repeat(2, 1fr); - gap: var(--card-gap); - margin-top: 20px; -} - -.start-grid article { - min-height: 152px; - padding: 24px 20px; - border: 0.5px solid var(--cs-border); - border-radius: 0; -} - -.start-grid h3 { - margin-bottom: 18px; - font-size: 20px; -} - -.start-grid p { - font-size: 16px; -} - -.docs-shell { - min-height: calc(100vh - var(--nav-height)); -} - -.docs-main { - width: min(var(--doc-main-width), calc(100vw - (2 * var(--page-gutter)))); - padding: 1.625rem 0 6rem; -} - -.docs-shell--navigation-collapsed .docs-main { - width: min(var(--frame), calc(100vw - (2 * var(--page-gutter)))); - margin-inline: auto; -} - .docs-body { display: grid; grid-template-columns: minmax(0, var(--doc-article-width)) var(--doc-right-gap) var(--doc-rail-width); @@ -678,82 +531,10 @@ p { grid-template-columns: minmax(0, 1fr); } -.docs-shell--navigation-collapsed .docs-body { - grid-template-columns: - minmax(0, calc(100% - var(--doc-right-gap) - var(--doc-rail-width))) - var(--doc-right-gap) - var(--doc-rail-width); -} - -.docs-shell--old-page .docs-main { - width: min(var(--frame), 100%); - margin-inline: auto; - /* The footer is the last child here; sit it flush at the bottom of the content - column instead of leaving the article's bottom padding below it. */ - padding-bottom: 0; -} - -/* Cloudscape gives the AppLayout content region a 40px bottom margin. With the footer living - inside it, that leaves a 40px strip below the footer; drop it so the footer sits flush at the - page bottom, matching the global site footer. */ -.docs-shell--old-page [class*='awsui_main_'] { - margin-bottom: 0 !important; -} - -.docs-shell--old-page .docs-body { - grid-template-columns: - minmax(0, calc(100% - var(--doc-right-gap) - var(--doc-rail-width))) - var(--doc-right-gap) - var(--doc-rail-width); -} - .docs-article { grid-column: 1; } -.docs-right { - grid-column: 3; - position: sticky; - /* Clear the sticky header (banner + nav), matching scroll-padding-top above. */ - top: calc(var(--nav-height) + 3.25rem); - margin-top: 1.5rem; -} - -/* The section divider in the docs side nav is inset by default (unlike the header - divider); pull it out to the drawer edges so it spans the full sidebar width. */ -.docs-shell [class*='awsui_divider-default'] { - margin-left: -28px !important; - margin-right: -24px !important; -} - -/* Cloudscape's expandable nav group renders the caret before the label, indenting the parent - past its sibling links, and indents the children 40px. Pull the caret into the left gutter so - the parent label lines up with its siblings, and tighten the children to one indent level. */ -.docs-shell [class*='awsui_expandable-link-group'] [class*='awsui_expand-button'] { - margin-left: -22px !important; -} - -.docs-shell [class*='awsui_list-variant-expandable-link-group'] { - padding-left: 20px !important; -} - -.docs-title { - margin-top: 1rem; - padding-bottom: 1.125rem; - border-bottom: 0.5px solid var(--cs-border); -} - -.docs-title h1 { - font-size: 36px; - line-height: 1.2; -} - -.docs-title p { - margin-top: 8px; - color: var(--cs-muted); - font-size: 18px; -} - .docs-section { padding: 1.5rem 0 3rem; border-bottom: 0.5px solid var(--cs-border); @@ -809,31 +590,6 @@ p { margin-top: 20px; } - -.doc-alternating img { - width: 100%; - aspect-ratio: 980 / 640; - height: auto; - object-fit: cover; - border-radius: var(--radius); -} - -.doc-alternating img.doc-diagram { - object-fit: contain; -} - -.doc-diagram--dark { - display: none; -} - -:root[data-theme='dark'] .doc-diagram--light { - display: none; -} - -:root[data-theme='dark'] .doc-diagram--dark { - display: block; -} - .doc-alternating h2 { margin-bottom: 12px; font-size: 28px; @@ -1256,13 +1012,7 @@ p { margin-top: 0; } -/* "Get started" deployment switcher. The Products-popup component reused as a vertical switcher: - a bordered selector panel on the left whose options carry a caption + description, with the - selected option filled by the brand gradient. Open-source is featured (bigger, with a live star - count and a Documentation button) and selected by default; dstack Sky / Factory are compact - rows. The right column shows the selection's detail — install code for open-source, an included- - features list + CTA for the hosted / self-hosted tiers. align-items: stretch keeps the selector - and the detail equal height and top-aligned. */ +/* Product links beside the dstack installation controls. */ .gs-deploy { display: grid; grid-template-columns: 1fr 2fr; /* selector 1/3, detail 2/3 */ @@ -1276,35 +1026,15 @@ p { display: flex; flex-direction: column; padding: 0; - position: relative; /* anchors the sliding selection indicator */ - border-radius: 8px; /* same radius as the rows and the indicator card */ + border-radius: 8px; overflow: hidden; /* clip the full-width group bars to the rounded corners */ background: transparent; /* the rows carry the panel ground, so the gap between groups is a true hole */ } -/* The selection indicator: a white card (0.5px solid ring) that slides up/down to the selected - row. Rows are transparent; bars sit above it so it passes underneath between groups. */ -.gs-rail__indicator { - position: absolute; - left: 0; - right: 0; - z-index: 2; /* above the rows' ground, beneath their content (3) and the bars (4) */ - background: var(--cs-bg); - border: 1px solid var(--cs-text); - border-radius: 8px; - pointer-events: none; -} -/* No z-index on the rows: a row with its own stacking context would trap its content beneath - the indicator card, so the content is lifted above the card explicitly instead. */ -.gs-opt__icwrap, -.gs-opt__ic, -.gs-opt__body { - position: relative; - z-index: 3; -} -/* In the Products popup the rows are links; render them exactly like the section's buttons. */ +/* Product links share the same styling in the section and popup. */ .gs-rail a.gs-opt { color: var(--cs-text); text-decoration: none; + cursor: pointer; } .gs-rail__group { position: relative; @@ -1320,8 +1050,6 @@ p { font: inherit; line-height: 1.15; /* pinned (not inherited) so the rail renders identically on the docs site */ text-align: left; - cursor: pointer; - transition: background-color 0.2s ease, border-color 0.2s ease; } /* Adjacent rows inside a group square off their shared edge, so the group's ground reads as one @@ -1334,6 +1062,11 @@ p { border-bottom-left-radius: 0; border-bottom-right-radius: 0; } +.gs-opt.gs-opt--on { + border-radius: 8px; + background: var(--cs-bg); + box-shadow: inset 0 0 0 1px var(--cs-text); +} /* Row names mirror the tab typography: 16px / 700. */ .gs-opt__name { @@ -1356,18 +1089,13 @@ p { } /* The first bar sits inside the rail's border corner: a 12px outer radius with a 1px border has an 11px inner curve, so the bar needs the nested radius to fill it without a sliver. */ -/* A group bar that follows a row stands off from it, so the groups read as separate - blocks inside the rail (keyed on adjacency, not :first-child — the indicator div - is the rail's first DOM child). */ +/* A group bar that follows a row stands off from it, so the groups read as separate blocks. */ .gs-opt + .gs-rail__group { margin-top: 10px; } .gs-rail__group:first-child { border-radius: 8px 8px 0 0; /* flush with the borderless rail's 8px clip */ } -/* A white inset ring around the state fill, like the active tab chip floating in its strip. - Drawn as an inset shadow so the rows' borders, radii, and the bar hug stay untouched. */ - /* Rows keep their full rounding. The bar hugs the first row's top corners with two inverted-radius notch shapes painted over the corners (nothing black sits behind the row's antialiased edge, so no hairline outline), and tucks 1px under the row to kill the seam. */ @@ -1404,29 +1132,13 @@ p { color: color-mix(in srgb, var(--cs-text) 68%, var(--cs-muted)); } -/* Open-source: the featured card — bigger, but now with an icon tile (like the Sky/Factory rows) - and the star count moved beneath the name, so all three controls read consistently. */ +/* Open-source product row. */ .gs-opt--feat { padding: 13px 14px; display: flex; align-items: flex-start; gap: 14px; } -/* Left column: the icon tile with the star count centered beneath it (the icon itself lays out - exactly like the Sky / Factory icon tiles). */ -.gs-opt__icwrap { - display: flex; - flex-direction: column; - align-items: center; - gap: 6px; - flex: 0 0 auto; -} -.gs-opt__stars { - font-size: 12px; - font-weight: 300; - font-variant-numeric: tabular-nums; - color: inherit; -} /* dstack Sky / Factory: compact rows with an icon. */ .gs-opt--row { @@ -1472,10 +1184,66 @@ p { border-radius: 12px; overflow: hidden; } +.gs-box--compute { + height: auto; + min-height: 360px; +} +.gs-box--compute.gs-box--offers { + height: 360px; +} +.gs-box--offers .gs-skybody { + overflow-y: auto; + padding-top: 0; +} +.sky-gpu-prices { + flex: 0 0 auto; + width: 100%; + border-collapse: collapse; + font-size: 13px; +} +.sky-gpu-prices th, +.sky-gpu-prices td { + padding: 6px 0 6px 14px; + text-align: right; + vertical-align: top; +} +.sky-gpu-prices tr > :first-child { + padding-left: 0; + text-align: left; +} +.sky-gpu-prices thead th { + padding-block: 14px; + position: sticky; + top: 0; + z-index: 1; + background: var(--cs-bg); + color: var(--cs-muted); + font-weight: 600; +} +.sky-gpu-prices td { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + white-space: nowrap; +} +@media (max-width: 480px) { + .gs-box--offers .gs-skybody { + padding-inline: 16px; + } + .sky-gpu-prices, + .sky-gpu-prices .gs-mkt__g { + font-size: 12px; + } + .sky-gpu-prices th, + .sky-gpu-prices td { + padding-left: 8px; + } + .gs-box--compute .gs-tabs, + .gs-box--compute .gs-tab { + padding-inline: 4px; + } +} .gs-boxfoot { display: flex; align-items: center; - /* Buttons only (the footer notes are gone) — keep them together at the right edge. */ justify-content: flex-end; gap: 14px; padding: 13px 16px 13px 20px; @@ -1487,17 +1255,13 @@ p { min-width: 0; /* allow the note to shrink/wrap so it shares the row with the CTA on narrow screens */ margin-inline-end: auto; /* note left, button(s) right — the footer itself is flex-end */ } -/* Footer note has a full (desktop) and a shorter (mobile) wording; the mobile breakpoint swaps them. */ -.gs-foot__short { - display: none; -} + /* The footer CTA(s) keep their width (never shrink/wrap mid-word); the note takes the rest. */ .gs-boxfoot > [class*='awsui_button'] { flex: 0 0 auto; } -/* Tab strip — shared by the open-source code box (uv / pip / Docker) and dstack Sky (GPU - marketplace / bring your own clouds), so both boxes read the same: tabs on top, content, footer. */ +/* Shared tabs for installation methods and Sky compute options. */ .gs-tabs { display: flex; gap: 2px; @@ -1570,34 +1334,7 @@ p { .gs-sky .gs-col ~ .gs-col { border-left: 1px solid color-mix(in srgb, var(--cs-border) 14%, transparent); } -.gs-col__list { - list-style: none; - margin: 0; - padding: 0 2px 14px 0; - flex: 1 1 auto; - min-height: 0; - display: flex; - flex-direction: column; - gap: 12px; - overflow-y: auto; - /* soften the cut-off row so the scroll reads cleanly into the footer */ - -webkit-mask-image: linear-gradient(to bottom, #000 calc(100% - 18px), transparent); - mask-image: linear-gradient(to bottom, #000 calc(100% - 18px), transparent); -} -/* Sky pane lists — multi-column grids sized so everything fits without scrolling, all - top-aligned under the tabs like the GPU marketplace. */ -.gs-col__list--grid { - display: grid; - column-gap: 28px; - align-content: start; - overflow-y: visible; - -webkit-mask-image: none; - mask-image: none; -} -.gs-col__list--offers { - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); -} /* The explore block's categorized clouds grid: one column per group (hyperscalers, top GPU neoclouds, smaller GPU clouds), each column an ordered list. Rows match .gs-cloud styling. */ .gs-cloudcols { @@ -1624,27 +1361,19 @@ p { } } -/* Sky tab pane body. The 44px top padding reproduces the offset the GPU marketplace had when - its near-full grid was vertically centered — now fixed, so every tab starts at the same line. */ +/* Sky compute tab content. */ .gs-skybody { flex: 1 1 auto; min-height: 0; display: flex; flex-direction: column; - padding: 36px 28px 18px; /* was 44px; trimmed so the 5-item Governance caps grid fits the box */ + padding: 36px 28px 18px; } -/* The on-prem tab reuses the Factory capability rows, top-aligned like the other Sky tabs. */ .gs-skybody .gs-caps { flex: 1 1 auto; align-content: start; } -.gs-mkt__row { - display: flex; - align-items: baseline; - justify-content: space-between; - gap: 12px; -} .gs-mkt__g { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 13px; @@ -1653,12 +1382,7 @@ p { .gs-mkt__name { font-weight: 600; /* emphasize the GPU name by weight, not size */ } -.gs-mkt__p { - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 13px; - font-weight: 400; /* price is not emphasized — only the GPU name is */ - white-space: nowrap; -} + .gs-cloud { display: flex; align-items: center; @@ -1684,14 +1408,9 @@ p { padding: 0; display: grid; grid-template-columns: 1fr 1fr; - gap: 12px 26px; /* row gap was 18px; tightened for the same reason */ -} -/* Column-flow variant (Governance): items fill the left column top-to-bottom, then the right — - so the array order is the reading order, on desktop and in the single mobile column alike. */ -.gs-caps--cols { - grid-auto-flow: column; - grid-template-rows: repeat(3, auto); + gap: 12px 26px; } + .gs-cap { display: flex; align-items: flex-start; @@ -1757,39 +1476,12 @@ p { .gs-sky .gs-col ~ .gs-col { border-left: 0; } /* Separate the second pane (its header follows the first list). */ .gs-sky .gs-col + .gs-skyhalf { margin-top: 12px; } - .gs-col__list { overflow-y: visible; -webkit-mask-image: none; mask-image: none; } .gs-caps { grid-template-columns: 1fr; } - .gs-caps--cols { grid-auto-flow: row; grid-template-rows: none; } /* Read-only install code is wider than the phone; let it scroll instead of clipping. */ .gs-codebody { overflow-x: auto; } - /* Footer stays a single row (note + CTA, like desktop), never split onto separate rows. The note - uses its shorter wording and the CTAs use tighter padding so the note keeps room beside them. */ + /* Keep footer notes and buttons on one row. */ .gs-boxfoot { align-items: center; gap: 10px; } .gs-boxfoot [class*='awsui_button'] { padding-inline: 14px !important; } - .gs-foot__full { display: none; } - .gs-foot__short { display: inline; } -} - -.right-rail-block { - margin-top: 32px; - padding-top: 28px; - border-top: 0.5px solid var(--cs-border); -} - -.docs-right > h2 { - margin-bottom: 0.5rem; - font-size: 1rem; - line-height: 1.4; -} - -.right-rail-block h2 { - margin-bottom: 8px; - font-size: 18px; -} - -.right-rail-block p { - margin-bottom: 10px; - color: var(--cs-muted); } .site-footer { @@ -1832,23 +1524,6 @@ p { font-size: 13px; } -/* The Old page renders the footer inside the AppLayout content column (so the side nav runs - full-height beside it). Fill the column instead of the centered site frame, and reserve - space on the right so the last link column isn't flush with the edge (matches the column gap). */ -.docs-main .footer-content { - width: auto; - max-width: none; - margin-inline: 0; -} - -/* Drop the divider line and the opaque block: Cloudscape paints the content area a slightly - different grey than --cs-bg, so an opaque footer reads as a mismatched rectangle (most - visible in dark mode). Going transparent + borderless lets it blend into the column. */ -.docs-main .site-footer { - border-top: 0; - background: transparent; -} - /* Top row: brand block (logo + social) on the left, the link group flush to the right edge (justify-content on this outer row), so the columns themselves keep a tight content width + fixed gap rather than being stretched apart. */ @@ -2004,7 +1679,6 @@ p { .home-stack a, .docs-article a, -.right-rail-block a, .site-footer a { text-decoration-line: underline !important; text-decoration-color: currentColor !important; @@ -2013,17 +1687,10 @@ p { .home-stack a:hover, .docs-article a:hover, -.right-rail-block a:hover, .site-footer a:hover { text-decoration-thickness: 2px !important; } -/* The rail's "Need help?" Discord CTA is a Cloudscape Button (renders an ); keep the - link-underline rules above from underlining it, matching the landing's Discord button. */ -.right-rail-block a[class*='awsui_button'] { - text-decoration-line: none !important; -} - /* Footer column links: full-strength text, same 16px as the column headings, no underline until hover. */ .footer-links a { @@ -2103,9 +1770,6 @@ p { top: 12px; } - .feature-card-grid, - .stats-grid, - .start-grid, .doc-card-grid, .product-grid, .concept-grid, @@ -2113,21 +1777,11 @@ p { grid-template-columns: 1fr; } - .image-text-row, - .image-text-row:not(.image-first), - .home-with-rail .image-text-row, - .home-with-rail .image-text-row:not(.image-first), - .docs-shell--old-page .image-text-row, - .docs-shell--old-page .image-text-row:not(.image-first), .doc-alternating, .doc-alternating:not(.image-first) { grid-template-columns: 1fr; } - .image-text-row.image-first .landing-image { - order: initial; - } - /* Stacked on mobile, always lead with the text block; the visual follows below (regardless of the desktop image-first / image-last alternation). */ .doc-alternating .doc-visual { @@ -2139,38 +1793,10 @@ p { margin-top: 32px; } - .landing-image, - .doc-alternating img { - width: 100%; - height: auto; - } - - .docs-shell { - display: block; - } - - .docs-main { - width: calc(100vw - 48px); - margin: 0 auto; - padding-top: 24px; - } - .docs-body { display: flex; flex-direction: column; } - - .docs-right { - display: block; - order: -1; - position: static; - width: 100%; - margin: 1.5rem 0 2rem; - } - - .right-rail-block { - display: none; - } } /* Switch to the burger nav at ≤1024px: the full menu (Documentation + the three blog links + @@ -2201,10 +1827,6 @@ p { flex: 0 0 28px; } - .site-desktop-trigger { - display: none; - } - .site-logo { position: absolute; left: 50%; @@ -2282,10 +1904,7 @@ p { text-align: center !important; } - .home-hero h1 { - font-size: 36px; - } - + .home-hero h1, .home-hero h2 { font-size: 32px; } @@ -2294,11 +1913,6 @@ p { padding-top: 0; } - .stat-card { - grid-template-columns: 1fr; - gap: 10px; - } - .figma-card, .figma-card img { width: 100%; diff --git a/website/vite.config.ts b/website/vite.config.ts index b4a54b2c88..18790fe849 100644 --- a/website/vite.config.ts +++ b/website/vite.config.ts @@ -1,3 +1,4 @@ +import { fileURLToPath } from 'node:url'; import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; @@ -7,11 +8,17 @@ import react from '@vitejs/plugin-react'; // `assetsDir` is namespaced to `website-assets/` (not Vite's default `assets/`) so the // build can be overlaid onto the MkDocs `site/` output without colliding with MkDocs's // own `/assets/...` tree. Public files live under `public/static/` for the same reason — -// after the overlay the only root file this app contributes is `index.html`. +// the HTML entries provide the landing and Sky product routes on static hosting. export default defineConfig({ base: process.env.BASE_PATH || '/', plugins: [react()], build: { assetsDir: 'website-assets', + rollupOptions: { + input: { + home: fileURLToPath(new URL('./index.html', import.meta.url)), + sky: fileURLToPath(new URL('./products/sky/index.html', import.meta.url)), + }, + }, }, }); From 979ac05b4445008804417474bf797b3ede969b8a Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 15 Sep 2026 07:04:28 +0000 Subject: [PATCH 588/640] Collect the last job logs before killing the container (#4289) The server pulled logs only from `_process_running`, which runs while the job is RUNNING. Once the job started terminating, `jobs_terminating` asked the runner to stop and killed the container ten seconds later without ever pulling again, so everything the runner had buffered since the last RUNNING pull died with it -- the job's own output as it shut down, and the runner logs describing the termination. The runner has always expected that last pull. It reports `has_more` to say whether it still holds logs, `/api/stop` deliberately leaves the state non-final so the real outcome can still be handed over, and the runner then waits up to five minutes for a pull that never came. The terminating pipeline now collects what is left before stopping the container, resuming from `runner_timestamp` the same way the RUNNING pull does, and stops when the runner reports no more logs or when the deadline passes. `remove_at` becomes that deadline, which is what its name and its original docstring always said it was. It is no longer a fetcher condition: the job has to be processed during the wait rather than withheld from the pipeline, and how long to wait before killing the container belongs in `_process_terminating_job`. Jobs that never started running have no logs to wait for and are terminated as before. `running_at` is the direct signal but only exists on jobs submitted since it was added, so `runner_timestamp` stands in for jobs that were already running before the upgrade. Fixes: https://github.com/dstackai/dstack/issues/4285 Co-authored-by: Claude Opus 5 (1M context) --- .../pipeline_tasks/jobs_terminating.py | 123 +++++++-- src/dstack/_internal/server/models.py | 5 +- src/dstack/_internal/server/schemas/runner.py | 4 + .../server/services/jobs/__init__.py | 5 - .../pipeline_tasks/test_terminating_jobs.py | 253 +++++++++++++++++- 5 files changed, 362 insertions(+), 28 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index c5cd9c7b6d..4b2ddf6e22 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -11,7 +11,7 @@ from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport -from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT +from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT from dstack._internal.core.errors import BackendError from dstack._internal.core.models.instances import InstanceStatus, InstanceTerminationReason from dstack._internal.core.models.runs import ( @@ -50,6 +50,7 @@ ) from dstack._internal.server.services import backends as backends_services from dstack._internal.server.services import events +from dstack._internal.server.services import logs as logs_services from dstack._internal.server.services.instances import ( emit_instance_status_change_event, get_instance_ssh_private_keys, @@ -81,6 +82,12 @@ logger = get_logger(__name__) +# How long a job is given to finish and hand over its last logs before its container is killed. +# Covers the runner's own termination staging -- SIGHUP after 5s, SIGKILL after 10s -- and the +# final log flush that follows it. +JOB_TERMINATION_DEADLINE = timedelta(seconds=30) + + @dataclass class JobTerminatingPipelineItem(PipelineItem): volumes_detached_at: Optional[datetime] @@ -173,10 +180,6 @@ async def fetch(self, limit: int) -> list[JobTerminatingPipelineItem]: select(JobModel) .where( JobModel.status == JobStatus.TERMINATING, - or_( - JobModel.remove_at.is_(None), - JobModel.remove_at < now, - ), or_( # Processing volumes detach can be less frequent since it may take time. and_( @@ -302,6 +305,7 @@ class _JobUpdateMap(ItemUpdateMap, total=False): termination_reason_message: Optional[str] instance_id: Optional[uuid.UUID] graceful_termination_attempts: int + runner_timestamp: Optional[int] volumes_detached_at: UpdateMapDateTime registered: bool remove_at: UpdateMapDateTime @@ -659,9 +663,28 @@ async def _process_terminating_job( result.job_update_map["status"] = _get_job_termination_status(job_model) return result - if job_model.graceful_termination_attempts == 0 and job_model.remove_at is None: - result.job_update_map = await _stop_job_gracefully(job_model, instance_model) - result.graceful_stop_event_message = "Graceful job stop requested" + if job_model.remove_at is None: + # The first terminating pass. `graceful_termination_attempts` decides whether the runner + # is asked to stop the job; `remove_at` decides when the container is killed regardless. + # The two are independent: a job that has already finished on its own is not asked to + # stop, but its logs are still drained below. + graceful = job_model.graceful_termination_attempts == 0 + if graceful: + await stop_runner(job_model=job_model, instance_model=instance_model) + result.job_update_map["graceful_termination_attempts"] = 1 + result.graceful_stop_event_message = "Graceful job stop requested" + if graceful or _has_logs_to_drain(job_model): + result.job_update_map["remove_at"] = get_current_datetime() + JOB_TERMINATION_DEADLINE + return result + # Nothing to wait for, stop the container right away + elif get_current_datetime() < job_model.remove_at and not await _drain_job_logs( + job_model=job_model, + instance_model=instance_model, + job_update_map=result.job_update_map, + ): + # The runner still has logs to hand over and there is time left to collect them. They + # must be collected before the container is stopped, since that destroys the runner + # along with everything it has buffered. return result jrd = get_job_runtime_data(job_model) @@ -718,18 +741,82 @@ async def _process_terminating_job( return result -async def _stop_job_gracefully( - job_model: JobModel, instance_model: InstanceModel -) -> _JobUpdateMap: +def _has_logs_to_drain(job_model: JobModel) -> bool: + """ + Whether the runner may still be holding logs for this job. + + `running_at` says the workload started, but it is only stamped by servers new enough to have + the column, so `runner_timestamp` -- advanced on every successful pull -- covers jobs that + were already running before the upgrade. A job that never started running has neither, and + waiting for logs it cannot have would only delay its termination. + """ + return job_model.running_at is not None or job_model.runner_timestamp is not None + + +async def _drain_job_logs( + job_model: JobModel, + instance_model: InstanceModel, + job_update_map: _JobUpdateMap, +) -> bool: """ - Tells the runner to stop the job's command. Records the first graceful-stop attempt and - sets `remove_at` so `_process_terminating_job()` stops the container on a later iteration. + Collects the logs the runner has buffered since the last pull. + + Returns whether the runner has nothing left to hand over, or cannot be asked at all -- the + caller keeps the container alive until then, or until `remove_at` passes. """ - job_update_map = _JobUpdateMap() - await stop_runner(job_model=job_model, instance_model=instance_model) - job_update_map["graceful_termination_attempts"] = 1 - job_update_map["remove_at"] = get_current_datetime() + timedelta(seconds=10) - return job_update_map + jpd = get_job_provisioning_data(job_model) + if jpd is None: + return True + jrd = get_job_runtime_data(job_model) + ssh_private_keys = get_instance_ssh_private_keys(instance_model) + try: + return await common.run_async( + _pull_job_logs, + ssh_private_keys, + jpd, + jrd, + job_model.run, + job_model, + job_update_map, + ) + except client.PeerConnectionError as e: + # An unreachable runner has nothing left to give, and waiting out the deadline would + # only delay terminating a job whose instance is likely gone already. + logger.debug("%s: can't collect the last logs: %s", fmt(job_model), e) + return True + except client.RunnerError as e: + # The runner answered, but not usefully. Collecting the logs is best-effort: the job is + # being terminated either way, and retrying until the deadline would only delay it. + logger.warning("%s: runner failed to hand over the last logs: %s", fmt(job_model), e) + return True + + +@runner_ssh_tunnel +def _pull_job_logs( + addresses: Mapping[int, client.LocalAddress], + run_model: RunModel, + job_model: JobModel, + job_update_map: _JobUpdateMap, +) -> bool: + runner_client = client.RunnerClient.from_address(addresses[DSTACK_RUNNER_HTTP_PORT]) + resp = runner_client.pull(job_model.runner_timestamp or 0) + try: + logs_services.write_logs( + project=run_model.project, + run_name=run_model.run_name, + job_submission_id=job_model.id, + runner_logs=resp.runner_logs, + job_logs=resp.job_logs, + ) + except logs_services.LogStorageError as e: + # `runner_timestamp` is not advanced, so the same logs are pulled again on the next pass + # instead of being lost. + logger.error("%s: failed to write the last logs: %s", fmt(job_model), e) + return False + job_update_map["runner_timestamp"] = resp.last_updated + # An old runner does not report `has_more`, and then there is no way to tell when the logs + # are exhausted -- take what this pull returned and stop. + return not resp.has_more async def _process_job_volumes_detaching( diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 8fb9113a95..a80f4922a4 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -568,8 +568,9 @@ class JobModel(PipelineModelMixin, BaseModel): * `>= 1` means at least one graceful stop attempt was sent. """ remove_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) - """`remove_at` is used to ensure the container/instance is killed after the job is gracefully finished. - Cannot kill the container/instance until `remove_at` is set. + """`remove_at` is when the job's container is killed, whether or not the runner has handed + over its last logs. `None` until the job starts terminating, and only set for jobs that are + given time to finish -- the rest are stopped on their first terminating pass. """ volumes_detached_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) instance_assigned: Mapped[bool] = mapped_column(Boolean, default=False) diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index 532a8e93fa..a9aa404921 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -47,6 +47,10 @@ class PullResponse(CoreModel): last_updated: int no_connections_secs: Optional[int] = None """`no_connections_secs` is optional for compatibility with old runners.""" + has_more: Optional[bool] = None + """`has_more` tells whether the runner may still have logs to hand over. + It is optional for compatibility with runners that do not report it. + """ class JobInfoResponse(CoreModel): diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index 4e3d26a5b9..ffd3078c21 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -1,7 +1,6 @@ import itertools import json from collections.abc import Mapping -from datetime import timedelta from typing import Dict, Iterable, List, Optional, Tuple from uuid import UUID @@ -331,10 +330,6 @@ def job_spec_updatable_in_place(old_job_spec: JobSpec, new_job_spec: JobSpec) -> return old_job_spec == new_job_spec -def delay_job_instance_termination(job_model: JobModel): - job_model.remove_at = common.get_current_datetime() + timedelta(seconds=15) - - def is_multinode_job(job: Job) -> bool: return job.job_spec.jobs_per_replica > 1 diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_terminating_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_terminating_jobs.py index f6f93613dd..80e062246d 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_terminating_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_terminating_jobs.py @@ -1,7 +1,8 @@ import asyncio import uuid from datetime import datetime, timedelta, timezone -from unittest.mock import AsyncMock, Mock, patch +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from sqlalchemy import select @@ -13,6 +14,7 @@ from dstack._internal.core.models.instances import InstanceStatus from dstack._internal.core.models.runs import JobStatus, JobTerminationReason from dstack._internal.core.models.volumes import VolumeStatus +from dstack._internal.core.services.ssh.tunnel import SSHTunnel from dstack._internal.server.background.pipeline_tasks.jobs_terminating import ( JobTerminatingFetcher, JobTerminatingPipeline, @@ -21,6 +23,12 @@ _get_related_instance_lock_owner, ) from dstack._internal.server.models import InstanceModel, JobModel, VolumeAttachmentModel +from dstack._internal.server.schemas.runner import LogEvent, PullResponse +from dstack._internal.server.services.runner.client import ( + PeerConnectionError, + RunnerClient, + RunnerError, +) from dstack._internal.server.testing.common import ( ComputeMockSpec, create_instance, @@ -46,6 +54,23 @@ def worker() -> JobTerminatingWorker: return JobTerminatingWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock()) +@pytest.fixture +def ssh_tunnel_mock(monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = MagicMock(spec_set=SSHTunnel) + monkeypatch.setattr("dstack._internal.server.services.runner.pool.SSHTunnel", mock) + return mock + + +@pytest.fixture +def runner_client_mock(monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = Mock(spec_set=RunnerClient) + monkeypatch.setattr( + "dstack._internal.server.services.runner.client.RunnerClient.from_address", + Mock(return_value=mock), + ) + return mock + + @pytest.fixture def fetcher() -> JobTerminatingFetcher: return JobTerminatingFetcher( @@ -107,6 +132,8 @@ async def test_fetch_selects_eligible_jobs_and_sets_lock_fields( past_remove_at.remove_at = stale past_remove_at.volumes_detached_at = stale - timedelta(seconds=30) + # `remove_at` is when the container is killed, not a condition for processing the job: + # the job is still fetched so that its logs can be collected in the meantime future_remove_at = await create_job( session=session, run=run, @@ -168,12 +195,14 @@ async def test_fetch_selects_eligible_jobs_and_sets_lock_fields( assert [item.id for item in items] == [ terminating.id, past_remove_at.id, + future_remove_at.id, expired_same_owner.id, recent_skip.id, ] assert {(item.id, item.volumes_detached_at) for item in items} == { (terminating.id, None), (past_remove_at.id, past_remove_at.volumes_detached_at), + (future_remove_at.id, None), (expired_same_owner.id, None), (recent_skip.id, None), } @@ -190,14 +219,19 @@ async def test_fetch_selects_eligible_jobs_and_sets_lock_fields( ]: await session.refresh(job) - fetched_jobs = [terminating, past_remove_at, expired_same_owner, recent_skip] + fetched_jobs = [ + terminating, + past_remove_at, + future_remove_at, + expired_same_owner, + recent_skip, + ] assert all(job.lock_owner == JobTerminatingPipeline.__name__ for job in fetched_jobs) assert all(job.lock_expires_at is not None for job in fetched_jobs) assert all(job.lock_token is not None for job in fetched_jobs) assert all(not job.skip_min_processing_interval for job in fetched_jobs) assert len({job.lock_token for job in fetched_jobs}) == 1 - assert future_remove_at.lock_owner is None assert non_terminating.lock_owner is None assert recent.lock_owner is None assert locked.lock_owner == "OtherPipeline" @@ -344,6 +378,219 @@ async def test_stops_job_gracefully_before_terminating_container( events = await list_events(session) assert any(event.message == "Graceful job stop requested" for event in events) + async def test_gives_finished_job_time_to_hand_over_logs( + self, test_db, session: AsyncSession, worker: JobTerminatingWorker + ): + """A job that finished on its own is not asked to stop, but its logs are still waited for.""" + project = await create_project(session=session) + user = await create_user(session=session) + instance = await create_instance( + session=session, + project=project, + status=InstanceStatus.BUSY, + ) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=JobTerminationReason.DONE_BY_RUNNER, + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + running_at=get_current_datetime() - timedelta(minutes=1), + ) + _lock_job(job) + await session.commit() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.stop_runner", + new=AsyncMock(), + ) as stop_runner, + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + stop_runner.assert_not_awaited() + stop_container.assert_not_awaited() + + await session.refresh(job) + assert job.status == JobStatus.TERMINATING + assert job.graceful_termination_attempts is None + assert job.remove_at is not None + + async def test_terminates_job_that_never_ran_without_waiting( + self, test_db, session: AsyncSession, worker: JobTerminatingWorker + ): + """There are no logs to wait for if the workload never started.""" + project = await create_project(session=session) + user = await create_user(session=session) + instance = await create_instance( + session=session, + project=project, + status=InstanceStatus.BUSY, + ) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=JobTerminationReason.TERMINATED_BY_SERVER, + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + ) + _lock_job(job) + await session.commit() + assert job.running_at is None and job.runner_timestamp is None + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.stop_runner", + new=AsyncMock(), + ) as stop_runner, + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + stop_runner.assert_not_awaited() + stop_container.assert_awaited_once() + + await session.refresh(job) + assert job.remove_at is None + + @pytest.mark.parametrize( + ("has_more", "container_stopped"), + [ + # The runner still has logs buffered, the container must stay up to hand them over + pytest.param(True, False, id="has_more"), + pytest.param(False, True, id="drained"), + # An old runner does not report `has_more`, so there is no way to wait for the end + pytest.param(None, True, id="not_reported"), + ], + ) + async def test_collects_last_logs_before_terminating_container( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + ssh_tunnel_mock: Mock, + runner_client_mock: Mock, + has_more: Optional[bool], + container_stopped: bool, + ): + project = await create_project(session=session) + user = await create_user(session=session) + instance = await create_instance( + session=session, + project=project, + status=InstanceStatus.BUSY, + ) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=JobTerminationReason.TERMINATED_BY_USER, + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + running_at=get_current_datetime() - timedelta(minutes=1), + ) + job.graceful_termination_attempts = 1 + job.remove_at = get_current_datetime() + timedelta(seconds=30) + job.runner_timestamp = 1 + _lock_job(job) + await session.commit() + + runner_client_mock.pull.return_value = PullResponse( + job_states=[], + job_logs=[LogEvent(timestamp=2, message=b"the tail")], + runner_logs=[LogEvent(timestamp=3, message=b"Job state changed")], + last_updated=3, + has_more=has_more, + ) + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.logs_services.write_logs" + ) as write_logs, + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + runner_client_mock.pull.assert_called_once_with(1) + write_logs.assert_called_once() + assert write_logs.call_args.kwargs["job_logs"][0].message == b"the tail" + assert stop_container.await_count == int(container_stopped) + + await session.refresh(job) + # Advanced, so the next pass does not collect the same logs again + assert job.runner_timestamp == 3 + + @pytest.mark.parametrize( + "error", + [ + pytest.param(RunnerError("runner is confused"), id="runner_error"), + pytest.param(PeerConnectionError("instance is gone"), id="unreachable"), + ], + ) + async def test_terminates_container_when_last_logs_cannot_be_collected( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + ssh_tunnel_mock: Mock, + runner_client_mock: Mock, + error: Exception, + ): + """Collecting the last logs is best-effort: a failure must not hold up the termination.""" + project = await create_project(session=session) + user = await create_user(session=session) + instance = await create_instance( + session=session, + project=project, + status=InstanceStatus.BUSY, + ) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=JobTerminationReason.TERMINATED_BY_USER, + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + running_at=get_current_datetime() - timedelta(minutes=1), + ) + job.graceful_termination_attempts = 1 + job.remove_at = get_current_datetime() + timedelta(seconds=30) + _lock_job(job) + await session.commit() + + runner_client_mock.pull.side_effect = error + with patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container: + await worker.process(_job_to_pipeline_item(job)) + + stop_container.assert_awaited_once() + + await session.refresh(job) + # The job must not be left locked for the pipeline to trip over + assert job.lock_owner is None + assert job.lock_token is None + async def test_terminates_gracefully_stopped_job_after_remove_at( self, test_db, session: AsyncSession, worker: JobTerminatingWorker ): From c12a77f146feaa1f1c1f0cd121e0c8b67a848529 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 15 Sep 2026 14:33:46 +0500 Subject: [PATCH 589/640] Improve DB connections handling (#4291) * Document how to work with DB connections * Use dedicated connection for server_init lock * Set command_timeout on Postgres * Fetch gateway stats outside DB session in runs pipeline * Poll migrations lock to avoid deadlock with concurrent index builds A replica blocked in pg_advisory_lock() holds a snapshot that another replica's CREATE INDEX CONCURRENTLY migration waits for, so both hang until Postgres kills the waiter. * Run migrations and server init on the connection holding the lock Holding a session-level advisory lock on a separate connection lets Postgres release it, e.g. via idle_in_transaction_session_timeout, while the work continues on another connection. Migrations now run on the lock connection, and server init uses a session bound to it. * Fix advisory lock documentation example * Drop extra docstring --- AGENTS.md | 4 +- contributing/DATABASE.md | 86 +++++++++++++++++++ contributing/LOCKING.md | 35 +++++++- mkdocs/docs/reference/env.md | 1 + src/dstack/_internal/server/app.py | 58 +++++++------ .../pipeline_tasks/runs/__init__.py | 35 +++----- src/dstack/_internal/server/db.py | 56 +++++++++--- .../server/services/gateways/__init__.py | 1 - src/dstack/_internal/server/settings.py | 4 + 9 files changed, 221 insertions(+), 59 deletions(-) create mode 100644 contributing/DATABASE.md diff --git a/AGENTS.md b/AGENTS.md index 6b7aec13cc..234464fbd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Repository Guidelines Before touching a subsystem, read the relevant notes in `contributing/`: `ARCHITECTURE.md`, -`PIPELINES.md`, `LOCKING.md`, `MIGRATIONS.md`, `RUNS-AND-JOBS.md`, `AUTOSCALING.md`, +`PIPELINES.md`, `LOCKING.md`, `DATABASE.md`, `MIGRATIONS.md`, `RUNS-AND-JOBS.md`, `AUTOSCALING.md`, `BACKENDS.md`, `GPUHUNT.md`, `PROXY.md`, `RUNNER-AND-SHIM.md`, `FRONTEND.md`, `DOCS.md`, `DEVELOPMENT.md`, `RELEASE.md`. @@ -29,7 +29,7 @@ Before touching a subsystem, read the relevant notes in `contributing/`: `ARCHIT - Prefer pydantic-style models in `core/models`. - Document attributes when the note adds behavior, compatibility, or semantic context that is not obvious from the name and type. Use attribute docstrings without leading newline. - Tests use `test_*.py` modules and `test_*` functions; fixtures live near usage. -- Never make network calls inside a DB session or transaction. Fetch what you need before opening the session, or commit and close it before the call. +- Never make network calls inside a DB session or transaction. Fetch what you need before opening the session, or commit and close it before the call. See `contributing/DATABASE.md`. - Don't use function-level (inner) imports to break circular imports. Inject the dependency or move the shared code to a lower-level module instead. - Never edit a migration that has already been applied or released; add a new migration instead. - Derive paths under `SERVER_DIR_PATH` on access (a `get_*` function), not as module-level constants, so that patching `settings.SERVER_DIR_PATH` redirects all of them. Tests rely on this to keep server state out of the real `~/.dstack`. diff --git a/contributing/DATABASE.md b/contributing/DATABASE.md new file mode 100644 index 0000000000..f70dffa084 --- /dev/null +++ b/contributing/DATABASE.md @@ -0,0 +1,86 @@ +# Database + +This document describes the `dstack` server rules of working with the database. Separately covered: + +* DB migrations in `MIGRATIONS.md` +* Resource locking in `LOCKING.md` +* Pipeline-based background processing in `PIPELINES.md`. + +## Connections + +DB connections are a scarce resource. For example, a minimal RDS Postgres instance comes with ~80 connections. Increasing the number of connections requires +scaling the DB instance and is expensive. This section describes how the `dstack` server should use DB connections to avoid running out of them. + +### Connection budget + +The server creates one SQLAlchemy async engine per process with an `AsyncAdaptedQueuePool` of `DSTACK_DB_POOL_SIZE` (default 20) persistent connections plus `DSTACK_DB_MAX_OVERFLOW` (default 20) overflow connections, i.e. up to 40 connections per server process. When all of them are checked out, the next checkout waits `pool_timeout` (30 seconds) and then fails with: + +``` +sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 +``` + +Far more coroutines compete for the slots than there are slots. This works because each of them holds a connection for a short time. + +A session does not hold a connection until it executes its first statement. From that moment until the session commits, rolls back, or closes, the connection is checked out and unavailable to anyone else. The invariant that follows: + +**Hold a connection only while doing DB work. Do not keep a session open across when doing heavy work.** + +On SQLite the same rule applies for a different reason: a long write transaction blocks every other writer for up to `busy_timeout` (30 seconds). + +### Awaits that must not run inside a session + +Anything whose completion depends on something other than the database: + +* SSH: opening tunnels, `aexec()`, `acheck()`, any subprocess. +* HTTP to runners, shims, gateways, or external services. +* Cloud SDK calls, including those wrapped in `run_async()`. The thread pool executor is shared, so a saturated executor makes even a fast call wait. +* Retry loops with sleeps, and waiting on asyncio locks or events that other tasks control. +* Streaming a response to a client. + +Fetch what you need before opening the session, or commit and close the session before the call and open a new one to apply the results: + +```python +async with get_session_ctx() as session: + # Load and lock what processing needs. + context = await _load_context(session, item) + +# No connection is held here. +result = await _do_network_work(context) + +async with get_session_ctx() as session: + # Apply results with a guarded update. + await _apply_result(session, item, result) +``` + +### Session lifetime by context + +**Pipelines** + +Pipeline workers follow load, process, apply with a short session for the load and apply phases and no session during processing. See `PIPELINES.md`. + +**API handlers** + +The request session is created before the handler runs. The authentication dependency executes a SELECT on it, so a connection is checked out before the handler starts and stays checked out until the handler returns and the session commits. + +This is acceptable because API handlers are assumed to be quick: read or write a few rows and return. A handler that needs to do slow work must not do it while holding the request session's connection. Either: + +* Commit the session before the slow work. Committing returns the connection to the pool. The next statement on the session checks out a new connection and starts a new transaction, so any locks taken earlier are released, and the handler must not rely on them across the gap. +* Make the API async and move the slow work out of the handler. + +The rule is not applied currently applied to API handlers strictly: a handler may make one or a few network calls while holding the connection. + +**Startup** + +Migrations and server initialization may hold a connection and a session-level advisory lock for long: they run once per process, before anything else competes for connections. Recurring cross-replica coordination should use the pipeline lock columns instead. + +### Lock waits + +A coroutine blocked on a DB-level lock holds its connection while it waits. This applies to `SELECT ... FOR UPDATE` and to `pg_advisory_lock` alike: the waiter occupies a pool slot on its replica until the holder releases the lock. A lock held for a long time thus costs one slot on the holder plus one per waiter, and a hung holder pins all of them. + +Keep locked regions short and bounded. See `LOCKING.md` for how to take advisory locks correctly. + +### Settings + +`DSTACK_DB_COMMAND_TIMEOUT` (default 300 seconds) bounds every operation on a Postgres connection, so a coroutine waiting on a connection whose socket died silently gets an error and releases the slot instead of holding it forever. It applies to migrations too, since they run on the same engine, so it must leave room for index builds and backfills. + +`DSTACK_DB_POOL_SIZE` and `DSTACK_DB_MAX_OVERFLOW` size the pool per process. Raising them lets more coroutines hold connections at once, which hides a long hold rather than fixing it, and each replica needs `pool_size + max_overflow` connections from the Postgres `max_connections` budget. diff --git a/contributing/LOCKING.md b/contributing/LOCKING.md index c89ad526a4..420c493c01 100644 --- a/contributing/LOCKING.md +++ b/contributing/LOCKING.md @@ -29,7 +29,7 @@ Locksets are an optimization. One can think of them as per-resource-id locks tha Postgres resource locking is implemented via standard SELECT FOR UPDATE. SQLAlchemy provides `.with_for_update()` that has no effect if SELECT FOR UPDATE is not supported as in SQLite. -There are few places that rely on advisory locks as when generating unique resource names. +There are few places that rely on advisory locks as when generating unique resource names or serializing server initialization across replicas. See **Advisory locks** below. ## Working with locks @@ -112,6 +112,39 @@ Note that: * This pattern works assuming that Postgres is using default isolation level Read Committed. By the time a transaction acquires the advisory lock, all other transactions that can take the name have committed, so their changes can be seen and a unique name is taken. * SQLite needs a commit before selecting taken names due to Snapshot Isolation as noted above. +**Advisory locks** + +Postgres has two kinds of advisory locks: + +* `pg_advisory_xact_lock` is released when the transaction ends. The unique names pattern above uses it. +* `pg_advisory_lock` is bound to the connection. It survives commit and rollback and is released only by `pg_advisory_unlock` on the same connection or when the connection closes. `advisory_lock_ctx()` in `services/locking.py` wraps this kind for work that must span several transactions, such as migrations or server initialization. + +Within a single transaction, prefer `pg_advisory_xact_lock` for automatic release on commit or rollback. When using `advisory_lock_ctx()`, the requirement is to acquire and release the lock on the same physical connection: + +* An `AsyncSession` is a valid bind when the locked block stays within one transaction. Do not commit, roll back, or close the session before releasing the lock. +* If the block spans multiple transactions, hold an `AsyncConnection` from `engine.connect()` and bind the lock and any sessions to it. Otherwise, a session's next transaction may use a different pooled connection, and `pg_advisory_unlock` goes to a connection that never held the lock. Postgres only returns `false` with a warning in this case, so the failure is silent: the lock stays on an idle pooled connection until the process exits, and every replica blocks forever on its next acquire. See https://github.com/dstackai/dstack/issues/3881 for an example. +* Run protected DB work on the connection holding the lock. When binding a new session to an explicit connection, end the lock statement's transaction first so that the session controls its own transactions. If the connection is lost, abort the protected operation instead of reconnecting and continuing without the lock. +* Keep the locked block short and bounded. Every waiter is blocked inside `pg_advisory_lock` holding a DB connection of its own, so a long or hung holder pins one connection per waiter. See `DATABASE.md`. + +For a block that spans multiple transactions: + +```python +async with get_db().engine.connect() as connection: + async with advisory_lock_ctx( + bind=connection, + dialect_name=get_db().dialect_name, + resource="server_init", + ): + # End the lock statement's transaction. The session-level lock survives the commit. + await connection.commit() + async with get_db().get_session(bind=connection) as session: + # Session commits keep using the connection that holds the lock. + ... + await session.commit() +``` + +A released connection goes back to the pool, so a lock that failed to release stays there too. `_release_advisory_lock()` tolerates failures because the common one is an invalidated connection, in which case Postgres has already dropped the lock. A release that fails on a live connection strands the lock. + **Use `AsyncExitStack`** In-memory locking typically requires taking lock for long (until commit). diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 5e4a80ee58..f00b0dd96f 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -141,6 +141,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_SERVER_GCS_BUCKET`{ #DSTACK_SERVER_GCS_BUCKET } - The bucket that repo diffs will be uploaded to if set. If unset, diffs are uploaded to the database. - `DSTACK_DB_POOL_SIZE`{ #DSTACK_DB_POOL_SIZE } - The client DB connections pool size. Defaults to `20`, - `DSTACK_DB_MAX_OVERFLOW`{ #DSTACK_DB_MAX_OVERFLOW } - The client DB connections pool allowed overflow. Defaults to `20`. +- `DSTACK_DB_COMMAND_TIMEOUT`{ #DSTACK_DB_COMMAND_TIMEOUT } - The timeout for a single DB operation in seconds, Postgres only. Set to `0` to disable. Defaults to `300`. - `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED } - Disables background processing if set to any value. Useful to run only web frontend and API server. - `DSTACK_SERVER_MAX_PROBES_PER_JOB`{ #DSTACK_SERVER_MAX_PROBES_PER_JOB } - Maximum number of probes allowed in a run configuration. Validated at apply time. - `DSTACK_SERVER_MAX_PROBE_TIMEOUT`{ #DSTACK_SERVER_MAX_PROBE_TIMEOUT } - Maximum allowed timeout for a probe. Validated at apply time. diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index da5877a277..904cb3285d 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -23,7 +23,7 @@ from dstack._internal.server.background.pipeline_tasks import start_pipeline_tasks from dstack._internal.server.background.scheduled_tasks import start_scheduled_tasks from dstack._internal.server.background.scheduled_tasks.probes import PROBES_SCHEDULER -from dstack._internal.server.db import get_db, get_session_ctx, migrate +from dstack._internal.server.db import get_db, migrate from dstack._internal.server.routers import ( auth, backends, @@ -130,33 +130,43 @@ async def lifespan(app: FastAPI): server_config_loaded = server_config_manager.load_config() # Encryption has to be configured before working with users and projects await server_config_manager.apply_encryption() - async with get_session_ctx() as session: + async with get_db().engine.connect() as connection: + # Running server init using a dedicated connection because there are multiple + # transactions/commits happening under the advisory lock + # and we need to guarantee the same connection releases the lock. async with advisory_lock_ctx( - bind=session, + bind=connection, dialect_name=get_db().dialect_name, resource="server_init", ): - admin, _ = await get_or_create_admin_user(session=session) - await get_or_create_default_project( - session=session, - user=admin, - ) - if server_config_manager is not None: - server_config_file_path = get_server_config_file_path() - server_config_dir = _get_server_config_dir() - if not server_config_loaded: - logger.info("Initializing the default configuration...", {"show_path": False}) - await server_config_manager.init_config(session=session) - logger.info( - f"Initialized the default configuration at [link=file://{server_config_file_path}]{server_config_dir}[/link]", - {"show_path": False}, - ) - else: - logger.info( - f"Applying [link=file://{server_config_file_path}]{server_config_dir}[/link]...", - {"show_path": False}, - ) - await server_config_manager.apply_config(session=session, owner=admin) + # End the lock statement's transaction so that the session controls its own transactions. + # The session-level lock survives the commit. + await connection.commit() + async with get_db().get_session(bind=connection) as session: + admin, _ = await get_or_create_admin_user(session=session) + await get_or_create_default_project( + session=session, + user=admin, + ) + if server_config_manager is not None: + server_config_file_path = get_server_config_file_path() + server_config_dir = _get_server_config_dir() + if not server_config_loaded: + logger.info( + "Initializing the default configuration...", {"show_path": False} + ) + await server_config_manager.init_config(session=session) + logger.info( + f"Initialized the default configuration at [link=file://{server_config_file_path}]{server_config_dir}[/link]", + {"show_path": False}, + ) + else: + logger.info( + f"Applying [link=file://{server_config_file_path}]{server_config_dir}[/link]...", + {"show_path": False}, + ) + await server_config_manager.apply_config(session=session, owner=admin) + await session.commit() update_default_project( project_name=DEFAULT_PROJECT_NAME, diff --git a/src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py b/src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py index 20461f0814..713828d27b 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py @@ -18,7 +18,8 @@ import dstack._internal.server.background.pipeline_tasks.runs.active as active import dstack._internal.server.background.pipeline_tasks.runs.pending as pending import dstack._internal.server.background.pipeline_tasks.runs.terminating as terminating -from dstack._internal.core.models.runs import JobStatus, RunStatus +from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus +from dstack._internal.proxy.gateway.schemas.stats import PerWindowStats from dstack._internal.server.background.pipeline_tasks.base import ( Fetcher, Heartbeater, @@ -300,6 +301,7 @@ async def _process_pending_item(item: RunPipelineItem) -> None: if context is None: return + context.gateway_stats = await _load_gateway_stats(context.run_model, context.run_spec) result = await pending.process_pending_run(context) if result is None: await _apply_noop_result( @@ -330,21 +332,11 @@ async def _load_pending_context( return None secrets = await get_project_secrets_mapping(session=session, project=run_model.project) run_spec = get_run_spec(run_model) - - gateway_stats = None - if run_spec.configuration.type == "service" and run_model.gateway is not None: - gateway_stats = await get_combined_gateway_stats( - get_gateway_replica_models(run_model.gateway), - run_model.project.name, - run_model.run_name, - ) - return pending.PendingContext( run_model=run_model, run_spec=run_spec, secrets=secrets, locked_job_ids=locked_job_ids, - gateway_stats=gateway_stats, ) @@ -506,6 +498,7 @@ async def _process_active_item(item: RunPipelineItem) -> None: return context = load_result + context.gateway_stats = await _load_gateway_stats(context.run_model, context.run_spec) result = await active.process_active_run(context) await _apply_active_result(item=item, context=context, result=result) @@ -532,21 +525,21 @@ async def _load_active_context( return None secrets = await get_project_secrets_mapping(session=session, project=run_model.project) run_spec = get_run_spec(run_model) - - gateway_stats = None - if run_spec.configuration.type == "service" and run_model.gateway is not None: - gateway_stats = await get_combined_gateway_stats( - get_gateway_replica_models(run_model.gateway), - run_model.project.name, - run_model.run_name, - ) - return active.ActiveContext( run_model=run_model, run_spec=run_spec, secrets=secrets, locked_job_ids=locked_job_ids, - gateway_stats=gateway_stats, + ) + + +async def _load_gateway_stats(run_model: RunModel, run_spec: RunSpec) -> Optional[PerWindowStats]: + if run_spec.configuration.type != "service" or run_model.gateway is None: + return None + return await get_combined_gateway_stats( + get_gateway_replica_models(run_model.gateway), + run_model.project.name, + run_model.run_name, ) diff --git a/src/dstack/_internal/server/db.py b/src/dstack/_internal/server/db.py index 3136b4e6b9..de1b21354c 100644 --- a/src/dstack/_internal/server/db.py +++ b/src/dstack/_internal/server/db.py @@ -1,10 +1,12 @@ +import asyncio from contextlib import asynccontextmanager from typing import Optional from alembic import command, config -from sqlalchemy import AsyncAdaptedQueuePool, event +from sqlalchemy import AsyncAdaptedQueuePool, event, make_url from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.ext.asyncio import ( + AsyncConnection, AsyncEngine, AsyncSession, async_sessionmaker, @@ -13,7 +15,7 @@ from sqlalchemy.pool import ConnectionPoolEntry from dstack._internal.server import settings -from dstack._internal.server.services.locking import advisory_lock_ctx +from dstack._internal.server.services.locking import try_advisory_lock_ctx class Database: @@ -28,6 +30,7 @@ def __init__(self, url: str, engine: Optional[AsyncEngine] = None): poolclass=AsyncAdaptedQueuePool, pool_size=settings.DB_POOL_SIZE, max_overflow=settings.DB_MAX_OVERFLOW, + connect_args=self._get_connect_args(self.url), ) self.session_maker = async_sessionmaker( bind=self.engine, # type: ignore[assignment] @@ -52,8 +55,22 @@ def set_sqlite_pragma(dbapi_connection: DBAPIConnection, _: ConnectionPoolEntry) def dialect_name(self) -> str: return self.engine.dialect.name - def get_session(self) -> AsyncSession: - return self.session_maker() + def get_session(self, bind: Optional[AsyncConnection] = None) -> AsyncSession: + """ + Returns a new session. If `bind` is given, the session runs on that connection + instead of checking connections out of the pool. The connection must not be in + a transaction, otherwise the session joins it and its commits do not commit. + """ + if bind is None: + return self.session_maker() + return self.session_maker(bind=bind) + + def _get_connect_args(self, url: str) -> dict: + if make_url(url).get_backend_name() == "postgresql": + # TODO: Consider setting "command_timeout" high for migrations + # and low for queries – requires a separate Database instance for migrations. + return {"command_timeout": settings.DB_COMMAND_TIMEOUT} + return {} def get_new_db() -> Database: @@ -80,15 +97,34 @@ def override_db(new_db: Database): _db = new_db +_MIGRATIONS_LOCK_POLL_INTERVAL = 1 +_MIGRATIONS_LOCK_MAX_ATTEMPTS = 600 + + async def migrate(): db = get_db() + # The lock is polled instead of awaited in pg_advisory_lock() to avoid this: + # migrations waiting for older snapshots to finish (e.g. CREATE INDEX CONCURRENTLY) + # wait for the blocked replicas waiting for the "migrations" lock, deadlocking both. async with db.engine.connect() as connection: - async with advisory_lock_ctx( - bind=connection, - dialect_name=db.dialect_name, - resource="migrations", - ): - await connection.run_sync(_run_alembic_upgrade) + for _ in range(_MIGRATIONS_LOCK_MAX_ATTEMPTS): + async with try_advisory_lock_ctx( + bind=connection, + dialect_name=db.dialect_name, + resource="migrations", + ) as locked: + # End the attempt's transaction so that no snapshot is held while waiting. + await connection.commit() + if locked: + await connection.run_sync(_run_alembic_upgrade) + return + await asyncio.sleep(_MIGRATIONS_LOCK_POLL_INTERVAL) + raise TimeoutError( + "Timed out waiting for the migrations lock." + " Another server replica may be running long migrations, or a replica that lost" + " connectivity may still hold it: check pg_locks for the advisory lock" + " and terminate the holder's backend if it is gone." + ) async def get_session(): diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 7b0705faee..e903bbfe07 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -677,7 +677,6 @@ async def generate_gateway_name(session: AsyncSession, project: ProjectModel) -> return name -# TODO: Connect to gateway outside session async def get_or_add_gateway_connections( gateway_replicas: Sequence[GatewayReplicaModel], ) -> List[GatewayConnection]: diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 3605725357..e7fe97124b 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -5,6 +5,7 @@ import os from enum import Enum from pathlib import Path +from typing import Optional from dstack._internal.server.utils.settings import parse_hostname_port from dstack._internal.utils.env import environ @@ -67,6 +68,9 @@ def get_database_url() -> str: # or increase client pool size to support more concurrent requests. DB_POOL_SIZE = int(os.getenv("DSTACK_DB_POOL_SIZE", 20)) DB_MAX_OVERFLOW = int(os.getenv("DSTACK_DB_MAX_OVERFLOW", 20)) +DB_COMMAND_TIMEOUT: Optional[float] = float(os.getenv("DSTACK_DB_COMMAND_TIMEOUT", 300)) or None +"""Bounds every operation on a Postgres connection so that a connection whose +socket died silently is released back to the pool instead of being held forever. 0 disables the timeout.""" SERVER_BACKGROUND_PROCESSING_DISABLED = ( os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None From c1271af41dfed5229641745acfec10f57b664de6 Mon Sep 17 00:00:00 2001 From: Ziming Wang <125807850+ZenAlexa@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:48:33 +0800 Subject: [PATCH 590/640] Prevent resuming saved presets (#4228) Signed-off-by: ZenAlexa --- .../_internal/cli/services/presets/output.py | 9 +++++--- .../_internal/cli/services/presets/session.py | 4 +++- .../_internal/cli/commands/test_preset.py | 23 +++++++++++++++++++ .../cli/services/presets/test_agent.py | 15 ++++++++++++ .../cli/services/presets/test_output.py | 5 ++-- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 05db1828ce..52df677d61 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -116,6 +116,7 @@ def get_presets_table( table.add_column("STATUS") table.add_column("SUBMITTED", style="secondary") presets_by_base: dict[str, list[AnyStoredPreset]] = defaultdict(list) + preset_ids = {preset.id for preset in presets} repo_to_base: dict[str, str] = {} for preset in presets: presets_by_base[preset.base].append(preset) @@ -123,9 +124,11 @@ def get_presets_table( sessions_by_model: dict[str, list[dict[str, Any]]] = defaultdict(list) creations_by_id: dict[str, dict[str, Any]] = {} for session in sessions or []: - if str(session.get("status")) == "success": - # A completed creation session decorates its preset row. - creations_by_id[str(session.get("id"))] = session + session_id = str(session.get("id")) + if session_id in preset_ids: + # The stored artifact is the completion record. Cleanup can be + # interrupted before session.json advances to `success`. + creations_by_id[session_id] = session continue model = str(session.get("model") or "unknown") sessions_by_model[repo_to_base.get(model, model)].append(session) diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index f92b70b824..a048e78030 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -343,7 +343,9 @@ def load_resumable_session(preset_id: str) -> PresetSession: state = session.read_state() if not path.is_dir() or state is None: raise CLIError(f"Unknown preset: {preset_id}") - if state.status == "success": + # `PresetStore.save()` commits this file before best-effort run cleanup. + # An interrupted cleanup must not turn that durable result back into work. + if (path / "preset.yml").is_file() or state.status == "success": raise CLIError(f"Preset {preset_id} is already created; nothing to resume") if state.status == "failed": raise CLIError(f"Preset {preset_id} creation failed and cannot be resumed") diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index d8a52097f5..bfc9581b27 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -108,6 +108,29 @@ def test_resume_uses_the_configuration_and_prompt_pinned_at_creation(self, tmp_p assert kwargs["configuration"].model.base == "Qwen/Qwen3.5-27B" assert kwargs["user_prompt"] == "go deep" + def test_resume_refuses_a_session_with_a_saved_preset(self, tmp_path, capsys): + preset = get_preset() + session_dir = tmp_path / ".dstack" / "presets" / preset.id + session_dir.mkdir(parents=True) + session_dir.joinpath("session.json").write_text( + get_session_state( + id=preset.id, + status="interrupted", + run=get_session_run(session_id="sid-1"), + ).model_dump_json(), + encoding="utf-8", + ) + PresetStore(session_dir.parent).save(preset) + + with patch("dstack._internal.cli.commands.preset.create_preset") as create: + exit_code = run_dstack_cli( + ["preset", "resume", preset.id], home_dir=tmp_path, repo_dir=tmp_path + ) + + assert exit_code == 1 + create.assert_not_called() + assert "already created; nothing to resume" in capsys.readouterr().out + def test_get_returns_the_requested_configuration_for_an_unfinished_preset( self, tmp_path, capsys ): diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index 6276a58c1a..29eba3dd1f 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -1544,6 +1544,21 @@ def test_treats_dead_running_session_as_resumable(self, tmp_path, monkeypatch): assert load_resumable_session("ab12cd34").preset_id == "ab12cd34" + def test_refuses_session_that_already_saved_a_preset(self, tmp_path, monkeypatch): + path = self._write_session( + tmp_path, + monkeypatch, + { + "id": "ab12cd34", + "status": "interrupted", + "run": get_session_run(session_id="sid-1"), + }, + ) + (path / "preset.yml").write_text("status: verified\n", encoding="utf-8") + + with pytest.raises(CLIError, match="already created; nothing to resume"): + load_resumable_session("ab12cd34") + @pytest.mark.parametrize( ("state", "match"), [ diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 379a6472bd..b362584b15 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -305,14 +305,15 @@ def test_sorts_all_rows_newest_first_without_grouping(self, monkeypatch): class TestDoneProgress: - def test_completed_creation_decorates_preset_row_without_extra_session_row(self, monkeypatch): + @pytest.mark.parametrize("session_status", ["success", "interrupted", "failed"]) + def test_saved_preset_owns_row_regardless_of_session_status(self, monkeypatch, session_status): buffer = StringIO() monkeypatch.setattr(output_module, "console", plain_console(buffer, width=200)) preset = get_preset() sessions = [ { "id": preset.id, - "status": "success", + "status": session_status, "model": preset.base, "trials_num": 4, "trials": {"count": 3}, From d0fad0eb24682a6cfc259a68be9eccee9ce58026 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 16 Sep 2026 06:49:38 +0000 Subject: [PATCH 591/640] Support Certificate Manager for GCP gateways (#4290) Allow to attach certificates managed by Google Cloud Certificate Manager to GCP gateways with an ALB. ```yaml type: gateway name: example-gateway backend: gcp region: europe-west4 domain: example.com replicas: 2 load_balancer: type: alb certificate: type: gcp-cm name: projects/my-project/locations/europe-west4/certificates/my-certificate ``` This enables HTTPS for services behind the gateway, with TLS terminated at the load balancer. The certificate must be a regional certificate created in the same region as the gateway. New permissions are required to create such gateways: ``` certificatemanager.certs.use compute.regionTargetHttpsProxies.create compute.regionTargetHttpsProxies.delete compute.regionTargetHttpsProxies.use ``` --- mkdocs/docs/concepts/backends.md | 9 ++ mkdocs/docs/concepts/gateways.md | 10 +- mkdocs/docs/reference/dstack.yml/gateway.md | 8 ++ .../_internal/core/backends/gcp/compute.py | 110 +++++++++++++----- .../_internal/core/backends/gcp/resources.py | 8 ++ src/dstack/_internal/core/models/gateways.py | 27 ++++- .../_internal/core/services/gateways.py | 12 ++ .../server/services/gateways/__init__.py | 20 +++- .../server/services/services/__init__.py | 15 ++- .../_internal/server/routers/test_gateways.py | 61 +++++++++- 10 files changed, 236 insertions(+), 44 deletions(-) diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index 7c91b27825..110c1b78a5 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -591,6 +591,15 @@ gcloud projects list --format="json(projectId)" compute.regionUrlMaps.use ``` + If you also plan to use gateways with `certificate: { type: gcp-cm }`, additional permissions are required: + + ``` + certificatemanager.certs.use + compute.regionTargetHttpsProxies.create + compute.regionTargetHttpsProxies.delete + compute.regionTargetHttpsProxies.use + ``` + If you plan to use TPUs, additional permissions are required: ``` diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md index 268b6bae6d..ca4cf3e2cd 100644 --- a/mkdocs/docs/concepts/gateways.md +++ b/mkdocs/docs/concepts/gateways.md @@ -122,7 +122,9 @@ Setting `load_balancer: { type: alb }` provisions an Application Load Balancer ( replicas: 2 load_balancer: type: alb - certificate: null + certificate: + type: gcp-cm + name: projects/my-project/locations/europe-west4/certificates/my-certificate ``` @@ -131,7 +133,7 @@ Setting `load_balancer: { type: alb }` provisions an Application Load Balancer ( An ALB gateway on `gcp` requires: - The `gcp` backend. - - `certificate: null`. + - Either `certificate: { type: gcp-cm, ... }` or `certificate: null`. - A VPC with a proxy-only subnet (`purpose: REGIONAL_MANAGED_PROXY`) in the target region — required by GCP for all Envoy-based regional load balancers. See [Proxy-only subnets](https://cloud.google.com/load-balancing/docs/proxy-only-subnets) for how to create one. The provisioned load balancer provides a hostname (or IP address) you can add to your DNS records. Replica hostnames do not need to be added to DNS. @@ -162,6 +164,8 @@ If you disable [public IP](#public-ip) (e.g. to make the gateway private) or if * `lets-encrypt` (default) — Automatic certificates via [Let's Encrypt](https://letsencrypt.org/). Requires a [public IP](#public-ip). * `acm` — Certificates managed by [AWS Certificate Manager](https://aws.amazon.com/certificate-manager/). AWS-only. TLS is terminated at the load balancer, not at the gateway, and HTTP requests are redirected to HTTPS by the ALB. Implies `load_balancer: { type: alb }`. + * `gcp-cm` — Certificates managed by [Google Cloud Certificate Manager](https://cloud.google.com/certificate-manager/docs/overview). GCP-only. TLS is terminated at the load balancer, not at the gateway, and the load balancer only serves HTTPS on port 443. + Requires `load_balancer: { type: alb }`. The certificate must be a regional certificate created in the same region as the gateway, referenced by its full resource name. * `null` — No certificate. Services will use HTTP. ### Public IP @@ -244,7 +248,7 @@ $ dstack gateway list !!! warning "Experimental" Replicated gateways are an experimental feature and currently have limitations: - - HTTPS is only supported for AWS gateways with the `acm` [certificate type](#certificate). For other gateways, use an external load balancer for TLS termination. + - HTTPS is only supported for AWS gateways with the `acm` [certificate type](#certificate) and GCP gateways with the `gcp-cm` [certificate type](#certificate). For other gateways, use an external load balancer for TLS termination. - All replicas are bound to the same backend and region. - At most 3 replicas are allowed per gateway. diff --git a/mkdocs/docs/reference/dstack.yml/gateway.md b/mkdocs/docs/reference/dstack.yml/gateway.md index 65a3409c53..d52fac9e0a 100644 --- a/mkdocs/docs/reference/dstack.yml/gateway.md +++ b/mkdocs/docs/reference/dstack.yml/gateway.md @@ -30,6 +30,14 @@ Set to `null` to disable certificates (e.g. for [private gateways](../../concept type: required: true +=== "GCP Certificate Manager" + + #SCHEMA# dstack._internal.core.models.gateways.GCPCertificateManagerGatewayCertificate + overrides: + show_root_heading: false + type: + required: true + ### `load_balancer` === "ALB" diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index 057cde7fe0..d406476048 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -117,7 +117,8 @@ class GCPGatewayBackendData(CoreModel): health_check_name: str backend_service_name: str url_map_name: str - target_http_proxy_name: str + target_http_proxy_name: Optional[str] = None + target_https_proxy_name: Optional[str] = None forwarding_rule_name: str @@ -158,6 +159,9 @@ def __init__(self, config: GCPConfig): credentials=self.credentials ) self.region_url_maps_client = compute_v1.RegionUrlMapsClient(credentials=self.credentials) + self.region_target_https_proxies_client = compute_v1.RegionTargetHttpsProxiesClient( + credentials=self.credentials + ) self.region_target_http_proxies_client = compute_v1.RegionTargetHttpProxiesClient( credentials=self.credentials ) @@ -691,7 +695,7 @@ def create_gateway_load_balancer( self, configuration: GatewayLoadBalancerConfiguration, ) -> GatewayLoadBalancerData: - assert configuration.certificate is None + assert configuration.certificate is None or configuration.certificate.type == "gcp-cm" zone = self._get_gateway_zone(configuration.region) @@ -732,7 +736,7 @@ def create_gateway_load_balancer( health_check_name = f"{name}-hc" backend_service_name = f"{name}-bs" url_map_name = f"{name}-um" - target_http_proxy_name = f"{name}-proxy" + target_proxy_name = f"{name}-proxy" forwarding_rule_name = f"{name}-fr" instance_group_resource_name = ( @@ -750,9 +754,12 @@ def create_gateway_load_balancer( f"projects/{self.config.project_id}/regions/{configuration.region}" f"/urlMaps/{url_map_name}" ) - target_http_proxy_resource_name = ( + target_proxy_kind = ( + "targetHttpProxies" if configuration.certificate is None else "targetHttpsProxies" + ) + target_proxy_resource_name = ( f"projects/{self.config.project_id}/regions/{configuration.region}" - f"/targetHttpProxies/{target_http_proxy_name}" + f"/{target_proxy_kind}/{target_proxy_name}" ) logger.debug("Creating instance group for gateway %s...", configuration.gateway_name) @@ -817,25 +824,48 @@ def create_gateway_load_balancer( gcp_resources.wait_for_extended_operation(operation, "URL map creation") logger.debug("Created URL map for gateway %s.", configuration.gateway_name) - logger.debug("Creating target HTTP proxy for gateway %s...", configuration.gateway_name) - target_http_proxy = compute_v1.TargetHttpProxy() - target_http_proxy.name = target_http_proxy_name - target_http_proxy.url_map = url_map_resource_name - operation = self.region_target_http_proxies_client.insert( - project=self.config.project_id, - region=configuration.region, - target_http_proxy_resource=target_http_proxy, - ) - gcp_resources.wait_for_extended_operation(operation, "target HTTP proxy creation") - logger.debug("Created target HTTP proxy for gateway %s.", configuration.gateway_name) + if configuration.certificate is None: + logger.debug( + "Creating target HTTP proxy for gateway %s...", configuration.gateway_name + ) + target_http_proxy = compute_v1.TargetHttpProxy() + target_http_proxy.name = target_proxy_name + target_http_proxy.url_map = url_map_resource_name + operation = self.region_target_http_proxies_client.insert( + project=self.config.project_id, + region=configuration.region, + target_http_proxy_resource=target_http_proxy, + ) + gcp_resources.wait_for_extended_operation(operation, "target HTTP proxy creation") + logger.debug("Created target HTTP proxy for gateway %s.", configuration.gateway_name) + else: + logger.debug( + "Creating target HTTPS proxy for gateway %s...", configuration.gateway_name + ) + target_https_proxy = compute_v1.TargetHttpsProxy() + target_https_proxy.name = target_proxy_name + target_https_proxy.url_map = url_map_resource_name + target_https_proxy.ssl_certificates = [ + gcp_resources.get_certificate_manager_certificate_url( + configuration.certificate.name + ) + ] + operation = self.region_target_https_proxies_client.insert( + project=self.config.project_id, + region=configuration.region, + target_https_proxy_resource=target_https_proxy, + ) + gcp_resources.wait_for_extended_operation(operation, "target HTTPS proxy creation") + logger.debug("Created target HTTPS proxy for gateway %s.", configuration.gateway_name) + # TODO: HTTP->HTTPS redirect? logger.debug("Creating forwarding rule for gateway %s...", configuration.gateway_name) forwarding_rule = compute_v1.ForwardingRule() forwarding_rule.name = forwarding_rule_name forwarding_rule.load_balancing_scheme = load_balancing_scheme forwarding_rule.I_p_protocol = compute_v1.ForwardingRule.IPProtocolEnum.TCP.name - forwarding_rule.port_range = "80" - forwarding_rule.target = target_http_proxy_resource_name + forwarding_rule.port_range = "80" if configuration.certificate is None else "443" + forwarding_rule.target = target_proxy_resource_name forwarding_rule.network = self.config.vpc_resource_name if subnetwork is not None: forwarding_rule.subnetwork = subnetwork @@ -860,7 +890,12 @@ def create_gateway_load_balancer( health_check_name=health_check_name, backend_service_name=backend_service_name, url_map_name=url_map_name, - target_http_proxy_name=target_http_proxy_name, + target_http_proxy_name=( + target_proxy_name if configuration.certificate is None else None + ), + target_https_proxy_name=( + target_proxy_name if configuration.certificate is not None else None + ), forwarding_rule_name=forwarding_rule_name, ).model_dump_json(), ) @@ -888,7 +923,7 @@ def terminate_gateway_load_balancer( logger.debug( "Deleting load balancer resources for gateway %s...", configuration.gateway_name ) - for delete_call, verbose_name in [ + delete_calls = [ ( lambda: self.forwarding_rules_client.delete( project=self.config.project_id, @@ -897,14 +932,30 @@ def terminate_gateway_load_balancer( ), "forwarding rule deletion", ), - ( - lambda: self.region_target_http_proxies_client.delete( - project=self.config.project_id, - region=configuration.region, - target_http_proxy=backend_data_parsed.target_http_proxy_name, - ), - "target HTTP proxy deletion", - ), + ] + if backend_data_parsed.target_http_proxy_name is not None: + delete_calls.append( + ( + lambda: self.region_target_http_proxies_client.delete( + project=self.config.project_id, + region=configuration.region, + target_http_proxy=backend_data_parsed.target_http_proxy_name, + ), + "target HTTP proxy deletion", + ) + ) + if backend_data_parsed.target_https_proxy_name is not None: + delete_calls.append( + ( + lambda: self.region_target_https_proxies_client.delete( + project=self.config.project_id, + region=configuration.region, + target_https_proxy=backend_data_parsed.target_https_proxy_name, + ), + "target HTTPS proxy deletion", + ) + ) + delete_calls += [ ( lambda: self.region_url_maps_client.delete( project=self.config.project_id, @@ -937,7 +988,8 @@ def terminate_gateway_load_balancer( ), "instance group deletion", ), - ]: + ] + for delete_call, verbose_name in delete_calls: try: operation = delete_call() gcp_resources.wait_for_extended_operation(operation, verbose_name) diff --git a/src/dstack/_internal/core/backends/gcp/resources.py b/src/dstack/_internal/core/backends/gcp/resources.py index ed1ace65a6..a9f1d23541 100644 --- a/src/dstack/_internal/core/backends/gcp/resources.py +++ b/src/dstack/_internal/core/backends/gcp/resources.py @@ -16,6 +16,8 @@ logger = get_logger(__name__) +CERTIFICATE_MANAGER_API_NAMESPACE = "//certificatemanager.googleapis.com" + DSTACK_INSTANCE_TAG = "dstack-runner-instance" DSTACK_GATEWAY_TAG = "dstack-gateway-instance" @@ -691,3 +693,9 @@ def instance_type_supports_persistent_disk(instance_type_name: str) -> bool: "g4-", ] ) + + +def get_certificate_manager_certificate_url(certificate_name: str) -> str: + if certificate_name.startswith(("//", "https://")): + return certificate_name + return f"{CERTIFICATE_MANAGER_API_NAMESPACE}/{certificate_name.lstrip('/')}" diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py index 623d87c07e..bff16d3e57 100644 --- a/src/dstack/_internal/core/models/gateways.py +++ b/src/dstack/_internal/core/models/gateways.py @@ -49,9 +49,34 @@ class ACMGatewayCertificate(CoreModel): ] +class GCPCertificateManagerGatewayCertificate(CoreModel): + type: Annotated[ + Literal["gcp-cm"], + Field( + description=( + "Certificates by Google Cloud Certificate Manager." + " Requires `load_balancer: { type: alb }`" + ) + ), + ] = "gcp-cm" + name: Annotated[ + str, + Field( + description=( + "The full resource name of the Certificate Manager certificate for the domain," + " e.g. `projects/my-project/locations/europe-west9/certificates/my-certificate`" + ) + ), + ] + + # TODO: Allow setting up custom ACME certificate (e.g. ZeroSSL) via GatewayConfiguration -AnyGatewayCertificate = Union[LetsEncryptGatewayCertificate, ACMGatewayCertificate] +AnyGatewayCertificate = Union[ + LetsEncryptGatewayCertificate, + ACMGatewayCertificate, + GCPCertificateManagerGatewayCertificate, +] class GatewayCertificate(RootModel[Annotated[AnyGatewayCertificate, Field(discriminator="type")]]): diff --git a/src/dstack/_internal/core/services/gateways.py b/src/dstack/_internal/core/services/gateways.py index 3fc244ffab..3ee31ac055 100644 --- a/src/dstack/_internal/core/services/gateways.py +++ b/src/dstack/_internal/core/services/gateways.py @@ -1,10 +1,13 @@ from dstack._internal.core.models.gateways import ( ALBGatewayLoadBalancer, + AnyGatewayCertificate, AnyGatewayLoadBalancer, GatewayConfiguration, ) from dstack._internal.core.services.diff import ModelDiff, diff_models +LOAD_BALANCER_TERMINATED_CERTIFICATE_TYPES = ("acm", "gcp-cm") + def diff_gateway_configurations(old: GatewayConfiguration, new: GatewayConfiguration) -> ModelDiff: return diff_models( @@ -15,11 +18,20 @@ def diff_gateway_configurations(old: GatewayConfiguration, new: GatewayConfigura ) +def is_tls_terminated_at_load_balancer( + certificate: AnyGatewayCertificate | None, +) -> bool: + return ( + certificate is not None and certificate.type in LOAD_BALANCER_TERMINATED_CERTIFICATE_TYPES + ) + + def get_effective_load_balancer( configuration: GatewayConfiguration, ) -> AnyGatewayLoadBalancer | None: if configuration.load_balancer is not None: return configuration.load_balancer + # `acm` implies an ALB for backward compatibility if configuration.certificate is not None and configuration.certificate.type == "acm": return ALBGatewayLoadBalancer() return None diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index e903bbfe07..b29d1dc513 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -1254,10 +1254,13 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): " or `certificate: { type: acm }`" ) elif configuration.backend == BackendType.GCP: - if configuration.certificate is not None: + if ( + configuration.certificate is not None + and configuration.certificate.type != "gcp-cm" + ): raise ServerClientError( "`load_balancer: { type: alb }` for the `gcp` backend can only be used" - " with `certificate: null`" + " with `certificate: null` or `certificate: { type: gcp-cm }`" ) else: raise ServerClientError( @@ -1271,6 +1274,15 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): ) if configuration.certificate.type == "acm" and configuration.backend != BackendType.AWS: raise ServerClientError("acm certificate type is supported for aws backend only") + if configuration.certificate.type == "gcp-cm": + if configuration.backend != BackendType.GCP: + raise ServerClientError( + "gcp-cm certificate type is supported for gcp backend only" + ) + if configuration.load_balancer is None or configuration.load_balancer.type != "alb": + raise ServerClientError( + "`certificate: { type: gcp-cm }` requires `load_balancer: { type: alb }`" + ) if configuration.certificate.type == "lets-encrypt" and replicas > 1: err = ( "The `lets-encrypt` certificate type is not supported for gateways with `replicas`" @@ -1280,4 +1292,8 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): ) if configuration.backend == BackendType.AWS: err += " or `certificate: { type: acm, arn: }` (AWS ACM)" + elif configuration.backend == BackendType.GCP: + err += ( + " or `certificate: { type: gcp-cm, name: }` (GCP Certificate Manager)" + ) raise ServerClientError(err) diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 48a55714df..91499880b7 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -17,6 +17,7 @@ from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus from dstack._internal.core.models.runs import RunSpec, ServiceModelSpec, ServiceSpec from dstack._internal.core.models.services import OpenAIChatModel +from dstack._internal.core.services.gateways import is_tls_terminated_at_load_balancer from dstack._internal.server import settings from dstack._internal.server.models import GatewayModel, RunModel from dstack._internal.server.services import events @@ -117,14 +118,12 @@ async def _assign_service_to_gateway( show_service_https = _should_show_service_https(run_spec, gateway_configuration) service_protocol = "https" if show_service_https else "http" - if ( - not show_service_https - and gateway_configuration.certificate is not None - and gateway_configuration.certificate.type == "acm" + if not show_service_https and is_tls_terminated_at_load_balancer( + gateway_configuration.certificate ): # SSL termination is done globally at load balancer so cannot runs only some services via http. raise ServerClientError( - "Cannot run HTTP service on gateway with ACM certificates configured" + "Cannot run HTTP service on gateway with load balancer certificates configured" ) if show_service_https and gateway_configuration.certificate is None: @@ -235,12 +234,12 @@ def should_configure_service_https_on_gateway( if https == "auto": if configuration.certificate is None: return False - if configuration.certificate.type == "acm": + if is_tls_terminated_at_load_balancer(configuration.certificate): return False return True if not https: return False - if configuration.certificate is not None and configuration.certificate.type == "acm": + if is_tls_terminated_at_load_balancer(configuration.certificate): return False return True @@ -261,7 +260,7 @@ def _should_show_service_https(run_spec: RunSpec, configuration: GatewayConfigur def get_gateway_https(configuration: GatewayConfiguration) -> bool: - if configuration.certificate is not None and configuration.certificate.type == "acm": + if is_tls_terminated_at_load_balancer(configuration.certificate): return False if configuration.certificate is not None and configuration.certificate.type == "lets-encrypt": return True diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index 98e614df9f..748cc8cfa7 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -792,9 +792,68 @@ async def test_create_gateway_with_invalid_domain_interpolation( "load_balancer": {"type": "alb"}, }, "`load_balancer: { type: alb }` for the `gcp` backend can only be used with" - " `certificate: null`", + " `certificate: null` or `certificate: { type: gcp-cm }`", id="gcp-load-balancer-with-lets-encrypt-cert", ), + pytest.param( + { + "type": "gateway", + "name": "test", + "backend": "aws", + "region": "us", + "certificate": { + "type": "gcp-cm", + "name": "projects/p/locations/us/certificates/c", + }, + }, + "gcp-cm certificate type is supported for gcp backend only", + id="gcp-cm-cert-with-aws-backend", + ), + pytest.param( + { + "type": "gateway", + "name": "test", + "backend": "gcp", + "region": "us", + "certificate": { + "type": "gcp-cm", + "name": "projects/p/locations/us/certificates/c", + }, + }, + "`certificate: { type: gcp-cm }` requires `load_balancer: { type: alb }`", + id="gcp-cm-cert-without-load-balancer", + ), + pytest.param( + { + "type": "gateway", + "name": "test", + "backend": "aws", + "region": "us", + "certificate": { + "type": "gcp-cm", + "name": "projects/p/locations/us/certificates/c", + }, + "load_balancer": {"type": "alb"}, + }, + "`load_balancer: { type: alb }` can only be used with `certificate: null` or" + " `certificate: { type: acm }`", + id="aws-load-balancer-with-gcp-cm-cert", + ), + pytest.param( + { + "type": "gateway", + "name": "test", + "backend": "gcp", + "region": "us", + "replicas": 2, + }, + "The `lets-encrypt` certificate type is not supported for gateways with `replicas`" + " greater than `1`. To create a replicated gateway, set the `certificate`" + " configuration property to one of the supported values, such as" + " `certificate: null` (no HTTPS)" + " or `certificate: { type: gcp-cm, name: }` (GCP Certificate Manager)", + id="gcp-multi-replica-with-letsencrypt-cert", + ), ], ) async def test_invalid_configuration_rejected( From abb8ce82f76b4cf0de453b6d8b1e31b41bf0f93d Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:26:50 +0000 Subject: [PATCH 592/640] =?UTF-8?q?Increase=20gateway=20replica=20count=20?= =?UTF-8?q?limit=20`3`=20=E2=86=92=20`9`=20(#4295)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And make configurable server-wide via the `DSTACK_SERVER_GATEWAY_MAX_REPLICAS`. --- mkdocs/docs/concepts/gateways.md | 7 +------ mkdocs/docs/reference/env.md | 1 + .../_internal/server/services/gateways/__init__.py | 9 +++------ src/dstack/_internal/server/settings.py | 2 ++ src/tests/_internal/server/routers/test_gateways.py | 4 ++-- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md index ca4cf3e2cd..c84c19af76 100644 --- a/mkdocs/docs/concepts/gateways.md +++ b/mkdocs/docs/concepts/gateways.md @@ -245,12 +245,7 @@ $ dstack gateway list -!!! warning "Experimental" - Replicated gateways are an experimental feature and currently have limitations: - - - HTTPS is only supported for AWS gateways with the `acm` [certificate type](#certificate) and GCP gateways with the `gcp-cm` [certificate type](#certificate). For other gateways, use an external load balancer for TLS termination. - - All replicas are bound to the same backend and region. - - At most 3 replicas are allowed per gateway. +Replicated gateways do not support automatic certificate issuance via Let's Encrypt (`certificate: { type: lets-encrypt }`). For TLS termination, use an external load balancer or one of the other [certificate types](#certificate). !!! info "Reference" For all gateway configuration options, refer to the [reference](../reference/dstack.yml/gateway.md). diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index f00b0dd96f..37b5c0a6fa 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -148,6 +148,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS`{ #DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS } – Maximum age of metrics samples for running jobs. - `DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS`{ #DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS } – Maximum age of metrics samples for finished jobs. - `DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS } – Maximum age of instance health checks. +- `DSTACK_SERVER_GATEWAY_MAX_REPLICAS`{ #DSTACK_SERVER_GATEWAY_MAX_REPLICAS } - Maximum number of replicas allowed for a gateway. Defaults to `9`. - `DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS } – Minimum time interval between consecutive health checks of the same instance. - `DSTACK_SERVER_EVENTS_TTL_SECONDS`{ #DSTACK_SERVER_EVENTS_TTL_SECONDS } - Maximum age of event records. Set to `0` to disable event storage. Defaults to 30 days. - `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY`{ #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY } – A default Docker registry to use for job images that do not specify an explicit registry. E.g., if set to `registry.example`, then `image: ubuntu` becomes equivalent to `image: registry.example/ubuntu`. **Note**: This setting should only be used for configuring registries that act as a pull-through cache for Docker Hub. The default `dstack` images are also pulled from the configured registry. diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index b29d1dc513..100a955a37 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -159,10 +159,6 @@ def emit_gateway_replica_status_change_event( GATEWAY_CONNECT_DELAY = 10 GATEWAY_CONFIGURE_ATTEMPTS = 50 GATEWAY_CONFIGURE_DELAY = 3 -# Artificial limit to avoid doing too many per-replica operations (gateway replica provisioning, -# service registration, etc) in a single pipeline tick. Can be lifted once the implementation is -# more mature. -GATEWAY_MAX_REPLICAS = 3 # documented in gateways.md, keep in sync async def list_project_gateways( @@ -1237,9 +1233,10 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): configuration.replicas if configuration.replicas is not None else GATEWAY_REPLICAS_DEFAULT ) - if replicas > GATEWAY_MAX_REPLICAS: + if replicas > settings.GATEWAY_MAX_REPLICAS: raise ServerClientError( - f"Cannot provision {replicas} gateway replicas. This server allows at most {GATEWAY_MAX_REPLICAS}" + f"Cannot provision {replicas} gateway replicas. This server allows at most" + f" {settings.GATEWAY_MAX_REPLICAS}" ) if configuration.load_balancer is not None: diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index e7fe97124b..efe6e320fc 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -83,6 +83,8 @@ def get_database_url() -> str: MAX_PROBES_PER_JOB = int(os.getenv("DSTACK_SERVER_MAX_PROBES_PER_JOB", 10)) MAX_PROBE_TIMEOUT = int(os.getenv("DSTACK_SERVER_MAX_PROBE_TIMEOUT", 60 * 5)) +GATEWAY_MAX_REPLICAS = int(os.getenv("DSTACK_SERVER_GATEWAY_MAX_REPLICAS", 9)) + SERVER_CONFIG_DISABLED = os.getenv("DSTACK_SERVER_CONFIG_DISABLED") is not None SERVER_CONFIG_ENABLED = not SERVER_CONFIG_DISABLED diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index 748cc8cfa7..211a5bed0a 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -753,9 +753,9 @@ async def test_create_gateway_with_invalid_domain_interpolation( "backend": "aws", "region": "us", "certificate": None, - "replicas": 4, + "replicas": 10, }, - "Cannot provision 4 gateway replicas. This server allows at most 3", + "Cannot provision 10 gateway replicas. This server allows at most 9", id="replicas-exceed-max", ), pytest.param( From 1462fa1de792aaf5d692bf4ded7517cbfc89db54 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Wed, 16 Sep 2026 11:54:48 +0000 Subject: [PATCH 593/640] Treat pre-releases as final releases in feature gates (#4296) `_parse_version` returned `None` for any pre-release or dev build, which `_check_min_version` interprets as "latest", opening every feature gate. Since we publish rc builds (e.g. `0.21.4rc1`), a shim or runner on an rc was assumed to support features it does not have. For example, a shim on `0.20.25rc1` passed the `(0, 21, 3)` gate for restart-safe RUNNING tasks, letting the server restart a shim with running jobs on it. Ignore the pre-release, dev, post-release, and local segments instead, so `0.20.1rc1` is treated as `0.20.1`. An rc is cut from the release branch, so it normally carries the features of the version it leads to. Bare run numbers of staging builds (e.g. `1234`) still resolve to "latest". --- .../server/services/runner/client.py | 15 ++++++------- .../server/services/runner/test_client.py | 21 ++++++++++++------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/dstack/_internal/server/services/runner/client.py b/src/dstack/_internal/server/services/runner/client.py index 8ade87d875..3407c7d8e3 100644 --- a/src/dstack/_internal/server/services/runner/client.py +++ b/src/dstack/_internal/server/services/runner/client.py @@ -855,20 +855,17 @@ def _is_json_response(response: requests.Response) -> bool: def _parse_version(version_string: str) -> Optional[_Version]: """ - Returns a (major, minor, micro) tuple if the version if final. - Returns `None`, which means "latest", if: - * the version is prerelease or dev build -- assuming that in most cases it's a build based on - the latest final release - * the version consists of only major part or not valid at all, e.g., staging builds have - GitHub run number (e.g., 1234) instead of the version -- assuming that it's a "bleeding edge", - not yet released version + Returns a (major, minor, micro) tuple for feature gating. The pre-release, dev, post-release, + and local segments are ignored, that is, `0.20.1rc1` is treated as `0.20.1` -- assuming that + a build carrying a version has the features released in that version. + Returns `None`, which means "latest", if the version consists of only major part or not valid + at all, e.g., staging builds have GitHub run number (e.g., 1234) instead of the version + -- assuming that it's a "bleeding edge", not yet released version. """ try: version = packaging.version.parse(version_string) except packaging.version.InvalidVersion: return None - if version.is_prerelease or version.is_devrelease: - return None release = version.release if len(release) <= 1: return None diff --git a/src/tests/_internal/server/services/runner/test_client.py b/src/tests/_internal/server/services/runner/test_client.py index ad253fbb22..dddd946c2f 100644 --- a/src/tests/_internal/server/services/runner/test_client.py +++ b/src/tests/_internal/server/services/runner/test_client.py @@ -241,10 +241,9 @@ class TestShimClientNegotiate(BaseShimClientTest): # invalid versions, assuming local builds with the latest version pytest.param(None, 2, marks=pytest.mark.shim_version("latest")), pytest.param(None, 2, marks=pytest.mark.shim_version("0.17.0-next")), - # even though this version is less than _FUTURE_API_MIN_VERSION, for the sake of - # simplicity we assume that any non-final version is the latest; normally, users - # should not use non-latest RC versions - pytest.param(None, 2, marks=pytest.mark.shim_version("0.17.0rc1")), + # pre-release versions are treated as the final version they lead to + pytest.param((0, 17, 0), 1, marks=pytest.mark.shim_version("0.17.0rc1")), + pytest.param((0, 18, 34), 2, marks=pytest.mark.shim_version("0.18.34rc1")), ], ) def test( @@ -687,9 +686,17 @@ class TestParseVersion: def test_valid_final(self, value: str, expected: tuple[int, int, int]): assert _parse_version(value) == expected - @pytest.mark.parametrize("value", ["1.12alpha1", "1.12.3rc1", "1.12.3.dev0"]) - def test_valid_pre_dev_local(self, value: str): - assert _parse_version(value) is None + @pytest.mark.parametrize( + ["value", "expected"], + [ + ["1.12alpha1", (1, 12, 0)], + ["1.12.3rc1", (1, 12, 3)], + ["1.12.3.dev0", (1, 12, 3)], + ["1.12.3.post1", (1, 12, 3)], + ], + ) + def test_valid_pre_dev_post(self, value: str, expected: tuple[int, int, int]): + assert _parse_version(value) == expected @pytest.mark.parametrize("value", ["1", "1234"]) def test_valid_major_only(self, value: str): From 242484bb2d332758a2a5d5e17f32b471e0681c07 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:04:58 +0000 Subject: [PATCH 594/640] Fix model endpoint protocol with LB-terminated TLS (#4299) If the gateway has a load balancer with a TLS certificate (`acm`, `gcp-cm`), it always serves the model endpoint over HTTPS, yet the run's `service.model.base_url` used to incorrectly contain a `http://` URL. This commit fixes `service.model.base_url` so that it contains an `https://` URL in such cases. --- .../pipeline_tasks/gateway_replicas.py | 12 ++++++---- .../server/services/services/__init__.py | 24 +++++++++---------- .../server/services/services/test_services.py | 14 +++++------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index ab8a7af11e..44fd687cda 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -63,8 +63,8 @@ from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.runs import get_run_spec from dstack._internal.server.services.services import ( - get_gateway_https, - should_configure_service_https_on_gateway, + should_configure_gateway_endpoint_https_on_gateway_replica, + should_configure_service_https_on_gateway_replica, ) from dstack._internal.server.utils import tracing from dstack._internal.utils.common import get_current_datetime, get_or_error, run_async @@ -1280,8 +1280,12 @@ async def _register_service( run_id=run_model.id, run_name=run_model.run_name, domain=domain, - service_https=should_configure_service_https_on_gateway(run_spec, gateway_configuration), - gateway_https=get_gateway_https(gateway_configuration), + service_https=should_configure_service_https_on_gateway_replica( + run_spec, gateway_configuration + ), + gateway_https=should_configure_gateway_endpoint_https_on_gateway_replica( + gateway_configuration + ), auth=run_spec.configuration.auth, client_max_body_size=settings.DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE, options=service_spec.options, diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 91499880b7..9b5868d8c4 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -131,9 +131,6 @@ async def _assign_service_to_gateway( "Cannot run HTTPS service on gateway with no SSL certificates configured" ) - gateway_https = get_gateway_https(gateway_configuration) - gateway_protocol = "https" if gateway_https else "http" - wildcard_domain = gateway.wildcard_domain.lstrip("*.") if gateway.wildcard_domain else None if wildcard_domain is None: raise ServerClientError("Domain is required for gateway") @@ -146,6 +143,7 @@ async def _assign_service_to_gateway( if isinstance(run_spec.configuration.model, OpenAIChatModel): model_url = service_url + run_spec.configuration.model.prefix else: + gateway_protocol = "https" if gateway_configuration.certificate is not None else "http" model_url = f"{gateway_protocol}://gateway.{wildcard_domain}" service_spec = _get_service_spec( configuration=run_spec.configuration, @@ -220,11 +218,11 @@ def _get_service_spec( return service_spec -def should_configure_service_https_on_gateway( +def should_configure_service_https_on_gateway_replica( run_spec: RunSpec, configuration: GatewayConfiguration ) -> bool: """ - Returns `True` if the gateway needs to serve the service with HTTPS. + Returns `True` if the gateway replica needs to serve the service with HTTPS. May be `False` for HTTPS services, e.g. SSL termination is done on a load balancer. """ assert run_spec.configuration.type == "service" @@ -244,6 +242,14 @@ def should_configure_service_https_on_gateway( return True +def should_configure_gateway_endpoint_https_on_gateway_replica( + configuration: GatewayConfiguration, +) -> bool: + return ( + configuration.certificate is not None and configuration.certificate.type == "lets-encrypt" + ) + + def _should_show_service_https(run_spec: RunSpec, configuration: GatewayConfiguration) -> bool: """ Returns `True` if the service needs to be accessed via https://. @@ -257,11 +263,3 @@ def _should_show_service_https(run_spec: RunSpec, configuration: GatewayConfigur return False return True return https - - -def get_gateway_https(configuration: GatewayConfiguration) -> bool: - if is_tls_terminated_at_load_balancer(configuration.certificate): - return False - if configuration.certificate is not None and configuration.certificate.type == "lets-encrypt": - return True - return False diff --git a/src/tests/_internal/server/services/services/test_services.py b/src/tests/_internal/server/services/services/test_services.py index 27854893be..1ce03c52a6 100644 --- a/src/tests/_internal/server/services/services/test_services.py +++ b/src/tests/_internal/server/services/services/test_services.py @@ -12,7 +12,7 @@ from dstack._internal.core.models.runs import RunSpec from dstack._internal.server.services.services import ( _should_show_service_https, - should_configure_service_https_on_gateway, + should_configure_service_https_on_gateway_replica, ) from dstack._internal.server.testing.common import get_run_spec @@ -55,36 +55,36 @@ class TestShouldConfigureServiceHttpsOnGateway: def test_auto_resolves_to_true_with_lets_encrypt_gateway(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config(certificate=LetsEncryptGatewayCertificate()) - assert should_configure_service_https_on_gateway(run_spec, gw) is True + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is True def test_auto_resolves_to_false_when_gateway_has_no_certificate(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config(certificate=None) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_auto_resolves_to_false_with_acm_gateway(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config( certificate=ACMGatewayCertificate(arn="arn:aws:acm:us-east-1:123:cert/abc") ) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_true_enables_https_when_gateway_has_no_certificate(self) -> None: run_spec = _service_run_spec(https=True) gw = _gateway_config(certificate=None) - assert should_configure_service_https_on_gateway(run_spec, gw) is True + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is True def test_false_disables_https_regardless_of_gateway_certificate(self) -> None: run_spec = _service_run_spec(https=False) gw = _gateway_config(certificate=LetsEncryptGatewayCertificate()) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_true_does_not_configure_https_on_acm_gateway(self) -> None: run_spec = _service_run_spec(https=True) gw = _gateway_config( certificate=ACMGatewayCertificate(arn="arn:aws:acm:us-east-1:123:cert/abc") ) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False class TestShouldShowServiceHttps: From b5610b1e5df5ef511c09705f23157e15808a7149 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Thu, 17 Sep 2026 08:43:08 +0000 Subject: [PATCH 595/640] Do not downgrade runner and shim by default (#4297) When several server replicas of different versions run at the same time, as during a rolling deployment, each of them tried to install the component version matching its own, fighting over the instance and reinstalling the binaries in a loop. The server now installs a component only if it can prove that the installed version is older than the expected one. Versions are compared as PyPA versions rather than as strings, so equivalent spellings such as `0.20.1` and `0.20.1.0` no longer trigger a reinstall. An unparseable version never leads to an install: on the installed side it means a dev build, which is assumed to be the newest one; on the expected side it's a server misconfiguration, which is logged as a warning. Downgrading is still possible via `DSTACK_RUNNER_ALLOW_DOWNGRADE` and `DSTACK_SHIM_ALLOW_DOWNGRADE`. They are meant for a server downgrade, which requires manual actions anyway, and are incompatible with rolling deployment. Also: * Drop the `DSTACK_SHIM_VERSION` fallback to `DSTACK_RUNNER_VERSION`, which was never documented and warned about its removal for a long time. * Document `DSTACK_SHIM_VERSION` and `DSTACK_{RUNNER,SHIM}_VERSION_URL`, and correct the `DSTACK_RUNNER_VERSION` description: it never defaulted to `latest`, that is only the download URL placeholder fallback when the version cannot be determined. Co-authored-by: Claude Opus 5 (1M context) --- mkdocs/docs/reference/env.md | 13 +- .../_internal/core/backends/base/compute.py | 7 - .../pipeline_tasks/instances/check.py | 201 +++++++++--- src/dstack/_internal/settings.py | 2 + .../pipeline_tasks/test_instances/conftest.py | 6 + .../test_instances/test_check.py | 309 ++++++++++-------- 6 files changed, 352 insertions(+), 186 deletions(-) diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 37b5c0a6fa..4a169fc22f 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -161,6 +161,8 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_SERVER_SSH_CONNECT_TIMEOUT`{ #DSTACK_SERVER_SSH_CONNECT_TIMEOUT } – The SSH `ConnectTimeout` for server-instance connections, in seconds. Defaults to `3`. Increase if there are high-latency links between the server and instances. - `DSTACK_SERVER_SSH_POOL_DISABLED`{ #DSTACK_SERVER_SSH_POOL_DISABLED } – Disables the reuse of server SSH connections to instances. If set, significantly decreases server RAM usage, but slows down processing and may cause CPU spikes due to frequent SSH-connection establishment. +- `DSTACK_RUNNER_ALLOW_DOWNGRADE`{ #DSTACK_RUNNER_ALLOW_DOWNGRADE } – When set to any value, allows the server to install an older `dstack-runner` version over a newer one. By default, the server skips installation if a newer version is already installed, so that replicas running different server versions don't reinstall the runner over each other during a rolling deployment. Set this variable only while downgrading the server, and unset it afterwards — leaving it set brings the reinstall loop back. +- `DSTACK_SHIM_ALLOW_DOWNGRADE`{ #DSTACK_SHIM_ALLOW_DOWNGRADE } – Same as `DSTACK_RUNNER_ALLOW_DOWNGRADE` but for `dstack-shim`. ??? info "Internal environment variables" The following environment variables are intended for development purposes: @@ -169,10 +171,15 @@ slows down processing and may cause CPU spikes due to frequent SSH-connection es * `DSTACK_SERVER_UVICORN_LOG_LEVEL` – Sets uvicorn logger log level. Defaults to `ERROR`. * `DSTACK_SERVER_MAX_OFFERS_TRIED` - Sets how many instance offers to try when starting a job. Setting a high value can degrade server performance. - * `DSTACK_RUNNER_VERSION` – Sets exact runner version for debug. Defaults to `latest`. Ignored if `DSTACK_RUNNER_DOWNLOAD_URL` is set. + * `DSTACK_RUNNER_VERSION` – Overrides the `dstack-runner` version the server installs on instances. Release builds of the server + default to their own version; dev builds have no default version. Must be a valid [PyPA version](https://packaging.python.org/en/latest/specifications/version-specifiers/), for example, `0.20.1`. + * `DSTACK_RUNNER_VERSION_URL` – URL to fetch the `dstack-runner` version from. The response body must be the version string, + see `DSTACK_RUNNER_VERSION` for the format. Used only if `DSTACK_RUNNER_VERSION` is not set. * `DSTACK_RUNNER_DOWNLOAD_URL` – Overrides `dstack-runner` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders, - where `{version}` is `dstack` version in the `X.Y.Z` format or `latest`, and `{arch}` is either `amd64` or `arm64`, for example, - `https://dstack.example.com/{arch}/{version}/dstack-runner`. + where `{version}` is the `dstack-runner` version (see `DSTACK_RUNNER_VERSION`), or `latest` if the version cannot be determined, + and `{arch}` is either `amd64` or `arm64`, for example, `https://dstack.example.com/{arch}/{version}/dstack-runner`. + * `DSTACK_SHIM_VERSION` – Same as `DSTACK_RUNNER_VERSION` but for `dstack-shim`. + * `DSTACK_SHIM_VERSION_URL` – Same as `DSTACK_RUNNER_VERSION_URL` but for `dstack-shim`. * `DSTACK_SHIM_DOWNLOAD_URL` – Overrides `dstack-shim` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders, see `DSTACK_RUNNER_DOWNLOAD_URL` for the details. * `DSTACK_GATEWAY_PACKAGE_URL` – Overrides the URL the `dstack` package is installed from on gateway instances (used as `dstack[gateway] @ `). diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 60ee0d6d01..9bf202d3f9 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -969,13 +969,6 @@ def get_dstack_shim_version() -> Optional[str]: return version if version := settings.DSTACK_SHIM_VERSION: return version - if version := settings.DSTACK_RUNNER_VERSION: - logger.warning( - "DSTACK_SHIM_VERSION is not set, using DSTACK_RUNNER_VERSION." - " Future versions will not fall back to DSTACK_RUNNER_VERSION." - " Set DSTACK_SHIM_VERSION to supress this warning." - ) - return version if version_url := settings.DSTACK_SHIM_VERSION_URL: return _fetch_version(version_url) if settings.DSTACK_USE_LATEST_FROM_BRANCH: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py index 8185621f88..0bd31d06c8 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/check.py @@ -5,11 +5,13 @@ from typing import Optional import gpuhunt +import packaging.version import requests from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload +from dstack._internal import settings from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.backends.base.compute import ( get_dstack_runner_download_url, @@ -46,6 +48,7 @@ from dstack._internal.server.schemas.instances import InstanceCheck from dstack._internal.server.schemas.runner import ( ComponentInfo, + ComponentName, ComponentStatus, InstanceHealthResponse, ) @@ -482,86 +485,114 @@ def _maybe_install_components( installation_requested = False if (runner_info := components.runner) is not None: - installation_requested |= _maybe_install_runner(instance_model, shim_client, runner_info) + installation_requested |= _maybe_install_runner( + instance_model=instance_model, + shim_client=shim_client, + runner_info=runner_info, + allow_downgrade=settings.DSTACK_RUNNER_ALLOW_DOWNGRADE, + ) else: - logger.debug("Instance %s: no runner info", instance_model.name) + logger.debug( + "Instance %s: %s: no component info", instance_model.name, ComponentName.RUNNER.value + ) if (shim_info := components.shim) is not None: if shim_info.status == ComponentStatus.INSTALLED: installed_shim_version = shim_info.version - installation_requested |= _maybe_install_shim(instance_model, shim_client, shim_info) + installation_requested |= _maybe_install_shim( + instance_model=instance_model, + shim_client=shim_client, + shim_info=shim_info, + allow_downgrade=settings.DSTACK_SHIM_ALLOW_DOWNGRADE, + ) else: - logger.debug("Instance %s: no shim info", instance_model.name) + logger.debug( + "Instance %s: %s: no component info", instance_model.name, ComponentName.SHIM.value + ) - # old shim without `dstack-shim` component and `/api/shutdown` support - # or the same version is already running - # or we just requested installation of at least one component - # or at least one component is already being installed - # or at least one shim task won't survive restart running_shim_version = shim_client.get_version_string() + # skip the restart if: if ( + # old shim without `dstack-shim` component and `/api/shutdown` support installed_shim_version is None + # the same version is already running -- a string comparison on purpose: both sides come + # from the same `Version` build-time variable, one read off the on-disk binary, the other + # reported by the running process, so the question is whether it's the same build, not + # which version is newer or installed_shim_version == running_shim_version + # we just requested installation of at least one component or installation_requested + # at least one component is already being installed or any(component.status == ComponentStatus.INSTALLING for component in components) + # at least one shim task won't survive restart or not shim_client.is_safe_to_restart() ): return if shim_client.shutdown(force=False): logger.debug( - "Instance %s: restarting shim %s -> %s", + "Instance %s: %s: restarting %r -> %r", instance_model.name, + ComponentName.SHIM.value, running_shim_version, installed_shim_version, ) else: - logger.debug("Instance %s: cannot restart shim", instance_model.name) + logger.debug( + "Instance %s: %s: cannot restart", instance_model.name, ComponentName.SHIM.value + ) def _maybe_install_runner( instance_model: InstanceModel, shim_client: runner_client.ShimClient, runner_info: ComponentInfo, + allow_downgrade: bool, ) -> bool: # For developers: # * To install the latest dev build for the current branch from the CI, # set DSTACK_USE_LATEST_FROM_BRANCH=1. # * To provide your own build, set DSTACK_RUNNER_VERSION_URL and DSTACK_RUNNER_DOWNLOAD_URL. - expected_version = get_dstack_runner_version() - if expected_version is None: - return False - + name = runner_info.name installed_version = runner_info.version + expected_version = get_dstack_runner_version() logger.debug( - "Instance %s: runner status=%s installed_version=%s", + "Instance %s: %s: status=%s installed_version=%r expected_version=%r", instance_model.name, + name, runner_info.status.value, - installed_version or "(no version)", + installed_version, + expected_version, ) - if runner_info.status == ComponentStatus.INSTALLING: - logger.debug("Instance %s: runner is already being installed", instance_model.name) + + if not expected_version: return False - if installed_version and installed_version == expected_version: - logger.debug("Instance %s: expected runner version already installed", instance_model.name) + + if not _should_install_component( + instance_model=instance_model, + component_info=runner_info, + expected_version=expected_version, + allow_downgrade=allow_downgrade, + ): return False - url = get_dstack_runner_download_url( + download_url = get_dstack_runner_download_url( arch=_get_instance_cpu_arch(instance_model), version=expected_version, ) logger.debug( - "Instance %s: installing runner %s -> %s from %s", + "Instance %s: %s: installing %r -> %r from %s", instance_model.name, - installed_version or "(no version)", + name, + installed_version, expected_version, - url, + download_url, ) try: - shim_client.install_runner(url) + shim_client.install_runner(download_url) return True - except requests.RequestException as exc: - logger.warning("Instance %s: shim.install_runner(): %s", instance_model.name, exc) + except requests.RequestException as e: + logger.warning("Instance %s: %s: failed to install: %s", instance_model.name, name, e) return False @@ -569,49 +600,129 @@ def _maybe_install_shim( instance_model: InstanceModel, shim_client: runner_client.ShimClient, shim_info: ComponentInfo, + allow_downgrade: bool, ) -> bool: # For developers: # * To install the latest dev build for the current branch from the CI, # set DSTACK_USE_LATEST_FROM_BRANCH=1. # * To provide your own build, set DSTACK_SHIM_VERSION_URL and DSTACK_SHIM_DOWNLOAD_URL. - expected_version = get_dstack_shim_version() - if expected_version is None: - return False - + name = shim_info.name installed_version = shim_info.version + expected_version = get_dstack_shim_version() logger.debug( - "Instance %s: shim status=%s installed_version=%s running_version=%s", + "Instance %s: %s: status=%s installed_version=%r expected_version=%r running_version=%r", instance_model.name, + name, shim_info.status.value, - installed_version or "(no version)", + installed_version, + expected_version, shim_client.get_version_string(), ) - if shim_info.status == ComponentStatus.INSTALLING: - logger.debug("Instance %s: shim is already being installed", instance_model.name) + + if not expected_version: return False - if installed_version and installed_version == expected_version: - logger.debug("Instance %s: expected shim version already installed", instance_model.name) + + if not _should_install_component( + instance_model=instance_model, + component_info=shim_info, + expected_version=expected_version, + allow_downgrade=allow_downgrade, + ): return False - url = get_dstack_shim_download_url( + download_url = get_dstack_shim_download_url( arch=_get_instance_cpu_arch(instance_model), version=expected_version, ) logger.debug( - "Instance %s: installing shim %s -> %s from %s", + "Instance %s: %s: installing %r -> %r from %s", instance_model.name, - installed_version or "(no version)", + name, + installed_version, expected_version, - url, + download_url, ) try: - shim_client.install_shim(url) + shim_client.install_shim(download_url) return True - except requests.RequestException as exc: - logger.warning("Instance %s: shim.install_shim(): %s", instance_model.name, exc) + except requests.RequestException as e: + logger.warning("Instance %s: %s: failed to install: %s", instance_model.name, name, e) return False +def _should_install_component( + instance_model: InstanceModel, + component_info: ComponentInfo, + expected_version: str, + allow_downgrade: bool, +) -> bool: + """ + Decides whether the component should be installed, comparing the installed and the expected + versions. Logs the reason if the installation is skipped. + + Unless `allow_downgrade` is set, the component is not installed if the installed version is + newer than the expected one. This keeps server replicas running different versions from + reinstalling the component over each other during a rolling deployment. + """ + name = component_info.name + + if component_info.status == ComponentStatus.INSTALLING: + logger.debug("Instance %s: %s: already being installed", instance_model.name, name) + return False + + expected_version_parsed = _parse_pypa_version(expected_version) + if expected_version_parsed is None: + logger.warning( + "Instance %s: %s: failed to parse expected_version: %r", + instance_model.name, + name, + expected_version, + ) + return False + + installed_version = component_info.version + if not installed_version: + return True + + installed_version_parsed = _parse_pypa_version(installed_version) + if installed_version_parsed is None: + # Dev builds report `latest`; treat any unparseable version as the newest one + if not allow_downgrade: + logger.debug( + "Instance %s: %s: cannot parse installed_version, skipping the install", + instance_model.name, + name, + ) + return False + elif installed_version_parsed == expected_version_parsed: + logger.debug( + "Instance %s: %s: expected version already installed", instance_model.name, name + ) + return False + elif installed_version_parsed > expected_version_parsed and not allow_downgrade: + logger.debug("Instance %s: %s: newer version already installed", instance_model.name, name) + return False + + return True + + +def _parse_pypa_version(version_string: str) -> Optional[packaging.version.Version]: + """ + Parses the version for comparing component versions, keeping the pre-release, dev, post-release, + and local segments, that is, `0.20.1rc1` is older than `0.20.1`. + Returns `None` if the version is not PyPA-conformant, e.g., `latest` reported by dev builds. + + Not to be confused with `ShimClient.get_version_tuple()`, which is based on + `dstack._internal.server.services.runner.client._parse_version` -- it truncates the version to + `(major, minor, micro)` and treats unparseable versions as the latest, suiting feature gating + but not version comparison. + """ + try: + return packaging.version.parse(version_string) + except packaging.version.InvalidVersion: + return None + + def _get_instance_cpu_arch(instance_model: InstanceModel) -> Optional[gpuhunt.CPUArchitecture]: job_provisioning_data = get_instance_provisioning_data(instance_model) if job_provisioning_data is None: diff --git a/src/dstack/_internal/settings.py b/src/dstack/_internal/settings.py index 7a37ba5cfb..b04b94b856 100644 --- a/src/dstack/_internal/settings.py +++ b/src/dstack/_internal/settings.py @@ -13,9 +13,11 @@ DSTACK_RUNNER_VERSION = os.getenv("DSTACK_RUNNER_VERSION") DSTACK_RUNNER_VERSION_URL = os.getenv("DSTACK_RUNNER_VERSION_URL") DSTACK_RUNNER_DOWNLOAD_URL = os.getenv("DSTACK_RUNNER_DOWNLOAD_URL") +DSTACK_RUNNER_ALLOW_DOWNGRADE = os.getenv("DSTACK_RUNNER_ALLOW_DOWNGRADE") is not None DSTACK_SHIM_VERSION = os.getenv("DSTACK_SHIM_VERSION") DSTACK_SHIM_VERSION_URL = os.getenv("DSTACK_SHIM_VERSION_URL") DSTACK_SHIM_DOWNLOAD_URL = os.getenv("DSTACK_SHIM_DOWNLOAD_URL") +DSTACK_SHIM_ALLOW_DOWNGRADE = os.getenv("DSTACK_SHIM_ALLOW_DOWNGRADE") is not None DSTACK_USE_LATEST_FROM_BRANCH = os.getenv("DSTACK_USE_LATEST_FROM_BRANCH") is not None DSTACK_GATEWAY_PACKAGE_URL = os.getenv("DSTACK_GATEWAY_PACKAGE_URL") diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_instances/conftest.py b/src/tests/_internal/server/background/pipeline_tasks/test_instances/conftest.py index bbb48134b2..c80449aca2 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_instances/conftest.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_instances/conftest.py @@ -15,6 +15,12 @@ from dstack._internal.server.schemas.instances import InstanceCheck +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", "allow_downgrade: allow the server to install an older component version" + ) + + @pytest.fixture def fetcher() -> InstanceFetcher: return InstanceFetcher( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_check.py b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_check.py index 4ff57e0fd6..e3d9e401da 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_check.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_check.py @@ -1,6 +1,6 @@ import datetime as dt import logging -from typing import Optional +from typing import ClassVar, Optional from unittest.mock import Mock import pytest @@ -10,6 +10,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from dstack._internal import settings from dstack._internal.core.models.common import validate_json_extra_ignore from dstack._internal.core.models.fleets import FleetNodesSpec from dstack._internal.core.models.health import HealthStatus @@ -633,31 +634,65 @@ def shim_client_mock( return mock -@pytest.mark.usefixtures("get_dstack_runner_version_mock") -class TestMaybeInstallRunner(BaseTestMaybeInstallComponents): +@pytest.mark.usefixtures("version_mock", "download_url_mock") +class BaseTestMaybeInstallComponent(BaseTestMaybeInstallComponents): + """ + Shared tests for `_maybe_install_runner` and `_maybe_install_shim`. The two must behave + identically -- only the version getter, the download URL getter, and the install call differ. + """ + + COMPONENT_NAME: ClassVar[ComponentName] + DOWNLOAD_URL: ClassVar[str] + ALLOW_DOWNGRADE_SETTING: ClassVar[str] + @pytest.fixture - def component_list(self) -> ComponentList: - components = ComponentList() - components.add( - ComponentInfo( - name=ComponentName.RUNNER, - version=self.EXPECTED_VERSION, - status=ComponentStatus.INSTALLED, - ), + def component_info(self) -> ComponentInfo: + return ComponentInfo( + name=self.COMPONENT_NAME, + version=self.EXPECTED_VERSION, + status=ComponentStatus.INSTALLED, ) + + @pytest.fixture + def component_list(self, component_info: ComponentInfo) -> ComponentList: + components = ComponentList() + components.add(component_info) return components @pytest.fixture - def get_dstack_runner_version_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: - mock = Mock(return_value=self.EXPECTED_VERSION) - monkeypatch.setattr(instances_check, "get_dstack_runner_version", mock) - return mock + def version_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + raise NotImplementedError @pytest.fixture - def get_dstack_runner_download_url_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: - mock = Mock(return_value="https://example.com/runner") - monkeypatch.setattr(instances_check, "get_dstack_runner_download_url", mock) - return mock + def download_url_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + raise NotImplementedError + + @pytest.fixture + def install_mock(self, shim_client_mock: Mock) -> Mock: + raise NotImplementedError + + @pytest.fixture(autouse=True) + def allow_downgrade_settings( + self, request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The settings are read from the environment at import time, so they must be pinned in + # both directions -- otherwise the tests fail for developers who have the variables set + # in their environment and run `pytest` directly, without env isolation. + monkeypatch.setattr(settings, "DSTACK_RUNNER_ALLOW_DOWNGRADE", False) + monkeypatch.setattr(settings, "DSTACK_SHIM_ALLOW_DOWNGRADE", False) + if request.node.get_closest_marker("allow_downgrade") is not None: + monkeypatch.setattr(settings, self.ALLOW_DOWNGRADE_SETTING, True) + + def assert_installed(self, install_mock: Mock, download_url_mock: Mock) -> None: + download_url_mock.assert_called_once_with(arch=None, version=self.EXPECTED_VERSION) + install_mock.assert_called_once_with(self.DOWNLOAD_URL) + + def assert_installing_logged(self, log: pytest.LogCaptureFixture, installed_version: str): + expected = ( + f"{self.COMPONENT_NAME.value}: installing {installed_version!r}" + f" -> {self.EXPECTED_VERSION!r} from {self.DOWNLOAD_URL}" + ) + assert expected in log.text async def test_cannot_determine_expected_version( self, @@ -665,206 +700,172 @@ async def test_cannot_determine_expected_version( instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_runner_version_mock: Mock, + install_mock: Mock, + version_mock: Mock, ): - get_dstack_runner_version_mock.return_value = None + version_mock.return_value = None instances_check._maybe_install_components(instance, shim_client_mock) shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_runner.assert_not_called() + install_mock.assert_not_called() - async def test_expected_version_already_installed( + async def test_cannot_parse_expected_version( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, + install_mock: Mock, + version_mock: Mock, ): - shim_client_mock.get_components.return_value.runner.version = self.EXPECTED_VERSION + version_mock.return_value = "latest" instances_check._maybe_install_components(instance, shim_client_mock) - assert "expected runner version already installed" in debug_task_log.text - shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_runner.assert_not_called() + assert ( + f"{self.COMPONENT_NAME.value}: failed to parse expected_version: 'latest'" + in debug_task_log.text + ) + install_mock.assert_not_called() - @pytest.mark.parametrize("status", [ComponentStatus.NOT_INSTALLED, ComponentStatus.ERROR]) - async def test_install_not_installed_or_error( + @pytest.mark.parametrize("installed_version", ["0.20.1", "0.20.1.0"]) + async def test_expected_version_already_installed( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_runner_download_url_mock: Mock, - status: ComponentStatus, + component_info: ComponentInfo, + install_mock: Mock, + installed_version: str, ): - shim_client_mock.get_components.return_value.runner.version = "" - shim_client_mock.get_components.return_value.runner.status = status + # `0.20.1.0` is the same version as `0.20.1` according to PyPA + component_info.version = installed_version instances_check._maybe_install_components(instance, shim_client_mock) - assert f"installing runner (no version) -> {self.EXPECTED_VERSION}" in debug_task_log.text - get_dstack_runner_download_url_mock.assert_called_once_with( - arch=None, - version=self.EXPECTED_VERSION, + assert ( + f"{self.COMPONENT_NAME.value}: expected version already installed" + in debug_task_log.text ) shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_runner.assert_called_once_with( - get_dstack_runner_download_url_mock.return_value - ) + install_mock.assert_not_called() - @pytest.mark.parametrize("installed_version", ["0.19.40", "0.21.0", "dev"]) - async def test_install_installed( + @pytest.mark.parametrize("status", [ComponentStatus.NOT_INSTALLED, ComponentStatus.ERROR]) + async def test_install_not_installed_or_error( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_runner_download_url_mock: Mock, - installed_version: str, + component_info: ComponentInfo, + install_mock: Mock, + download_url_mock: Mock, + status: ComponentStatus, ): - shim_client_mock.get_components.return_value.runner.version = installed_version + component_info.version = "" + component_info.status = status instances_check._maybe_install_components(instance, shim_client_mock) - assert ( - f"installing runner {installed_version} -> {self.EXPECTED_VERSION}" - in debug_task_log.text - ) - get_dstack_runner_download_url_mock.assert_called_once_with( - arch=None, - version=self.EXPECTED_VERSION, - ) + self.assert_installing_logged(debug_task_log, "") shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_runner.assert_called_once_with( - get_dstack_runner_download_url_mock.return_value - ) + self.assert_installed(install_mock, download_url_mock) - async def test_already_installing( + async def test_install_older_version( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, + component_info: ComponentInfo, + install_mock: Mock, + download_url_mock: Mock, ): - shim_client_mock.get_components.return_value.runner.version = "dev" - shim_client_mock.get_components.return_value.runner.status = ComponentStatus.INSTALLING + component_info.version = "0.19.40" instances_check._maybe_install_components(instance, shim_client_mock) - assert "runner is already being installed" in debug_task_log.text + self.assert_installing_logged(debug_task_log, "0.19.40") shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_runner.assert_not_called() - + self.assert_installed(install_mock, download_url_mock) -@pytest.mark.usefixtures("get_dstack_shim_version_mock") -class TestMaybeInstallShim(BaseTestMaybeInstallComponents): - @pytest.fixture - def component_list(self) -> ComponentList: - components = ComponentList() - components.add( - ComponentInfo( - name=ComponentName.SHIM, - version=self.EXPECTED_VERSION, - status=ComponentStatus.INSTALLED, - ), - ) - return components - - @pytest.fixture - def get_dstack_shim_version_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: - mock = Mock(return_value=self.EXPECTED_VERSION) - monkeypatch.setattr(instances_check, "get_dstack_shim_version", mock) - return mock - - @pytest.fixture - def get_dstack_shim_download_url_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: - mock = Mock(return_value="https://example.com/shim") - monkeypatch.setattr(instances_check, "get_dstack_shim_download_url", mock) - return mock - - async def test_cannot_determine_expected_version( + async def test_skips_newer_version( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_shim_version_mock: Mock, + component_info: ComponentInfo, + install_mock: Mock, ): - get_dstack_shim_version_mock.return_value = None + component_info.version = "0.21.0" instances_check._maybe_install_components(instance, shim_client_mock) + assert ( + f"{self.COMPONENT_NAME.value}: newer version already installed" in debug_task_log.text + ) shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_shim.assert_not_called() + install_mock.assert_not_called() - async def test_expected_version_already_installed( + @pytest.mark.allow_downgrade + async def test_installs_newer_version_if_downgrade_allowed( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, + component_info: ComponentInfo, + install_mock: Mock, + download_url_mock: Mock, ): - shim_client_mock.get_components.return_value.shim.version = self.EXPECTED_VERSION + component_info.version = "0.21.0" instances_check._maybe_install_components(instance, shim_client_mock) - assert "expected shim version already installed" in debug_task_log.text - shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_shim.assert_not_called() + self.assert_installing_logged(debug_task_log, "0.21.0") + self.assert_installed(install_mock, download_url_mock) - @pytest.mark.parametrize("status", [ComponentStatus.NOT_INSTALLED, ComponentStatus.ERROR]) - async def test_install_not_installed_or_error( + async def test_skips_unparsable_installed_version( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_shim_download_url_mock: Mock, - status: ComponentStatus, + component_info: ComponentInfo, + install_mock: Mock, ): - shim_client_mock.get_components.return_value.shim.version = "" - shim_client_mock.get_components.return_value.shim.status = status + # dev builds report `latest`, assuming that it's the newest version + component_info.version = "latest" instances_check._maybe_install_components(instance, shim_client_mock) - assert f"installing shim (no version) -> {self.EXPECTED_VERSION}" in debug_task_log.text - get_dstack_shim_download_url_mock.assert_called_once_with( - arch=None, - version=self.EXPECTED_VERSION, + assert ( + f"{self.COMPONENT_NAME.value}: cannot parse installed_version, skipping the install" + in debug_task_log.text ) shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_shim.assert_called_once_with( - get_dstack_shim_download_url_mock.return_value - ) + install_mock.assert_not_called() - @pytest.mark.parametrize("installed_version", ["0.19.40", "0.21.0", "dev"]) - async def test_install_installed( + @pytest.mark.allow_downgrade + async def test_installs_unparsable_installed_version_if_downgrade_allowed( self, test_db, instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, - get_dstack_shim_download_url_mock: Mock, - installed_version: str, + component_info: ComponentInfo, + install_mock: Mock, + download_url_mock: Mock, ): - shim_client_mock.get_components.return_value.shim.version = installed_version + component_info.version = "latest" instances_check._maybe_install_components(instance, shim_client_mock) - assert ( - f"installing shim {installed_version} -> {self.EXPECTED_VERSION}" - in debug_task_log.text - ) - get_dstack_shim_download_url_mock.assert_called_once_with( - arch=None, - version=self.EXPECTED_VERSION, - ) - shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_shim.assert_called_once_with( - get_dstack_shim_download_url_mock.return_value - ) + self.assert_installing_logged(debug_task_log, "latest") + self.assert_installed(install_mock, download_url_mock) async def test_already_installing( self, @@ -872,15 +873,61 @@ async def test_already_installing( instance: InstanceModel, debug_task_log: pytest.LogCaptureFixture, shim_client_mock: Mock, + component_info: ComponentInfo, + install_mock: Mock, ): - shim_client_mock.get_components.return_value.shim.version = "dev" - shim_client_mock.get_components.return_value.shim.status = ComponentStatus.INSTALLING + component_info.version = "0.19.40" + component_info.status = ComponentStatus.INSTALLING instances_check._maybe_install_components(instance, shim_client_mock) - assert "shim is already being installed" in debug_task_log.text + assert f"{self.COMPONENT_NAME.value}: already being installed" in debug_task_log.text shim_client_mock.get_components.assert_called_once() - shim_client_mock.install_shim.assert_not_called() + install_mock.assert_not_called() + + +class TestMaybeInstallRunner(BaseTestMaybeInstallComponent): + COMPONENT_NAME = ComponentName.RUNNER + DOWNLOAD_URL = "https://example.com/runner" + ALLOW_DOWNGRADE_SETTING = "DSTACK_RUNNER_ALLOW_DOWNGRADE" + + @pytest.fixture + def version_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = Mock(return_value=self.EXPECTED_VERSION) + monkeypatch.setattr(instances_check, "get_dstack_runner_version", mock) + return mock + + @pytest.fixture + def download_url_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = Mock(return_value=self.DOWNLOAD_URL) + monkeypatch.setattr(instances_check, "get_dstack_runner_download_url", mock) + return mock + + @pytest.fixture + def install_mock(self, shim_client_mock: Mock) -> Mock: + return shim_client_mock.install_runner + + +class TestMaybeInstallShim(BaseTestMaybeInstallComponent): + COMPONENT_NAME = ComponentName.SHIM + DOWNLOAD_URL = "https://example.com/shim" + ALLOW_DOWNGRADE_SETTING = "DSTACK_SHIM_ALLOW_DOWNGRADE" + + @pytest.fixture + def version_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = Mock(return_value=self.EXPECTED_VERSION) + monkeypatch.setattr(instances_check, "get_dstack_shim_version", mock) + return mock + + @pytest.fixture + def download_url_mock(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + mock = Mock(return_value=self.DOWNLOAD_URL) + monkeypatch.setattr(instances_check, "get_dstack_shim_download_url", mock) + return mock + + @pytest.fixture + def install_mock(self, shim_client_mock: Mock) -> Mock: + return shim_client_mock.install_shim @pytest.mark.usefixtures("maybe_install_runner_mock", "maybe_install_shim_mock") From 06d8dc4796f5be363c85c4fcc557a88235f474a5 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:35:48 +0000 Subject: [PATCH 596/640] Discourage automatic PRs by autonomous agents (#4292) --- CONTRIBUTING.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdbca3eeb4..cf398ba0cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,11 +2,24 @@ We appreciate your interest in contributing to `dstack`! This document will help you get up to speed with `dstack` codebase and guide you through the contribution process. +## Who can contribute + +We'd be happy to see your contribution if you are a `dstack` user, a partner integrating with `dstack`, or any other person genuinely interested in `dstack`. + ## Using AI assistance -You may use AI assistance when contributing to `dstack`. -The contributor is responsible for the quality of the change: make sure you understand -what you're submitting, and hold it to the same standard as code you'd write by hand. +You may use AI assistance when contributing to `dstack`. However, we expect you, the human contributor, to initiate and validate the contribution, ensuring it meets the same standard as something you'd write by hand. + +Contributions made entirely without a human in the loop (e.g., PRs filed by an autonomous agent against open issues) are discouraged and will be declined. + +## Accepted changes + +* Bug fixes that address a bug you've hit while using `dstack`. Include steps to reproduce in the linked issue or the PR. +* New features that improve `dstack` for you. Before submitting a feature PR, please describe your use case and proposal in a GitHub issue to discuss it with the core team. +* Minor fixes such as typos. +* [Examples](examples). + +Existing open issues are typically picked up by the core team — please only contribute to one if it actually affects how you use `dstack`. ## Set up your development environment @@ -23,13 +36,6 @@ If you make a non-trivial change to `dstack`, we recommend you learn about `dsta 3. Commit your changes. 4. Open a PR. [Link the PR to the issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) (if you are solving one). -### Accepted changes - -* Bug fixes that address a clearly defined bug. Include steps to reproduce in the linked issue or the PR. -* New features. Before submitting a feature PR, create an issue with a proposal to discuss it with the core team and other interested parties. -* Minor fixes such as typos. -* [Examples](examples). - ### Before pushing your changes We use [`ruff`](https://docs.astral.sh/ruff/) to format Python code and to sort Python imports. Before committing your changes, run: From c7945c54a642599ffcfd4eed429b503afe603b85 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 21 Sep 2026 11:47:56 +0500 Subject: [PATCH 597/640] Refactor dstack for dstack Factory extension (#4303) * Parametrize register_routes with the statics package Lets extending servers reuse the statics mount and 404 handler instead of copying them. * Rename the factory UI flavor to enterprise * Add register_plugin extension point Lets extending servers register plugins that survive load_plugins, which replaces the plugin list from the server config. * Read the active Alembic config on each migration invocation * Add Factory UI flavor and shared provider-aware login --- contributing/FRONTEND.md | 15 ++ frontend/jest.auth.config.cjs | 6 +- frontend/package.json | 2 + frontend/product.config.cjs | 27 +++ .../src/App/Login/LoginByGithub/index.tsx | 79 ++------- .../App/Login/LoginByGithubCallback/index.tsx | 3 +- .../src/App/Login/SelfHostedLogin/index.tsx | 65 -------- frontend/src/App/Login/TokenLogin/index.tsx | 44 +---- frontend/src/App/Login/index.tsx | 98 +++++++++++ frontend/src/App/auth.test.tsx | 26 ++- frontend/src/App/index.tsx | 5 +- frontend/src/App/login.test.tsx | 156 ++++++++++++++++++ frontend/src/PublicApp/index.tsx | 8 +- frontend/src/analytics.ts | 4 +- frontend/src/api.ts | 1 + .../layouts/AppLayout/TutorialPanel/hooks.ts | 3 +- frontend/src/layouts/AppLayout/hooks.ts | 7 +- frontend/src/layouts/AppLayout/index.tsx | 13 +- frontend/src/locale/en.json | 3 +- .../src/pages/Fleets/Details/Events/index.tsx | 9 +- .../pages/Instances/Details/Events/index.tsx | 9 +- .../Project/Backends/Table/constants.tsx | 5 +- .../pages/Project/Backends/Table/index.tsx | 3 +- .../Project/Backends/YAMLForm/constants.tsx | 4 +- .../pages/Project/Backends/YAMLForm/index.tsx | 3 +- .../pages/Project/Details/Settings/index.tsx | 22 ++- frontend/src/pages/Project/Details/index.tsx | 3 +- .../pages/Runs/Details/Events/List/index.tsx | 9 +- .../pages/Runs/Details/Jobs/Events/index.tsx | 3 +- frontend/src/pages/User/Details/index.tsx | 5 +- frontend/src/pages/User/List/hooks.tsx | 3 +- frontend/src/product.ts | 3 + frontend/src/router.tsx | 27 ++- frontend/src/services/auth.ts | 4 + frontend/src/types/global.d.ts | 2 +- frontend/webpack/env.js | 7 +- src/dstack/_internal/server/app.py | 22 ++- src/dstack/_internal/server/migrations/env.py | 12 +- .../_internal/server/services/plugins.py | 14 +- 39 files changed, 463 insertions(+), 271 deletions(-) create mode 100644 frontend/product.config.cjs delete mode 100644 frontend/src/App/Login/SelfHostedLogin/index.tsx create mode 100644 frontend/src/App/Login/index.tsx create mode 100644 frontend/src/App/login.test.tsx create mode 100644 frontend/src/product.ts diff --git a/contributing/FRONTEND.md b/contributing/FRONTEND.md index 19b623b1d4..afecdaa29d 100644 --- a/contributing/FRONTEND.md +++ b/contributing/FRONTEND.md @@ -54,3 +54,18 @@ The `webpack` dev server expects the API to be running on `http://127.0.0.1:8000 ```shell dstack server --port 8000 ``` + +## Product flavors + +The shared frontend supports OSS, Enterprise, Factory, and Sky. From `frontend/`, use +`npm run build`, `npm run build-enterprise`, `npm run build-factory`, or +`npm run build-sky`; the corresponding development commands are `start`, +`start-enterprise`, `start-factory`, and `start-sky`. + +`frontend/product.config.cjs` defines product names and inherited UI features for both +webpack and the application. Components use `product` from `src/product.ts` instead of +checking `UI_VERSION` directly. Factory and Sky share billing and presets; hosted-service +content such as Sky terms and onboarding uses `product.isSky`. + +All flavors use the same login component. It displays supported providers enabled by +`/api/auth/list_providers`, independently of the UI flavor, and always offers token login. diff --git a/frontend/jest.auth.config.cjs b/frontend/jest.auth.config.cjs index fc57723338..c8616f64a9 100644 --- a/frontend/jest.auth.config.cjs +++ b/frontend/jest.auth.config.cjs @@ -3,7 +3,11 @@ module.exports = { clearMocks: true, testEnvironment: 'node', moduleDirectories: ['node_modules', 'src'], - testMatch: ['/src/App/auth.test.tsx', '/src/services/preset.test.tsx'], + testMatch: [ + '/src/App/auth.test.tsx', + '/src/App/login.test.tsx', + '/src/services/preset.test.tsx', + ], transform: { '\\.[jt]sx?$': 'babel-jest', }, diff --git a/frontend/package.json b/frontend/package.json index de869ace8e..9b55459c1e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,9 +10,11 @@ "start": "cross-env NODE_ENV=development webpack serve --config webpack.config.js", "start-sky": "cross-env NODE_ENV=development UI_VERSION=sky webpack serve --config webpack.config.js", "start-factory": "cross-env NODE_ENV=development UI_VERSION=factory webpack serve --config webpack.config.js", + "start-enterprise": "cross-env NODE_ENV=development UI_VERSION=enterprise webpack serve --config webpack.config.js", "build": "cross-env NODE_ENV=production webpack build --config webpack.config.js", "build-sky": "cross-env NODE_ENV=production UI_VERSION=sky webpack build --config webpack.config.js", "build-factory": "cross-env NODE_ENV=production UI_VERSION=factory webpack build --config webpack.config.js", + "build-enterprise": "cross-env NODE_ENV=production UI_VERSION=enterprise webpack build --config webpack.config.js", "eslint": "eslint ./src --ext .js,.jsx,.ts,.tsx", "eslint-fix": "eslint ./src --ext .js,.jsx,.ts,.tsx --fix", "test": "jest", diff --git a/frontend/product.config.cjs b/frontend/product.config.cjs new file mode 100644 index 0000000000..da846b9f15 --- /dev/null +++ b/frontend/product.config.cjs @@ -0,0 +1,27 @@ +/* global module */ +const oss = { + id: 'oss', + name: 'dstack', + hasEvents: false, + hasBilling: false, + hasPresets: false, + isSky: false, +}; +const enterprise = { ...oss, id: 'enterprise', name: 'dstack Enterprise', hasEvents: true }; +const factory = { ...enterprise, id: 'factory', name: 'dstack Factory', hasBilling: true, hasPresets: true }; +const sky = { ...factory, id: 'sky', name: 'dstack Sky', isSky: true }; + +function getProductConfig(version) { + switch (version) { + case 'enterprise': + return enterprise; + case 'factory': + return factory; + case 'sky': + return sky; + default: + return oss; + } +} + +module.exports = { getProductConfig }; diff --git a/frontend/src/App/Login/LoginByGithub/index.tsx b/frontend/src/App/Login/LoginByGithub/index.tsx index 6748af523b..84f23be85c 100644 --- a/frontend/src/App/Login/LoginByGithub/index.tsx +++ b/frontend/src/App/Login/LoginByGithub/index.tsx @@ -1,17 +1,9 @@ import React from 'react'; -import { Navigate } from 'react-router-dom'; -import { colorBackgroundHomeHeader } from '@cloudscape-design/design-tokens'; -import { Alert, Box, Button, Container, ContentLayout, Header, Link, NavigateLink, SpaceBetween } from 'components'; +import { Alert, Button } from 'components'; -import { useAppSelector } from 'hooks'; import { goToUrl } from 'libs'; -import { ROUTES } from 'routes'; import { useGithubAuthorizeMutation } from 'services/auth'; -import { useGetUserDataQuery } from 'services/user'; - -import { Loading } from 'App/Loading'; -import { selectAuthToken } from 'App/slice'; const GitHubIcon: React.FC = () => ( @@ -25,71 +17,26 @@ const GitHubIcon: React.FC = () => ( ); export const LoginByGithub: React.FC = () => { - const token = useAppSelector(selectAuthToken); - const localStorageIsAvailable = 'localStorage' in window; - const { - currentData: userData, - error, - isFetching, - } = useGetUserDataQuery({ token }, { skip: !token || !localStorageIsAvailable }); const [githubAuthorize, { isLoading, isError }] = useGithubAuthorizeMutation(); - const signInClick = () => { githubAuthorize() .unwrap() .then((data) => goToUrl(data.authorization_url)) .catch(() => undefined); }; - - if (token && localStorageIsAvailable && isFetching && !userData) return ; - if (token && localStorageIsAvailable && userData?.username && !error) { - return ; - } - return ( - - Welcome to dstack Sky - - } - > - -
Sign in
- - } + <> + {isError && Unable to start GitHub authentication} + - - By continuing with GitHub, you agree to the{' '} - - Terms - {' '} - and{' '} - - Privacy policy - - - Sign in with a token - - - + Continue with GitHub + + ); }; diff --git a/frontend/src/App/Login/LoginByGithubCallback/index.tsx b/frontend/src/App/Login/LoginByGithubCallback/index.tsx index cc97cb88fa..07eac4ee51 100644 --- a/frontend/src/App/Login/LoginByGithubCallback/index.tsx +++ b/frontend/src/App/Login/LoginByGithubCallback/index.tsx @@ -1,6 +1,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useNavigate, useSearchParams } from 'react-router-dom'; +import { product } from 'product'; import { NavigateLink } from 'components'; import { UnauthorizedLayout } from 'layouts/UnauthorizedLayout'; @@ -41,7 +42,7 @@ export const LoginByGithubCallback: React.FC = () => { .unwrap() .then(async ({ creds: { token } }) => { dispatch(setAuthData({ token })); - if (process.env.UI_VERSION === 'sky') { + if (product.hasPresets) { const result = await getProjects({}).unwrap(); if (result.data.length === 0) { navigate(ROUTES.PROJECT.ADD); diff --git a/frontend/src/App/Login/SelfHostedLogin/index.tsx b/frontend/src/App/Login/SelfHostedLogin/index.tsx deleted file mode 100644 index 406e591ec0..0000000000 --- a/frontend/src/App/Login/SelfHostedLogin/index.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import React from 'react'; -import { useTranslation } from 'react-i18next'; -import { PublicApp } from 'PublicApp'; -import { colorBackgroundHomeHeader } from '@cloudscape-design/design-tokens'; - -import { Box, Container, ContentLayout, Header, NavigateLink, SpaceBetween, Spinner } from 'components'; - -import { ROUTES } from 'routes'; -import { useGetEntraInfoQuery, useGetGoogleInfoQuery, useGetOktaInfoQuery } from 'services/auth'; - -import { LoginByEntraID } from '../EntraID/LoginByEntraID'; -import { LoginByGoogle } from '../LoginByGoogle'; -import { LoginByOkta } from '../LoginByOkta'; -import { LoginByTokenForm } from '../LoginByTokenForm'; - -export const SelfHostedLogin: React.FC<{ tokenOnly?: boolean }> = ({ tokenOnly = false }) => { - const { t } = useTranslation(); - const { data: oktaData, isLoading: isLoadingOkta } = useGetOktaInfoQuery(); - const { data: entraData, isLoading: isLoadingEntra } = useGetEntraInfoQuery(); - const { data: googleData, isLoading: isLoadingGoogle } = useGetGoogleInfoQuery(); - - const oktaEnabled = oktaData?.enabled; - const entraEnabled = entraData?.enabled; - const googleEnabled = googleData?.enabled; - const isLoading = isLoadingOkta || isLoadingEntra || isLoadingGoogle; - const hasSSO = oktaEnabled || entraEnabled || googleEnabled; - const showTokenForm = tokenOnly || (!isLoading && !hasSSO); - - return ( - - - {t('auth.sign_in_to_dstack')} - - } - > - -
{showTokenForm ? 'Sign in with a token' : t('common.login')}
- - } - > - - {showTokenForm && } - {!tokenOnly && isLoading && } - {!tokenOnly && !isLoading && oktaEnabled && } - {!tokenOnly && !isLoading && entraEnabled && } - {!tokenOnly && !isLoading && googleEnabled && } - {!isLoading && hasSSO && ( - - {tokenOnly ? t('auth.another_login_methods') : 'Sign in with a token'} - - )} - - - - - ); -}; diff --git a/frontend/src/App/Login/TokenLogin/index.tsx b/frontend/src/App/Login/TokenLogin/index.tsx index fbf45622af..8c033dd469 100644 --- a/frontend/src/App/Login/TokenLogin/index.tsx +++ b/frontend/src/App/Login/TokenLogin/index.tsx @@ -1,45 +1,5 @@ import React from 'react'; -import { useTranslation } from 'react-i18next'; -import { colorBackgroundHomeHeader } from '@cloudscape-design/design-tokens'; -import { Box, Container, ContentLayout, Header, NavigateLink, SpaceBetween } from 'components'; +import { Login } from '../index'; -import { ROUTES } from 'routes'; - -import { LoginByTokenForm } from '../LoginByTokenForm'; -import { SelfHostedLogin } from '../SelfHostedLogin'; - -export const TokenLogin: React.FC = () => { - const { t } = useTranslation(); - - if (process.env.UI_VERSION === 'sky') { - return ( - - {t('auth.sign_in_to_dstack_sky')} - - } - > - -
Sign in with a token
- - } - > - - - {t('auth.another_login_methods')} - - - - ); - } - - return ; -}; +export const TokenLogin: React.FC = () => ; diff --git a/frontend/src/App/Login/index.tsx b/frontend/src/App/Login/index.tsx new file mode 100644 index 0000000000..98b242070d --- /dev/null +++ b/frontend/src/App/Login/index.tsx @@ -0,0 +1,98 @@ +import React from 'react'; +import { useTranslation } from 'react-i18next'; +import { Navigate } from 'react-router-dom'; +import { product } from 'product'; +import { PublicApp } from 'PublicApp'; +import { colorBackgroundHomeHeader } from '@cloudscape-design/design-tokens'; + +import { Box, Container, ContentLayout, Header, Link, NavigateLink, SpaceBetween, Spinner } from 'components'; + +import { useAppSelector } from 'hooks'; +import { ROUTES } from 'routes'; +import { useGetAuthProvidersQuery } from 'services/auth'; +import { useGetUserDataQuery } from 'services/user'; + +import { Loading } from 'App/Loading'; +import { selectAuthToken } from 'App/slice'; + +import { LoginByEntraID } from './EntraID/LoginByEntraID'; +import { LoginByGithub } from './LoginByGithub'; +import { LoginByGoogle } from './LoginByGoogle'; +import { LoginByOkta } from './LoginByOkta'; +import { LoginByTokenForm } from './LoginByTokenForm'; + +const providerButtons = { + github: LoginByGithub, + okta: LoginByOkta, + entra: LoginByEntraID, + google: LoginByGoogle, +}; + +export const Login: React.FC<{ tokenOnly?: boolean }> = ({ tokenOnly = false }) => { + const { t } = useTranslation(); + const token = useAppSelector(selectAuthToken); + const localStorageIsAvailable = 'localStorage' in window; + const { + currentData: userData, + error, + isFetching, + } = useGetUserDataQuery({ token }, { skip: tokenOnly || !token || !localStorageIsAvailable }); + const { data: providers, isLoading } = useGetAuthProvidersQuery(undefined, { skip: tokenOnly }); + const enabledProviders = Object.entries(providerButtons).filter(([name]) => + providers?.some((provider) => provider.name === name && provider.enabled), + ); + const showTokenForm = tokenOnly || (!isLoading && enabledProviders.length === 0); + + if (!tokenOnly && token && localStorageIsAvailable) { + if (isFetching && !userData) return ; + if (userData?.username && !error) return ; + } + + const content = ( + + {t('auth.welcome', { product: product.name })} + + } + > + +
{showTokenForm ? 'Sign in with a token' : t('common.login')}
+ + } + > + + {showTokenForm && } + {!tokenOnly && isLoading && } + {!tokenOnly && + !isLoading && + enabledProviders.map(([name, ProviderButton]) => )} + {!tokenOnly && !isLoading && enabledProviders.length > 0 && product.isSky && ( + + By continuing, you agree to the{' '} + + Terms + {' '} + and{' '} + + Privacy policy + + + )} + {tokenOnly ? ( + {t('auth.another_login_methods')} + ) : ( + !showTokenForm && Sign in with a token + )} + + + + ); + return product.hasPresets ? content : {content}; +}; diff --git a/frontend/src/App/auth.test.tsx b/frontend/src/App/auth.test.tsx index ebc49993cf..67f195e06e 100644 --- a/frontend/src/App/auth.test.tsx +++ b/frontend/src/App/auth.test.tsx @@ -6,7 +6,7 @@ import { act, create, ReactTestRenderer } from 'react-test-renderer'; import { ROUTES } from 'routes'; import { useGetUserDataQuery } from 'services/user'; -import { LoginByGithub } from 'App/Login/LoginByGithub'; +import { Login } from 'App/Login'; const mockDispatch = jest.fn(); const mockPrivateQuery = jest.fn(); @@ -26,6 +26,7 @@ jest.mock('hooks', () => ({ jest.mock('libs', () => ({ goToUrl: jest.fn() })); jest.mock('services/auth', () => ({ + useGetAuthProvidersQuery: () => ({ data: [{ name: 'github', enabled: true }], isLoading: false }), useGithubAuthorizeMutation: () => [jest.fn(), { isLoading: false }], })); @@ -38,7 +39,11 @@ jest.mock('./slice', () => ({ setUserData: (payload: unknown) => ({ type: 'app/setUserData', payload }), })); -jest.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) })); +jest.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, options?: { product: string }) => (options ? `Welcome to ${options.product}` : key), + }), +})); jest.mock('layouts/AppLayout', () => ({ __esModule: true, @@ -61,13 +66,24 @@ jest.mock('components', () => { Container: Wrapper, Header: Wrapper, NavigateLink: Wrapper, - ContentLayout: ({ header }: { header: React.ReactNode }) => <>{header}, + ContentLayout: ({ header, children }: { header: React.ReactNode; children: React.ReactNode }) => ( + <> + {header} + {children} + + ), + Spinner: Wrapper, Link: Wrapper, SpaceBetween: Wrapper, }; }); -jest.mock('./Login/SelfHostedLogin', () => ({ SelfHostedLogin: () =>

Server login

})); +jest.mock('product', () => ({ product: { ...jest.requireActual('../../product.config.cjs').getProductConfig('sky') } })); +jest.mock('PublicApp', () => ({ PublicApp: ({ children }: { children: React.ReactNode }) => <>{children} })); +jest.mock('./Login/EntraID/LoginByEntraID', () => ({ LoginByEntraID: () => })); +jest.mock('./Login/LoginByGoogle', () => ({ LoginByGoogle: () => })); +jest.mock('./Login/LoginByOkta', () => ({ LoginByOkta: () => })); +jest.mock('./Login/LoginByTokenForm', () => ({ LoginByTokenForm: () =>
Token
})); jest.mock('./Loading', () => ({ Loading: () =>

Loading

})); jest.mock('./AuthErrorMessage', () => ({ AuthErrorMessage: () =>

Storage unavailable

})); @@ -82,7 +98,7 @@ const renderApp = (path: string) => { rendered = create( - } /> + } /> }> Runs} /> diff --git a/frontend/src/App/index.tsx b/frontend/src/App/index.tsx index e6b6de8f80..78afd5bc15 100644 --- a/frontend/src/App/index.tsx +++ b/frontend/src/App/index.tsx @@ -1,16 +1,17 @@ import React, { useEffect } from 'react'; import { useTranslation } from 'react-i18next'; import { Navigate, Outlet, useLocation } from 'react-router-dom'; +import { product } from 'product'; import AppLayout from 'layouts/AppLayout'; import { useAppDispatch, useAppSelector } from 'hooks'; import { useGetUserDataQuery } from 'services/user'; -import { SelfHostedLogin } from './Login/SelfHostedLogin'; import { ROUTES } from '../routes'; import { AuthErrorMessage } from './AuthErrorMessage'; import { Loading } from './Loading'; +import { Login } from './Login'; import { selectAuthToken, setUserData } from './slice'; const IGNORED_AUTH_PATHS = [ @@ -21,7 +22,7 @@ const IGNORED_AUTH_PATHS = [ ROUTES.AUTH.TOKEN, ]; -const LoginFormComponent = process.env.UI_VERSION === 'sky' ? () => : SelfHostedLogin; +const LoginFormComponent = product.hasPresets ? () => : Login; const App: React.FC = () => { const { t } = useTranslation(); diff --git a/frontend/src/App/login.test.tsx b/frontend/src/App/login.test.tsx new file mode 100644 index 0000000000..39e127d7f2 --- /dev/null +++ b/frontend/src/App/login.test.tsx @@ -0,0 +1,156 @@ +/** @jest-environment node */ +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; +import { act, create, ReactTestRenderer } from 'react-test-renderer'; +import { product } from 'product'; + +import { ROUTES } from 'routes'; +import { useGetAuthProvidersQuery } from 'services/auth'; + +import { getProductConfig } from '../../product.config.cjs'; +import { Login } from './Login'; + +type ProvidersResult = { + data?: { name: string; enabled: boolean }[]; + isLoading: boolean; + isError?: boolean; +}; +let mockProviders: ProvidersResult; +let rendered: ReactTestRenderer | undefined; +const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window'); + +jest.mock('product', () => ({ product: { ...jest.requireActual('../../product.config.cjs').getProductConfig('factory') } })); +jest.mock('hooks', () => ({ useAppSelector: () => undefined })); +jest.mock('services/auth', () => ({ useGetAuthProvidersQuery: jest.fn(() => mockProviders) })); +jest.mock('services/user', () => ({ useGetUserDataQuery: () => ({ isFetching: false }) })); +jest.mock('./slice', () => ({ selectAuthToken: jest.fn() })); +jest.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, options?: { product: string }) => (options ? `Welcome to ${options.product}` : key), + }), +})); +jest.mock('PublicApp', () => ({ PublicApp: ({ children }: { children: React.ReactNode }) => <>{children} })); +jest.mock('@cloudscape-design/design-tokens', () => ({ colorBackgroundHomeHeader: 'transparent' })); +jest.mock('components', () => { + const Wrapper = ({ children }: { children: React.ReactNode }) => <>{children}; + const Anchor = ({ href, children }: { href: string; children: React.ReactNode }) => {children}; + return { + Box: ({ variant, children }: { variant?: string; children: React.ReactNode }) => + variant === 'h1' ?

{children}

: <>{children}, + Container: Wrapper, + Header: Wrapper, + ContentLayout: ({ header, children }: { header: React.ReactNode; children: React.ReactNode }) => ( + <> + {header} + {children} + + ), + Link: Anchor, + NavigateLink: Anchor, + SpaceBetween: Wrapper, + Spinner: () =>

Loading providers

, + }; +}); +jest.mock('./Login/LoginByGithub', () => ({ LoginByGithub: () => })); +jest.mock('./Login/EntraID/LoginByEntraID', () => ({ LoginByEntraID: () => })); +jest.mock('./Login/LoginByGoogle', () => ({ LoginByGoogle: () => })); +jest.mock('./Login/LoginByOkta', () => ({ LoginByOkta: () => })); +jest.mock('./Login/LoginByTokenForm', () => ({ LoginByTokenForm: () =>
Token
})); +jest.mock('./Loading', () => ({ Loading: () =>

Loading user

})); + +const renderLogin = (tokenOnly = false) => { + act(() => { + rendered = create( + + + , + ); + }); + return rendered!.root; +}; + +beforeEach(() => { + Object.defineProperty(globalThis, 'window', { configurable: true, value: { localStorage: {} } }); + Object.assign(product, getProductConfig('factory')); + mockProviders = { data: [], isLoading: false }; +}); +afterEach(() => { + act(() => rendered?.unmount()); + rendered = undefined; +}); +afterAll(() => { + if (originalWindow) Object.defineProperty(globalThis, 'window', originalWindow); + else Reflect.deleteProperty(globalThis, 'window'); +}); + +test.each([ + ['oss', 'dstack'], + ['enterprise', 'dstack Enterprise'], + ['factory', 'dstack Factory'], + ['sky', 'dstack Sky'], +])('%s login uses its branding and the server provider list', (version, name) => { + Object.assign(product, getProductConfig(version)); + mockProviders.data = [ + { name: 'google', enabled: true }, + { name: 'github', enabled: false }, + ]; + const view = renderLogin(); + expect(view.findByType('h1').children).toEqual([`Welcome to ${name}`]); + expect(view.findAllByType('button').map((button) => button.children)).toEqual([['Google']]); + expect(view.findByProps({ href: ROUTES.AUTH.TOKEN })).toBeDefined(); +}); + +test('Factory supports all configured providers without Sky terms', () => { + mockProviders.data = ['github', 'okta', 'entra', 'google'].map((name) => ({ name, enabled: true })); + const view = renderLogin(); + expect(view.findAllByType('button').map((button) => button.children)).toEqual([ + ['GitHub'], + ['Okta'], + ['Entra'], + ['Google'], + ]); + expect(view.findAllByProps({ href: 'https://dstack.ai/terms/' })).toHaveLength(0); +}); + +test('Sky keeps its terms with GitHub login', () => { + Object.assign(product, getProductConfig('sky')); + mockProviders.data = [{ name: 'github', enabled: true }]; + const view = renderLogin(); + expect(view.findByType('button').children).toEqual(['GitHub']); + expect(view.findByProps({ href: 'https://dstack.ai/terms/' })).toBeDefined(); +}); + +test.each([ + { data: [], isLoading: false }, + { data: [{ name: 'github', enabled: false }], isLoading: false }, + { data: [{ name: 'unknown', enabled: true }], isLoading: false }, + { isError: true, isLoading: false }, +])('falls back to token login when no supported provider is available: %j', (result) => { + mockProviders = result; + const view = renderLogin(); + expect(view.findByType('form').children).toEqual(['Token']); + expect(view.findAllByType('button')).toHaveLength(0); +}); + +test('token login remains accessible while provider discovery loads', () => { + mockProviders = { isLoading: true }; + const view = renderLogin(); + expect(view.findByProps({ role: 'status' }).children).toEqual(['Loading providers']); + expect(view.findByProps({ href: ROUTES.AUTH.TOKEN })).toBeDefined(); +}); + +test('the token page does not wait for provider discovery', () => { + mockProviders = { isLoading: true }; + const view = renderLogin(true); + expect(view.findByType('form').children).toEqual(['Token']); + expect(useGetAuthProvidersQuery).toHaveBeenCalledWith(undefined, { skip: true }); + expect(view.findByProps({ href: ROUTES.BASE })).toBeDefined(); +}); + +test('Factory inherits Enterprise features and Sky inherits Factory features', () => { + expect(getProductConfig('enterprise')).toMatchObject({ hasEvents: true, hasBilling: false, hasPresets: false }); + for (const version of ['factory', 'sky']) { + expect(getProductConfig(version)).toMatchObject({ hasEvents: true, hasBilling: true, hasPresets: true }); + } + expect(getProductConfig(undefined)).toMatchObject({ id: 'oss', hasEvents: false, hasBilling: false, hasPresets: false }); +}); diff --git a/frontend/src/PublicApp/index.tsx b/frontend/src/PublicApp/index.tsx index 17fd98ba24..33f29c95c7 100644 --- a/frontend/src/PublicApp/index.tsx +++ b/frontend/src/PublicApp/index.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { createPortal } from 'react-dom'; import { useTranslation } from 'react-i18next'; import { Outlet, useLocation, useNavigate } from 'react-router-dom'; +import { product } from 'product'; import enMessages from '@cloudscape-design/components/i18n/messages/all.en.json'; import { applyMode, Mode } from '@cloudscape-design/global-styles'; @@ -46,7 +47,6 @@ const HeaderPortal = ({ children }: PortalProps) => { }; export const PublicApp: React.FC = ({ children }) => { - const isSky = process.env.UI_VERSION === 'sky'; const { t } = useTranslation(); const dispatch = useAppDispatch(); const navigate = useNavigate(); @@ -115,7 +115,7 @@ export const PublicApp: React.FC = ({ children }) => { }, ]} /> - {isSky && !isAuth && ( + {product.hasPresets && !isAuth && (
- ) - } - > +
{t('common.full_view')}}> {t('navigation.events')}
} diff --git a/frontend/src/pages/Instances/Details/Events/index.tsx b/frontend/src/pages/Instances/Details/Events/index.tsx index 52294acbc9..dc328954cf 100644 --- a/frontend/src/pages/Instances/Details/Events/index.tsx +++ b/frontend/src/pages/Instances/Details/Events/index.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { useTranslation } from 'react-i18next'; import { useNavigate, useParams } from 'react-router-dom'; +import { product } from 'product'; import Button from '@cloudscape-design/components/button'; import { Header, Loader, Table } from 'components'; @@ -46,13 +47,7 @@ export const EventsList = () => { loading={isLoading} loadingText={t('common.loading')} header={ -
{t('common.full_view')} - ) - } - > +
{t('common.full_view')}}> {t('navigation.events')}
} diff --git a/frontend/src/pages/Project/Backends/Table/constants.tsx b/frontend/src/pages/Project/Backends/Table/constants.tsx index 67e22a0f79..1a118dc6f8 100644 --- a/frontend/src/pages/Project/Backends/Table/constants.tsx +++ b/frontend/src/pages/Project/Backends/Table/constants.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { product } from 'product'; export const BACKENDS_HELP_SKY = { header:

Backends

, @@ -9,8 +10,8 @@ export const BACKENDS_HELP_SKY = {

Marketplace

- By default, dstack Sky includes a preset of backends that let you access compute from the{' '} - dstack marketplace and pay through your dstack Sky user billing. + By default, {product.name} includes a preset of backends that let you access compute from the{' '} + dstack marketplace and pay through your {product.name} user billing.

Your own cloud accounts

diff --git a/frontend/src/pages/Project/Backends/Table/index.tsx b/frontend/src/pages/Project/Backends/Table/index.tsx index fcac9b446a..850b5a6cec 100644 --- a/frontend/src/pages/Project/Backends/Table/index.tsx +++ b/frontend/src/pages/Project/Backends/Table/index.tsx @@ -1,5 +1,6 @@ import React from 'react'; import { useTranslation } from 'react-i18next'; +import { product } from 'product'; import { Button, ButtonWithConfirmation, Header, InfoLink, ListEmptyMessage, SpaceBetween, Table } from 'components'; @@ -9,7 +10,7 @@ import { BACKENDS_HELP_SELF_HOSTED, BACKENDS_HELP_SKY } from './constants'; import { useColumnsDefinitions } from './hooks'; import { IProps } from './types'; -const INFO = process.env.UI_VERSION === 'sky' ? BACKENDS_HELP_SKY : BACKENDS_HELP_SELF_HOSTED; +const INFO = product.hasBilling ? BACKENDS_HELP_SKY : BACKENDS_HELP_SELF_HOSTED; export const BackendsTable: React.FC = ({ backends, diff --git a/frontend/src/pages/Project/Backends/YAMLForm/constants.tsx b/frontend/src/pages/Project/Backends/YAMLForm/constants.tsx index f8686b246e..2ec1377c29 100644 --- a/frontend/src/pages/Project/Backends/YAMLForm/constants.tsx +++ b/frontend/src/pages/Project/Backends/YAMLForm/constants.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { product } from 'product'; export const CONFIG_YAML_HELP_SKY = { header:

Backend config

, @@ -11,7 +12,8 @@ export const CONFIG_YAML_HELP_SKY = {

Marketplace

If you set creds's type to dstack, you'll get compute from{' '} - dstack's marketplace and will pay for it via your dstack Sky user billing. Example: + dstack's marketplace and will pay for it via your {product.name} user billing. + Example: 

diff --git a/frontend/src/pages/Project/Backends/YAMLForm/index.tsx b/frontend/src/pages/Project/Backends/YAMLForm/index.tsx
index f14e8b1f3a..8f5b1c3e39 100644
--- a/frontend/src/pages/Project/Backends/YAMLForm/index.tsx
+++ b/frontend/src/pages/Project/Backends/YAMLForm/index.tsx
@@ -1,6 +1,7 @@
 import React, { useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
+import { product } from 'product';
 
 import { Button, FormCodeEditor, FormUI, InfoLink, SpaceBetween } from 'components';
 
@@ -11,7 +12,7 @@ import { CONFIG_YAML_HELP_SELF_HOSTED, CONFIG_YAML_HELP_SKY } from './constants'
 
 import { FieldPath } from 'react-hook-form/dist/types/path';
 
-const INFO = process.env.UI_VERSION === 'sky' ? CONFIG_YAML_HELP_SKY : CONFIG_YAML_HELP_SELF_HOSTED;
+const INFO = product.hasBilling ? CONFIG_YAML_HELP_SKY : CONFIG_YAML_HELP_SELF_HOSTED;
 
 export interface IProps {
     initialValues?: IBackendConfigYaml;
diff --git a/frontend/src/pages/Project/Details/Settings/index.tsx b/frontend/src/pages/Project/Details/Settings/index.tsx
index db3a6fe339..a2f0b5f4e8 100644
--- a/frontend/src/pages/Project/Details/Settings/index.tsx
+++ b/frontend/src/pages/Project/Details/Settings/index.tsx
@@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next';
 import { useDispatch } from 'react-redux';
 import { useLocation, useNavigate, useParams } from 'react-router-dom';
 import { debounce } from 'lodash';
+import { product } from 'product';
 import { ExpandableSection, Tabs } from '@cloudscape-design/components';
 import { FetchBaseQueryError } from '@reduxjs/toolkit/query';
 
@@ -115,7 +116,6 @@ export const ProjectSettings: React.FC = () => {
         value: data?.owner.username,
     };
 
-    const isSky = process.env.UI_VERSION === 'sky';
     const visibilityOptions = [
         { label: t('projects.edit.visibility.private'), value: 'private' },
         { label: t('projects.edit.visibility.public'), value: 'public' },
@@ -213,7 +213,7 @@ export const ProjectSettings: React.FC = () => {
     };
 
     const openChangeVisibilityDialog = () => {
-        setVisibilityEnabled(isSky ? (data?.public_presets ?? false) : !!data?.isPublic);
+        setVisibilityEnabled(product.hasPresets ? (data?.public_presets ?? false) : !!data?.isPublic);
         setIsChangeVisibilityVisible(true);
     };
 
@@ -221,7 +221,7 @@ export const ProjectSettings: React.FC = () => {
         if (!data || !isProjectAdmin(data) || isUpdatingVisibility) return;
 
         try {
-            if (isSky) {
+            if (product.hasPresets) {
                 await updateProjectPublicPresets({
                     project_name: paramProjectName,
                     public_presets: visibilityEnabled,
@@ -233,7 +233,9 @@ export const ProjectSettings: React.FC = () => {
             setIsChangeVisibilityVisible(false);
             pushNotification({
                 type: 'success',
-                content: t(isSky ? 'projects.edit.update_presets_success' : 'projects.edit.update_visibility_success'),
+                content: t(
+                    product.hasPresets ? 'projects.edit.update_presets_success' : 'projects.edit.update_visibility_success',
+                ),
             });
         } catch (error: unknown) {
             pushNotification({ type: 'error', content: getApiErrorMessage(error) });
@@ -511,18 +513,20 @@ export const ProjectSettings: React.FC = () => {
                                         
                                     )}
 
-                                    {(isSky || isAvailableProjectManaging) && (
+                                    {(product.hasPresets || isAvailableProjectManaging) && (
                                         <>
                                             

                                                 
                                                     {t(
-                                                        isSky
+                                                        product.hasPresets
                                                             ? 'projects.edit.presets_settings'
                                                             : 'projects.edit.project_visibility_settings',
                                                     )}
                                                 
                                                  openHelpPanel(isSky ? PRESETS_INFO : VISIBILITY_INFO)}
+                                                    onFollow={() =>
+                                                        openHelpPanel(product.hasPresets ? PRESETS_INFO : VISIBILITY_INFO)
+                                                    }
                                                 />
                                             

 
@@ -607,12 +611,12 @@ export const ProjectSettings: React.FC = () => {
                 visible={isChangeVisibilityVisible}
                 onDiscard={() => setIsChangeVisibilityVisible(false)}
                 onConfirm={confirmChangeVisibility}
-                title={t(isSky ? 'projects.edit.presets_settings' : 'projects.edit.project_visibility_settings')}
+                title={t(product.hasPresets ? 'projects.edit.presets_settings' : 'projects.edit.project_visibility_settings')}
                 confirmButtonLabel={t('projects.edit.change_visibility')}
                 content={
                      {
             id: 'settings',
             href: ROUTES.PROJECT.DETAILS.SETTINGS.FORMAT(paramProjectName),
         },
-        (process.env.UI_VERSION === 'factory' || process.env.UI_VERSION === 'sky') && {
+        product.hasEvents && {
             label: t('projects.events'),
             id: 'events',
             href: ROUTES.PROJECT.DETAILS.EVENTS.FORMAT(paramProjectName),
diff --git a/frontend/src/pages/Runs/Details/Events/List/index.tsx b/frontend/src/pages/Runs/Details/Events/List/index.tsx
index 178ad717c7..05c3015a10 100644
--- a/frontend/src/pages/Runs/Details/Events/List/index.tsx
+++ b/frontend/src/pages/Runs/Details/Events/List/index.tsx
@@ -2,6 +2,7 @@ import React from 'react';
 import { useListener } from 'react-bus';
 import { useTranslation } from 'react-i18next';
 import { useNavigate, useParams } from 'react-router-dom';
+import { product } from 'product';
 import Button from '@cloudscape-design/components/button';
 
 import { Header, Loader, Table } from 'components';
@@ -51,13 +52,7 @@ export const EventsList = () => {
             loading={isLoading}
             loadingText={t('common.loading')}
             header={
-                
{t('common.full_view')}
-                        )
-                    }
-                >
+                
{t('common.full_view')}}>
                     {t('navigation.events')}
                 

             }
diff --git a/frontend/src/pages/Runs/Details/Jobs/Events/index.tsx b/frontend/src/pages/Runs/Details/Jobs/Events/index.tsx
index 3693f067d5..3f79de57c5 100644
--- a/frontend/src/pages/Runs/Details/Jobs/Events/index.tsx
+++ b/frontend/src/pages/Runs/Details/Jobs/Events/index.tsx
@@ -1,6 +1,7 @@
 import React, { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useNavigate, useParams } from 'react-router-dom';
+import { product } from 'product';
 import Button from '@cloudscape-design/components/button';
 
 import { Header, Loader, Table } from 'components';
@@ -64,7 +65,7 @@ export const EventsList = () => {
             header={
                 

                                 {t('common.full_view')}
                             
diff --git a/frontend/src/pages/User/Details/index.tsx b/frontend/src/pages/User/Details/index.tsx
index 62cff381e4..aaaca99f5f 100644
--- a/frontend/src/pages/User/Details/index.tsx
+++ b/frontend/src/pages/User/Details/index.tsx
@@ -1,6 +1,7 @@
 import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Outlet, useNavigate, useParams } from 'react-router-dom';
+import { product } from 'product';
 
 import { Box, ConfirmationDialog, ContentLayout, Tabs } from 'components';
 import { DetailsHeader } from 'components';
@@ -72,12 +73,12 @@ export const UserDetails: React.FC = () => {
             id: UserDetailsTabTypeEnum.PROJECTS,
             href: ROUTES.USER.PROJECTS.FORMAT(paramUserName),
         },
-        (process.env.UI_VERSION === 'factory' || process.env.UI_VERSION === 'sky') && {
+        product.hasEvents && {
             label: t('users.events'),
             id: UserDetailsTabTypeEnum.EVENTS,
             href: ROUTES.USER.EVENTS.FORMAT(paramUserName),
         },
-        process.env.UI_VERSION === 'sky' && {
+        product.hasBilling && {
             label: t('billing.title'),
             id: UserDetailsTabTypeEnum.BILLING,
             href: ROUTES.USER.BILLING.LIST.FORMAT(paramUserName),
diff --git a/frontend/src/pages/User/List/hooks.tsx b/frontend/src/pages/User/List/hooks.tsx
index 7f9f24549c..25bb74b849 100644
--- a/frontend/src/pages/User/List/hooks.tsx
+++ b/frontend/src/pages/User/List/hooks.tsx
@@ -1,6 +1,7 @@
 import React, { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { format } from 'date-fns';
+import { product } from 'product';
 
 import { Link, NavigateLink } from 'components';
 
@@ -29,7 +30,7 @@ export const useColumnDefinitions = () => {
                 header: t('users.global_role'),
                 cell: (item: IUser) => t(`roles.${item.global_role}`),
             },
-            process.env.UI_VERSION === 'sky' && {
+            product.hasBilling && {
                 id: 'created_at',
                 header: t('users.created_at'),
                 cell: (item: IUser) => format(new Date(item.created_at), DATE_TIME_FORMAT),
diff --git a/frontend/src/product.ts b/frontend/src/product.ts
new file mode 100644
index 0000000000..562e5fd65a
--- /dev/null
+++ b/frontend/src/product.ts
@@ -0,0 +1,3 @@
+import { getProductConfig } from '../product.config.cjs';
+
+export const product = getProductConfig(process.env.UI_VERSION);
diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx
index 702bc6193d..0617436182 100644
--- a/frontend/src/router.tsx
+++ b/frontend/src/router.tsx
@@ -2,12 +2,13 @@ import React from 'react';
 import type { RouteObject } from 'react-router-dom';
 import { createBrowserRouter } from 'react-router-dom';
 import { Navigate } from 'react-router-dom';
+import { product } from 'product';
 import { PublicApp } from 'PublicApp';
 import { PresetApp } from 'PublicApp/PresetApp';
 
 import App from 'App';
+import { Login } from 'App/Login';
 import { LoginByEntraIDCallback } from 'App/Login/EntraID/LoginByEntraIDCallback';
-import { LoginByGithub } from 'App/Login/LoginByGithub';
 import { LoginByGithubCallback } from 'App/Login/LoginByGithubCallback';
 import { LoginByGoogleCallback } from 'App/Login/LoginByGoogleCallback';
 import { LoginByOktaCallback } from 'App/Login/LoginByOktaCallback';
@@ -42,13 +43,13 @@ import { VolumeList } from './pages/Volumes';
 import { ROUTES } from './routes';
 
 export const router = createBrowserRouter([
-    ...(process.env.UI_VERSION === 'sky'
+    ...(product.hasPresets
         ? [
               {
                   element: ,
                   errorElement: ,
                   children: [
-                      { path: ROUTES.BASE, element:  },
+                      { path: ROUTES.BASE, element:  },
                       { path: ROUTES.AUTH.TOKEN, element:  },
                   ],
               },
@@ -96,11 +97,9 @@ export const router = createBrowserRouter([
                 path: ROUTES.AUTH.GOOGLE_CALLBACK,
                 element: ,
             },
-            ...(process.env.UI_VERSION !== 'sky' ? [{ path: ROUTES.AUTH.TOKEN, element:  }] : []),
+            ...(!product.hasPresets ? [{ path: ROUTES.AUTH.TOKEN, element:  }] : []),
             // hubs
-            ...(process.env.UI_VERSION !== 'sky'
-                ? [{ path: ROUTES.BASE, element:  }]
-                : []),
+            ...(!product.hasPresets ? [{ path: ROUTES.BASE, element:  }] : []),
             {
                 path: ROUTES.PROJECT.LIST,
                 element: ,
@@ -113,7 +112,7 @@ export const router = createBrowserRouter([
                         index: true,
                         element: ,
                     },
-                    (process.env.UI_VERSION === 'factory' || process.env.UI_VERSION === 'sky') && {
+                    product.hasEvents && {
                         path: ROUTES.PROJECT.DETAILS.EVENTS.TEMPLATE,
                         element: ,
                     },
@@ -185,11 +184,11 @@ export const router = createBrowserRouter([
             },
 
             ...([
-                process.env.UI_VERSION !== 'sky' && {
+                !product.hasBilling && {
                     path: ROUTES.PROJECT.ADD,
                     element: ,
                 },
-                process.env.UI_VERSION === 'sky' && {
+                product.hasBilling && {
                     path: ROUTES.PROJECT.ADD,
                     element: ,
                 },
@@ -222,9 +221,9 @@ export const router = createBrowserRouter([
                 element: ,
             },
 
-            // Events, Factory and Sky only
+            // Events, Enterprise and above
             ...([
-                (process.env.UI_VERSION === 'factory' || process.env.UI_VERSION === 'sky') && {
+                product.hasEvents && {
                     path: ROUTES.EVENTS.LIST,
                     element: ,
                 },
@@ -309,7 +308,7 @@ export const router = createBrowserRouter([
                         path: ROUTES.USER.PROJECTS.TEMPLATE,
                         element: ,
                     },
-                    (process.env.UI_VERSION === 'factory' || process.env.UI_VERSION === 'sky') && {
+                    product.hasEvents && {
                         path: ROUTES.USER.EVENTS.TEMPLATE,
                         element: ,
                     },
@@ -318,7 +317,7 @@ export const router = createBrowserRouter([
                         path: ROUTES.USER.PUBLIC_KEYS.TEMPLATE,
                         element: ,
                     },
-                    process.env.UI_VERSION === 'sky' && {
+                    product.hasBilling && {
                         path: ROUTES.USER.BILLING.LIST.TEMPLATE,
                         element: ,
                     },
diff --git a/frontend/src/services/auth.ts b/frontend/src/services/auth.ts
index 2512ed0a7d..7a8e29b110 100644
--- a/frontend/src/services/auth.ts
+++ b/frontend/src/services/auth.ts
@@ -12,6 +12,9 @@ export const authApi = createApi({
     tagTypes: ['Auth'],
 
     endpoints: (builder) => ({
+        getAuthProviders: builder.query<{ name: string; enabled: boolean }[], void>({
+            query: () => ({ url: API.AUTH.LIST_PROVIDERS(), method: 'POST' }),
+        }),
         getNextRedirect: builder.mutation<{ redirect_url?: string }, { code: string; state: string }>({
             query: (body) => ({
                 url: API.AUTH.NEXT_REDIRECT(),
@@ -111,6 +114,7 @@ export const authApi = createApi({
 });
 
 export const {
+    useGetAuthProvidersQuery,
     useGetNextRedirectMutation,
     useGithubAuthorizeMutation,
     useGithubCallbackMutation,
diff --git a/frontend/src/types/global.d.ts b/frontend/src/types/global.d.ts
index 0709a1bd4d..a35aa9f9d4 100644
--- a/frontend/src/types/global.d.ts
+++ b/frontend/src/types/global.d.ts
@@ -30,7 +30,7 @@ declare namespace NodeJS {
     interface ProcessEnv {
         readonly NODE_ENV: 'development' | 'production' | 'test';
         readonly GA_MEASUREMENT_ID: string;
-        readonly UI_VERSION: 'sky' | 'factory' | 'oss';
+        readonly UI_VERSION: 'sky' | 'factory' | 'enterprise' | 'oss';
         readonly PUBLIC_URL: string;
         readonly API_URL: string;
     }
diff --git a/frontend/webpack/env.js b/frontend/webpack/env.js
index f87077c2cb..76703da6a4 100644
--- a/frontend/webpack/env.js
+++ b/frontend/webpack/env.js
@@ -1,4 +1,5 @@
 const { join } = require('path');
+const { getProductConfig } = require('../product.config.cjs');
 
 const apiURLs = '/api';
 
@@ -16,9 +17,9 @@ const publicDir = join(__dirname, '../public');
 const apiUrl = process.env.API_URL || apiURLs;
 const publicUrl = process.env.PUBLIC_URL || publicURLs;
 const gaMeasurementId = process.env.GA_MEASUREMENT_ID || '';
-const uiVersion = ['sky', 'factory'].includes(process.env.UI_VERSION) ? process.env.UI_VERSION : 'oss';
-
-const title = uiVersion === 'sky' ? 'dstack Sky' : 'dstack';
+const product = getProductConfig(process.env.UI_VERSION);
+const uiVersion = product.id;
+const title = product.name;
 const description =
     'Get GPUs at the best prices and availability from a wide range of providers. No cloud ' +
     'account of your own is required.\n';
diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py
index 904cb3285d..083435cb32 100644
--- a/src/dstack/_internal/server/app.py
+++ b/src/dstack/_internal/server/app.py
@@ -4,7 +4,6 @@
 import time
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import asynccontextmanager
-from pathlib import Path
 from typing import Annotated, Awaitable, Callable, List, Optional
 
 import sentry_sdk
@@ -245,7 +244,14 @@ def add_no_api_version_check_routes(paths: List[str]):
     _NO_API_VERSION_CHECK_ROUTES.extend(paths)
 
 
-def register_routes(app: FastAPI, ui: bool = True):
+def register_routes(app: FastAPI, statics_package: Optional[str] = "dstack._internal.server"):
+    """
+    Registers dstack server routes on `app`.
+
+    `statics_package` is the package whose `statics` directory holds the frontend build.
+    Passing `None` or a package without `statics` disables the UI and redirects `/`
+    to the API docs.
+    """
     app.include_router(server.router)
     app.include_router(users.router)
     app.include_router(auth.router)
@@ -338,10 +344,8 @@ async def profile_request(request: Request, call_next):
     async def healthcheck():
         return CustomJSONResponse(content={"status": "running"})
 
-    if ui and Path(__file__).parent.joinpath("statics").exists():
-        app.mount(
-            "/", CustomStaticFiles(packages=["dstack._internal.server"], html=True), name="statics"
-        )
+    if statics_package is not None and _statics_exist(statics_package):
+        app.mount("/", CustomStaticFiles(packages=[statics_package], html=True), name="statics")
 
         @app.exception_handler(404)
         async def custom_http_exception_handler(request, exc):
@@ -356,7 +360,7 @@ async def custom_http_exception_handler(request, exc):
                 )
             else:
                 return HTMLResponse(
-                    importlib.resources.files("dstack._internal.server")
+                    importlib.resources.files(statics_package)
                     .joinpath("statics/index.html")
                     .read_text()
                 )
@@ -381,6 +385,10 @@ def _check_client_version(
         )
 
 
+def _statics_exist(statics_package: str) -> bool:
+    return importlib.resources.files(statics_package).joinpath("statics").is_dir()
+
+
 def _is_proxy_request(request: Request) -> bool:
     if request.url.path.startswith("/proxy"):
         return True
diff --git a/src/dstack/_internal/server/migrations/env.py b/src/dstack/_internal/server/migrations/env.py
index 0ab4b713a8..3c7eacb9bf 100644
--- a/src/dstack/_internal/server/migrations/env.py
+++ b/src/dstack/_internal/server/migrations/env.py
@@ -9,11 +9,6 @@
 from dstack._internal.server.models import BaseModel, EnumAsString
 from dstack._internal.server.settings import init_server_data_dir
 
-config = context.config
-
-if config.config_file_name is not None and config.attributes.get("configure_logging", True):
-    fileConfig(config.config_file_name)
-
 target_metadata = BaseModel.metadata
 
 
@@ -57,7 +52,7 @@ def run_migrations_online():
     In this scenario we need to create an Engine
     and associate a connection with the context.
     """
-    connection = config.attributes.get("connection", None)
+    connection = context.config.attributes.get("connection", None)
     if connection is None:
         asyncio.run(run_async_migrations())
     else:
@@ -103,6 +98,11 @@ async def run_async_migrations():
 
 
 def main():
+    # Extending servers import this module once and reuse it across Alembic commands.
+    # Read the active configuration on each invocation rather than caching it at import.
+    config = context.config
+    if config.config_file_name is not None and config.attributes.get("configure_logging", True):
+        fileConfig(config.config_file_name)
     if context.is_offline_mode():
         run_migrations_offline()
     else:
diff --git a/src/dstack/_internal/server/services/plugins.py b/src/dstack/_internal/server/services/plugins.py
index d40b84b36d..9e3915923d 100644
--- a/src/dstack/_internal/server/services/plugins.py
+++ b/src/dstack/_internal/server/services/plugins.py
@@ -13,6 +13,10 @@
 
 _PLUGINS: list[Plugin] = []
 
+# Plugins registered by extending servers. Unlike `_PLUGINS`, they are not affected by
+# `load_plugins`, so they stay active regardless of the plugins enabled in the server config.
+_REGISTERED_PLUGINS: list[Plugin] = []
+
 _BUILTIN_PLUGINS: Dict[str, str] = {"rest_plugin": "dstack.plugins.builtin.rest_plugin:RESTPlugin"}
 
 
@@ -91,6 +95,14 @@ def load_plugins(enabled_plugins: list[str]):
         logger.warning("Enabled plugins not found: %s", plugins_to_load)
 
 
+def register_plugin(plugin: Plugin):
+    """
+    Extension point for alternative dstack versions.
+    Registers a plugin that is always loaded, independently of `load_plugins`.
+    """
+    _REGISTERED_PLUGINS.append(plugin)
+
+
 async def apply_plugin_policies(user: str, project: str, spec: ApplySpec) -> ApplySpec:
     policies = _get_apply_policies()
     for policy in policies:
@@ -105,4 +117,4 @@ async def apply_plugin_policies(user: str, project: str, spec: ApplySpec) -> App
 
 
 def _get_apply_policies() -> list[ApplyPolicy]:
-    return list(itertools.chain(*[p.get_apply_policies() for p in _PLUGINS]))
+    return list(itertools.chain(*[p.get_apply_policies() for p in _REGISTERED_PLUGINS + _PLUGINS]))

From 3c0d04871e2ff414d70b94e3a8ce3048522b861e Mon Sep 17 00:00:00 2001
From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com>
Date: Tue, 22 Sep 2026 22:12:30 +0200
Subject: [PATCH 598/640] [Website] Updated product pages (#4304)

* [Website] Update dstack Sky page

* [Website] Enable React Router transition flag

* [Website] Use thinner product selection outline

* [Website] Align product links and docs hover states

* [Website] Add product popup hover feedback

* [Docs] Open dstack Sky in a new tab

* [Website] Clarify dstack Sky hero description

* [Website] Swap dstack Sky hero and marketplace titles

* [Website] Refine dstack Sky marketplace heading

* [Website] Shorten dstack Sky hero description

* [Website] Align dstack Sky metadata and product copy

* [Website] Refine hero copy and Get started links

* [Website] Add dstack Factory landing page

* [Website] Trim contributor notes

---------

Co-authored-by: Andrey Cheptsov 
---
 contributing/DOCS.md                          |   6 +-
 mkdocs/assets/stylesheets/cloudscape-docs.css |  10 +-
 mkdocs/docs/concepts/presets.md               |   2 +-
 mkdocs/index.md                               |   5 +-
 mkdocs/overrides/header-2.html                |  18 +-
 mkdocs/overrides/sitemap.xml                  |  17 ++
 website/index.html                            |  14 +-
 website/products/factory/index.html           |  63 +++++++
 website/products/sky/index.html               |  20 +--
 website/src/App/index.tsx                     |  36 +++-
 website/src/components/ComputeSourcesTabs.tsx |  63 +++++++
 website/src/components/ContentTabs.tsx        |  71 ++++++++
 website/src/components/HeroSquircle.tsx       |  56 +++---
 website/src/components/SiteNavigation.tsx     |  74 +++++---
 website/src/components/highlightTerms.tsx     |   2 +-
 website/src/index.tsx                         |   2 +-
 website/src/pages/Factory/CapacityTabs.tsx    |  46 +++++
 website/src/pages/Factory/FactoryPage.css     |   4 +
 website/src/pages/Factory/FactoryPage.tsx     | 167 ++++++++++++++++++
 website/src/pages/Factory/InferenceTabs.tsx   |  31 ++++
 .../src/pages/Factory/OrchestrationTabs.tsx   |  48 +++++
 website/src/pages/Factory/TenantTabs.tsx      |  53 ++++++
 website/src/pages/Home/ExploreSection.tsx     |  77 +-------
 website/src/pages/Home/FaqSection.tsx         |  36 ++--
 website/src/pages/Home/GetStartedSection.tsx  |  18 +-
 website/src/pages/Home/HomePage.tsx           |   6 +-
 website/src/pages/Sky/MarketplaceTabs.tsx     |  92 ++++++++++
 website/src/pages/Sky/SkyPage.tsx             | 167 +++++-------------
 website/src/router.tsx                        |   2 +
 website/src/routes.ts                         |   3 +-
 website/src/styles.css                        |  48 +++--
 website/vite.config.ts                        |   3 +-
 32 files changed, 916 insertions(+), 344 deletions(-)
 create mode 100644 mkdocs/overrides/sitemap.xml
 create mode 100644 website/products/factory/index.html
 create mode 100644 website/src/components/ComputeSourcesTabs.tsx
 create mode 100644 website/src/components/ContentTabs.tsx
 create mode 100644 website/src/pages/Factory/CapacityTabs.tsx
 create mode 100644 website/src/pages/Factory/FactoryPage.css
 create mode 100644 website/src/pages/Factory/FactoryPage.tsx
 create mode 100644 website/src/pages/Factory/InferenceTabs.tsx
 create mode 100644 website/src/pages/Factory/OrchestrationTabs.tsx
 create mode 100644 website/src/pages/Factory/TenantTabs.tsx
 create mode 100644 website/src/pages/Sky/MarketplaceTabs.tsx

diff --git a/contributing/DOCS.md b/contributing/DOCS.md
index c47bb269ca..aa07d9673f 100644
--- a/contributing/DOCS.md
+++ b/contributing/DOCS.md
@@ -72,8 +72,8 @@ just website-dev            # Vite dev server on http://127.0.0.1:5173
 Docs/blog links on the landing resolve same-origin (`/docs`, `/blog`), which 404 in standalone
 dev. Point them at a live site while iterating: `just website-dev https://dstack.ai`.
 
-The Sky product page is served at `/products/sky/`. Google Analytics and the social/OG image
-reuse the same property and MkDocs-generated card as the rest of the site.
+Keep page metadata in the HTML entries and `website/src/App/index.tsx` in sync.
+When adding or moving product pages, update the [sitemap template](../mkdocs/overrides/sitemap.xml).
 
 ## Building the whole site
 
@@ -196,7 +196,7 @@ mkdocs/                         # docs_dir for the mkdocs site
 
 website/                        # React (Vite) landing page — served at "/"
 ├── index.html                  # Entry; title, OG/meta, Google Analytics
-├── src/                        # App, pages (Home, Sky), components, routes
+├── src/                        # App, pages (Home, Sky, Factory), components, routes
 └── public/static/              # Landing assets (namespaced to avoid clashing with /assets)
 
 scripts/docs/
diff --git a/mkdocs/assets/stylesheets/cloudscape-docs.css b/mkdocs/assets/stylesheets/cloudscape-docs.css
index 2a9caf2289..3f4105ad1a 100644
--- a/mkdocs/assets/stylesheets/cloudscape-docs.css
+++ b/mkdocs/assets/stylesheets/cloudscape-docs.css
@@ -292,9 +292,8 @@
     margin-top: 0;
 }
 
-/* The switcher rail — copied from the landing (website/src/styles.css, .gs-rail block): grouped
-   rows under solid bars, with a selected card. The docs and the landing are separate
-   builds, so changes to the rail styles must be mirrored here (and vice versa). */
+/* Shared rail visuals match website/src/styles.css. Docs have no current product,
+   so rows are highlighted only on hover or keyboard focus. */
 .gs-rail {
     display: flex;
     flex-direction: column;
@@ -329,10 +328,11 @@
     border-bottom-left-radius: 0;
     border-bottom-right-radius: 0;
 }
-.gs-opt.gs-opt--on {
+.gs-opt:hover,
+.gs-opt:focus-visible {
     border-radius: 8px;
     background: var(--cs-bg);
-    box-shadow: inset 0 0 0 1px var(--cs-text);
+    box-shadow: inset 0 0 0 0.5px var(--cs-text);
 }
 .gs-opt--feat {
     padding: 13px 14px;
diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md
index 6dd8757145..5a0ef995da 100644
--- a/mkdocs/docs/concepts/presets.md
+++ b/mkdocs/docs/concepts/presets.md
@@ -272,7 +272,7 @@ Presets are pushed to and pulled from the registry hosted at [dstack Sky](https:
 
 Presets are private by default. Set **Presets** to **Public** in the project settings to let anyone browse and pull its presets without signing in, including users of a self-hosted `dstack` server. Only project admins and global admins can change this setting; enabling it does not let anyone join the project.
 
-A self-hosted registry is part of [dstack Factory](https://calendly.com/dstackai/discovery-call){ target="_blank" }.
+A self-hosted registry is part of [dstack Factory](/products/factory/){ target="_blank" }.
 
 ## Export a preset
 
diff --git a/mkdocs/index.md b/mkdocs/index.md
index 50475eafce..0fcf89bffd 100644
--- a/mkdocs/index.md
+++ b/mkdocs/index.md
@@ -1,8 +1,7 @@
 ---
 # The landing page ("/") is now the React app in website/. When the docs are served on
-# their own (e.g. `mkdocs serve`), this page redirects to /docs/. The title/description
-# below are only used to render the social card that the React landing reuses as its OG
-# image (assets/images/social/index.png).
+# their own (e.g. `mkdocs serve`), this page redirects to /docs/. The React landing has
+# its own metadata and social preview, separate from this redirect page.
 template: redirect.html
 title: The orchestration stack for AI infrastructure
 description: dstack is a unified control plane for GPU provisioning and orchestration that works with any GPU cloud, Kubernetes, or on-prem clusters.
diff --git a/mkdocs/overrides/header-2.html b/mkdocs/overrides/header-2.html
index cdea2fe8ab..86a97c4b11 100644
--- a/mkdocs/overrides/header-2.html
+++ b/mkdocs/overrides/header-2.html
@@ -83,7 +83,7 @@
       
-      }
+      visual={}
       title="Bring your own compute"
       imageFirst
     >
@@ -118,7 +55,10 @@ function BringComputeBlock() {
   );
 }
 
-export function KeyConceptsBlock({ children }: { children?: ReactNode }) {
+export function KeyConceptsBlock({ children, imageFirst = false }: {
+  children?: ReactNode;
+  imageFirst?: boolean;
+}) {
   return (
     
       }
       title="AI-native orchestration"
+      imageFirst={imageFirst}
     >
       {children ?? <>
         Managing AI infrastructure requires first-class primitives for compute management, training, inference, and observability that support heterogeneous AI compute.
diff --git a/website/src/pages/Home/FaqSection.tsx b/website/src/pages/Home/FaqSection.tsx
index 7172498f6e..a51dc60c3c 100644
--- a/website/src/pages/Home/FaqSection.tsx
+++ b/website/src/pages/Home/FaqSection.tsx
@@ -1,4 +1,4 @@
-import { useState } from 'react';
+import { ReactNode, useState } from 'react';
 import Button from '@cloudscape-design/components/button';
 import ExpandableSection from '@cloudscape-design/components/expandable-section';
 import SpaceBetween from '@cloudscape-design/components/space-between';
@@ -6,7 +6,12 @@ import { mainButtonStyle } from '../../cloudscape-theme';
 import { AlternatingDocBlock } from '../../components/AlternatingDocBlock';
 import { highlightTerms } from '../../components/highlightTerms';
 
-const faqItems = [
+type FaqItem = {
+  q: string;
+  a: string | string[];
+};
+
+const faqItems: FaqItem[] = [
   {
     q: 'How does dstack differ from Slurm?',
     a: 'Slurm is a battle-tested workload manager with decades of production use in HPC environments. dstack is a unified orchestration layer built for containerized AI workloads and heterogeneous AI compute. While both support batch jobs and distributed training, dstack also provides first-class primitives for compute management, inference, and observability, including native cloud provisioning.',
@@ -26,15 +31,19 @@ const faqItems = [
 ];
 
 // FAQ block: a single-open accordion of questions beside contact actions.
-export function FaqSection({ items = faqItems, imageFirst = false }: {
-  items?: typeof faqItems;
+export function FaqSection({ items = faqItems, showContact = true, imageFirst = false, action, children }: {
+  items?: FaqItem[];
+  showContact?: boolean;
   imageFirst?: boolean;
+  action?: ReactNode;
+  children?: ReactNode;
 }) {
   const [openQuestion, setOpenQuestion] = useState(null);
 
   return (
     

       
             {items.map(item => (
@@ -45,25 +54,28 @@ export function FaqSection({ items = faqItems, imageFirst = false }: {
                 expanded={openQuestion === item.q}
                 onChange={({ detail }) => setOpenQuestion(detail.expanded ? item.q : null)}
               >
-                {highlightTerms(item.a)}
+                {(Array.isArray(item.a) ? item.a : [item.a]).map((paragraph, index) => (
+                  
{highlightTerms(paragraph)}

+                ))}
               
             ))}
} title="FAQ" - imageFirst={imageFirst} - action={ + action={action ?? ( - + {showContact && ( + + )} - } + )} > - Have questions, or need help? Reach out to us on Discord or directly. + {children ?? <>Have questions, or need help? Reach out to us on Discord{showContact && ' or directly'}.} ); diff --git a/website/src/pages/Home/GetStartedSection.tsx b/website/src/pages/Home/GetStartedSection.tsx index edd2c35604..a07fb08e75 100644 --- a/website/src/pages/Home/GetStartedSection.tsx +++ b/website/src/pages/Home/GetStartedSection.tsx @@ -7,7 +7,7 @@ import { mainButtonStyle } from '../../cloudscape-theme'; import { installMethods, maxInstallLines, padYamlToLines } from '../../data/snippets'; import { DOCS_URL, ROUTES, docsUrl } from '../../routes'; -const BoxGlyph = () => ( +export const BoxGlyph = () => ( @@ -17,7 +17,7 @@ export const CloudGlyph = () => ( ); -const LayersGlyph = () => ( +export const LayersGlyph = () => ( @@ -103,33 +103,31 @@ export function GetStartedSection() {
diff --git a/website/src/pages/Home/HomePage.tsx b/website/src/pages/Home/HomePage.tsx index 2b9a8788a8..7bc27e100b 100644 --- a/website/src/pages/Home/HomePage.tsx +++ b/website/src/pages/Home/HomePage.tsx @@ -27,9 +27,9 @@ export function HomePage() {

{highlightTerms( - 'dstack is an open-source orchestration layer for heterogeneous AI compute. ' + - 'It standardizes how workloads run across GPU clouds, Kubernetes, and on-prem clusters, ' + - 'across NVIDIA, AMD, Tenstorrent, and TPU accelerators.', + 'dstack is an open-source orchestration layer for AI workloads on heterogeneous accelerators. ' + + 'It standardizes how you manage compute and run training and inference on GPU clouds, ' + + 'Kubernetes, VMs, or bare-metal clusters.', )}

diff --git a/website/src/pages/Sky/MarketplaceTabs.tsx b/website/src/pages/Sky/MarketplaceTabs.tsx new file mode 100644 index 0000000000..852d431bac --- /dev/null +++ b/website/src/pages/Sky/MarketplaceTabs.tsx @@ -0,0 +1,92 @@ +import Button from '@cloudscape-design/components/button'; +import { mainButtonStyle } from '../../cloudscape-theme'; +import { ContentTabs } from '../../components/ContentTabs'; +import { ChipGlyph, CloudGlyph } from '../Home/GetStartedSection'; +import { gpuPrices } from './pricing'; + +// Default marketplace clouds from dstack Sky's /api/backends/list_base_types. +const partnerGroups = [ + ['AWS', 'GCP'], + ['Lambda', 'Runpod'], + ['Verda', 'Nebius'], +]; + +export function MarketplaceTabs() { + return ( + + + + GPU + On-demand + Spot + + + + {gpuPrices.map(gpu => ( + + + + + {gpu.name}{' '}{gpu.memory} + + + {gpu.onDemand} + {gpu.spot ?? '—'} + + ))} + + + ), + footer: ( + + One account and unified billing across GPU clouds + + ), + }, + { + id: 'partners', + label: 'Partners', + content: ( +
+ {partnerGroups.map(group => ( +
    + {group.map(partner => ( +
  • + + {partner} +
    • + ))} +
+ ))} +
+ ), + footer: ( + <> + + Offer your GPU capacity through dstack Sky + + + + ), + }, + ]} + /> + ); +} diff --git a/website/src/pages/Sky/SkyPage.tsx b/website/src/pages/Sky/SkyPage.tsx index a11ec22af4..0248322df1 100644 --- a/website/src/pages/Sky/SkyPage.tsx +++ b/website/src/pages/Sky/SkyPage.tsx @@ -1,35 +1,17 @@ -import { KeyboardEvent, useRef, useState } from 'react'; import Button from '@cloudscape-design/components/button'; import { useLayoutContext } from '../../App'; -import { heroButtonStyle, mainButtonStyle } from '../../cloudscape-theme'; +import { heroButtonStyle } from '../../cloudscape-theme'; import { AlternatingDocBlock } from '../../components/AlternatingDocBlock'; +import { ComputeSourcesTabs } from '../../components/ComputeSourcesTabs'; import { highlightTerms } from '../../components/highlightTerms'; import { KeyConceptsBlock } from '../Home/ExploreSection'; import { FaqSection } from '../Home/FaqSection'; -import { CapList, ChipGlyph, CloudGlyph, KubernetesGlyph, ServerGlyph } from '../Home/GetStartedSection'; import { TrustedBySection } from '../Home/TrustedBySection'; import { HeroSquircle } from './HeroSquircle'; -import { gpuPrices } from './pricing'; - -const computeTabs = ['On-demand', 'Reserved clusters', 'BYOC']; +import { MarketplaceTabs } from './MarketplaceTabs'; export function SkyPage() { const { theme } = useLayoutContext(); - const [computeTab, setComputeTab] = useState(0); - const tabRefs = useRef<(HTMLButtonElement | null)[]>([]); - - const onTabKeyDown = (event: KeyboardEvent, index: number) => { - const nextIndex = { - ArrowRight: (index + 1) % computeTabs.length, - ArrowLeft: (index + computeTabs.length - 1) % computeTabs.length, - Home: 0, - End: computeTabs.length - 1, - }[event.key]; - if (nextIndex === undefined) return; - event.preventDefault(); - setComputeTab(nextIndex); - tabRefs.current[nextIndex]?.focus(); - }; return (
@@ -40,12 +22,12 @@ export function SkyPage() {
-

The AI cloud with AI-native orchestration

+

One account across GPU clouds

{highlightTerms( - 'dstack Sky is a heterogeneous AI cloud with NVIDIA Blackwell, Hopper, and AMD Instinct GPUs. ' + - 'Built on dstack, it combines compute management, training, inference, ' + - 'and observability in one interface.', + 'dstack Sky combines GPU capacity from multiple cloud partners with competitive pricing, ' + + 'unified billing, and AI-native orchestration. We host and manage the dstack server for you, ' + + 'whether you use the marketplace or bring your own cloud and on-prem compute.', )}

@@ -68,122 +50,53 @@ export function SkyPage() {
-
- {computeTabs.map((label, index) => ( - - ))} -
-
- {computeTab === 0 && ( - - - - - - - - - - {gpuPrices.map(gpu => ( - - - - - - ))} - -
GPUOn-demandSpot
- - - {gpu.name}{' '}{gpu.memory} - - {gpu.onDemand}{gpu.spot ?? '—'}
- )} - {computeTab === 1 && ( - , title: 'GPU clusters', sub: 'Reserve for a fixed period at a price agreed in advance' }, - ]} /> - )} - {computeTab === 2 && ( - , title: 'GPU clouds', sub: 'AWS, GCP, Azure, Nebius, Runpod, and more' }, - { icon: , title: 'SSH fleets', sub: 'Pre-provisioned VMs or bare-metal' }, - { icon: , title: 'Kubernetes', sub: 'Existing Kubernetes clusters' }, - ]} /> - )} -
-
- {computeTab !== 1 && ( - - {computeTab === 0 ? 'Prices in USD per GPU-hour' : 'Bring your own compute'} - - )} - {computeTab === 1 && ( - - )} -
-
- } - title="On-demand GPUs for AI workloads" + visual={} + title="Better prices, unified billing" imageFirst > - Access affordable on-demand and spot GPUs with pay-as-you-go pricing. + The dstack Sky marketplace offers on-demand and spot GPUs from a wide set of partners. + You get competitive pricing and capacity across regions, without depending on a + single cloud. +
+
+ One dstack Sky account gives you access to all marketplace partners. Billing is unified, + with no separate cloud accounts to manage. + + + } + title="Bring your own compute" + > + dstack Sky provisions and manages compute in your own GPU cloud accounts using your + credentials. Compute is billed directly to those accounts.

- You can also bring compute from your own cloud accounts or on-prem infrastructure - and use it alongside Sky GPUs. + You can also bring on-prem compute, including Kubernetes clusters, pre-provisioned + VMs, and bare-metal clusters. We host and maintain the dstack server for you. - + dstack Sky provides first-class primitives for compute management, training, - inference, and observability across heterogeneous AI compute. Use one interface - to efficiently utilize cloud GPUs, manage your own clusters, or run your own - AI token factory at scale. + inference, and observability across heterogeneous AI compute.

- Sky is built on dstack and uses the same CLI and YAML configurations as a - self-hosted dstack server. + dstack Sky includes all the features of open-source dstack and uses the same CLI and + YAML configurations. It adds a built-in gateway with an HTTPS domain, advanced + observability, and usage metering. - diff --git a/website/src/router.tsx b/website/src/router.tsx index fb46c14ddd..6c9e810c80 100644 --- a/website/src/router.tsx +++ b/website/src/router.tsx @@ -1,5 +1,6 @@ import { Navigate, createBrowserRouter } from 'react-router-dom'; import { App } from './App'; +import { FactoryPage } from './pages/Factory/FactoryPage'; import { HomePage } from './pages/Home'; import { SkyPage } from './pages/Sky/SkyPage'; import { ROUTES } from './routes'; @@ -12,6 +13,7 @@ export const router = createBrowserRouter([ children: [ { index: true, element: }, { path: ROUTES.SKY, element: }, + { path: ROUTES.FACTORY, element: }, { path: '*', element: }, ], }, diff --git a/website/src/routes.ts b/website/src/routes.ts index 4d759478cc..8a1309ac44 100644 --- a/website/src/routes.ts +++ b/website/src/routes.ts @@ -1,6 +1,6 @@ // Central route table + cross-links to the MkDocs-served parts of the site. // -// This app owns `/` and `/products/sky`. Docs and blog are served by MkDocs on the +// This app owns `/` and `/products/*`. Docs and blog are served by MkDocs on the // SAME origin in production (`/docs`, `/blog`). For standalone landing development you // can point those links at the live site by setting VITE_DOCS_BASE, e.g. // VITE_DOCS_BASE=https://dstack.ai npm run dev @@ -9,6 +9,7 @@ const SITE_BASE = (import.meta.env.VITE_DOCS_BASE ?? '').replace(/\/+$/, ''); export const ROUTES = { HOME: '/', SKY: '/products/sky', + FACTORY: '/products/factory', } as const; export type Route = (typeof ROUTES)[keyof typeof ROUTES]; diff --git a/website/src/styles.css b/website/src/styles.css index 13d8fa9197..fd7e718a81 100644 --- a/website/src/styles.css +++ b/website/src/styles.css @@ -382,6 +382,10 @@ p { font-size: 17px; } +.home-hero .highlight { + font-weight: 600; +} + /* Hero CTAs sit side by side, each sized to its own label (not forced to equal width). */ .home-hero__actions { margin-top: 28px; @@ -582,6 +586,7 @@ p { Higher specificity than `.docs-article a` so the reset wins regardless of source order. */ .docs-article .doc-action a, .docs-article .figma-card a, +.docs-article .gs-boxfoot a, .docs-article .gs-deploy a { text-decoration: none !important; } @@ -992,6 +997,14 @@ p { color: var(--cs-text) !important; } +.faq-list p { + font-size: inherit; +} + +.faq-list p + p { + margin-top: 1em; +} + .testimonial-person { display: flex; @@ -1062,10 +1075,12 @@ p { border-bottom-left-radius: 0; border-bottom-right-radius: 0; } -.gs-opt.gs-opt--on { +.gs-rail:not(:has(a.gs-opt:hover, a.gs-opt:focus-visible)) .gs-opt.gs-opt--on, +.gs-rail a.gs-opt:hover, +.gs-rail a.gs-opt:focus-visible { border-radius: 8px; background: var(--cs-bg); - box-shadow: inset 0 0 0 1px var(--cs-text); + box-shadow: inset 0 0 0 0.5px var(--cs-text); } /* Row names mirror the tab typography: 16px / 700. */ @@ -1184,15 +1199,15 @@ p { border-radius: 12px; overflow: hidden; } -.gs-box--compute { - height: auto; - min-height: 360px; -} -.gs-box--compute.gs-box--offers { +.gs-box.gs-box--compute, +.gs-box.gs-box--compute-sources { height: 360px; } -.gs-box--offers .gs-skybody { +.gs-box--compute .gs-skybody, +.gs-box--compute-sources .gs-skybody { overflow-y: auto; +} +.gs-box--offers .gs-skybody { padding-top: 0; } .sky-gpu-prices { @@ -1236,10 +1251,6 @@ p { .sky-gpu-prices td { padding-left: 8px; } - .gs-box--compute .gs-tabs, - .gs-box--compute .gs-tab { - padding-inline: 4px; - } } .gs-boxfoot { display: flex; @@ -1261,6 +1272,15 @@ p { flex: 0 0 auto; } +@media (max-width: 480px) { + .gs-box--compute .gs-boxfoot { + flex-wrap: wrap; + } + .gs-box--compute .gs-foot__note { + flex-basis: 100%; + } +} + /* Shared tabs for installation methods and Sky compute options. */ .gs-tabs { display: flex; @@ -1896,6 +1916,10 @@ p { grid-template-columns: 1fr 1fr; width: 100%; } + + .home-hero__actions--single { + grid-template-columns: max-content; + } /* Each button fills its half-column; center the label and trim the desktop 34px side padding so both fit. !important: Cloudscape sets text-align:start via a boosted rule. */ .home-hero__actions [class*='awsui_button'] { diff --git a/website/vite.config.ts b/website/vite.config.ts index 18790fe849..ff84e085ac 100644 --- a/website/vite.config.ts +++ b/website/vite.config.ts @@ -8,7 +8,7 @@ import react from '@vitejs/plugin-react'; // `assetsDir` is namespaced to `website-assets/` (not Vite's default `assets/`) so the // build can be overlaid onto the MkDocs `site/` output without colliding with MkDocs's // own `/assets/...` tree. Public files live under `public/static/` for the same reason — -// the HTML entries provide the landing and Sky product routes on static hosting. +// the HTML entries provide the landing and product routes on static hosting. export default defineConfig({ base: process.env.BASE_PATH || '/', plugins: [react()], @@ -18,6 +18,7 @@ export default defineConfig({ input: { home: fileURLToPath(new URL('./index.html', import.meta.url)), sky: fileURLToPath(new URL('./products/sky/index.html', import.meta.url)), + factory: fileURLToPath(new URL('./products/factory/index.html', import.meta.url)), }, }, }, From 40772a96fec7ad5c2a6337b715687d4dd95b6104 Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:15:03 +0000 Subject: [PATCH 599/640] Fix GCP `a3-edgegpu-8g` and `a3-highgpu-8g` (#4307) Upgrade the COS image used for these instance types to the currently [recommended](https://docs.cloud.google.com/compute/docs/gpus/gpudirect#create-high-edge-instances) `cos-121-lts` family that comes with a GLIBC version (2.37) compatible with `dstack-shim`. --- src/dstack/_internal/core/backends/gcp/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index d406476048..8ab25584f6 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -1580,7 +1580,7 @@ def _get_image(instance_type_name: str, gpu_name: Optional[str]) -> GCPImage: is_ufw_installed = False elif instance_type_name in ["a3-edgegpu-8g", "a3-highgpu-8g"]: return GCPImage( - id="projects/cos-cloud/global/images/cos-105-17412-535-78", + id="projects/cos-cloud/global/images/family/cos-121-lts", is_ufw_installed=False, ) elif gpu_name is not None: From 39b28c22b2ea63526d4b1a3cc3501bfcd0313724 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Thu, 24 Sep 2026 09:46:58 +0000 Subject: [PATCH 600/640] Fix errors in service router worker sync (#4305) * Fixed two `DetachedInstanceError` due to missing `RunModel.run_name` and `JobModel.job_name`. * Fixed `Could not fetch server_info for worker ...: ReadError('')` logged as an error with a traceback, repeating until the worker is ready. An SSH tunnel to a port nobody listens on yet fails with `httpx.ReadError`, so this is the normal startup path. Both `_probe_http_worker()` and `_get_router_workers()` now catch `httpx.RequestError` and log it at the debug level. Also: * `_get_router_workers()` returns `None` instead of `[]` when the response cannot be read. `[]` means "the router has no workers", so an unexpected status, an unparsable body, or a missing `workers` key made the sync re-register every worker. * Bare `except Exception` clauses are replaced with the exceptions actually expected, so a programming error is no longer reported as an unready replica. `sync_router_workers_for_run_model()` logs such errors itself instead of letting them escape to the pipeline worker, which leaves the sync row locked until the lock expires. * `_get_worker()` takes the replica address instead of two pre-built URLs, and each probe builds the URL it registers the worker under. `_get_http_worker()`, `_get_grpc_worker()`, and `_discover_grpc_server_info()` are folded into loops over the connection modes and the runtime types to probe. Fixes: https://github.com/dstackai/dstack/issues/4300 --- .../service_router_worker_sync.py | 16 +- .../services/runs/router_worker_sync.py | 350 +++++------- .../test_service_router_worker_sync.py | 122 ++++- .../services/runs/test_router_worker_sync.py | 506 ++++++++++++------ 4 files changed, 616 insertions(+), 378 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py b/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py index 48fe522108..ca42017571 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py @@ -201,7 +201,14 @@ async def process(self, item: ServiceRouterWorkerSyncPipelineItem) -> None: ServiceRouterWorkerSyncModel.id == item.id, ServiceRouterWorkerSyncModel.lock_token == item.lock_token, ) - .options(selectinload(ServiceRouterWorkerSyncModel.run)) + .options( + joinedload(ServiceRouterWorkerSyncModel.run).load_only( + RunModel.id, + RunModel.deleted, + RunModel.status, + RunModel.run_spec, + ) + ) ) sync_row = res.unique().scalar_one_or_none() if sync_row is None: @@ -229,7 +236,11 @@ async def process(self, item: ServiceRouterWorkerSyncPipelineItem) -> None: select(RunModel) .where(RunModel.id == item.run_id) .options( - load_only(RunModel.id, RunModel.run_spec), + load_only( + RunModel.id, + RunModel.run_name, + RunModel.run_spec, + ), selectinload( RunModel.jobs.and_( JobModel.status == JobStatus.RUNNING, @@ -238,6 +249,7 @@ async def process(self, item: ServiceRouterWorkerSyncPipelineItem) -> None: ) .load_only( JobModel.id, + JobModel.job_name, JobModel.status, JobModel.job_spec_data, JobModel.job_provisioning_data, diff --git a/src/dstack/_internal/server/services/runs/router_worker_sync.py b/src/dstack/_internal/server/services/runs/router_worker_sync.py index e8d13fa4c6..918711be1e 100644 --- a/src/dstack/_internal/server/services/runs/router_worker_sync.py +++ b/src/dstack/_internal/server/services/runs/router_worker_sync.py @@ -6,14 +6,7 @@ import grpc from google.protobuf.json_format import MessageToDict -from httpx import ( - AsyncClient, - ConnectError, - ConnectTimeout, - ReadTimeout, - RemoteProtocolError, - Response, -) +from httpx import AsyncClient, RequestError, Response from smg_grpc_proto import ( sglang_scheduler_pb2, sglang_scheduler_pb2_grpc, @@ -30,7 +23,6 @@ from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_spec from dstack._internal.server.services.jobs.job_replica_grpc_client import ( get_service_replica_grpc_channel_over_uds, - get_service_replica_grpc_client, ) from dstack._internal.server.services.jobs.job_replica_http_client import ( get_service_replica_client, @@ -45,13 +37,14 @@ logger = get_logger(__name__) -_ROUTER_HTTP = "http://dstack" -_ROUTER_HTTP_TIMEOUT = 10.0 +# Requests are made over a UDS tunnel to the replica, so the authority is a placeholder. +_HTTP_BASE_URL = "http://dstack" +_HTTP_TIMEOUT = 10.0 _MAX_SERVER_INFO_RESPONSE_BYTES = 256 * 1024 _MAX_WORKERS_RESPONSE_BYTES = 2 * 1024 * 1024 _MAX_WORKERS_COMMAND_ACK_BYTES = 64 * 1024 _MAX_WORKERS_LIST_ITEMS = 8192 -_GRPC_DISCOVERY_TIMEOUT = 30.0 +_GRPC_TIMEOUT = 30.0 class _ResponseTooLargeError(Exception): @@ -75,7 +68,7 @@ async def _request_json_limited( max_response_bytes: int, ok_statuses: set[int], json_body: Optional[dict] = None, - timeout: float = _ROUTER_HTTP_TIMEOUT, + timeout: float = _HTTP_TIMEOUT, ) -> Any: kwargs: dict[str, Any] = {"timeout": timeout} if json_body is not None: @@ -105,7 +98,7 @@ async def _request_json_limited( return None -class _TargetWorker(TypedDict): +class _Worker(TypedDict): url: str worker_type: str bootstrap_port: NotRequired[Optional[int]] @@ -115,14 +108,13 @@ class _TargetWorker(TypedDict): kv_role: NotRequired[str] -class _WorkerPayloadResult(TypedDict): - status: Literal["ready", "not_ready"] - worker: Optional[_TargetWorker] - - _ConnectionMode = Literal["grpc", "http"] +# The order does matter -- we discover connection modes in the specified order +_CONNECTION_MODES: tuple[_ConnectionMode, ...] = ("http", "grpc") + _RuntimeType = Literal["sglang", "vllm"] -_GRPC_RUNTIME_TYPES: tuple[_RuntimeType, ...] = ("sglang", "vllm") +# The order does matter -- we discover runtime types in the specified order +_RUNTIME_TYPES: tuple[_RuntimeType, ...] = ("sglang", "vllm") def run_model_has_sglang_router_replica_group(run_model: RunModel) -> bool: @@ -183,7 +175,7 @@ def _get_runtime_type_from_workers( if worker.get("connection_mode") != "grpc": continue runtime_type = worker.get("runtime_type") - if isinstance(runtime_type, str) and runtime_type in _GRPC_RUNTIME_TYPES: + if isinstance(runtime_type, str) and runtime_type in _RUNTIME_TYPES: runtimes.add(runtime_type) if runtimes == {"sglang"}: return "sglang" @@ -192,45 +184,25 @@ def _get_runtime_type_from_workers( return None -def _is_expected_router_workers_fetch_error(error: Exception) -> bool: - """SMG router may not accept HTTP yet during startup.""" - if isinstance( - error, - ( - RemoteProtocolError, - ConnectError, - ConnectTimeout, - ReadTimeout, - TimeoutError, - ), - ): - return True - if isinstance(error, OSError) and error.errno in {61, 111}: - return True - return False - - -def _log_router_workers_fetch_failure(error: Exception) -> None: - if _is_expected_router_workers_fetch_error(error): - logger.debug("Router /workers not ready yet: %r", error) - return - logger.exception("Error getting router /workers") - - -async def _get_router_workers(client: AsyncClient) -> List[dict]: +async def _get_router_workers(client: AsyncClient) -> Optional[List[dict]]: try: data = await _request_json_limited( client, "GET", - f"{_ROUTER_HTTP}/workers", + f"{_HTTP_BASE_URL}/workers", max_response_bytes=_MAX_WORKERS_RESPONSE_BYTES, ok_statuses={200}, ) if not isinstance(data, dict): - return [] - workers = data.get("workers", []) + # Non-200 status or unparsable response or response is not a JSON object + return None + workers = data.get("workers") if not isinstance(workers, list): - return [] + # Unexpected response structure -- `workers` is missing or is not an array + return None + # TODO: Truncating a long list and/or dropping unexpectedly shaped items doesn't seem + # right. We should add validation (an item must be a dict with some required fields, see + # _update_workers_in_router_replica) and decide what to do with a partially valid list if len(workers) > _MAX_WORKERS_LIST_ITEMS: logger.warning( "Router /workers list exceeds %s items, truncating", @@ -240,9 +212,9 @@ async def _get_router_workers(client: AsyncClient) -> List[dict]: return [w for w in workers if isinstance(w, dict)] except _ResponseTooLargeError: logger.warning("Router /workers response exceeded size limit") - except Exception as e: - _log_router_workers_fetch_failure(e) - return [] + except RequestError as e: + logger.debug("Router /workers not ready yet: %r", e) + return None async def _add_worker_to_router( @@ -271,18 +243,20 @@ async def _add_worker_to_router( body = await _request_json_limited( client, "POST", - f"{_ROUTER_HTTP}/workers", + f"{_HTTP_BASE_URL}/workers", max_response_bytes=_MAX_WORKERS_COMMAND_ACK_BYTES, ok_statuses={202}, json_body=payload, ) - return isinstance(body, dict) and body.get("status") == "accepted" + added = isinstance(body, dict) and body.get("status") == "accepted" + if not added: + logger.warning("Unexpected add-worker response for %s: %s", url, body) + return added except _ResponseTooLargeError: logger.warning("Router add-worker response exceeded size limit for %s", url) - return False - except Exception: - logger.exception("Error adding worker %s", url) - return False + except RequestError as e: + logger.warning("Error adding worker %s: %r", url, e) + return False async def _remove_worker_from_router_by_id( @@ -292,22 +266,24 @@ async def _remove_worker_from_router_by_id( body = await _request_json_limited( client, "DELETE", - f"{_ROUTER_HTTP}/workers/{worker_id}", + f"{_HTTP_BASE_URL}/workers/{worker_id}", max_response_bytes=_MAX_WORKERS_COMMAND_ACK_BYTES, ok_statuses={202}, ) - return isinstance(body, dict) and body.get("status") == "accepted" + removed = isinstance(body, dict) and body.get("status") == "accepted" + if not removed: + logger.warning("Unexpected remove-worker response for %s: %s", worker_url, body) + return removed except _ResponseTooLargeError: logger.warning("Router remove-worker response exceeded size limit for %s", worker_url) - return False - except Exception: - logger.exception("Error removing worker %s", worker_url) - return False + except RequestError as e: + logger.warning("Error removing worker %s: %r", worker_url, e) + return False async def _update_workers_in_router_replica( client: AsyncClient, - target_workers: List[_TargetWorker], + target_workers: List[_Worker], *, current_workers: List[dict], ) -> None: @@ -339,7 +315,7 @@ async def _update_workers_in_router_replica( kv_role=tw.get("kv_role"), ) if not ok: - logger.warning("Failed to add worker %s, continuing with others", tw["url"]) + logger.debug("Failed to add worker %s, continuing with others", tw["url"]) for url in to_remove: wid = current_ids_by_norm_url.get(url) if not wid: @@ -348,7 +324,7 @@ async def _update_workers_in_router_replica( else: ok = await _remove_worker_from_router_by_id(client, wid, worker_url=url) if not ok: - logger.warning("Failed to remove worker %s, continuing with others", url) + logger.debug("Failed to remove worker %s, continuing with others", url) def _vllm_kv_role_to_worker_type(kv_role: str) -> str: @@ -359,33 +335,33 @@ def _vllm_kv_role_to_worker_type(kv_role: str) -> str: return "regular" -def _is_expected_grpc_discovery_error(error: Exception) -> bool: +def _is_expected_grpc_error(error: grpc.aio.AioRpcError) -> bool: """Expected while a gRPC worker is still starting or the wrong stub is probed.""" - if isinstance(error, grpc.aio.AioRpcError): - return error.code() in ( - grpc.StatusCode.UNAVAILABLE, - grpc.StatusCode.DEADLINE_EXCEEDED, - grpc.StatusCode.UNIMPLEMENTED, - ) - return False + return error.code() in ( + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.UNIMPLEMENTED, + ) -async def _probe_http_worker(client: AsyncClient, *, worker_url: str) -> _WorkerPayloadResult: +async def _probe_http_worker(client: AsyncClient, *, address: str) -> Optional[_Worker]: + # The request goes over the tunnel, `worker_url` is the address the router itself dials. + worker_url = f"http://{address}" try: data = await _request_json_limited( client, "GET", - f"{_ROUTER_HTTP}/server_info", + f"{_HTTP_BASE_URL}/server_info", max_response_bytes=_MAX_SERVER_INFO_RESPONSE_BYTES, ok_statuses={200}, ) if isinstance(data, dict): if data.get("status") != "ready": - return {"status": "not_ready", "worker": None} + return None mode = data.get("disaggregation_mode", "") if mode == "prefill": bootstrap_port = data.get("disaggregation_bootstrap_port") - worker: _TargetWorker = { + worker: _Worker = { "url": worker_url, "worker_type": "prefill", "connection_mode": "http", @@ -393,38 +369,25 @@ async def _probe_http_worker(client: AsyncClient, *, worker_url: str) -> _Worker } if bootstrap_port is not None: worker["bootstrap_port"] = bootstrap_port - return {"status": "ready", "worker": worker} + return worker if mode == "decode": return { - "status": "ready", - "worker": { - "url": worker_url, - "worker_type": "decode", - "connection_mode": "http", - "runtime_type": "sglang", - }, - } - return { - "status": "ready", - "worker": { "url": worker_url, - "worker_type": "regular", + "worker_type": "decode", "connection_mode": "http", "runtime_type": "sglang", - }, + } + return { + "url": worker_url, + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", } except _ResponseTooLargeError: logger.warning("server_info response too large for worker %s", worker_url) - except RemoteProtocolError as e: - logger.debug("HTTP server_info not available for worker %s: %r", worker_url, e) - except Exception as e: - logger.exception("Could not fetch server_info for worker %s: %r", worker_url, e) - return {"status": "not_ready", "worker": None} - - -async def _get_http_worker(job_model: JobModel, *, worker_url: str) -> _WorkerPayloadResult: - async with get_service_replica_client(job_model) as client: - return await _probe_http_worker(client, worker_url=worker_url) + except RequestError as e: + logger.debug("Could not fetch server_info for worker %s: %r", worker_url, e) + return None async def _get_grpc_server_info( @@ -437,33 +400,18 @@ async def _get_grpc_server_info( else: stub = vllm_engine_pb2_grpc.VllmEngineStub(channel) request = vllm_engine_pb2.GetServerInfoRequest() - return await stub.GetServerInfo(request, timeout=_GRPC_DISCOVERY_TIMEOUT) - - -async def _discover_grpc_server_info( - channel: grpc.aio.Channel, -) -> tuple[Optional[_RuntimeType], Optional[Any]]: - # Bootstrap only: router workers list has no runtime_type yet. - for runtime_type in _GRPC_RUNTIME_TYPES: - try: - response = await _get_grpc_server_info(channel, runtime_type) - except Exception as e: - if _is_expected_grpc_discovery_error(e): - continue - raise - return runtime_type, response - return None, None + return await stub.GetServerInfo(request, timeout=_GRPC_TIMEOUT) def _grpc_server_info_to_worker( worker_url: str, runtime_type: _RuntimeType, response: Any, -) -> _TargetWorker: +) -> _Worker: if runtime_type == "vllm": kv_role = response.kv_role or "" kv_connector = response.kv_connector or "" - worker: _TargetWorker = { + worker: _Worker = { "url": worker_url, "connection_mode": "grpc", "runtime_type": runtime_type, @@ -498,81 +446,67 @@ def _grpc_server_info_to_worker( async def _probe_grpc_worker( channel: grpc.aio.Channel, *, - worker_url: str, + address: str, runtime_type: Optional[_RuntimeType] = None, -) -> _WorkerPayloadResult: - if runtime_type is not None: +) -> Optional[_Worker]: + # The RPC goes over the tunnel, `worker_url` is the address the router itself dials. + worker_url = f"grpc://{address}" + runtime_types: tuple[_RuntimeType, ...] + if runtime_type is None: + # Bootstrap only: router workers list has no runtime_type yet, should try all + runtime_types = _RUNTIME_TYPES + else: + runtime_types = (runtime_type,) + for runtime_type in runtime_types: try: response = await _get_grpc_server_info(channel, runtime_type) - except Exception as e: - if _is_expected_grpc_discovery_error(e): - logger.debug("gRPC worker %s not ready (GetServerInfo)", worker_url) - return {"status": "not_ready", "worker": None} + break + except grpc.aio.AioRpcError as e: + if _is_expected_grpc_error(e): + continue raise else: - runtime_type, response = await _discover_grpc_server_info(channel) - if runtime_type is None or response is None: - logger.debug("gRPC worker %s not ready (GetServerInfo)", worker_url) - return {"status": "not_ready", "worker": None} - - worker = _grpc_server_info_to_worker(worker_url, runtime_type, response) - return {"status": "ready", "worker": worker} - - -async def _get_grpc_worker( - job_model: JobModel, - *, - worker_url: str, - runtime_type: Optional[_RuntimeType] = None, -) -> _WorkerPayloadResult: - try: - async with get_service_replica_grpc_client(job_model) as channel: - return await _probe_grpc_worker( - channel, worker_url=worker_url, runtime_type=runtime_type - ) - except Exception as e: - logger.exception( - "Could not fetch gRPC GetServerInfo for worker %s: %r", - worker_url, - e, - ) - return {"status": "not_ready", "worker": None} + logger.debug("gRPC worker %s not ready (GetServerInfo)", worker_url) + return None + return _grpc_server_info_to_worker(worker_url, runtime_type, response) async def _get_worker( job_model: JobModel, *, - http_worker_url: str, - grpc_worker_url: str, + address: str, connection_mode: Optional[_ConnectionMode] = None, runtime_type: Optional[_RuntimeType] = None, -) -> _WorkerPayloadResult: - if connection_mode == "grpc": - return await _get_grpc_worker( - job_model, worker_url=grpc_worker_url, runtime_type=runtime_type - ) - if connection_mode == "http": - return await _get_http_worker(job_model, worker_url=http_worker_url) - # Router workers list is empty and no connection_mode discovered. - async with get_service_replica_tunnel(job_model) as uds_path: - async with get_service_replica_http_client_over_uds(uds_path) as client: - result = await _probe_http_worker(client, worker_url=http_worker_url) - if result["status"] == "ready": - return result - async with get_service_replica_grpc_channel_over_uds(uds_path) as channel: - try: - return await _probe_grpc_worker( - channel, - worker_url=grpc_worker_url, - runtime_type=runtime_type, - ) - except Exception as e: - logger.exception( - "Could not fetch gRPC GetServerInfo for worker %s: %r", - grpc_worker_url, - e, - ) - return {"status": "not_ready", "worker": None} +) -> Optional[_Worker]: + connection_modes: tuple[_ConnectionMode, ...] + if connection_mode is None: + # No connection_mode discovered -- should probe all + connection_modes = _CONNECTION_MODES + else: + connection_modes = (connection_mode,) + try: + async with get_service_replica_tunnel(job_model) as uds_path: + for connection_mode in connection_modes: + if connection_mode == "grpc": + async with get_service_replica_grpc_channel_over_uds(uds_path) as channel: + worker = await _probe_grpc_worker( + channel, address=address, runtime_type=runtime_type + ) + elif connection_mode == "http": + async with get_service_replica_http_client_over_uds(uds_path) as client: + worker = await _probe_http_worker(client, address=address) + if worker is not None: + return worker + except SSHError as e: + # An unreachable worker is reported as not ready rather than aborting the sync, so that + # one dead replica cannot hold back registration of the healthy ones. The cost is that a + # transient failure deregisters a healthy worker until the next sync re-adds it. + # TODO: `_update_workers_in_router_replica` cannot tell "not serving" from "could not be + # reached" -- both mean "absent from the target list", hence "remove". A third, unknown + # outcome should be excluded from both `to_add` and `to_remove`, leaving an unreachable + # worker as the router last saw it. + logger.warning("%s: failed to connect to worker replica: %r", fmt(job_model), e) + return None async def _build_target_workers( @@ -582,8 +516,8 @@ async def _build_target_workers( *, connection_mode: Optional[_ConnectionMode] = None, runtime_type: Optional[_RuntimeType] = None, -) -> List[_TargetWorker]: - workers: List[_TargetWorker] = [] +) -> List[_Worker]: + workers: List[_Worker] = [] config = run_spec.configuration if not isinstance(config, ServiceConfiguration): return workers @@ -601,28 +535,21 @@ async def _build_target_workers( jpd = get_job_provisioning_data(job) if jpd is None: continue - ip = jpd.internal_ip or jpd.hostname - if not ip: + hostname = jpd.internal_ip or jpd.hostname + if not hostname: continue job_spec = get_job_spec(job) port = get_service_port(job_spec, config) - http_worker_url = f"http://{ip}:{port}" - grpc_worker_url = f"grpc://{ip}:{port}" - result = await _get_worker( + worker = await _get_worker( job, - http_worker_url=http_worker_url, - grpc_worker_url=grpc_worker_url, + address=f"{hostname}:{port}", connection_mode=connection_mode, runtime_type=runtime_type, ) - if result["status"] == "ready" and result["worker"]: - workers.append(result["worker"]) - elif result["status"] == "not_ready": - logger.debug( - "Worker not ready http=%s grpc=%s", - http_worker_url, - grpc_worker_url, - ) + if worker is not None: + workers.append(worker) + else: + logger.debug("%s: worker replica not ready", fmt(job)) return workers @@ -644,9 +571,20 @@ async def sync_router_workers_for_run_model(run_model: RunModel) -> None: router_group.name, ) return + # A tunnel is opened here for the router, and inside `_build_target_workers` for every + # worker. Only the router being unreachable aborts the sync -- without a client there is + # nothing to reconcile against. An unreachable worker is skipped instead, see `_get_worker`. try: async with get_service_replica_client(router_job) as client: current_workers = await _get_router_workers(client) + if current_workers is None: + logger.debug( + "%s: failed to get current workers from the router in group %s," + " skipping worker sync", + fmt(run_model), + router_group.name, + ) + return # connection_mode can be grpc or http, runtime_type can be sglang or vllm. connection_mode = _get_connection_mode_from_workers(current_workers) runtime_type = _get_runtime_type_from_workers(current_workers) @@ -664,8 +602,14 @@ async def sync_router_workers_for_run_model(run_model: RunModel) -> None: client, target_workers, current_workers=current_workers ) except SSHError as e: + # Only the router's own tunnel reaches here, worker tunnels are handled in + # `_get_worker`. Warning is the right level: a job only reaches `RUNNING` after the + # server has talked to its runner over SSH, so an unreachable replica is always a + # regression, never a replica that has not started yet. logger.warning( "%s: failed to sync workers with router: %r", fmt(router_job), e, ) + except Exception: + logger.exception("%s: unexpected error when syncing workers with router", fmt(run_model)) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py b/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py index 5827b14966..26633a8baf 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py @@ -1,13 +1,15 @@ import asyncio +import logging import uuid from datetime import timedelta -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from sqlalchemy.ext.asyncio import AsyncSession +from dstack._internal.core.errors import SSHError from dstack._internal.core.models.configurations import parse_run_configuration -from dstack._internal.core.models.runs import RunStatus +from dstack._internal.core.models.runs import JobStatus, RunStatus from dstack._internal.server.background.pipeline_tasks.service_router_worker_sync import ( ServiceRouterWorkerSyncFetcher, ServiceRouterWorkerSyncPipeline, @@ -15,13 +17,19 @@ ServiceRouterWorkerSyncWorker, ) from dstack._internal.server.models import RunModel, ServiceRouterWorkerSyncModel +from dstack._internal.server.services.runs import router_worker_sync from dstack._internal.server.testing.common import ( + create_instance, + create_job, create_project, create_repo, create_run, create_user, get_run_spec, ) +from dstack._internal.server.testing.common import ( + get_job_provisioning_data as make_job_provisioning_data, +) from dstack._internal.utils.common import get_current_datetime @@ -451,3 +459,113 @@ async def test_process_calls_sync_and_unlocks_on_success( assert sync_row.lock_expires_at is None assert sync_row.lock_owner is None assert sync_row.last_processed_at is not None + + async def test_process_skips_sync_when_router_replica_not_ready( + self, + test_db, + session: AsyncSession, + worker: ServiceRouterWorkerSyncWorker, + caplog: pytest.LogCaptureFixture, + ): + caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + status=RunStatus.RUNNING, + run_spec=_router_service_run_spec(repo.name), + ) + instance = await create_instance(session=session, project=project) + # The router replica is still starting up, the worker replica is already serving. + await create_job( + session=session, + run=run, + instance=instance, + status=JobStatus.RUNNING, + ready=False, + replica_group_name="router", + job_provisioning_data=make_job_provisioning_data(), + ) + await create_job( + session=session, + run=run, + instance=instance, + status=JobStatus.RUNNING, + ready=True, + replica_num=1, + replica_group_name="worker", + job_provisioning_data=make_job_provisioning_data(), + ) + sync_row = await _add_service_router_worker_sync_row(session, run.id) + sync_row.lock_token = uuid.uuid4() + sync_row.lock_expires_at = get_current_datetime() + timedelta(seconds=30) + sync_row.lock_owner = ServiceRouterWorkerSyncPipeline.__name__ + await session.commit() + item = _sync_row_to_pipeline_item(sync_row) + + await worker.process(item) + + assert "no ready router job in group router, skipping worker sync" in caplog.text + # The run stays eligible for the next sync attempt. + await session.refresh(sync_row) + assert sync_row.deleted is False + assert sync_row.lock_token is None + + async def test_process_logs_router_job_when_router_connection_fails( + self, + test_db, + session: AsyncSession, + worker: ServiceRouterWorkerSyncWorker, + caplog: pytest.LogCaptureFixture, + ): + caplog.set_level(level=logging.WARNING, logger=router_worker_sync.__name__) + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + status=RunStatus.RUNNING, + run_spec=_router_service_run_spec(repo.name), + ) + instance = await create_instance(session=session, project=project) + router_job = await create_job( + session=session, + run=run, + instance=instance, + status=JobStatus.RUNNING, + ready=True, + replica_group_name="router", + # `dockerized` makes `get_container_ssh_credentials()` also read + # `instance.project` and `job_runtime_data`. + job_provisioning_data=make_job_provisioning_data(dockerized=True), + ) + sync_row = await _add_service_router_worker_sync_row(session, run.id) + sync_row.lock_token = uuid.uuid4() + sync_row.lock_expires_at = get_current_datetime() + timedelta(seconds=30) + sync_row.lock_owner = ServiceRouterWorkerSyncPipeline.__name__ + await session.commit() + item = _sync_row_to_pipeline_item(sync_row) + + # Patching the tunnel itself keeps `get_container_ssh_credentials()` real, so the test + # covers the job/instance/project attributes it reads. + tunnel_mock = MagicMock() + ssh_error = SSHError("connection refused") + tunnel_mock.return_value.__aenter__ = AsyncMock(side_effect=ssh_error) + tunnel_mock.return_value.__aexit__ = AsyncMock(return_value=False) + with patch("dstack._internal.server.services.ssh.SSHTunnel", tunnel_mock): + await worker.process(item) + + assert ( + f"job({router_job.id.hex[:6]}){router_job.job_name}:" + f" failed to sync workers with router: {ssh_error!r}" in caplog.text + ) + await session.refresh(sync_row) + assert sync_row.deleted is False + assert sync_row.lock_token is None diff --git a/src/tests/_internal/server/services/runs/test_router_worker_sync.py b/src/tests/_internal/server/services/runs/test_router_worker_sync.py index 0932939942..80a9137e0e 100644 --- a/src/tests/_internal/server/services/runs/test_router_worker_sync.py +++ b/src/tests/_internal/server/services/runs/test_router_worker_sync.py @@ -1,15 +1,25 @@ -from contextlib import asynccontextmanager, contextmanager +import json +import logging +from contextlib import contextmanager from pathlib import Path +from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch +import grpc +import httpx import pytest +from httpx import AsyncClient +from dstack._internal.core.errors import SSHError +from dstack._internal.server.services.runs import router_worker_sync from dstack._internal.server.services.runs.router_worker_sync import ( _get_connection_mode_from_workers, - _get_grpc_worker, + _get_router_workers, _get_runtime_type_from_workers, _get_worker, _grpc_server_info_to_worker, + _probe_grpc_worker, + _probe_http_worker, ) @@ -76,10 +86,127 @@ def test_sglang_prefill(self): } +def _router_client(handler) -> AsyncClient: + return AsyncClient(transport=httpx.MockTransport(handler)) + + +def _json_response(status_code: int, payload) -> httpx.Response: + return httpx.Response(status_code, content=json.dumps(payload).encode()) + + +@pytest.mark.asyncio +class TestGetRouterWorkers: + """ + `[]` must mean "the router really has no workers", so every response that does not carry a + usable worker list has to come back as `None`. Otherwise the caller reconciles against a + fabricated empty list and re-adds every worker, see + https://github.com/dstackai/dstack/issues/4300. + """ + + async def test_returns_workers(self): + payload = {"workers": [{"id": "1", "url": "http://10.0.0.1:8000"}]} + async with _router_client(lambda _: _json_response(200, payload)) as client: + assert await _get_router_workers(client) == payload["workers"] + + async def test_returns_empty_list_when_router_has_no_workers(self): + async with _router_client(lambda _: _json_response(200, {"workers": []})) as client: + assert await _get_router_workers(client) == [] + + async def test_returns_none_on_unexpected_status(self): + async with _router_client(lambda _: _json_response(500, {"workers": []})) as client: + assert await _get_router_workers(client) is None + + async def test_returns_none_on_unparsable_body(self): + async with _router_client(lambda _: httpx.Response(200, content=b"not json")) as client: + assert await _get_router_workers(client) is None + + async def test_returns_none_when_body_is_not_an_object(self): + async with _router_client(lambda _: _json_response(200, ["a"])) as client: + assert await _get_router_workers(client) is None + + async def test_returns_none_when_workers_key_is_missing(self): + async with _router_client(lambda _: _json_response(200, {})) as client: + assert await _get_router_workers(client) is None + + async def test_returns_none_when_workers_is_not_an_array(self): + async with _router_client(lambda _: _json_response(200, {"workers": "oops"})) as client: + assert await _get_router_workers(client) is None + + async def test_returns_none_on_request_error(self, caplog: pytest.LogCaptureFixture): + def handler(_): + # What an SSH tunnel to a port nobody listens on yet produces. + raise httpx.ReadError("") + + caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) + async with _router_client(handler) as client: + assert await _get_router_workers(client) is None + # A router that has not bound its port yet is expected, not an error. + assert "Router /workers not ready yet" in caplog.text + assert not [r for r in caplog.records if r.levelno > logging.DEBUG] + + +@pytest.mark.asyncio +class TestProbeHttpWorker: + """ + The probe talks to the replica over the tunnel, but the `url` it reports is the address the + router dials. It must stay byte-identical to what the router echoes back in `/workers`, + otherwise `_update_workers_in_router_replica` re-registers every worker on each sync. + """ + + async def test_regular_worker(self): + async with _router_client(lambda _: _json_response(200, {"status": "ready"})) as client: + assert await _probe_http_worker(client, address="10.0.0.1:8000") == { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", + } + + async def test_prefill_worker(self): + payload = { + "status": "ready", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + } + async with _router_client(lambda _: _json_response(200, payload)) as client: + assert await _probe_http_worker(client, address="10.0.0.1:8000") == { + "url": "http://10.0.0.1:8000", + "worker_type": "prefill", + "connection_mode": "http", + "runtime_type": "sglang", + "bootstrap_port": 8998, + } + + async def test_decode_worker(self): + payload = {"status": "ready", "disaggregation_mode": "decode"} + async with _router_client(lambda _: _json_response(200, payload)) as client: + worker = await _probe_http_worker(client, address="10.0.0.1:8000") + assert worker is not None + assert worker["worker_type"] == "decode" + + async def test_returns_none_when_not_ready(self): + payload = {"status": "starting"} + async with _router_client(lambda _: _json_response(200, payload)) as client: + assert await _probe_http_worker(client, address="10.0.0.1:8000") is None + + async def test_returns_none_on_request_error(self, caplog: pytest.LogCaptureFixture): + def handler(_): + # A gRPC-only worker, or one that has not bound its port yet. + raise httpx.ReadError("") + + caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) + async with _router_client(handler) as client: + assert await _probe_http_worker(client, address="10.0.0.1:8000") is None + # Repeats every sync until the worker is up, so it must not be logged as an error, + # see https://github.com/dstackai/dstack/issues/4300. + assert "Could not fetch server_info for worker http://10.0.0.1:8000" in caplog.text + assert not [r for r in caplog.records if r.levelno > logging.DEBUG] + + @contextmanager -def _fake_vllm_grpc_proto(*, server_info: MagicMock): +def _fake_vllm_grpc_proto(*, server_info=None, error: Optional[Exception] = None): stub = MagicMock() - stub.GetServerInfo = AsyncMock(return_value=server_info) + stub.GetServerInfo = AsyncMock(return_value=server_info, side_effect=error) pb2 = MagicMock(GetServerInfoRequest=MagicMock(return_value="req")) pb2_grpc = MagicMock(VllmEngineStub=MagicMock(return_value=stub)) with ( @@ -96,9 +223,9 @@ def _fake_vllm_grpc_proto(*, server_info: MagicMock): @contextmanager -def _fake_sglang_grpc_proto(*, server_info: MagicMock): +def _fake_sglang_grpc_proto(*, server_info=None, error: Optional[Exception] = None): stub = MagicMock() - stub.GetServerInfo = AsyncMock(return_value=server_info) + stub.GetServerInfo = AsyncMock(return_value=server_info, side_effect=error) pb2 = MagicMock(GetServerInfoRequest=MagicMock(return_value="req")) pb2_grpc = MagicMock(SglangSchedulerStub=MagicMock(return_value=stub)) with ( @@ -114,195 +241,232 @@ def _fake_sglang_grpc_proto(*, server_info: MagicMock): yield -@pytest.mark.asyncio -async def test_get_grpc_worker_ready(): - job = MagicMock() - channel = MagicMock() - - @asynccontextmanager - async def _fake_grpc_client(_job): - yield channel - - server_info = MagicMock(kv_role="kv_producer", kv_connector="NixlConnector") - - with ( - _fake_vllm_grpc_proto(server_info=server_info), - patch( - "dstack._internal.server.services.runs.router_worker_sync.get_service_replica_grpc_client", - _fake_grpc_client, - ), - ): - result = await _get_grpc_worker( - job, - worker_url="grpc://10.0.0.1:50051", - runtime_type="vllm", - ) - - assert result["status"] == "ready" - assert result["worker"] == { - "url": "grpc://10.0.0.1:50051", - "worker_type": "prefill", - "connection_mode": "grpc", - "runtime_type": "vllm", - "kv_connector": "NixlConnector", - "kv_role": "kv_producer", - } - - -@pytest.mark.asyncio -async def test_get_grpc_worker_not_ready_on_error(): - job = MagicMock() - - @asynccontextmanager - async def _failing_client(_job): - raise OSError("ssh failed") - yield # pragma: no cover - - with patch( - "dstack._internal.server.services.runs.router_worker_sync.get_service_replica_grpc_client", - _failing_client, - ): - result = await _get_grpc_worker(job, worker_url="grpc://10.0.0.1:50051") - - assert result == {"status": "not_ready", "worker": None} +def _rpc_error(code: grpc.StatusCode) -> grpc.aio.AioRpcError: + return grpc.aio.AioRpcError(code, grpc.aio.Metadata(), grpc.aio.Metadata(), details=code.name) @pytest.mark.asyncio -async def test_get_grpc_worker_sglang_bootstrap(): - job = MagicMock() - channel = MagicMock() - sglang_server_info = MagicMock(server_args=MagicMock()) - - @asynccontextmanager - async def _fake_grpc_client(_job): - yield channel - - with ( - _fake_sglang_grpc_proto(server_info=sglang_server_info), - patch( - "dstack._internal.server.services.runs.router_worker_sync.MessageToDict", - return_value={ - "disaggregation_mode": "prefill", - "disaggregation_bootstrap_port": 8998, - }, - ), - patch( - "dstack._internal.server.services.runs.router_worker_sync" - ".get_service_replica_grpc_client", - _fake_grpc_client, - ), - ): - result = await _get_grpc_worker(job, worker_url="grpc://10.0.0.1:8000") - - assert result["status"] == "ready" - assert result["worker"] == { - "url": "grpc://10.0.0.1:8000", - "worker_type": "prefill", - "connection_mode": "grpc", - "runtime_type": "sglang", - "bootstrap_port": 8998, - } - - -@pytest.mark.asyncio -async def test_get_worker_grpc_preference_skips_http(): - job = MagicMock() - grpc_not_ready = {"status": "not_ready", "worker": None} - - with ( - patch( - "dstack._internal.server.services.runs.router_worker_sync._get_grpc_worker", - new_callable=AsyncMock, - return_value=grpc_not_ready, - ) as grpc_mock, - patch( - "dstack._internal.server.services.runs.router_worker_sync._get_http_worker", - new_callable=AsyncMock, - ) as http_mock, - ): - result = await _get_worker( - job, - http_worker_url="http://10.0.0.1:8000", - grpc_worker_url="grpc://10.0.0.1:8000", - connection_mode="grpc", - ) - - assert result == grpc_not_ready - grpc_mock.assert_awaited_once() - http_mock.assert_not_awaited() - +class TestProbeGrpcWorker: + async def test_known_runtime_type(self): + server_info = MagicMock(kv_role="kv_producer", kv_connector="NixlConnector") + with _fake_vllm_grpc_proto(server_info=server_info): + worker = await _probe_grpc_worker( + MagicMock(), address="10.0.0.1:50051", runtime_type="vllm" + ) + assert worker == { + "url": "grpc://10.0.0.1:50051", + "worker_type": "prefill", + "connection_mode": "grpc", + "runtime_type": "vllm", + "kv_connector": "NixlConnector", + "kv_role": "kv_producer", + } -@pytest.mark.asyncio -async def test_get_worker_bootstrap_uses_single_tunnel(): - job = MagicMock() - uds_path = Path("/tmp/replica.sock") - grpc_ready: dict = { - "status": "ready", - "worker": { + async def test_bootstrap_tries_sglang_first(self): + with ( + _fake_sglang_grpc_proto(server_info=MagicMock(server_args=MagicMock())), + patch( + "dstack._internal.server.services.runs.router_worker_sync.MessageToDict", + return_value={ + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + }, + ), + ): + worker = await _probe_grpc_worker(MagicMock(), address="10.0.0.1:8000") + assert worker == { "url": "grpc://10.0.0.1:8000", "worker_type": "prefill", "connection_mode": "grpc", - "runtime_type": "vllm", - }, - } + "runtime_type": "sglang", + "bootstrap_port": 8998, + } + + async def test_bootstrap_falls_back_to_vllm(self): + # A vLLM worker does not implement the SGLang scheduler service. + with ( + _fake_sglang_grpc_proto(error=_rpc_error(grpc.StatusCode.UNIMPLEMENTED)), + _fake_vllm_grpc_proto( + server_info=MagicMock(kv_role="kv_consumer", kv_connector="NixlConnector") + ), + ): + worker = await _probe_grpc_worker(MagicMock(), address="10.0.0.1:8000") + assert worker is not None + assert worker["runtime_type"] == "vllm" + assert worker["worker_type"] == "decode" + + @pytest.mark.parametrize( + "code", + [ + # No listener yet, or an SSH tunnel to a dead remote port, or an HTTP-only worker. + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.UNIMPLEMENTED, + ], + ) + async def test_returns_none_on_expected_error(self, code: grpc.StatusCode): + with _fake_vllm_grpc_proto(error=_rpc_error(code)): + worker = await _probe_grpc_worker( + MagicMock(), address="10.0.0.1:8000", runtime_type="vllm" + ) + assert worker is None + + async def test_reraises_unexpected_error(self): + error = _rpc_error(grpc.StatusCode.PERMISSION_DENIED) + with _fake_vllm_grpc_proto(error=error), pytest.raises(grpc.aio.AioRpcError) as exc_info: + await _probe_grpc_worker(MagicMock(), address="10.0.0.1:8000", runtime_type="vllm") + assert exc_info.value is error + + async def test_returns_none_when_no_runtime_type_matches(self): + with ( + _fake_sglang_grpc_proto(error=_rpc_error(grpc.StatusCode.UNAVAILABLE)), + _fake_vllm_grpc_proto(error=_rpc_error(grpc.StatusCode.UNAVAILABLE)), + ): + worker = await _probe_grpc_worker(MagicMock(), address="10.0.0.1:8000") + assert worker is None + + +_HTTP_WORKER = { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", +} +_GRPC_WORKER = { + "url": "grpc://10.0.0.1:8000", + "worker_type": "prefill", + "connection_mode": "grpc", + "runtime_type": "vllm", +} - tunnel_cm = AsyncMock() - tunnel_cm.__aenter__.return_value = uds_path - tunnel_cm.__aexit__.return_value = None - http_cm = AsyncMock() - http_cm.__aenter__.return_value = MagicMock() - http_cm.__aexit__.return_value = None +def _async_cm(enter_value=None, enter_error: Optional[Exception] = None) -> AsyncMock: + cm = AsyncMock() + cm.__aenter__ = AsyncMock(return_value=enter_value, side_effect=enter_error) + cm.__aexit__ = AsyncMock(return_value=False) + return cm - grpc_cm = AsyncMock() - grpc_cm.__aenter__.return_value = MagicMock() - grpc_cm.__aexit__.return_value = None +@contextmanager +def _fake_replica_transports( + *, + uds_path: Path = Path("/tmp/replica.sock"), + tunnel_error: Optional[Exception] = None, + http_worker=None, + grpc_worker=None, +): + """Patch the tunnel and both probes, leaving `_get_worker`'s own logic intact.""" + mocks = MagicMock() with ( patch( "dstack._internal.server.services.runs.router_worker_sync.get_service_replica_tunnel", - return_value=tunnel_cm, - ) as tunnel_mock, + return_value=_async_cm(uds_path, tunnel_error), + ) as mocks.tunnel, patch( "dstack._internal.server.services.runs.router_worker_sync" ".get_service_replica_http_client_over_uds", - return_value=http_cm, - ) as http_over_uds_mock, + return_value=_async_cm(MagicMock()), + ) as mocks.http_client, patch( "dstack._internal.server.services.runs.router_worker_sync" ".get_service_replica_grpc_channel_over_uds", - return_value=grpc_cm, - ) as grpc_over_uds_mock, + return_value=_async_cm(MagicMock()), + ) as mocks.grpc_channel, patch( "dstack._internal.server.services.runs.router_worker_sync._probe_http_worker", new_callable=AsyncMock, - return_value={"status": "not_ready", "worker": None}, - ) as http_probe_mock, + return_value=http_worker, + ) as mocks.http_probe, patch( "dstack._internal.server.services.runs.router_worker_sync._probe_grpc_worker", new_callable=AsyncMock, - return_value=grpc_ready, - ) as grpc_probe_mock, - patch( - "dstack._internal.server.services.runs.router_worker_sync._get_http_worker", - new_callable=AsyncMock, - ) as get_http_mock, - patch( - "dstack._internal.server.services.runs.router_worker_sync._get_grpc_worker", - new_callable=AsyncMock, - ) as get_grpc_mock, + return_value=grpc_worker, + ) as mocks.grpc_probe, + ): + yield mocks + + +@pytest.mark.asyncio +class TestGetWorker: + async def test_connection_mode_grpc_skips_http(self): + with _fake_replica_transports(grpc_worker=_GRPC_WORKER) as mocks: + worker = await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + connection_mode="grpc", + ) + assert worker == _GRPC_WORKER + mocks.grpc_probe.assert_awaited_once() + mocks.http_probe.assert_not_awaited() + + async def test_connection_mode_http_skips_grpc(self): + with _fake_replica_transports(http_worker=_HTTP_WORKER) as mocks: + worker = await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + connection_mode="http", + ) + assert worker == _HTTP_WORKER + mocks.http_probe.assert_awaited_once() + mocks.grpc_probe.assert_not_awaited() + + async def test_bootstrap_probes_http_first(self): + # An HTTP worker must not pay for two gRPC `GetServerInfo` timeouts first. + with _fake_replica_transports(http_worker=_HTTP_WORKER, grpc_worker=_GRPC_WORKER) as mocks: + worker = await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + ) + assert worker == _HTTP_WORKER + mocks.http_probe.assert_awaited_once() + mocks.grpc_probe.assert_not_awaited() + + async def test_bootstrap_falls_back_to_grpc_over_one_tunnel(self): + job = MagicMock() + uds_path = Path("/tmp/replica.sock") + with _fake_replica_transports(uds_path=uds_path, grpc_worker=_GRPC_WORKER) as mocks: + worker = await _get_worker( + job, + address="10.0.0.1:8000", + ) + assert worker == _GRPC_WORKER + mocks.tunnel.assert_called_once_with(job) + mocks.http_client.assert_called_once_with(uds_path) + mocks.grpc_channel.assert_called_once_with(uds_path) + mocks.http_probe.assert_awaited_once() + mocks.grpc_probe.assert_awaited_once() + + async def test_returns_none_when_no_mode_reports_ready(self): + with _fake_replica_transports() as mocks: + worker = await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + ) + assert worker is None + mocks.http_probe.assert_awaited_once() + mocks.grpc_probe.assert_awaited_once() + + async def test_unreachable_worker_is_skipped_not_raised( + self, caplog: pytest.LogCaptureFixture ): - result = await _get_worker( - job, - http_worker_url="http://10.0.0.1:8000", - grpc_worker_url="grpc://10.0.0.1:8000", - ) - - assert result == grpc_ready - tunnel_mock.assert_called_once_with(job) - http_over_uds_mock.assert_called_once_with(uds_path) - grpc_over_uds_mock.assert_called_once_with(uds_path) - http_probe_mock.assert_awaited_once() - grpc_probe_mock.assert_awaited_once() - get_http_mock.assert_not_awaited() - get_grpc_mock.assert_not_awaited() + # One dead replica must not abort the sync for the healthy ones. + caplog.set_level(level=logging.WARNING, logger=router_worker_sync.__name__) + ssh_error = SSHError("connection refused") + with _fake_replica_transports(tunnel_error=ssh_error) as mocks: + worker = await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + ) + assert worker is None + mocks.http_probe.assert_not_awaited() + mocks.grpc_probe.assert_not_awaited() + assert f"failed to connect to worker replica: {ssh_error!r}" in caplog.text + + async def test_unexpected_tunnel_error_propagates(self): + # Only `SSHError` means "unreachable"; anything else is a bug and must not be swallowed. + with _fake_replica_transports(tunnel_error=RuntimeError("boom")): + with pytest.raises(RuntimeError, match="boom"): + await _get_worker( + MagicMock(), + address="10.0.0.1:8000", + ) From e3705ce05eed72aa2c27820da839caa7c76982f5 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:08:20 +0200 Subject: [PATCH 601/640] Clarify Kubernetes volume context selection (#4311) --- mkdocs/docs/concepts/volumes.md | 9 +++++++-- mkdocs/docs/reference/dstack.yml/volume.md | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/concepts/volumes.md b/mkdocs/docs/concepts/volumes.md index da0c0254c4..19be81952d 100644 --- a/mkdocs/docs/concepts/volumes.md +++ b/mkdocs/docs/concepts/volumes.md @@ -43,13 +43,17 @@ size: 100GB If you use this configuration, `dstack` will create a new volume based on the specified options. ??? info "Kubernetes" - With the `kubernetes` backend, you don't have to specify `region`, but you can optionally specify `storage_class_name` and/or `access_modes`: + Set `region` to a kubeconfig context name enabled in the [backend configuration](backends.md#kubernetes). + Omit it only for legacy configurations without `contexts`. + + You can optionally specify `storage_class_name` and/or `access_modes`:
```yaml type: volume backend: kubernetes + region: gpu-cluster-a name: my-volume size: 100GB @@ -110,13 +114,14 @@ If you register an existing volume, you must ensure the volume already has a fil ??? info "Kubernetes" - With the `kubernetes` backend, to reuse an existing `PersistentVolumeClaim`, specify its name in `claim_name`: + To reuse an existing `PersistentVolumeClaim`, specify `claim_name` and `region`.
```yaml type: volume backend: kubernetes + region: gpu-cluster-a name: my-volume claim_name: existing-pvc diff --git a/mkdocs/docs/reference/dstack.yml/volume.md b/mkdocs/docs/reference/dstack.yml/volume.md index d3f851c8c0..23eccccd11 100644 --- a/mkdocs/docs/reference/dstack.yml/volume.md +++ b/mkdocs/docs/reference/dstack.yml/volume.md @@ -30,11 +30,14 @@ The `volume` configuration type allows creating, registering, and updating [volu Kubernetes backend volumes are mapped to [`PersistentVolumeClaim`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) objects. + With `contexts` configured in the [backend](../../concepts/backends.md#kubernetes), set `region` to a kubeconfig context name. + To create a new claim, specify `size` and optionally `storage_class_name` and/or `access_modes`: ```yaml type: volume backend: kubernetes + region: gpu-cluster-a name: new-volume size: 100GB # By default, storage_class_name is not set, and the decision is delegated to @@ -51,6 +54,7 @@ The `volume` configuration type allows creating, registering, and updating [volu ```yaml type: volume backend: kubernetes + region: gpu-cluster-a name: existing-volume claim_name: existing-pvc ``` From fdfce0afed3158c55ec77cbd1d6f5577e2910ab7 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:02:22 +0200 Subject: [PATCH 602/640] [Website] Updated Slurm info on landing (#4312) --- mkdocs/overrides/header-2.html | 8 +- website/public/static/logos/crusoe.svg | 5 + website/public/static/logos/slurm.svg | 5 + .../src/components/ArchitectureDiagram.tsx | 82 +++--- website/src/components/Capabilities.tsx | 79 ++++++ website/src/components/ComputeSourcesTabs.tsx | 5 +- website/src/components/SiteNavigation.tsx | 10 +- website/src/pages/Factory/CapacityTabs.tsx | 27 +- .../pages/Factory/FactoryGetStartedTabs.tsx | 38 +++ website/src/pages/Factory/InferenceTabs.tsx | 18 +- .../src/pages/Factory/OrchestrationTabs.tsx | 2 +- website/src/pages/Factory/TenantTabs.tsx | 40 +-- website/src/pages/Factory/capabilities.tsx | 69 +++++ website/src/pages/Home/ExploreSection.tsx | 10 +- website/src/pages/Home/FaqSection.tsx | 38 ++- website/src/pages/Home/GetStartedSection.tsx | 263 ++++++++---------- website/src/pages/Sky/MarketplaceTabs.tsx | 12 +- website/src/styles.css | 156 +++++++---- 18 files changed, 542 insertions(+), 325 deletions(-) create mode 100644 website/public/static/logos/crusoe.svg create mode 100644 website/public/static/logos/slurm.svg create mode 100644 website/src/components/Capabilities.tsx create mode 100644 website/src/pages/Factory/FactoryGetStartedTabs.tsx create mode 100644 website/src/pages/Factory/capabilities.tsx diff --git a/mkdocs/overrides/header-2.html b/mkdocs/overrides/header-2.html index 86a97c4b11..5b3183284a 100644 --- a/mkdocs/overrides/header-2.html +++ b/mkdocs/overrides/header-2.html @@ -74,7 +74,7 @@ {% include "partials/search.html" %} {% endif %} - {# Keep product descriptions in sync with SiteNavigation.tsx and GetStartedSection.tsx. #} + {# Keep menu descriptions in sync with SiteNavigation.tsx. GetStartedSection.tsx uses a longer dstack description. #}
diff --git a/website/public/static/logos/crusoe.svg b/website/public/static/logos/crusoe.svg new file mode 100644 index 0000000000..6de594c0c1 --- /dev/null +++ b/website/public/static/logos/crusoe.svg @@ -0,0 +1,5 @@ + + + + diff --git a/website/public/static/logos/slurm.svg b/website/public/static/logos/slurm.svg new file mode 100644 index 0000000000..5d4de19ea9 --- /dev/null +++ b/website/public/static/logos/slurm.svg @@ -0,0 +1,5 @@ + + + + diff --git a/website/src/components/ArchitectureDiagram.tsx b/website/src/components/ArchitectureDiagram.tsx index c8d4ab118f..4bb2082d1f 100644 --- a/website/src/components/ArchitectureDiagram.tsx +++ b/website/src/components/ArchitectureDiagram.tsx @@ -1,4 +1,5 @@ import { asset } from '../asset'; +import { docsUrl } from '../routes'; // Layered "vendor-agnostic" architecture diagram, rebuilt as HTML/CSS (replaces the previous // static SVG). Logos are recolored to the current text color via CSS masking (see .arch-logo in @@ -32,16 +33,28 @@ const MODELS: Logo[] = [ const DOCKER: Logo = { key: 'docker', label: 'Docker', src: logoSrc('docker.svg') }; -const GPU_CLOUDS: Logo[] = [ - { key: 'aws', label: 'AWS', src: logoSrc('aws.svg') }, - { key: 'gcp', label: 'Google Cloud', src: logoSrc('gcp.svg') }, - { key: 'lambda', label: 'Lambda', src: logoSrc('lambda.svg') }, +const ORCHESTRATION = [ + { label: 'Fleets', href: docsUrl('concepts/fleets/') }, + { label: 'Tasks', href: docsUrl('concepts/tasks/') }, + { label: 'Services', href: docsUrl('concepts/services/') }, + { label: 'Presets', href: docsUrl('concepts/presets/') }, + { label: 'Gateways', href: docsUrl('concepts/gateways/') }, +]; + +const PLATFORMS = [ + { label: 'VMs', href: docsUrl('concepts/fleets/#ssh-fleets') }, + { label: 'Bare-metal', href: docsUrl('concepts/fleets/#ssh-fleets') }, + { label: 'K8S', href: docsUrl('concepts/backends/#kubernetes') }, + { label: 'Slurm', href: docsUrl('concepts/backends/#slurm') }, +]; + +const CLOUDS: Logo[] = [ { key: 'nebius', label: 'Nebius', src: logoSrc('nebius.svg') }, + { key: 'crusoe', label: 'Crusoe', src: logoSrc('crusoe.svg') }, { key: 'runpod', label: 'RunPod', src: logoSrc('runpod.svg') }, + { key: 'aws', label: 'AWS', src: logoSrc('aws.svg') }, ]; -const KUBERNETES: Logo = { key: 'kubernetes', label: 'Kubernetes', src: logoSrc('kubernetes.svg') }; - const HARDWARE: Logo[] = [ { key: 'nvidia', label: 'NVIDIA', src: logoSrc('nvidia.svg') }, { key: 'amd', label: 'AMD', src: logoSrc('amd.webp') }, @@ -80,7 +93,7 @@ function LogoRow({ logos }: { logos: Logo[] }) { export function ArchitectureDiagram() { return (
-
+
{/* Top: what plugs in on top of the orchestration layer. */}
@@ -93,38 +106,41 @@ export function ArchitectureDiagram() {
- {/* Middle: the orchestration layer itself */} -
-
- dstack orchestration - +
+ {/* The orchestration layer and the infrastructure it runs on. */} +
+
+ dstack orchestration + +
+
+ {ORCHESTRATION.map(({ label, href }) => ( + + {label} + + ))} +
-
- {['Fleets', 'Tasks', 'Services', 'Presets', 'Gateways'].map(name => ( -
- {name} -
- ))} -
-
- {/* Bottom: where workloads run */} -
-
- - Clouds -
-
- - Kubernetes -
-
- VMs or bare-metal +
+ + Any cloud + + + {PLATFORMS.map(({ label, href }) => ( + + {label} + + ))}
- Any hardware + Any accelerator
diff --git a/website/src/components/Capabilities.tsx b/website/src/components/Capabilities.tsx new file mode 100644 index 0000000000..280ee1d74c --- /dev/null +++ b/website/src/components/Capabilities.tsx @@ -0,0 +1,79 @@ +import { ReactNode } from 'react'; +import { asset } from '../asset'; + +// Icon, title, and description rows shared by the landing and product pages. +export function CapList({ items }: { items: { icon: ReactNode; title: string; sub: string }[] }) { + return ( +
    + {items.map(cap => ( +
  • + {cap.icon} + + {cap.title} + {cap.sub} + +
    • + ))} +
+ ); +} + +export const BoxGlyph = () => ( + +); +export const CloudGlyph = () => ( + +); +export const LayersGlyph = () => ( + +); +// GPU chip glyph for the marketplace rows. +export const ChipGlyph = () => ( + +); +// On-prem glyphs (shared with the landing's "bring your own compute" block). +export const SlurmGlyph = () => ( + +); +export const ServerGlyph = () => ( + +); +// The Kubernetes helm wheel from /static/logos/kubernetes.svg, inlined with currentColor so it +// takes the capability-icon color in both themes (the asset has hardcoded black fills). +export const KubernetesGlyph = () => ( + +); diff --git a/website/src/components/ComputeSourcesTabs.tsx b/website/src/components/ComputeSourcesTabs.tsx index 7723eb449b..41b1fd7b11 100644 --- a/website/src/components/ComputeSourcesTabs.tsx +++ b/website/src/components/ComputeSourcesTabs.tsx @@ -1,4 +1,4 @@ -import { CapList, CloudGlyph, KubernetesGlyph, ServerGlyph } from '../pages/Home/GetStartedSection'; +import { CapList, CloudGlyph, KubernetesGlyph, ServerGlyph, SlurmGlyph } from './Capabilities'; import { ContentTabs } from './ContentTabs'; const cloudGroups = [ @@ -10,6 +10,7 @@ const cloudGroups = [ const onPremItems = [ { icon: , title: 'SSH fleets', sub: 'Connect VMs or bare-metal clusters over SSH' }, { icon: , title: 'Kubernetes', sub: 'Connect your existing Kubernetes clusters' }, + { icon: , title: 'Slurm (experimental)', sub: 'Connect your existing Slurm clusters' }, ]; export function OnPremCapabilities() { @@ -48,7 +49,7 @@ export function ComputeSourcesTabs({ cloudsFirst = false }: { content: , footer: ( - Use your existing Kubernetes clusters, VMs, or bare-metal clusters + Use your existing Kubernetes or Slurm clusters, VMs, or bare-metal ), }; diff --git a/website/src/components/SiteNavigation.tsx b/website/src/components/SiteNavigation.tsx index 796ae7600b..ceefad16ad 100644 --- a/website/src/components/SiteNavigation.tsx +++ b/website/src/components/SiteNavigation.tsx @@ -50,12 +50,12 @@ type ProductLink = { // The products. products[0] (open-source) is featured at the top of the "Products" menu; the rest // follow as rows. Reused by the standalone top-nav hover menu and the mobile nav's "Products" // section. -// NOTE: the descriptions are duplicated in GetStartedSection.tsx (the product list) and -// mkdocs/overrides/header-2.html — keep all three in sync. +// Keep menu descriptions in sync with mkdocs/overrides/header-2.html. +// GetStartedSection.tsx uses a longer dstack description. const products: ProductLink[] = [ - { id: 'open-source', text: 'dstack', secondaryText: 'The open-source control plane for AI-native orchestration.', href: docsUrl('installation'), icon: , badge: 'Self-hosted', external: true }, - { id: 'factory', text: 'dstack Factory', secondaryText: 'A heterogeneous orchestration stack for AI token factories.', href: asset(ROUTES.FACTORY), icon: , badge: 'Self-hosted' }, - { id: 'sky-product', text: 'dstack Sky', secondaryText: 'One account across GPU clouds. Better prices and unified billing.', href: asset(ROUTES.SKY), icon: , badge: 'Hosted by us' }, + { id: 'open-source', text: 'dstack', secondaryText: 'A unified orchestration interface across GPU clouds, Kubernetes, VMs, and bare-metal.', href: docsUrl('installation'), icon: , badge: 'Self-hosted' }, + { id: 'factory', text: 'dstack Factory', secondaryText: 'A multi-tenant orchestration stack for AI labs, data centers, and AI token factories.', href: asset(ROUTES.FACTORY), icon: , badge: 'Self-hosted' }, + { id: 'sky-product', text: 'dstack Sky', secondaryText: 'Unified access to GPU clouds. Better prices and one billing.', href: asset(ROUTES.SKY), icon: , badge: 'Hosted by us' }, ]; // Items for the mobile slide-out navigation. The blog categories are top-level links (mirroring diff --git a/website/src/pages/Factory/CapacityTabs.tsx b/website/src/pages/Factory/CapacityTabs.tsx index 10739e705e..b1079643d3 100644 --- a/website/src/pages/Factory/CapacityTabs.tsx +++ b/website/src/pages/Factory/CapacityTabs.tsx @@ -1,28 +1,7 @@ +import { CapList } from '../../components/Capabilities'; import { OnPremCapabilities } from '../../components/ComputeSourcesTabs'; import { ContentTabs } from '../../components/ContentTabs'; -import { CapList } from '../Home/GetStartedSection'; - -const observabilityItems = [ - { - icon: ( - - ), - title: 'Events', - sub: 'Track resource state changes and operations', - }, - { - icon: ( - - ), - title: 'Metrics', - sub: 'Monitor utilization, accelerator health, and network health', - }, -]; +import { eventsCapability, metricsCapability } from './capabilities'; export function CapacityTabs() { return ( @@ -38,7 +17,7 @@ export function CapacityTabs() { { id: 'observability', label: 'Observability', - content: , + content: , }, ]} /> diff --git a/website/src/pages/Factory/FactoryGetStartedTabs.tsx b/website/src/pages/Factory/FactoryGetStartedTabs.tsx new file mode 100644 index 0000000000..d28b4cb957 --- /dev/null +++ b/website/src/pages/Factory/FactoryGetStartedTabs.tsx @@ -0,0 +1,38 @@ +import { ReactNode } from 'react'; +import { CapList } from '../../components/Capabilities'; +import { ContentTabs } from '../../components/ContentTabs'; +import { + billingCapability, + eventsCapability, + exportsCapability, + registryCapability, +} from './capabilities'; + +export function FactoryGetStartedTabs({ footer }: { footer?: ReactNode }) { + return ( + , + footer, + }, + { + id: 'observability', + label: 'Observability', + content: , + footer, + }, + { + id: 'optimized-inference', + label: 'Serverless inference', + content: , + footer, + }, + ]} + /> + ); +} diff --git a/website/src/pages/Factory/InferenceTabs.tsx b/website/src/pages/Factory/InferenceTabs.tsx index 0a57fc5c58..860b347255 100644 --- a/website/src/pages/Factory/InferenceTabs.tsx +++ b/website/src/pages/Factory/InferenceTabs.tsx @@ -1,18 +1,6 @@ +import { CapList } from '../../components/Capabilities'; import { ContentTabs } from '../../components/ContentTabs'; -import { BoxGlyph, CapList, LayersGlyph } from '../Home/GetStartedSection'; - -const inferenceItems = [ - { - icon: , - title: 'Presets', - sub: 'Agent-based optimization and kernel generation', - }, - { - icon: , - title: 'Registry', - sub: 'Day-0 optimized presets for frontier open models', - }, -]; +import { presetsCapability, registryCapability } from './capabilities'; export function InferenceTabs() { return ( @@ -23,7 +11,7 @@ export function InferenceTabs() { { id: 'inference-optimization', label: 'Inference optimization', - content: , + content: , }, ]} /> diff --git a/website/src/pages/Factory/OrchestrationTabs.tsx b/website/src/pages/Factory/OrchestrationTabs.tsx index e8d8bec93e..1cd46aeeb1 100644 --- a/website/src/pages/Factory/OrchestrationTabs.tsx +++ b/website/src/pages/Factory/OrchestrationTabs.tsx @@ -1,5 +1,5 @@ +import { CapList, ServerGlyph } from '../../components/Capabilities'; import { ContentTabs } from '../../components/ContentTabs'; -import { CapList, ServerGlyph } from '../Home/GetStartedSection'; const orchestrationItems = [ { diff --git a/website/src/pages/Factory/TenantTabs.tsx b/website/src/pages/Factory/TenantTabs.tsx index 967b80cdce..09d421d05c 100644 --- a/website/src/pages/Factory/TenantTabs.tsx +++ b/website/src/pages/Factory/TenantTabs.tsx @@ -1,40 +1,6 @@ +import { CapList } from '../../components/Capabilities'; import { ContentTabs } from '../../components/ContentTabs'; -import { CapList } from '../Home/GetStartedSection'; - -const tenantItems = [ - { - icon: ( - - ), - title: 'Projects', - sub: 'Tenant isolation and resource access control', - }, - { - icon: ( - - ), - title: 'Exports', - sub: 'Share selected fleets and gateways across projects', - }, - { - icon: ( - - ), - title: 'Billing', - sub: 'Usage metering and automated customer billing', - }, -]; +import { billingCapability, exportsCapability, projectsCapability } from './capabilities'; export function TenantTabs() { return ( @@ -45,7 +11,7 @@ export function TenantTabs() { { id: 'usage-and-billing', label: 'Usage and billing', - content: , + content: , }, ]} /> diff --git a/website/src/pages/Factory/capabilities.tsx b/website/src/pages/Factory/capabilities.tsx new file mode 100644 index 0000000000..871f2ea464 --- /dev/null +++ b/website/src/pages/Factory/capabilities.tsx @@ -0,0 +1,69 @@ +import { BoxGlyph, LayersGlyph } from '../../components/Capabilities'; + +export const eventsCapability = { + icon: ( + + ), + title: 'Events', + sub: 'Audit user actions and resource lifecycle events across projects.', +}; + +export const metricsCapability = { + icon: ( + + ), + title: 'Metrics', + sub: 'Monitor utilization, accelerator health, and network health', +}; + +export const projectsCapability = { + icon: ( + + ), + title: 'Projects', + sub: 'Tenant isolation and resource access control', +}; + +export const exportsCapability = { + icon: ( + + ), + title: 'Exports', + sub: 'Share selected fleets and gateways across projects', +}; + +export const billingCapability = { + icon: ( + + ), + title: 'Billing', + sub: 'Compute and token metering and billing automation', +}; + +export const presetsCapability = { + icon: , + title: 'Presets', + sub: 'Agent-based optimization and kernel generation', +}; + +export const registryCapability = { + icon: , + title: 'Registry', + sub: 'Day-0 optimized presets for frontier open models', +}; diff --git a/website/src/pages/Home/ExploreSection.tsx b/website/src/pages/Home/ExploreSection.tsx index d49664ee1c..0bc57fb4a4 100644 --- a/website/src/pages/Home/ExploreSection.tsx +++ b/website/src/pages/Home/ExploreSection.tsx @@ -1,4 +1,5 @@ import { ReactNode } from 'react'; +import Link from '@cloudscape-design/components/link'; import { AlternatingDocBlock } from '../../components/AlternatingDocBlock'; import { ArchitectureDiagram } from '../../components/ArchitectureDiagram'; import { ComputeSourcesTabs } from '../../components/ComputeSourcesTabs'; @@ -23,7 +24,7 @@ export function ExploreSection() { dstack gives cloud tenants and data-center operators a unified control plane for managing compute and orchestrating AI workloads.

- It improves operational efficiency and removes vendor lock-in. No more hassle of building your own compute stack on top of Kubernetes or Slurm. + It improves operational efficiency and removes vendor lock-in. Use your existing infrastructure without building and maintaining your own AI compute stack. @@ -43,9 +44,10 @@ function BringComputeBlock() { imageFirst > Have bare-metal servers or VMs with SSH access? Point dstack to those hosts and provide SSH - credentials to create an SSH fleet. Have an existing Kubernetes cluster? Point dstack's - Kubernetes backend to the kubeconfig. dstack will schedule workloads on them alongside cloud - clusters. + credentials to create an SSH fleet. Have an existing Kubernetes or Slurm cluster? Connect it + through the Kubernetes backend or the {highlightTerms('experimental Slurm backend')}. + dstack provides a unified workload interface while Kubernetes or Slurm handles scheduling + within the cluster.

dstack natively integrates with the major GPU clouds and automates provisioning of clusters. diff --git a/website/src/pages/Home/FaqSection.tsx b/website/src/pages/Home/FaqSection.tsx index a51dc60c3c..c9aa7c7310 100644 --- a/website/src/pages/Home/FaqSection.tsx +++ b/website/src/pages/Home/FaqSection.tsx @@ -1,32 +1,58 @@ import { ReactNode, useState } from 'react'; import Button from '@cloudscape-design/components/button'; import ExpandableSection from '@cloudscape-design/components/expandable-section'; +import Link from '@cloudscape-design/components/link'; import SpaceBetween from '@cloudscape-design/components/space-between'; import { mainButtonStyle } from '../../cloudscape-theme'; import { AlternatingDocBlock } from '../../components/AlternatingDocBlock'; import { highlightTerms } from '../../components/highlightTerms'; +import { docsUrl } from '../../routes'; type FaqItem = { q: string; - a: string | string[]; + a: ReactNode; }; const faqItems: FaqItem[] = [ { - q: 'How does dstack differ from Slurm?', - a: 'Slurm is a battle-tested workload manager with decades of production use in HPC environments. dstack is a unified orchestration layer built for containerized AI workloads and heterogeneous AI compute. While both support batch jobs and distributed training, dstack also provides first-class primitives for compute management, inference, and observability, including native cloud provisioning.', + q: 'How does dstack compare to Slurm?', + a: [ + 'Slurm is a battle-tested workload manager used in HPC and large-scale AI training. dstack is a unified orchestration layer built for containerized AI workloads and heterogeneous AI compute, with first-class primitives for compute management, training, inference, and observability.', + 'dstack can be used instead of Slurm to orchestrate AI workloads directly on cloud GPUs, pre-provisioned VMs, or bare-metal. It can also run on top of existing Slurm clusters through the Slurm backend.', + ], }, { q: 'How does dstack compare to Kubernetes?', - a: 'Kubernetes is a general-purpose container orchestrator. dstack also orchestrates containers, but provides a lightweight, streamlined interface purpose-built for AI workloads, with first-class primitives for compute management, training, inference, and observability. It can use Kubernetes as a compute backend or work directly with cloud GPUs, pre-provisioned VMs, or bare-metal, helping cloud tenants and data-center operators improve utilization without building and maintaining their own AI orchestration stack.', + a: [ + 'Kubernetes is a general-purpose container orchestrator. dstack also orchestrates containers, but provides a lightweight, streamlined interface purpose-built for AI workloads, with first-class primitives for compute management, training, inference, and observability.', + 'dstack can be used instead of Kubernetes to orchestrate AI workloads directly on cloud GPUs, pre-provisioned VMs, or bare-metal. It can also run on top of existing Kubernetes clusters through the Kubernetes backend.', + ], }, { q: 'Can I use dstack with Kubernetes?', - a: 'Yes. Connect existing Kubernetes clusters through the Kubernetes backend, and dstack will schedule AI workloads on them alongside cloud GPUs, VMs, and bare-metal. Use the Kubernetes backend when your GPUs already run on Kubernetes or your team relies on its ecosystem and tooling. Otherwise, cloud backends are often simpler for cloud GPUs, and SSH fleets for pre-provisioned VMs or bare-metal.', + a: [ + <> + {highlightTerms('Yes. Connect existing Kubernetes clusters through the ')} + {highlightTerms('Kubernetes backend')} + {highlightTerms(', and dstack will schedule AI workloads on them alongside cloud GPUs, VMs, and bare-metal.')} + , + 'Use the Kubernetes backend when your GPUs already run on Kubernetes or your team relies on its ecosystem and tooling. Otherwise, cloud backends are often simpler for cloud GPUs, and SSH fleets for pre-provisioned VMs or bare-metal.', + ], + }, + { + q: 'Can I use dstack with Slurm?', + a: [ + <> + {highlightTerms('Yes. Connect existing Slurm clusters through the experimental ')} + {highlightTerms('Slurm backend')} + {highlightTerms('. dstack connects to each cluster’s login node over SSH and submits runs as Slurm jobs.')} + , + 'Slurm retains resource allocation and scheduling, while dstack provides the same interface for development, training, and inference used with other backends. The cluster must have Pyxis and enroot installed and configured.', + ], }, { q: 'When should I use dstack?', - a: 'Use dstack as a unified orchestration layer for cluster management, training, and inference, built for heterogeneous AI compute. It is designed for cloud tenants, data-center operators, and teams running their own AI token factory at planet scale.', + a: 'Use dstack as a unified orchestration layer for cluster management, training, and inference, built for heterogeneous AI compute. It is designed for cloud tenants, data-center operators, and AI token factories.', }, ]; diff --git a/website/src/pages/Home/GetStartedSection.tsx b/website/src/pages/Home/GetStartedSection.tsx index a07fb08e75..31ee42f1e9 100644 --- a/website/src/pages/Home/GetStartedSection.tsx +++ b/website/src/pages/Home/GetStartedSection.tsx @@ -1,164 +1,147 @@ -import { ReactNode, useState } from 'react'; +import { Fragment, useId, useState } from 'react'; import CodeView from '@cloudscape-design/code-view/code-view'; import shHighlight from '@cloudscape-design/code-view/highlight/sh'; import Button from '@cloudscape-design/components/button'; import { asset } from '../../asset'; import { mainButtonStyle } from '../../cloudscape-theme'; +import { BoxGlyph, CloudGlyph, LayersGlyph } from '../../components/Capabilities'; +import { ContentTabs } from '../../components/ContentTabs'; +import { highlightTerms } from '../../components/highlightTerms'; import { installMethods, maxInstallLines, padYamlToLines } from '../../data/snippets'; import { DOCS_URL, ROUTES, docsUrl } from '../../routes'; +import { FactoryGetStartedTabs } from '../Factory/FactoryGetStartedTabs'; +import { MarketplaceTabs } from '../Sky/MarketplaceTabs'; -export const BoxGlyph = () => ( - -); -export const CloudGlyph = () => ( - -); -export const LayersGlyph = () => ( - -); -// GPU chip glyph for the marketplace rows. -export const ChipGlyph = () => ( - -); -// On-prem glyphs (shared with the landing's "bring your own compute" block). -export const ServerGlyph = () => ( - -); -// The Kubernetes helm wheel from /static/logos/kubernetes.svg, inlined with currentColor so it -// takes the capability-icon color in both themes (the asset has hardcoded black fills). -export const KubernetesGlyph = () => ( - -); +// Product menus in SiteNavigation.tsx and mkdocs/overrides/header-2.html use a shorter +// dstack description. +const products = [ + { + id: 'dstack', + name: 'dstack', + description: 'A unified orchestration interface across GPU clouds, Kubernetes, VMs, and bare-metal clusters.', + group: 'Self-hosted', + icon: , + }, + { + id: 'factory', + name: 'dstack Factory', + description: 'A multi-tenant orchestration stack for AI labs, data centers, and AI token factories.', + group: 'Self-hosted', + icon: , + }, + { + id: 'sky', + name: 'dstack Sky', + description: 'Unified access to GPU clouds. Better prices and one billing.', + group: 'Hosted by us', + icon: , + }, +] as const; -// Read-only shell snippet. Line wrapping is left off so padded snippets stay -// equal height across tabs (see padYamlToLines). -function ShellCode({ content }: { content: string }) { - return ( -
- -
- ); -} +type ProductId = (typeof products)[number]['id']; -// Icon, title, and description rows shared by the Home and Sky pages. -export function CapList({ items }: { items: { icon: ReactNode; title: string; sub: string }[] }) { - return ( -
    - {items.map(cap => ( -
  • - {cap.icon} - - {cap.title} - {cap.sub} - -
    • - ))} -
- ); -} - -// NOTE: the three product descriptions below are duplicated in SiteNavigation.tsx (products) -// and mkdocs/overrides/header-2.html — keep all three in sync. export function GetStartedSection() { - const [method, setMethod] = useState<(typeof installMethods)[number]['id']>(installMethods[0].id); - - const activeInstall = installMethods.find(m => m.id === method) ?? installMethods[0]; + const [product, setProduct] = useState('dstack'); + const id = useId(); + const footer = ; return (

Get started

-
-
Self-hosted
- - - - dstack - The open-source control plane for AI-native orchestration. - - - - - - - dstack Factory - A heterogeneous orchestration stack for AI token factories. - - - -
Hosted by us
- - - - dstack Sky - One account across GPU clouds. Better prices and unified billing. - - +
+ {products.map((option, index) => ( + + {option.group !== products[index - 1]?.group && ( +
{option.group}
+ )} + + + ))}
- {/* Open-source: install-method tabs + read-only code + footer CTA bar. */} -
-
-
- {installMethods.map(m => ( - - ))} -
-
- -
-
- - -
-
+
+ {product === 'dstack' && ( + ({ + id: method.id, + label: method.label, + content: , + footer, + }))} + /> + )} + {product === 'factory' && } + {product === 'sky' && }
); } + +function ProductFooter({ product }: { product: ProductId }) { + if (product === 'dstack') { + return ( + <> + + {highlightTerms('Set up the dstack server and bring your own compute')} + +
+ + +
+ + ); + } + + const isSky = product === 'sky'; + return ( + <> + + {highlightTerms( + isSky + ? 'All in dstack Factory, plus on-demand GPUs from our marketplace' + : 'All in dstack, plus multi-tenancy, billing, and serverless', + )} + +
+ + +
+ + ); +} + +// Keep padded snippets the same height across installation tabs. +function ShellCode({ content }: { content: string }) { + return ( +
+ +
+ ); +} diff --git a/website/src/pages/Sky/MarketplaceTabs.tsx b/website/src/pages/Sky/MarketplaceTabs.tsx index 852d431bac..7153858c48 100644 --- a/website/src/pages/Sky/MarketplaceTabs.tsx +++ b/website/src/pages/Sky/MarketplaceTabs.tsx @@ -1,7 +1,8 @@ +import { ReactNode } from 'react'; import Button from '@cloudscape-design/components/button'; import { mainButtonStyle } from '../../cloudscape-theme'; +import { ChipGlyph, CloudGlyph } from '../../components/Capabilities'; import { ContentTabs } from '../../components/ContentTabs'; -import { ChipGlyph, CloudGlyph } from '../Home/GetStartedSection'; import { gpuPrices } from './pricing'; // Default marketplace clouds from dstack Sky's /api/backends/list_base_types. @@ -11,7 +12,10 @@ const partnerGroups = [ ['Verda', 'Nebius'], ]; -export function MarketplaceTabs() { +export function MarketplaceTabs({ marketplaceOnly = false, footer }: { + marketplaceOnly?: boolean; + footer?: ReactNode; +}) { return ( ), - footer: ( + footer: footer ?? ( One account and unified billing across GPU clouds @@ -86,7 +90,7 @@ export function MarketplaceTabs() { ), }, - ]} + ].filter(tab => !marketplaceOnly || tab.id === 'marketplace')} /> ); } diff --git a/website/src/styles.css b/website/src/styles.css index fd7e718a81..282b26eb3c 100644 --- a/website/src/styles.css +++ b/website/src/styles.css @@ -628,11 +628,21 @@ p { .arch-diagram { display: flex; flex-direction: column; - gap: 2.66cqw; /* 24u between the four layers */ + gap: 2.66cqw; /* 24u between the diagram blocks */ width: 100%; color: var(--cs-text); --arch-border-width: max(1px, 0.22cqw); /* placeholder badge stroke */ --arch-gradient: var(--cs-text); + --arch-label-padding: 2cqw; +} + +.arch-stack { + display: flex; + flex-direction: column; + overflow: hidden; + border: 0.5px dotted currentColor; + border-radius: 12px; + background: var(--cs-panel); } .arch-row { @@ -645,9 +655,29 @@ p { grid-template-columns: repeat(2, minmax(0, 1fr)); } -/* Keep the logo-heavy cloud block wide, followed by Kubernetes and VMs/bare-metal. */ +/* Give cloud examples the remaining space after the compact compute labels. */ .arch-row--compute { - grid-template-columns: minmax(0, 1fr) max-content max-content; + grid-template-columns: minmax(0, 1fr) repeat(4, max-content); + gap: 1.26cqw; + padding: 3.2cqw var(--arch-label-padding) 3.48cqw; +} + +.arch-row--compute .arch-cell { + justify-content: center; + gap: 0.55cqw; + min-height: calc(8cqw + 6px); + padding-inline: 1.5cqw; + border: 1px dotted currentColor; + border-radius: 12px; +} + +.arch-row--compute .arch-cell--cloud { + justify-content: space-between; + gap: 1.55cqw; +} + +.arch-cell--cloud .arch-logos { + flex-shrink: 0; } /* Pin both input labels left and their logo groups right. */ @@ -661,7 +691,7 @@ p { align-items: center; gap: 1.55cqw; min-height: 9.83cqw; /* 88.7u */ - padding: 0 2cqw; + padding: 0 var(--arch-label-padding); border: 1px solid currentColor; border-radius: 12px; } @@ -680,17 +710,6 @@ p { min-height: 9.83cqw; } -/* "Any GPU cloud": logos left, label pushed to the right edge. */ -.arch-cell--gpu { - justify-content: space-between; - gap: 0.9cqw; - padding-inline: 2.1cqw; -} - -.arch-cell--gpu .arch-logos { - gap: 0.8cqw; -} - .arch-cell--platform { justify-content: center; gap: 0.55cqw; @@ -699,20 +718,17 @@ p { .arch-cell--platform .arch-cell__label { font-size: 2.55cqw; + text-align: center; + white-space: normal; } -.arch-cell--platform .arch-logo--kubernetes { - width: 4.8cqw; - height: 4.8cqw; -} - -/* "Any hardware": label pinned left, logos centered across the full row. */ +/* "Any accelerator": label pinned left, logos centered across the full row. */ .arch-cell--hw { justify-content: center; } .arch-cell--hw > .arch-cell__label { position: absolute; - left: 2cqw; + left: var(--arch-label-padding); } .arch-cell__label { @@ -722,15 +738,14 @@ p { white-space: nowrap; } -/* Orchestration layer — gradient band. Title gets more room above the cells than below - (≈73 vs ≈31u); cells inset ≈5% each side. */ +/* Orchestration and infrastructure share an outer border and aligned label padding. */ .arch-orchestration { display: flex; flex-direction: column; align-items: center; gap: 3cqw; - padding: 3.2cqw 3.2cqw 3.48cqw; - border-radius: 1.6cqw; + padding: 3.2cqw var(--arch-label-padding) 3.48cqw; + border-radius: 0 0 12px 12px; background: var(--arch-gradient); color: var(--cs-bg); } @@ -766,7 +781,17 @@ p { font-size: 2.25cqw; font-weight: 500; line-height: 1.2; - border: 1px solid currentColor; + border: 1px dotted currentColor; +} + +.arch-diagram .arch-link { + color: inherit; + text-decoration: none !important; +} + +.arch-diagram .arch-link:focus-visible { + outline: 2px solid currentColor; + outline-offset: 2px; } .arch-logos { @@ -790,14 +815,12 @@ p { mask-size: contain; } -/* Per-logo height (cqw, so it scales) + aspect from each source's viewBox. Wide marks (AWS, - NVIDIA) are set shorter so they don't dominate; tall ones (PyTorch) a touch taller. */ -.arch-logo--aws { height: 4.05cqw; aspect-ratio: 304 / 182; } -.arch-logo--gcp { height: 4.55cqw; aspect-ratio: 256 / 206; } -.arch-logo--lambda { height: 4.9cqw; aspect-ratio: 1; } +/* Per-logo height (cqw, so it scales) + aspect from each source's viewBox. Wide marks (NVIDIA) + are set shorter so they don't dominate; tall ones (PyTorch) a touch taller. */ .arch-logo--nebius { height: 4.6cqw; aspect-ratio: 1; } -.arch-logo--runpod { height: 5.04cqw; aspect-ratio: 120 / 130; } -.arch-logo--kubernetes { height: 4.98cqw; aspect-ratio: 1; } +.arch-logo--crusoe { height: 4.8cqw; aspect-ratio: 1; } +.arch-logo--runpod { height: calc(5.04cqw - 1.5px); aspect-ratio: 120 / 130; } +.arch-logo--aws { height: calc(4.05cqw - 1.5px); aspect-ratio: 304 / 182; } .arch-logo--pytorch { height: 4.9cqw; aspect-ratio: 90.3 / 109.1; } .arch-logo--vllm { height: 4.8cqw; aspect-ratio: 1; } .arch-logo--sglang { height: 4.8cqw; aspect-ratio: 596 / 683; } @@ -805,7 +828,7 @@ p { .arch-logo--nvidia { height: 4cqw; aspect-ratio: 271.7 / 179.7; } .arch-logo--amd { height: 4.8cqw; margin-left: 2px; aspect-ratio: 1600 / 1408; } .arch-logo--tenstorrent { height: 4.8cqw; aspect-ratio: 170 / 169; } -.arch-logo--tpu { height: 4.2cqw; aspect-ratio: 256 / 206; } /* shares the GCP mark, sized down a touch */ +.arch-logo--tpu { height: calc(4.2cqw - 1px); aspect-ratio: 256 / 206; } /* shares the GCP mark, sized down a touch */ /* "Any model" marks (top row) — square icons; plus the Docker marker on the orchestration band. */ /* Heights tuned per-logo for similar optical weight, compensating for differing padding and visual density inside each source viewBox. */ @@ -839,26 +862,26 @@ p { color: var(--cs-muted); } -/* Safety net for very narrow containers (phones): stop shrinking, stack the rows, drop the +/* Safety net for narrow containers (phones): stop shrinking, stack the rows, drop the primitives to two columns, and switch to fixed legible type. */ -@container (max-width: 360px) { - .arch-diagram { gap: 12px; } - /* Two columns: the wide first cell ("Any framework" / "Any cloud") spans the full width on its - own row, and the two narrower cells pair up on the row below. */ +@container (max-width: 480px) { + .arch-diagram { gap: 12px; --arch-label-padding: 14px; } + .arch-stack { border-radius: 8px; } + /* Stack the inputs and cloud examples, then pair the other compute sources. */ .arch-row { grid-template-columns: 1fr 1fr; gap: 8px; } .arch-row > :first-child { grid-column: 1 / -1; } .arch-row--inputs > .arch-cell { grid-column: 1 / -1; } - .arch-row--compute { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .arch-row--compute { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; padding: 16px var(--arch-label-padding); } .arch-row--compute > :first-child { grid-column: 1 / -1; } - .arch-cell { min-height: 52px; padding: 10px 14px; gap: 12px; border-width: 1px; border-radius: 8px; } + .arch-row--compute .arch-cell { min-height: 54px; padding: 11px 8px; border-radius: 8px; } + .arch-cell { min-height: 52px; padding: 10px var(--arch-label-padding); gap: 12px; border-width: 1px; border-radius: 8px; } .arch-cell__label { font-size: 16px; white-space: normal; } .arch-cell--platform { min-height: 52px; gap: 6px; padding: 8px 4px; } .arch-cell--platform .arch-cell__label { font-size: 16px; } - .arch-cell--platform .arch-logo--kubernetes { width: 22px; height: 22px; } - /* "Any hardware": label left, logos pushed to the right (the desktop layout centers them). */ + /* "Any accelerator": label left, logos pushed to the right (the desktop layout centers them). */ .arch-cell--hw { justify-content: space-between; } .arch-cell--hw > .arch-cell__label { position: static; } - .arch-orchestration { gap: 12px; padding: 16px; border-radius: 12px; } + .arch-orchestration { gap: 12px; padding: 16px var(--arch-label-padding); border-radius: 0 0 8px 8px; } .arch-orchestration__title { font-size: 15px; } .arch-orchestration__cells { grid-template-columns: repeat(2, 1fr); gap: 8px; } .arch-orchestration__cells > :last-child { grid-column: 1 / -1; } @@ -866,6 +889,9 @@ p { .arch-logos { gap: 12px; } /* !important to beat the per-logo cqw heights. */ .arch-logo { height: 22px !important; } + .arch-logo--runpod, + .arch-logo--aws { height: 20.5px !important; } + .arch-logo--tpu { height: 21px !important; } .arch-logo--placeholder { height: 22px !important; min-width: 22px !important; font-size: 12px; border-width: 1px; border-radius: 5px; } } @@ -1025,7 +1051,7 @@ p { margin-top: 0; } -/* Product links beside the dstack installation controls. */ +/* Product selector and its matching preview. */ .gs-deploy { display: grid; grid-template-columns: 1fr 2fr; /* selector 1/3, detail 2/3 */ @@ -1057,12 +1083,14 @@ p { position: relative; display: block; width: 100%; + border: 0; border-radius: 8px; /* same radius as the tab chips */ background: var(--cs-panel); /* each row paints its own ground (the rail is transparent) */ color: var(--cs-text); font: inherit; line-height: 1.15; /* pinned (not inherited) so the rail renders identically on the docs site */ text-align: left; + cursor: pointer; } /* Adjacent rows inside a group square off their shared edge, so the group's ground reads as one @@ -1082,6 +1110,10 @@ p { background: var(--cs-bg); box-shadow: inset 0 0 0 0.5px var(--cs-text); } +.gs-rail button.gs-opt:focus-visible { + outline: 2px solid var(--cs-text); + outline-offset: -3px; +} /* Row names mirror the tab typography: 16px / 700. */ .gs-opt__name { @@ -1187,6 +1219,13 @@ p { .gs-detail { min-width: 0; } +.gs-detail .gs-tabs { + flex-shrink: 0; + overflow-x: auto; +} +.gs-detail .gs-tab { + white-space: nowrap; +} /* The detail box — shared shell for all three tabs: a bordered container with a fixed height, holding a tab strip, content, and a footer CTA bar. The height is tuned a touch taller than the rail, giving the content room to breathe and matching the tabbed code cards higher up the page; all three tabs reuse this shell, so they stay equal height. */ @@ -1266,6 +1305,25 @@ p { min-width: 0; /* allow the note to shrink/wrap so it shares the row with the CTA on narrow screens */ margin-inline-end: auto; /* note left, button(s) right — the footer itself is flex-end */ } +.gs-foot__actions { + display: flex; + flex: 0 0 auto; + gap: 14px; +} + +@media (max-width: 640px) { + .gs-detail .gs-boxfoot { + flex-wrap: wrap; + } + .gs-detail .gs-foot__note { + flex-basis: 100%; + } + .gs-foot__actions { + flex-wrap: wrap; + gap: 10px; + max-width: 100%; + } +} /* The footer CTA(s) keep their width (never shrink/wrap mid-word); the note takes the rest. */ .gs-boxfoot > [class*='awsui_button'] { @@ -1310,11 +1368,11 @@ p { } /* Open-source: read-only code fills the body. */ -.gs-codebody { +.gs-box--install .gs-skybody { flex: 1 1 auto; min-height: 0; padding: 10px 12px; - overflow: hidden; + overflow: auto; } /* dstack Sky: two panes (GPU marketplace | bring your own clouds). Source order is header→list, @@ -1497,8 +1555,6 @@ p { /* Separate the second pane (its header follows the first list). */ .gs-sky .gs-col + .gs-skyhalf { margin-top: 12px; } .gs-caps { grid-template-columns: 1fr; } - /* Read-only install code is wider than the phone; let it scroll instead of clipping. */ - .gs-codebody { overflow-x: auto; } /* Keep footer notes and buttons on one row. */ .gs-boxfoot { align-items: center; gap: 10px; } .gs-boxfoot [class*='awsui_button'] { padding-inline: 14px !important; } From 7160f6bc1a542172e92c55df50a897e56d68976b Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 25 Sep 2026 12:47:43 +0000 Subject: [PATCH 603/640] Set upper bound of SQLAlchemy version to < 2.1 (#4314) See: https://github.com/kvesteri/sqlalchemy-utils/issues/800 --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 961c9211a4..4c623306c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,7 +230,8 @@ server = [ "httpx>=0.28.0", "jinja2", "watchfiles", - "sqlalchemy[asyncio]>=2.0.0", + # sqlalchemy_utils is not compatible with SQLAlchemy >= 2.1.0 + "sqlalchemy[asyncio]>=2.0.0,<2.1", "sqlalchemy_utils>=0.40.0", "alembic>=1.16.0", "aiosqlite", From 229f535b6471945c7d827f0385d297b7f076b0cc Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 25 Sep 2026 13:01:22 +0000 Subject: [PATCH 604/640] Don't require probes to pass in router worker sync (#4313) The sync only picked up router and worker jobs with `JobModel.ready`, i.e. with all probes passed. A router probe that depends on registered workers, such as `/health_generate`, never passed, since workers are only registered once the router is picked up, so the service deadlocked. The sync now takes all running jobs, filtered by `job_num == 0` instead. Workers are registered once the sync's own check (`/server_info` or `GetServerInfo`) reports them ready, so `probes` can no longer delay worker registration. As probes are configured per service, not per replica group, such gating only worked when the router and workers share the transport and API. The `job_num == 0` condition is a no-op for now, as services don't support multi-node replicas. Previously, it was implied by `ready`, which is only set for the first node. Fixes: https://github.com/dstackai/dstack/issues/4310 Co-authored-by: Claude Opus 5.5 (1M context) --- .../service_router_worker_sync.py | 11 ++++- .../services/jobs/configurators/base.py | 8 ++-- .../services/runs/router_worker_sync.py | 2 +- .../test_service_router_worker_sync.py | 47 ++++++++++++++----- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py b/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py index ca42017571..f946688a16 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/service_router_worker_sync.py @@ -244,7 +244,16 @@ async def process(self, item: ServiceRouterWorkerSyncPipelineItem) -> None: selectinload( RunModel.jobs.and_( JobModel.status == JobStatus.RUNNING, - JobModel.ready == True, + # `JobModel.ready` is deliberately not checked. A router only + # passes probes that depend on workers (e.g. `/health_generate`) + # after workers are registered, so waiting for it would deadlock. + # Workers are registered as soon as the sync's own check + # (`/server_info` or `GetServerInfo`) reports them ready, so + # `probes` cannot delay registration. + # Only the first node of a replica is synced. Services don't + # support multi-node replicas yet, so this is a no-op kept for + # forward compatibility. + JobModel.job_num == 0, ) ) .load_only( diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index c2da607979..c4c5bfd602 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -519,11 +519,9 @@ def _probes(self) -> list[ProbeSpec]: return [_openai_model_probe_spec(model.name, model.prefix)] group = self._replica_group() if group is not None and group.router is not None: - # A router only answers chat completions once dstack has registered workers - # with it, and registration skips routers that are not ready yet. Probing - # chat completions here would deadlock: readiness would wait on registration - # while registration waits on readiness. Probe the router's own liveness - # endpoint instead, which does not depend on any worker. + # Probe the router's own liveness endpoint, which does not depend on any + # worker. Workers get no default probe: they may not serve HTTP at all + # (gRPC workers), and they don't receive traffic directly. return [_router_health_probe_spec()] return [] diff --git a/src/dstack/_internal/server/services/runs/router_worker_sync.py b/src/dstack/_internal/server/services/runs/router_worker_sync.py index 918711be1e..100597dc0f 100644 --- a/src/dstack/_internal/server/services/runs/router_worker_sync.py +++ b/src/dstack/_internal/server/services/runs/router_worker_sync.py @@ -566,7 +566,7 @@ async def sync_router_workers_for_run_model(run_model: RunModel) -> None: router_job = _get_router_job(run_model, router_group) if router_job is None: logger.debug( - "%s: no ready router job in group %s, skipping worker sync", + "%s: no running router job in group %s, skipping worker sync", fmt(run_model), router_group.name, ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py b/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py index 26633a8baf..98a4c3998b 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_service_router_worker_sync.py @@ -460,14 +460,12 @@ async def test_process_calls_sync_and_unlocks_on_success( assert sync_row.lock_owner is None assert sync_row.last_processed_at is not None - async def test_process_skips_sync_when_router_replica_not_ready( + async def test_process_syncs_running_first_node_jobs_regardless_of_probes( self, test_db, session: AsyncSession, worker: ServiceRouterWorkerSyncWorker, - caplog: pytest.LogCaptureFixture, ): - caplog.set_level(level=logging.DEBUG, logger=router_worker_sync.__name__) project = await create_project(session=session) user = await create_user(session=session) repo = await create_repo(session=session, project_id=project.id) @@ -480,8 +478,9 @@ async def test_process_skips_sync_when_router_replica_not_ready( run_spec=_router_service_run_spec(repo.name), ) instance = await create_instance(session=session, project=project) - # The router replica is still starting up, the worker replica is already serving. - await create_job( + # The router's probes may depend on registered workers, so it must be synced before + # they pass. Otherwise, neither the probes nor the sync could make progress. + router_job = await create_job( session=session, run=run, instance=instance, @@ -490,16 +489,36 @@ async def test_process_skips_sync_when_router_replica_not_ready( replica_group_name="router", job_provisioning_data=make_job_provisioning_data(), ) + worker_job = await create_job( + session=session, + run=run, + instance=instance, + status=JobStatus.RUNNING, + ready=False, + replica_num=1, + replica_group_name="worker", + job_provisioning_data=make_job_provisioning_data(), + ) + # Not the first node of the replica await create_job( session=session, run=run, instance=instance, status=JobStatus.RUNNING, - ready=True, + job_num=1, replica_num=1, replica_group_name="worker", job_provisioning_data=make_job_provisioning_data(), ) + # Not running + await create_job( + session=session, + run=run, + instance=instance, + status=JobStatus.PROVISIONING, + replica_num=2, + replica_group_name="worker", + ) sync_row = await _add_service_router_worker_sync_row(session, run.id) sync_row.lock_token = uuid.uuid4() sync_row.lock_expires_at = get_current_datetime() + timedelta(seconds=30) @@ -507,13 +526,17 @@ async def test_process_skips_sync_when_router_replica_not_ready( await session.commit() item = _sync_row_to_pipeline_item(sync_row) - await worker.process(item) + with patch( + "dstack._internal.server.background.pipeline_tasks.service_router_worker_sync" + ".sync_router_workers_for_run_model", + new_callable=AsyncMock, + ) as sync_mock: + await worker.process(item) - assert "no ready router job in group router, skipping worker sync" in caplog.text - # The run stays eligible for the next sync attempt. - await session.refresh(sync_row) - assert sync_row.deleted is False - assert sync_row.lock_token is None + sync_mock.assert_awaited_once() + assert sync_mock.await_args is not None + called_run = sync_mock.await_args.args[0] + assert {j.id for j in called_run.jobs} == {router_job.id, worker_job.id} async def test_process_logs_router_job_when_router_connection_fails( self, From 9204e15e2f7077e771331beb4f473feabe30c0f8 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:57:23 +0200 Subject: [PATCH 605/640] Polish landing pages (#4319) Co-authored-by: Andrey Cheptsov --- mkdocs/assets/stylesheets/cloudscape-docs.css | 73 ++++-- mkdocs/overrides/header-2.html | 76 +++++-- mkdocs/overrides/partials/tx-footer.html | 2 +- website/src/cloudscape-overrides.css | 15 +- website/src/components/Capabilities.tsx | 31 ++- website/src/components/ComputeSourcesTabs.tsx | 60 +++-- website/src/components/ConceptIcons.tsx | 124 ++++++++++ website/src/components/SiteFooter.tsx | 2 +- website/src/components/SiteNavigation.tsx | 109 ++++++--- website/src/components/ViewDocsButton.tsx | 39 ++++ .../pages/Factory/FactoryGetStartedTabs.tsx | 15 +- website/src/pages/Factory/FactoryPage.tsx | 6 +- website/src/pages/Factory/InferenceTabs.tsx | 2 +- .../src/pages/Factory/OrchestrationTabs.tsx | 21 +- website/src/pages/Factory/TenantTabs.tsx | 4 +- website/src/pages/Factory/capabilities.tsx | 42 ++-- website/src/pages/Home/ExploreSection.tsx | 50 +++-- website/src/pages/Home/FaqSection.tsx | 14 +- website/src/pages/Home/GetStartedSection.tsx | 7 +- website/src/pages/Home/HomePage.tsx | 35 ++- website/src/pages/Sky/SkyPage.tsx | 58 +++-- website/src/styles.css | 211 ++++++++++-------- website/vite.config.ts | 1 + 23 files changed, 685 insertions(+), 312 deletions(-) create mode 100644 website/src/components/ConceptIcons.tsx create mode 100644 website/src/components/ViewDocsButton.tsx diff --git a/mkdocs/assets/stylesheets/cloudscape-docs.css b/mkdocs/assets/stylesheets/cloudscape-docs.css index 3f4105ad1a..1513f3f434 100644 --- a/mkdocs/assets/stylesheets/cloudscape-docs.css +++ b/mkdocs/assets/stylesheets/cloudscape-docs.css @@ -141,6 +141,13 @@ } } +/* Both button groups share the header center; clear landing.css's legacy top padding. */ +.md-header__inner .cs-products-menu, +.md-header__inner .md-header__buttons { + align-self: center; + padding-block: 0; +} + .md-header__buttons .md-button { margin-top: 0; margin-bottom: 0; @@ -171,6 +178,7 @@ /* Header action buttons → /old pills. Prefixed with the primary color attr to out-specify landing.css's button rules. */ +.cs-products-menu__trigger, [data-md-color-primary=white] .md-header__buttons .md-button--primary, [data-md-color-primary=white] .md-header__buttons .md-button--primary:hover { box-sizing: border-box; @@ -239,6 +247,9 @@ /* Products menu matches website/src/styles.css. */ .cs-products-menu { + --products-arrow-gap: 10px; + --cs-hover: rgba(22, 25, 31, 0.03); + --products-focus-color: #006ce0; position: relative; display: inline-flex; align-items: center; @@ -246,28 +257,54 @@ font-size: 16px; line-height: 1.15; } +[data-md-color-scheme="slate"] .cs-products-menu { + --cs-hover: rgba(242, 243, 243, 0.05); + --products-focus-color: #42b4ff; +} .cs-products-menu__trigger { - display: inline-flex; - align-items: center; - gap: 4px; - min-height: 42px; - padding: 10px 0; - border: 0; - border-radius: 21px; + position: relative; + gap: var(--products-arrow-gap); + padding-inline: 18px var(--products-arrow-gap); + border: 1px solid var(--cs-border); background: transparent; color: var(--cs-text); - font: inherit; - font-weight: 700; + font-family: inherit; + letter-spacing: 0.005em; cursor: pointer; } -.cs-products-menu__trigger:hover { - text-decoration: underline; - text-underline-offset: 0.2em; +.cs-products-menu__trigger:hover, +.cs-products-menu__trigger:active { + background: var(--cs-hover); +} +.cs-products-menu__trigger:focus { + outline: none; } +/* Match the secondary Cloudscape button's keyboard-focus ring. */ .cs-products-menu__trigger:focus-visible { - background: var(--cs-panel); + outline: 2px dotted transparent; + outline-offset: 3px; +} +.cs-products-menu__trigger:focus-visible::before { + content: ''; + position: absolute; + inset-inline-start: -4px; + inset-block-start: -4px; + inline-size: calc(100% + 8px); + block-size: calc(100% + 8px); + border-radius: 4px; + box-shadow: 0 0 0 2px var(--products-focus-color); + pointer-events: none; +} +.cs-products-menu__icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 12px; + height: 12px; + flex: 0 0 auto; } .cs-products-menu__caret { + display: block; width: 12px; height: 12px; transition: transform 0.15s ease; @@ -276,6 +313,7 @@ transform: rotate(180deg); } .cs-products-menu__popup { + box-sizing: border-box; display: none; position: absolute; top: calc(100% + 6px); @@ -283,11 +321,16 @@ z-index: 1000; width: 360px; background: var(--cs-panel); + border: 1px solid var(--cs-border); border-radius: 8px; + overflow: hidden; } .cs-products-menu--open .cs-products-menu__popup { display: block; } +.cs-products-menu__popup .gs-rail { + border-radius: 7px; +} .cs-products-menu__popup .gs-opt + .gs-rail__group { margin-top: 0; } @@ -334,6 +377,10 @@ background: var(--cs-bg); box-shadow: inset 0 0 0 0.5px var(--cs-text); } +.cs-products-menu__popup .gs-opt:hover, +.cs-products-menu__popup .gs-opt:focus-visible { + box-shadow: none; +} .gs-opt--feat { padding: 13px 14px; gap: 14px; diff --git a/mkdocs/overrides/header-2.html b/mkdocs/overrides/header-2.html index 5b3183284a..341e28f10c 100644 --- a/mkdocs/overrides/header-2.html +++ b/mkdocs/overrides/header-2.html @@ -76,14 +76,16 @@ {# Keep menu descriptions in sync with SiteNavigation.tsx. GetStartedSection.tsx uses a longer dstack description. #}
- - diff --git a/website/src/pages/Sky/SkyPage.tsx b/website/src/pages/Sky/SkyPage.tsx index 0248322df1..51585b1eff 100644 --- a/website/src/pages/Sky/SkyPage.tsx +++ b/website/src/pages/Sky/SkyPage.tsx @@ -1,6 +1,6 @@ import Button from '@cloudscape-design/components/button'; import { useLayoutContext } from '../../App'; -import { heroButtonStyle } from '../../cloudscape-theme'; +import { heroButtonStyle, mainButtonStyle } from '../../cloudscape-theme'; import { AlternatingDocBlock } from '../../components/AlternatingDocBlock'; import { ComputeSourcesTabs } from '../../components/ComputeSourcesTabs'; import { highlightTerms } from '../../components/highlightTerms'; @@ -86,21 +86,49 @@ export function SkyPage() { - - + + + + +
+ } + items={[ + { + q: 'How is dstack Sky different from dstack?', + a: [ + 'With dstack Sky, we host and maintain the server for you. ' + + 'With open-source dstack, you host the server yourself.', + 'dstack Sky includes all open-source features and adds a built-in gateway with an HTTPS domain, advanced observability, and usage metering.', + 'dstack Sky also provides a GPU marketplace with unified billing. ' + + 'Both support your own cloud accounts and on-prem compute. ' + + 'The CLI and YAML configurations are the same.', + ], + }, + ]} + />
diff --git a/website/src/styles.css b/website/src/styles.css index 282b26eb3c..f82cf21f18 100644 --- a/website/src/styles.css +++ b/website/src/styles.css @@ -163,20 +163,25 @@ p { transform: translateX(3px); } -/* "Get started" CTA arrow — the banner's thin arrow + hover slide, reused on the primary CTAs - (top-nav and hero). Cloudscape's Button takes no className, so a wrapper span carries the - hover: hovering the button hovers the wrapper, which slides the arrow. */ +/* Compact buttons give the arrow a little extra room at the button edge. + Hero buttons supply their own symmetric padding around the whole label. */ .cta-with-arrow { + --cta-padding-end: 10px; display: inline-flex; + flex: 0 0 auto; } -.cta-arrow { +.cta-with-arrow__label { display: inline-flex; - vertical-align: middle; - margin-inline-start: 6px; - transition: transform 0.2s ease; + align-items: center; + gap: 6px; + white-space: nowrap; + vertical-align: top; } -.cta-with-arrow:hover .cta-arrow { - transform: translateX(3px); +.cta-with-arrow__icon { + display: block; + flex: 0 0 12px; + width: 12px; + height: 12px; } /* Banner + nav stick to the top together. Sticking the wrapper (rather than each child) @@ -227,8 +232,7 @@ p { padding: 14px 24px; } -.site-logo, -.site-menu-button { +.site-logo { border: 0; background: transparent; color: var(--cs-text); @@ -270,7 +274,6 @@ p { display: none; } -.site-menu-button, .site-menu-link { display: inline-flex; align-items: center; @@ -286,47 +289,46 @@ p { text-decoration: none; } -.site-menu-button:hover, .site-menu-link:hover { text-decoration: underline; text-underline-offset: 0.2em; } -/* Keyboard focus only on the top-menu buttons. These are our own elements (not Cloudscape +/* Keyboard focus only on the top-menu links. These are our own elements (not Cloudscape components), so use plain values rather than Cloudscape's internal token variables. */ -.site-menu-button:focus-visible, .site-menu-link:focus-visible { background: var(--cs-panel); color: var(--cs-text); } -.site-menu-button.active { - border: 2px solid #879596; -} - -/* "Products" top-nav hover menu. The trigger (.site-hover-menu__trigger) inherits the plain - text-link look from .site-menu-button; the popup opens on hover and is anchored to the - trigger. Continuity across the trigger→popup gap is handled by a close delay in JS - (hover-intent), so no visual bridge is needed here. */ -.site-hover-menu { +/* The Products secondary button anchors the click-to-open popup. */ +.site-products-dropdown { + --products-arrow-gap: 10px; position: relative; display: inline-flex; align-items: center; } +/* The text-to-caret gap and trailing button padding use the same value. */ +.site-products-dropdown__label { + display: inline-flex; + align-items: center; + gap: var(--products-arrow-gap); + vertical-align: top; +} /* Dropdown caret on the Products trigger — rotates 180° while the menu is open. */ -.site-hover-menu__caret { +.site-products-dropdown__caret { + display: block; width: 12px; height: 12px; opacity: 1; /* full text colour — the caret reads as black like the menu label */ transition: transform 0.15s ease; } -.site-hover-menu__trigger[aria-expanded='true'] .site-hover-menu__caret { +.site-products-dropdown[data-open='true'] .site-products-dropdown__caret { transform: rotate(180deg); } -/* "Products" popup: renders the exact same grouped switcher rail as the Get-started section - (.gs-rail — see that block below for the visuals). This wrapper only anchors and sizes it. */ +/* The Products popup frames the shared grouped rail with one outer border. */ .site-products-menu { position: absolute; top: calc(100% + 6px); @@ -336,7 +338,12 @@ p { /* A floating popup can't show a see-through hole between the groups (arbitrary page content sits behind it) — back the rail with the unselected-row color so the gap reads as ground. */ background: var(--cs-panel); + border: 1px solid var(--cs-border); border-radius: 8px; + overflow: hidden; +} +.site-products-menu .gs-rail { + border-radius: 7px; } /* And no hole at all in the popup: the groups sit flush, like one continuous menu. */ .site-products-menu .gs-opt + .gs-rail__group { @@ -386,7 +393,7 @@ p { font-weight: 600; } -/* Hero CTAs sit side by side, each sized to its own label (not forced to equal width). */ +/* Paired actions share equal columns; single actions keep their natural width. */ .home-hero__actions { margin-top: 28px; display: inline-flex; @@ -394,6 +401,14 @@ p { gap: 8px; } +.cta-pair { + display: inline-grid; + grid-auto-flow: column; + grid-auto-columns: minmax(0, 1fr); + gap: 8px; + max-width: 100%; +} + .home-hero__art { position: absolute; inset: 0; @@ -483,9 +498,8 @@ p { border-bottom: 0; } -/* 1px outer frame on the FAQ block with rounded (12px) corners. ExpandableSection - has no radius token, so we border + clip the wrapper; overflow:hidden trims the items' square - corners to the radius. Internal dividers between questions stay 0.5px (see cloudscape-overrides). */ +/* One outer frame for the FAQ; individual questions only draw internal dividers + (see cloudscape-overrides). Clip their backgrounds to the shared rounded corners. */ .faq-list { border: 1px solid var(--cs-border); border-radius: 12px; @@ -605,16 +619,6 @@ p { font-size: 17px; } -/* Concept-card descriptions: muted, 14px (a step down from the standard small text) since they - sit in a dense card grid. Higher specificity than `.doc-alternating p` so it wins (the cards - sit inside an alternating block). */ -.concept-grid .media-card p { - padding: 0; - font-size: 14px; - line-height: 1.5; - color: var(--cs-muted); -} - /* "Vendor-agnostic, open-source" architecture diagram (components/ArchitectureDiagram.tsx), rebuilt from a static SVG into HTML/CSS and sized 1:1 with it. Every dimension is in cqw (% of the diagram's own width, resolved against the .arch-diagram-wrap container) so the @@ -665,10 +669,11 @@ p { .arch-row--compute .arch-cell { justify-content: center; gap: 0.55cqw; - min-height: calc(8cqw + 6px); + min-height: calc(8cqw - 6px); padding-inline: 1.5cqw; - border: 1px dotted currentColor; - border-radius: 12px; + border: 1px solid currentColor; + border-radius: 9px; + background: var(--cs-bg); } .arch-row--compute .arch-cell--cloud { @@ -774,7 +779,7 @@ p { display: flex; align-items: center; justify-content: center; - min-height: 5.54cqw; /* 50u */ + min-height: 6cqw; padding: 0 0.8cqw; border-radius: 12px; text-align: center; @@ -873,7 +878,7 @@ p { .arch-row--inputs > .arch-cell { grid-column: 1 / -1; } .arch-row--compute { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; padding: 16px var(--arch-label-padding); } .arch-row--compute > :first-child { grid-column: 1 / -1; } - .arch-row--compute .arch-cell { min-height: 54px; padding: 11px 8px; border-radius: 8px; } + .arch-row--compute .arch-cell { min-height: 54px; padding: 11px 8px; } .arch-cell { min-height: 52px; padding: 10px var(--arch-label-padding); gap: 12px; border-width: 1px; border-radius: 8px; } .arch-cell__label { font-size: 16px; white-space: normal; } .arch-cell--platform { min-height: 52px; gap: 6px; padding: 8px 4px; } @@ -948,46 +953,6 @@ p { font-size: var(--font-small); } -/* Core-concept cards: three columns inside the AlternatingDocBlock visual slot. */ -.concept-grid-wrap { - container-type: inline-size; - width: 100%; -} - -.concept-grid { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - grid-auto-rows: 1fr; /* equal-height cards across both rows */ - column-gap: 1.72cqw; /* match .arch-row */ - row-gap: 2.66cqw; /* match .arch-diagram */ -} - -/* Each concept card is a whole-card link to the docs with a simple 1px outline. */ -.concept-grid .media-card { - display: block; - color: var(--cs-text); - text-decoration: none !important; - background: transparent; - border: 1px solid currentColor; - border-radius: 12px; - padding: 16px 18px 18px; - transition: background-color 0.15s ease; -} - -.concept-grid .media-card:hover { - background: var(--cs-hover); -} - -.concept-grid .media-card h3 { - display: flex; - align-items: center; - margin: 0 0 6px; - padding: 0; - font-size: 17px; - font-weight: 600; - letter-spacing: -0.01em; -} - /* Quote author name reuses the card-heading size. */ .testimonial-name { font-size: var(--font-card-heading) !important; @@ -1108,7 +1073,12 @@ p { .gs-rail a.gs-opt:focus-visible { border-radius: 8px; background: var(--cs-bg); - box-shadow: inset 0 0 0 0.5px var(--cs-text); + box-shadow: inset 0 0 0 1px var(--cs-text); +} +.site-products-menu .gs-rail:not(:has(a.gs-opt:hover, a.gs-opt:focus-visible)) .gs-opt.gs-opt--on, +.site-products-menu .gs-rail a.gs-opt:hover, +.site-products-menu .gs-rail a.gs-opt:focus-visible { + box-shadow: none; } .gs-rail button.gs-opt:focus-visible { outline: 2px solid var(--cs-text); @@ -1252,7 +1222,8 @@ p { .sky-gpu-prices { flex: 0 0 auto; width: 100%; - border-collapse: collapse; + border-collapse: separate; + border-spacing: 0; font-size: 13px; } .sky-gpu-prices th, @@ -1278,6 +1249,19 @@ p { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; white-space: nowrap; } +.sky-gpu-prices tbody th, +.sky-gpu-prices tbody td { + transition: background-color 0.15s ease; +} +.sky-gpu-prices tbody tr:hover > * { + background: var(--cs-panel); +} +.sky-gpu-prices tbody tr > :first-child { + border-radius: 8px 0 0 8px; +} +.sky-gpu-prices tbody tr > :last-child { + border-radius: 0 8px 8px 0; +} @media (max-width: 480px) { .gs-box--offers .gs-skybody { padding-inline: 16px; @@ -1331,10 +1315,12 @@ p { } @media (max-width: 480px) { - .gs-box--compute .gs-boxfoot { + .gs-box--compute .gs-boxfoot, + .gs-box--compute-sources .gs-boxfoot { flex-wrap: wrap; } - .gs-box--compute .gs-foot__note { + .gs-box--compute .gs-foot__note, + .gs-box--compute-sources .gs-foot__note { flex-basis: 100%; } } @@ -1494,6 +1480,41 @@ p { align-items: flex-start; gap: 11px; } +.gs-cap, +.gs-cloud { + position: relative; + isolation: isolate; + border-radius: 8px; +} +.docs-article a.gs-cap, +.docs-article a.gs-cloud { + color: var(--cs-text); + text-decoration: none !important; +} +/* Extend the hover fill without changing list spacing or text alignment. */ +.gs-cap::before, +.gs-cloud::before { + content: ''; + position: absolute; + inset: -8px -12px; + z-index: -1; + border-radius: inherit; + background: var(--cs-panel); + opacity: 0; + pointer-events: none; + transition: opacity 0.15s ease; +} +.gs-cap:hover::before, +.gs-cloud:hover::before, +.gs-cap:focus-visible::before, +.gs-cloud:focus-visible::before { + opacity: 1; +} +.docs-article a.gs-cap:focus-visible, +.docs-article a.gs-cloud:focus-visible { + outline: 2px solid var(--cs-text); + outline-offset: 4px; +} .gs-cap__ic { flex: 0 0 auto; margin-top: 1px; @@ -1557,7 +1578,7 @@ p { .gs-caps { grid-template-columns: 1fr; } /* Keep footer notes and buttons on one row. */ .gs-boxfoot { align-items: center; gap: 10px; } - .gs-boxfoot [class*='awsui_button'] { padding-inline: 14px !important; } + .gs-boxfoot [class*='awsui_button'] { padding-inline: 14px var(--cta-padding-end, 14px) !important; } } .site-footer { @@ -1848,7 +1869,6 @@ p { .doc-card-grid, .product-grid, - .concept-grid, .testimonial-grid { grid-template-columns: 1fr; } @@ -1976,8 +1996,11 @@ p { .home-hero__actions--single { grid-template-columns: max-content; } - /* Each button fills its half-column; center the label and trim the desktop 34px side + /* Each button fills its half-column; center the label and trim the desktop side padding so both fit. !important: Cloudscape sets text-align:start via a boosted rule. */ + .cta-pair > a { + padding-inline: 16px !important; + } .home-hero__actions [class*='awsui_button'] { width: 100%; padding-inline: 16px !important; diff --git a/website/vite.config.ts b/website/vite.config.ts index ff84e085ac..ca0e5f13b3 100644 --- a/website/vite.config.ts +++ b/website/vite.config.ts @@ -12,6 +12,7 @@ import react from '@vitejs/plugin-react'; export default defineConfig({ base: process.env.BASE_PATH || '/', plugins: [react()], + esbuild: { legalComments: 'inline' }, build: { assetsDir: 'website-assets', rollupOptions: { From 8d9559ad59cea6e4229825d0bf31f9b65bacc60d Mon Sep 17 00:00:00 2001 From: Fede Kamelhar <209537060+fede-kamel@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:16:54 -0400 Subject: [PATCH 606/640] OCI: resolve availability domains from the configured compartment (#4309) Co-authored-by: Jvst Me --- src/dstack/_internal/core/backends/oci/region.py | 15 ++++++++++++--- .../_internal/core/backends/oci/resources.py | 6 +++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/core/backends/oci/region.py b/src/dstack/_internal/core/backends/oci/region.py index 806ed76fd1..8967e9ce80 100644 --- a/src/dstack/_internal/core/backends/oci/region.py +++ b/src/dstack/_internal/core/backends/oci/region.py @@ -1,8 +1,10 @@ +import threading from dataclasses import dataclass from functools import cached_property from typing import Any, Dict, Iterable, List, Mapping, Set import oci +from cachetools import Cache, cachedmethod from dstack._internal.core.backends.oci.auth import get_client_config from dstack._internal.core.backends.oci.models import AnyOCICreds @@ -15,6 +17,8 @@ class OCIRegionClient: def __init__(self, client_config: Mapping[str, Any]): self.client_config = client_config + self._availability_domains_cache = Cache(maxsize=10) + self._availability_domains_lock = threading.Lock() @property def name(self) -> str: @@ -44,9 +48,14 @@ def virtual_network_client(self) -> oci.core.VirtualNetworkClient: def work_request_client(self) -> oci.work_requests.WorkRequestClient: return oci.work_requests.WorkRequestClient(self.client_config) - @cached_property - def availability_domains(self) -> List[oci.identity.models.AvailabilityDomain]: - return self.identity_client.list_availability_domains(self.client_config["tenancy"]).data + @cachedmethod( + cache=lambda self: self._availability_domains_cache, + lock=lambda self: self._availability_domains_lock, + ) + def availability_domains_in( + self, compartment_id: str + ) -> List[oci.identity.models.AvailabilityDomain]: + return self.identity_client.list_availability_domains(compartment_id).data def make_region_client(region_name: str, creds: AnyOCICreds) -> OCIRegionClient: diff --git a/src/dstack/_internal/core/backends/oci/resources.py b/src/dstack/_internal/core/backends/oci/resources.py index 91e1d890f0..413395a4ca 100644 --- a/src/dstack/_internal/core/backends/oci/resources.py +++ b/src/dstack/_internal/core/backends/oci/resources.py @@ -140,7 +140,7 @@ def list_shapes_in_region(region: OCIRegionClient, compartment_id: str) -> Dict[ """ result = {} - for availability_domain in region.availability_domains: + for availability_domain in region.availability_domains_in(compartment_id): result[availability_domain.name] = list_shapes_in_domain( availability_domain.name, region.compute_client, compartment_id ) @@ -214,7 +214,7 @@ def check_availability_per_domain( all_shapes = set(shape_names) available_shapes_per_domain = {} - for availability_domain in region.availability_domains: + for availability_domain in region.availability_domains_in(compartment_id): shapes_to_check = { shape for shape in all_shapes @@ -277,7 +277,7 @@ def get_available_domains( `shape_name` is available and within `shapes_quota`. """ domains = [] - for domain in region.availability_domains: + for domain in region.availability_domains_in(compartment_id): if shapes_quota.is_within_domain_quota( shape_name, domain.name ) and check_availability_in_domain( From 6e360246081f6a03a7893536cfd5d74826aa6b65 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 29 Sep 2026 06:32:08 +0000 Subject: [PATCH 607/640] Sync workers with every running router (#4320) During a rolling deployment, the replacement router runs alongside the old one, but the worker sync only picked the first running router job, which is always the old one. The replacement had no workers until the old router was scaled down and could not serve requests routed to it. The sync now reads the workers of every running router, probes the workers once, and updates each router that is out of sync, so an unreachable router no longer blocks the others. Router tunnels are no longer held while workers are probed, which may take minutes: each router is read first, and connected to again only if it needs an update, re-reading its workers before applying the changes. Also: * `_Worker` is renamed back to `_TargetWorker`, its fields are typed after SMG's `WorkerSpec`, and `_add_worker_to_router()` sends it as the `POST /workers` body as is. Fixes: https://github.com/dstackai/dstack/issues/4315 Co-authored-by: Claude Opus 5.5 (1M context) --- .../services/runs/router_worker_sync.py | 264 ++++++++------- .../services/runs/test_router_worker_sync.py | 300 +++++++++++++++++- 2 files changed, 441 insertions(+), 123 deletions(-) diff --git a/src/dstack/_internal/server/services/runs/router_worker_sync.py b/src/dstack/_internal/server/services/runs/router_worker_sync.py index 100597dc0f..d1e8a45c19 100644 --- a/src/dstack/_internal/server/services/runs/router_worker_sync.py +++ b/src/dstack/_internal/server/services/runs/router_worker_sync.py @@ -1,6 +1,7 @@ """Reconcile SGLang router /workers with dstack's ready worker replicas (async, SSH-tunneled).""" import json +from dataclasses import dataclass from typing import Any, List, Literal, Optional, TypedDict from urllib.parse import urlsplit, urlunsplit @@ -98,20 +99,30 @@ async def _request_json_limited( return None -class _Worker(TypedDict): +# https://github.com/smg-project/smg/blob/3be823a700fabaff3add8a390cf78f163479d686/crates/protocols/src/worker.rs#L601 +# Only fields used to register a worker (POST /workers) are included +class _TargetWorker(TypedDict): url: str - worker_type: str - bootstrap_port: NotRequired[Optional[int]] - connection_mode: NotRequired[str] - runtime_type: NotRequired[str] + worker_type: "_WorkerType" + connection_mode: "_ConnectionMode" + runtime_type: "_RuntimeType" + bootstrap_port: NotRequired[int] kv_connector: NotRequired[str] kv_role: NotRequired[str] -_ConnectionMode = Literal["grpc", "http"] +# https://github.com/smg-project/smg/blob/3be823a700fabaff3add8a390cf78f163479d686/crates/protocols/src/worker.rs#L29 +# Only types we support are included +_WorkerType = Literal["regular", "prefill", "decode"] + +# https://github.com/smg-project/smg/blob/3be823a700fabaff3add8a390cf78f163479d686/crates/protocols/src/worker.rs#L76 +# Only modes we support are included +_ConnectionMode = Literal["http", "grpc"] # The order does matter -- we discover connection modes in the specified order _CONNECTION_MODES: tuple[_ConnectionMode, ...] = ("http", "grpc") +# https://github.com/smg-project/smg/blob/3be823a700fabaff3add8a390cf78f163479d686/crates/protocols/src/worker.rs#L227 +# Only types we support are included _RuntimeType = Literal["sglang", "vllm"] # The order does matter -- we discover runtime types in the specified order _RUNTIME_TYPES: tuple[_RuntimeType, ...] = ("sglang", "vllm") @@ -122,20 +133,17 @@ def run_model_has_sglang_router_replica_group(run_model: RunModel) -> bool: return run_spec_has_sglang_router_replica_group(run_spec) -def _get_router_job(run_model: RunModel, router_group: ReplicaGroup) -> Optional[JobModel]: +def _get_router_jobs(run_model: RunModel, router_group: ReplicaGroup) -> List[JobModel]: group_name = router_group.name assert group_name is not None, "Replica group name is set by validation" - router_jobs = [ + # The router group is validated to have `replicas: 1`, but a rolling deployment runs the + # replacement router alongside the old one until the old one is scaled down. Every running + # router is synced, otherwise the replacement has no workers until the old one is gone. + return [ j for j in run_model.jobs if job_belongs_to_group(j, group_name) and j.status == JobStatus.RUNNING ] - if not router_jobs: - return None - # Router replica group is currently validated to have count=1, so we assume a single active - # router job here. When we support multiple router replicas for HA, this should be updated - # to handle syncing across all active router jobs. - return router_jobs[0] def _normalize_worker_url(url: str) -> str: @@ -202,7 +210,7 @@ async def _get_router_workers(client: AsyncClient) -> Optional[List[dict]]: return None # TODO: Truncating a long list and/or dropping unexpectedly shaped items doesn't seem # right. We should add validation (an item must be a dict with some required fields, see - # _update_workers_in_router_replica) and decide what to do with a partially valid list + # _get_workers_diff) and decide what to do with a partially valid list if len(workers) > _MAX_WORKERS_LIST_ITEMS: logger.warning( "Router /workers list exceeds %s items, truncating", @@ -217,36 +225,16 @@ async def _get_router_workers(client: AsyncClient) -> Optional[List[dict]]: return None -async def _add_worker_to_router( - client: AsyncClient, - url: str, - worker_type: str = "regular", - bootstrap_port: Optional[int] = None, - *, - connection_mode: Optional[str] = None, - runtime_type: Optional[str] = None, - kv_connector: Optional[str] = None, - kv_role: Optional[str] = None, -) -> bool: +async def _add_worker_to_router(client: AsyncClient, worker: _TargetWorker) -> bool: + url = worker["url"] try: - payload: dict = {"url": url, "worker_type": worker_type} - if bootstrap_port is not None: - payload["bootstrap_port"] = bootstrap_port - if connection_mode is not None: - payload["connection_mode"] = connection_mode - if runtime_type is not None: - payload["runtime_type"] = runtime_type - if kv_connector is not None: - payload["kv_connector"] = kv_connector - if kv_role is not None: - payload["kv_role"] = kv_role body = await _request_json_limited( client, "POST", f"{_HTTP_BASE_URL}/workers", max_response_bytes=_MAX_WORKERS_COMMAND_ACK_BYTES, ok_statuses={202}, - json_body=payload, + json_body=dict(worker), ) added = isinstance(body, dict) and body.get("status") == "accepted" if not added: @@ -281,53 +269,60 @@ async def _remove_worker_from_router_by_id( return False -async def _update_workers_in_router_replica( - client: AsyncClient, - target_workers: List[_Worker], - *, - current_workers: List[dict], -) -> None: - current_urls: set[str] = set() - current_ids_by_norm_url: dict[str, str] = {} +@dataclass +class _WorkersDiff: + to_add: List[_TargetWorker] + to_remove: dict[str, Optional[str]] + """Normalized worker URL to the router's worker id, `None` if the router reported none""" + + def is_empty(self) -> bool: + return not self.to_add and not self.to_remove + + +def _get_workers_diff( + target_workers: List[_TargetWorker], current_workers: List[dict] +) -> _WorkersDiff: + current_ids_by_norm_url: dict[str, Optional[str]] = {} for w in current_workers: u = w.get("url") if not isinstance(u, str) or not u: continue norm_u = _normalize_worker_url(u) - current_urls.add(norm_u) wid = w.get("id") if isinstance(wid, str) and wid: current_ids_by_norm_url[norm_u] = wid - target_by_norm = {_normalize_worker_url(t["url"]): t for t in target_workers} - target_urls = set(target_by_norm.keys()) - to_add = sorted(target_urls - current_urls) - to_remove = sorted(current_urls - target_urls) - for norm_url in to_add: - tw = target_by_norm[norm_url] - ok = await _add_worker_to_router( - client, - tw["url"], - tw["worker_type"], - tw.get("bootstrap_port"), - connection_mode=tw.get("connection_mode"), - runtime_type=tw.get("runtime_type"), - kv_connector=tw.get("kv_connector"), - kv_role=tw.get("kv_role"), - ) - if not ok: - logger.debug("Failed to add worker %s, continuing with others", tw["url"]) - for url in to_remove: - wid = current_ids_by_norm_url.get(url) - if not wid: - logger.error("No worker id found for url %s", url) - ok = False else: - ok = await _remove_worker_from_router_by_id(client, wid, worker_url=url) - if not ok: - logger.debug("Failed to remove worker %s, continuing with others", url) + current_ids_by_norm_url.setdefault(norm_u, None) + target_by_norm_url = {_normalize_worker_url(t["url"]): t for t in target_workers} + to_add = sorted(target_by_norm_url.keys() - current_ids_by_norm_url.keys()) + to_remove = sorted(current_ids_by_norm_url.keys() - target_by_norm_url.keys()) + return _WorkersDiff( + to_add=[target_by_norm_url[u] for u in to_add], + to_remove={u: current_ids_by_norm_url[u] for u in to_remove}, + ) -def _vllm_kv_role_to_worker_type(kv_role: str) -> str: +async def _apply_workers_diff( + client: AsyncClient, diff: _WorkersDiff, *, router_job: JobModel +) -> None: + for worker in diff.to_add: + if not await _add_worker_to_router(client, worker): + logger.debug( + "%s: failed to add worker %s, continuing with others", + fmt(router_job), + worker["url"], + ) + for url, worker_id in diff.to_remove.items(): + if worker_id is None: + logger.error("%s: no worker id found for url %s", fmt(router_job), url) + continue + if not await _remove_worker_from_router_by_id(client, worker_id, worker_url=url): + logger.debug( + "%s: failed to remove worker %s, continuing with others", fmt(router_job), url + ) + + +def _vllm_kv_role_to_worker_type(kv_role: str) -> _WorkerType: if kv_role == "kv_producer": return "prefill" if kv_role == "kv_consumer": @@ -344,7 +339,7 @@ def _is_expected_grpc_error(error: grpc.aio.AioRpcError) -> bool: ) -async def _probe_http_worker(client: AsyncClient, *, address: str) -> Optional[_Worker]: +async def _probe_http_worker(client: AsyncClient, *, address: str) -> Optional[_TargetWorker]: # The request goes over the tunnel, `worker_url` is the address the router itself dials. worker_url = f"http://{address}" try: @@ -361,7 +356,7 @@ async def _probe_http_worker(client: AsyncClient, *, address: str) -> Optional[_ mode = data.get("disaggregation_mode", "") if mode == "prefill": bootstrap_port = data.get("disaggregation_bootstrap_port") - worker: _Worker = { + worker: _TargetWorker = { "url": worker_url, "worker_type": "prefill", "connection_mode": "http", @@ -407,11 +402,11 @@ def _grpc_server_info_to_worker( worker_url: str, runtime_type: _RuntimeType, response: Any, -) -> _Worker: +) -> _TargetWorker: if runtime_type == "vllm": kv_role = response.kv_role or "" kv_connector = response.kv_connector or "" - worker: _Worker = { + worker: _TargetWorker = { "url": worker_url, "connection_mode": "grpc", "runtime_type": runtime_type, @@ -448,7 +443,7 @@ async def _probe_grpc_worker( *, address: str, runtime_type: Optional[_RuntimeType] = None, -) -> Optional[_Worker]: +) -> Optional[_TargetWorker]: # The RPC goes over the tunnel, `worker_url` is the address the router itself dials. worker_url = f"grpc://{address}" runtime_types: tuple[_RuntimeType, ...] @@ -477,7 +472,7 @@ async def _get_worker( address: str, connection_mode: Optional[_ConnectionMode] = None, runtime_type: Optional[_RuntimeType] = None, -) -> Optional[_Worker]: +) -> Optional[_TargetWorker]: connection_modes: tuple[_ConnectionMode, ...] if connection_mode is None: # No connection_mode discovered -- should probe all @@ -501,7 +496,7 @@ async def _get_worker( # An unreachable worker is reported as not ready rather than aborting the sync, so that # one dead replica cannot hold back registration of the healthy ones. The cost is that a # transient failure deregisters a healthy worker until the next sync re-adds it. - # TODO: `_update_workers_in_router_replica` cannot tell "not serving" from "could not be + # TODO: `_get_workers_diff` cannot tell "not serving" from "could not be # reached" -- both mean "absent from the target list", hence "remove". A third, unknown # outcome should be excluded from both `to_add` and `to_remove`, leaving an unreachable # worker as the router last saw it. @@ -516,8 +511,8 @@ async def _build_target_workers( *, connection_mode: Optional[_ConnectionMode] = None, runtime_type: Optional[_RuntimeType] = None, -) -> List[_Worker]: - workers: List[_Worker] = [] +) -> List[_TargetWorker]: + workers: List[_TargetWorker] = [] config = run_spec.configuration if not isinstance(config, ServiceConfiguration): return workers @@ -563,53 +558,80 @@ async def sync_router_workers_for_run_model(run_model: RunModel) -> None: if router_group is None: return - router_job = _get_router_job(run_model, router_group) - if router_job is None: + router_jobs = _get_router_jobs(run_model, router_group) + if not router_jobs: logger.debug( "%s: no running router job in group %s, skipping worker sync", fmt(run_model), router_group.name, ) return - # A tunnel is opened here for the router, and inside `_build_target_workers` for every - # worker. Only the router being unreachable aborts the sync -- without a client there is - # nothing to reconcile against. An unreachable worker is skipped instead, see `_get_worker`. + # Probing the workers may take minutes, so no router tunnel is held meanwhile. Each router + # is read first, and connected to again only if it needs updating, which in most syncs it + # doesn't. An unreachable router is skipped, like an unreachable worker, see `_get_worker`. + try: + current_workers_by_router: List[tuple[JobModel, List[dict]]] = [] + for router_job in router_jobs: + current_workers = await _get_router_replica_workers(router_job) + if current_workers is not None: + current_workers_by_router.append((router_job, current_workers)) + if not current_workers_by_router: + logger.debug( + "%s: no router in group %s returned its workers, skipping worker sync", + fmt(run_model), + router_group.name, + ) + return + # The hints spare probing connection modes and runtime types that no registered worker + # uses. They are taken from all routers, as a router started by a rolling deployment has + # no workers yet, which alone would mean "probe everything". + all_current_workers = [w for _, workers in current_workers_by_router for w in workers] + target_workers = await _build_target_workers( + run_model, + run_spec, + replica_groups, + connection_mode=_get_connection_mode_from_workers(all_current_workers), + runtime_type=_get_runtime_type_from_workers(all_current_workers), + ) + for router_job, current_workers in current_workers_by_router: + if _get_workers_diff(target_workers, current_workers).is_empty(): + continue + await _update_workers_in_router_replica(router_job, target_workers) + except Exception: + logger.exception("%s: unexpected error when syncing workers with router", fmt(run_model)) + + +async def _get_router_replica_workers(router_job: JobModel) -> Optional[List[dict]]: + try: + async with get_service_replica_client(router_job) as client: + current_workers = await _get_router_workers(client) + except SSHError as e: + _log_router_replica_unreachable(router_job, e) + return None + if current_workers is None: + logger.debug("%s: failed to get current workers from the router", fmt(router_job)) + return current_workers + + +async def _update_workers_in_router_replica( + router_job: JobModel, target_workers: List[_TargetWorker] +) -> None: try: async with get_service_replica_client(router_job) as client: + # Read again: the list this update was decided on was fetched before the workers + # were probed, possibly minutes ago. current_workers = await _get_router_workers(client) if current_workers is None: - logger.debug( - "%s: failed to get current workers from the router in group %s," - " skipping worker sync", - fmt(run_model), - router_group.name, - ) + logger.debug("%s: failed to get current workers from the router", fmt(router_job)) return - # connection_mode can be grpc or http, runtime_type can be sglang or vllm. - connection_mode = _get_connection_mode_from_workers(current_workers) - runtime_type = _get_runtime_type_from_workers(current_workers) - # Empty current_workers on first sync is expected. First syncprobes both connection_mode and - # runtime_type. Subsequent syncs don't need to probe again because connection_mode and runtime_type - # is already set in current_workers. - target_workers = await _build_target_workers( - run_model, - run_spec, - replica_groups, - connection_mode=connection_mode, - runtime_type=runtime_type, - ) - await _update_workers_in_router_replica( - client, target_workers, current_workers=current_workers - ) + diff = _get_workers_diff(target_workers, current_workers) + await _apply_workers_diff(client, diff, router_job=router_job) except SSHError as e: - # Only the router's own tunnel reaches here, worker tunnels are handled in - # `_get_worker`. Warning is the right level: a job only reaches `RUNNING` after the - # server has talked to its runner over SSH, so an unreachable replica is always a - # regression, never a replica that has not started yet. - logger.warning( - "%s: failed to sync workers with router: %r", - fmt(router_job), - e, - ) - except Exception: - logger.exception("%s: unexpected error when syncing workers with router", fmt(run_model)) + _log_router_replica_unreachable(router_job, e) + + +def _log_router_replica_unreachable(router_job: JobModel, error: SSHError) -> None: + # Warning is the right level: a job only reaches `RUNNING` after the server has talked to + # its runner over SSH, so an unreachable replica is always a regression, never a replica + # that has not started yet. + logger.warning("%s: failed to sync workers with router: %r", fmt(router_job), error) diff --git a/src/tests/_internal/server/services/runs/test_router_worker_sync.py b/src/tests/_internal/server/services/runs/test_router_worker_sync.py index 80a9137e0e..1277f040a4 100644 --- a/src/tests/_internal/server/services/runs/test_router_worker_sync.py +++ b/src/tests/_internal/server/services/runs/test_router_worker_sync.py @@ -1,6 +1,7 @@ import json import logging -from contextlib import contextmanager +import uuid +from contextlib import asynccontextmanager, contextmanager from pathlib import Path from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -9,17 +10,34 @@ import httpx import pytest from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.errors import SSHError +from dstack._internal.core.models.configurations import parse_run_configuration +from dstack._internal.core.models.runs import JobStatus, RunStatus +from dstack._internal.server.models import JobModel, RunModel +from dstack._internal.server.services.logging import fmt from dstack._internal.server.services.runs import router_worker_sync from dstack._internal.server.services.runs.router_worker_sync import ( + _add_worker_to_router, _get_connection_mode_from_workers, _get_router_workers, _get_runtime_type_from_workers, _get_worker, + _get_workers_diff, _grpc_server_info_to_worker, _probe_grpc_worker, _probe_http_worker, + _TargetWorker, + sync_router_workers_for_run_model, +) +from dstack._internal.server.testing.common import ( + create_job, + create_project, + create_repo, + create_run, + create_user, + get_run_spec, ) @@ -145,12 +163,72 @@ def handler(_): assert not [r for r in caplog.records if r.levelno > logging.DEBUG] +@pytest.mark.asyncio +class TestAddWorkerToRouter: + """ + `_TargetWorker` is sent as the `POST /workers` body as is. `TypedDict` does not reject extra + keys at runtime, and the router silently ignores unknown ones, so this test is what catches a + field that should not go over the wire. + """ + + @pytest.mark.parametrize( + "worker", + [ + pytest.param( + { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", + }, + id="http-regular", + ), + pytest.param( + { + "url": "grpc://10.0.0.1:8000", + "worker_type": "prefill", + "connection_mode": "grpc", + "runtime_type": "vllm", + "bootstrap_port": 8998, + "kv_connector": "NixlConnector", + "kv_role": "kv_producer", + }, + id="grpc-prefill", + ), + ], + ) + async def test_posts_worker_as_is(self, worker: _TargetWorker): + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return _json_response(202, {"status": "accepted"}) + + async with _router_client(handler) as client: + assert await _add_worker_to_router(client, worker) is True + assert len(requests) == 1 + assert requests[0].method == "POST" + assert requests[0].url.path == "/workers" + assert json.loads(requests[0].content) == worker + + async def test_returns_false_when_not_accepted(self, caplog: pytest.LogCaptureFixture): + worker: _TargetWorker = { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", + } + async with _router_client(lambda _: _json_response(202, {"status": "rejected"})) as client: + assert await _add_worker_to_router(client, worker) is False + assert "Unexpected add-worker response for http://10.0.0.1:8000" in caplog.text + + @pytest.mark.asyncio class TestProbeHttpWorker: """ The probe talks to the replica over the tunnel, but the `url` it reports is the address the router dials. It must stay byte-identical to what the router echoes back in `/workers`, - otherwise `_update_workers_in_router_replica` re-registers every worker on each sync. + otherwise `_get_workers_diff` re-registers every worker on each sync. """ async def test_regular_worker(self): @@ -470,3 +548,221 @@ async def test_unexpected_tunnel_error_propagates(self): MagicMock(), address="10.0.0.1:8000", ) + + +class TestGetWorkersDiff: + def test_adds_missing_and_removes_extra_workers(self): + kept: _TargetWorker = { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", + } + added: _TargetWorker = {**kept, "url": "http://10.0.0.2:8000"} + current = [ + # The router may echo the URL back with a trailing slash + {"id": "1", "url": "http://10.0.0.1:8000/"}, + {"id": "2", "url": "http://10.0.0.3:8000"}, + ] + diff = _get_workers_diff([kept, added], current) + assert diff.to_add == [added] + assert diff.to_remove == {"http://10.0.0.3:8000": "2"} + assert not diff.is_empty() + + def test_is_empty_when_router_is_in_sync(self): + worker: _TargetWorker = { + "url": "http://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "http", + "runtime_type": "sglang", + } + diff = _get_workers_diff([worker], [{"id": "1", "url": "http://10.0.0.1:8000"}]) + assert diff.is_empty() + + def test_removed_worker_without_id(self): + diff = _get_workers_diff([], [{"url": "http://10.0.0.1:8000"}]) + assert diff.to_remove == {"http://10.0.0.1:8000": None} + + +class _FakeRouter: + """An in-memory router `/workers` API that counts the connections made to it.""" + + def __init__(self, workers: Optional[list[dict]] = None, *, unreachable: bool = False): + self.workers = list(workers or []) + self.unreachable = unreachable + self.connections = 0 + self.added: list[dict] = [] + self.removed_ids: list[str] = [] + + def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "GET" and request.url.path == "/workers": + return _json_response(200, {"workers": self.workers}) + if request.method == "POST" and request.url.path == "/workers": + worker = json.loads(request.content) + self.added.append(worker) + self.workers.append({"id": f"id-{len(self.added)}", **worker}) + return _json_response(202, {"status": "accepted"}) + if request.method == "DELETE" and request.url.path.startswith("/workers/"): + worker_id = request.url.path.removeprefix("/workers/") + self.removed_ids.append(worker_id) + self.workers = [w for w in self.workers if w["id"] != worker_id] + return _json_response(202, {"status": "accepted"}) + return httpx.Response(404) + + +@contextmanager +def _fake_router_replicas(routers: dict[uuid.UUID, _FakeRouter]): + """Route each router job's client to its fake router, keyed by the job id.""" + + @asynccontextmanager + async def get_service_replica_client(job: JobModel): + router = routers[job.id] + router.connections += 1 + if router.unreachable: + raise SSHError("connection refused") + async with AsyncClient(transport=httpx.MockTransport(router.handle)) as client: + yield client + + with patch( + "dstack._internal.server.services.runs.router_worker_sync.get_service_replica_client", + get_service_replica_client, + ): + yield + + +async def _create_router_service_run(session: AsyncSession, router_count: int) -> RunModel: + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + conf = parse_run_configuration( + { + "type": "service", + "port": 8000, + "gateway": False, + "groups": [ + { + "name": "router", + "replicas": 1, + "commands": ["smg launch"], + "router": {"type": "sglang"}, + }, + {"name": "worker", "replicas": 1, "commands": ["worker"]}, + ], + } + ) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + status=RunStatus.RUNNING, + run_spec=get_run_spec(repo_id=repo.name, run_name="test-run", configuration=conf), + ) + # More than one running router means a rolling deployment is replacing the router + for replica_num in range(router_count): + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + replica_num=replica_num, + replica_group_name="router", + ) + await session.refresh(run, attribute_names=["jobs"]) + return run + + +_VLLM_WORKER: _TargetWorker = { + "url": "grpc://10.0.0.1:8000", + "worker_type": "regular", + "connection_mode": "grpc", + "runtime_type": "vllm", +} + + +def _patch_build_target_workers(**kwargs): + return patch( + "dstack._internal.server.services.runs.router_worker_sync._build_target_workers", + new_callable=AsyncMock, + **kwargs, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestSyncRouterWorkersForRunModel: + async def test_syncs_replacement_router_without_touching_up_to_date_one( + self, test_db, session: AsyncSession + ): + run = await _create_router_service_run(session, router_count=2) + old_router = _FakeRouter([{"id": "1", **_VLLM_WORKER}]) + new_router = _FakeRouter() + routers = {run.jobs[0].id: old_router, run.jobs[1].id: new_router} + + with ( + _fake_router_replicas(routers), + _patch_build_target_workers(return_value=[_VLLM_WORKER]) as build_mock, + ): + await sync_router_workers_for_run_model(run) + + assert new_router.added == [_VLLM_WORKER] + assert old_router.added == [] + assert old_router.removed_ids == [] + # Read once; reconnected only for the router that needed an update + assert old_router.connections == 1 + assert new_router.connections == 2 + # Workers are probed once for all routers, with hints taken from all of them: the + # replacement router's empty list alone would mean "probe everything" + build_mock.assert_awaited_once() + assert build_mock.await_args is not None + assert build_mock.await_args.kwargs["connection_mode"] == "grpc" + assert build_mock.await_args.kwargs["runtime_type"] == "vllm" + + async def test_unreachable_router_does_not_block_others( + self, test_db, session: AsyncSession, caplog: pytest.LogCaptureFixture + ): + caplog.set_level(level=logging.WARNING, logger=router_worker_sync.__name__) + run = await _create_router_service_run(session, router_count=2) + unreachable_router = _FakeRouter(unreachable=True) + reachable_router = _FakeRouter() + routers = {run.jobs[0].id: unreachable_router, run.jobs[1].id: reachable_router} + + with ( + _fake_router_replicas(routers), + _patch_build_target_workers(return_value=[_VLLM_WORKER]), + ): + await sync_router_workers_for_run_model(run) + + assert reachable_router.added == [_VLLM_WORKER] + assert unreachable_router.connections == 1 + assert f"{fmt(run.jobs[0])}: failed to sync workers with router" in caplog.text + + async def test_skips_probing_workers_when_no_router_is_reachable( + self, test_db, session: AsyncSession + ): + run = await _create_router_service_run(session, router_count=1) + routers = {run.jobs[0].id: _FakeRouter(unreachable=True)} + + with _fake_router_replicas(routers), _patch_build_target_workers() as build_mock: + await sync_router_workers_for_run_model(run) + + build_mock.assert_not_awaited() + + async def test_rereads_router_workers_before_updating(self, test_db, session: AsyncSession): + run = await _create_router_service_run(session, router_count=1) + router = _FakeRouter([{"id": "1", **_VLLM_WORKER}]) + routers = {run.jobs[0].id: router} + + def reregister_worker(*args, **kwargs): + # Worker ids are assigned by the router. Here, the worker got a new one while the + # workers were being probed, so the id read before probing is stale. + router.workers = [{"id": "2", **_VLLM_WORKER}] + return [] + + with ( + _fake_router_replicas(routers), + _patch_build_target_workers(side_effect=reregister_worker), + ): + await sync_router_workers_for_run_model(run) + + assert router.removed_ids == ["2"] + assert router.workers == [] From f3de2e5ece98c19101892b4ceb656e7702cfe48f Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 29 Sep 2026 11:45:55 +0000 Subject: [PATCH 608/640] Restore model probe for service routers (#4322) Since #4270, a router replica of a service with `model` and no `probes` was probed with `GET /health` instead of the chat completions request. The latter deadlocked: the router only answers it once workers are registered, and the worker sync skipped routers that were not ready. #4313 dropped the readiness check from the sync, so this deadlock is gone. `/health` passes before the router has any workers. In a rolling deployment, the replacement router was considered ready, and the old one was scaled down, before the replacement could serve requests. Now that #4320 syncs workers with every running router, the replacement passes the chat completions probe once it has workers. Workers behind a router still get no default probe, as they may not serve HTTP at all (gRPC workers). With a Dynamo router, a router rolling deployment now waits for the replacement indefinitely instead of scaling down the old router, as workers stay attached to the old router's etcd/NATS. Such deployments were broken before as well and are to be fixed separately. Co-authored-by: Claude Opus 5.5 (1M context) --- .../_internal/core/models/configurations.py | 1 - .../services/jobs/configurators/base.py | 36 ++++++------------- .../jobs/configurators/test_service.py | 13 ++++--- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 13150805f4..81d5d6226a 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -72,7 +72,6 @@ DEFAULT_PROBE_READY_AFTER = 1 DEFAULT_PROBE_METHOD = "get" DEFAULT_PROBE_UNTIL_READY = False -ROUTER_HEALTH_PROBE_URL = "/health" MAX_PROBE_URL_LEN = 2048 DEFAULT_REPLICA_GROUP_NAME = "0" OPENAI_MODEL_PROBE_TIMEOUT = 30 diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index c4c5bfd602..1a1fadf375 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -27,7 +27,6 @@ DEFAULT_REPLICA_GROUP_NAME, LEGACY_REPO_DIR, OPENAI_MODEL_PROBE_TIMEOUT, - ROUTER_HEALTH_PROBE_URL, HTTPHeaderSpec, NodeGroup, PortMapping, @@ -513,17 +512,17 @@ def _probes(self) -> list[ProbeSpec]: model = conf.model if not isinstance(model, OpenAIChatModel): return [] - if all(group.router is None for group in conf.replica_groups): - # No router: every replica serves the model itself, so a chat completions - # request is a genuine end-to-end readiness check. - return [_openai_model_probe_spec(model.name, model.prefix)] - group = self._replica_group() - if group is not None and group.router is not None: - # Probe the router's own liveness endpoint, which does not depend on any - # worker. Workers get no default probe: they may not serve HTTP at all - # (gRPC workers), and they don't receive traffic directly. - return [_router_health_probe_spec()] - return [] + if any(group.router is not None for group in conf.replica_groups): + group = self._replica_group() + if group is None or group.router is None: + # Workers get no default probe: they may not serve HTTP at all (gRPC + # workers), and they don't receive traffic directly. + return [] + # The router answers chat completions only once it has workers, which it gets + # regardless of its own readiness. For SGLang routers, dstack registers workers + # via `ServiceRouterWorkerSyncWorker`, which doesn't check router readiness. + # Dynamo workers register themselves with the router via etcd/NATS. + return [_openai_model_probe_spec(model.name, model.prefix)] def interpolate_job_volumes( @@ -598,19 +597,6 @@ def _openai_model_probe_spec(model_name: str, prefix: str) -> ProbeSpec: ) -def _router_health_probe_spec() -> ProbeSpec: - # Both supported routers (SGLang/SMG and Dynamo) serve `/health` independently of - # whether any worker is registered. - return ProbeSpec( - type="http", - method=DEFAULT_PROBE_METHOD, - url=ROUTER_HEALTH_PROBE_URL, - timeout=DEFAULT_PROBE_TIMEOUT, - interval=DEFAULT_PROBE_INTERVAL, - ready_after=DEFAULT_PROBE_READY_AFTER, - ) - - def _join_shell_commands(commands: List[str]) -> str: for i, cmd in enumerate(commands): cmd = cmd.strip() diff --git a/src/tests/_internal/server/services/jobs/configurators/test_service.py b/src/tests/_internal/server/services/jobs/configurators/test_service.py index 8a962542a3..3f0de12890 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_service.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_service.py @@ -6,7 +6,6 @@ from dstack._internal import settings from dstack._internal.core.models.configurations import ( OPENAI_MODEL_PROBE_TIMEOUT, - ROUTER_HEALTH_PROBE_URL, ProbeConfig, PythonVersion, ReplicaGroup, @@ -116,9 +115,9 @@ def _router_worker_configuration() -> ServiceConfiguration: ], ) - async def test_router_group_gets_health_probe(self): - """The router must not be probed with chat completions: it only answers those once - dstack has registered workers, and registration requires the router to be ready.""" + async def test_router_group_gets_model_probe(self): + """The router is probed with chat completions like a replica serving the model + itself: worker registration doesn't wait for the router to be ready.""" run_spec = get_run_spec( run_name="run", repo_id="id", configuration=self._router_worker_configuration() ) @@ -128,9 +127,9 @@ async def test_router_group_gets_health_probe(self): probes = job_specs[0].probes assert len(probes) == 1 - assert probes[0].url == ROUTER_HEALTH_PROBE_URL - assert probes[0].method == "get" - assert probes[0].body is None + assert probes[0].method == "post" + assert probes[0].url == "/v1/chat/completions" + assert "meta-llama/Meta-Llama-3.1-8B-Instruct" in (probes[0].body or "") async def test_worker_group_gets_no_derived_probe(self): """Workers behind a router may speak gRPC, so no probe can be derived from `model`.""" From 85886c4e3e78925273287b6e700111a01485f6c1 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Tue, 29 Sep 2026 17:52:30 +0545 Subject: [PATCH 609/640] [Presets] Update system_prompt for router usage (#4294) * [Presets] Update system_prompt.md for router usage * Resolve Review Comments * Require router on cluster fleets with a CPU-only instance --------- Co-authored-by: Bihan Rana --- skills/dstack-prototyping/SKILL.md | 22 +++++++++++++---- .../presets/resources/system_prompt.md | 24 ++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/skills/dstack-prototyping/SKILL.md b/skills/dstack-prototyping/SKILL.md index 552d00dad2..f0fc1060ee 100644 --- a/skills/dstack-prototyping/SKILL.md +++ b/skills/dstack-prototyping/SKILL.md @@ -101,19 +101,33 @@ command, resources, cache, or model behavior needs to change, go back to a task. If the tested serving setup is still right and only the dstack service configuration is wrong, fix the configuration and submit the service again. -## PD disaggregation +## Router -If the intention is to use PD disaggregation: +If a fleet has `placement: cluster` and a CPU-only instance, you must use a +configuration with the router on the CPU-only instance, regardless of whether +the workers are aggregated or PD disaggregated. +Whenever possible, connect the workers over gRPC, not HTTP: with a gRPC +router, request parsing, serialization, and tokenization move from the +serving engine to the router, so latency improves just by introducing it. + +When using a router: - Use node groups for the task and replica groups for the service: tasks' node groups are the equivalent of services' replica groups. -- In both cases, you run a router and prefill/decode workers separately, and - you need to use a fleet with an interconnect (`placement: cluster`). - With tasks, still use `sleep infinity` even when using `groups` (set it in each group's `commands`; top-level `commands` is not allowed with `groups`), and run the actual commands on each node interactively over SSH. - When testing inference, call the router endpoint, not the workers directly (unless you want to test if they are alive). +- Look for "Prototyping services" in `https://dstack.ai/docs/concepts/tasks.md` + and "Router" in `https://dstack.ai/docs/concepts/services.md`. + +## PD disaggregation + +If the intention is to use PD disaggregation: + +- Follow `## Router`: the router and the prefill/decode workers run as + separate groups, and the fleet needs an interconnect (`placement: cluster`). - Look for "Node groups" and "PD disaggregation" in `https://dstack.ai/docs/concepts/tasks.md` and "Replica groups" and "PD disaggregation" in `https://dstack.ai/docs/concepts/services.md`. diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 101191b945..691df34b33 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -260,9 +260,9 @@ mindful of which specific change was the root cause. {"resources": {...}, "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} ``` -2. In case the task is using node groups (e.g. for PD disaggregation), - instead of a single `resources` it includes `groups`, and the fields are - these and no others: +2. In case the task is using node groups (a router group plus worker + groups, see `## Cluster Placement`), instead of a single `resources` it + includes `groups`, and the fields are these and no others: ``` {"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} @@ -321,9 +321,9 @@ serving engine: use `dataset` and `concurrency``co `shared_prefix_tokens` from `constraints.json` and measure all trials the same way so that their results are comparable with each other. -In case the task is using PD disaggregation, run benchmarks via SSH inside -the router node, directly against the router engine. Never benchmark prefill -or decode workers — each handles only part of a request. +In case the task is using a router (see `## Cluster Placement`), run benchmarks +via SSH inside the router node, directly against the router engine, never against a +worker, so that results stay comparable across trials. Before any benchmark, reset the serving engine's prefix cache, or restart the @@ -506,6 +506,12 @@ Use these offers when selecting fleet, backend, and hardware. To classify each backend's capabilities, fetch `https://dstack.ai/docs/concepts/backends.md` and classify from the fetched document, not from memory. VM-based backends are listed under `## VM-based` (they support idle instances and instance volumes). Kubernetes backend is listed under `## Container-based`, but supports instance volumes and thus is preferred over other container-based backends. SSH fleets can be treated as VM-based backends as they support both idle instances (its equivalent) and instance volumes. +## Cluster Placement + +If a fleet has `placement: cluster`, follow `## Router` in the +`$/dstack-prototyping` skill before +trial 1 and report the resulting configuration via `progress`. + # Final Service Once the trials are over, pick the best trial that has not been verified yet @@ -565,9 +571,9 @@ trial benchmarks so that the results are comparable with each other. Attach to the service with `dstack attach `, which enables `ssh ` into the replica. -In case the service is using PD disaggregation, run the final benchmark -inside the router replica, directly against the router engine. Attach to it -via `dstack attach --replica --job `. +In case the service is using a router (see `## Cluster Placement`), run the +final benchmark inside the router replica, directly against the router engine. +Attach to it via `dstack attach --replica --job `. If the service or its benchmark cannot be completed, stop that service, pick the next-best trial, and repeat, until a service is verified or there From e7eb5f067adab319f0f64e3ff64de627d7e381d1 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 29 Sep 2026 17:29:14 +0500 Subject: [PATCH 610/640] Parse Crusoe quota counters returned as strings (#4323) Fixes #4318 --- src/dstack/_internal/core/backends/crusoe/compute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/backends/crusoe/compute.py b/src/dstack/_internal/core/backends/crusoe/compute.py index 13e54eef67..bbe4db95ed 100644 --- a/src/dstack/_internal/core/backends/crusoe/compute.py +++ b/src/dstack/_internal/core/backends/crusoe/compute.py @@ -176,7 +176,8 @@ def _get_quota_map(self) -> dict[str, int]: result = {} for q in quotas: prog_name = q.get("programmatic_name", "") - available = q.get("available", 0) + # The API documents counters as integers but returns them as strings + available = int(q.get("available", 0)) category = q.get("category", "") if "Instance" in category: result[prog_name] = available From a3496d2f977da460649a559306bc618d96f5d869 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 29 Sep 2026 12:38:59 +0000 Subject: [PATCH 611/640] [UI] Fix job metrics tab (#4324) Previously, all jobs displayed metrics for replica_num=0 --- frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx | 1 + frontend/src/types/run.d.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx b/frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx index db89033984..2ea3d6b68a 100644 --- a/frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx +++ b/frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx @@ -46,6 +46,7 @@ export const JobMetrics: React.FC = () => { run_name: runData?.run_spec.run_name ?? '', run_id: runData?.id ?? '', job_num: jobData?.job_spec.job_num ?? 0, + replica_num: jobData?.job_spec.replica_num ?? 0, limit: 1000, }); diff --git a/frontend/src/types/run.d.ts b/frontend/src/types/run.d.ts index a47a7ae571..9c3f3b1f98 100644 --- a/frontend/src/types/run.d.ts +++ b/frontend/src/types/run.d.ts @@ -248,6 +248,7 @@ declare interface IJobSpec { image_name: string; job_name: string; job_num: number; + replica_num: number; max_duration?: number; working_dir: string; probes?: IJobProbe[]; From deac3096d1522a028c05f6fb7516c9fb8c968c88 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 30 Sep 2026 11:47:30 +0500 Subject: [PATCH 612/640] Disable gateway proxy buffering for model services (#4326) --- .../proxy/gateway/resources/nginx/service.jinja2 | 3 +++ src/dstack/_internal/proxy/gateway/services/nginx.py | 1 + src/dstack/_internal/proxy/gateway/services/registry.py | 2 ++ src/dstack/_internal/proxy/lib/models.py | 3 +++ .../_internal/proxy/gateway/routers/test_registry.py | 8 ++++++++ 5 files changed, 17 insertions(+) diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 index 7c8a5fcc92..4f2adda389 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 @@ -89,6 +89,9 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; proxy_read_timeout 300s; + {% if not proxy_buffering %} + proxy_buffering off; + {% endif %} {% else %} return 503; {% endif %} diff --git a/src/dstack/_internal/proxy/gateway/services/nginx.py b/src/dstack/_internal/proxy/gateway/services/nginx.py index 74581c6477..1486ea0c22 100644 --- a/src/dstack/_internal/proxy/gateway/services/nginx.py +++ b/src/dstack/_internal/proxy/gateway/services/nginx.py @@ -68,6 +68,7 @@ class ServiceConfig(SiteConfig): replicas: list[ReplicaConfig] has_router_replica: bool = False cors_enabled: bool = False + proxy_buffering: bool = True class ModelEntrypointConfig(SiteConfig): diff --git a/src/dstack/_internal/proxy/gateway/services/registry.py b/src/dstack/_internal/proxy/gateway/services/registry.py index 592a964924..c69b57136a 100644 --- a/src/dstack/_internal/proxy/gateway/services/registry.py +++ b/src/dstack/_internal/proxy/gateway/services/registry.py @@ -62,6 +62,7 @@ async def register_service( replicas=(), has_router_replica=has_router_replica, cors_enabled=cors_enabled, + proxy_buffering=model is None, ) async with lock: @@ -407,6 +408,7 @@ async def get_nginx_service_config( replicas=sorted(replicas, key=lambda r: r.id), # sort for reproducible configs has_router_replica=service.has_router_replica, cors_enabled=service.cors_enabled, + proxy_buffering=service.proxy_buffering, ) diff --git a/src/dstack/_internal/proxy/lib/models.py b/src/dstack/_internal/proxy/lib/models.py index 53eb13e742..511f27d70c 100644 --- a/src/dstack/_internal/proxy/lib/models.py +++ b/src/dstack/_internal/proxy/lib/models.py @@ -63,6 +63,9 @@ class Service(ImmutableModel): replicas: tuple[Replica, ...] has_router_replica: bool = False cors_enabled: bool = False # only used on gateways; enabled for openai-format models + proxy_buffering: bool = True + """Only used on gateways. Disabled for services with a model so that streamed responses + reach the client as the replica emits them.""" @model_validator(mode="before") @classmethod diff --git a/src/tests/_internal/proxy/gateway/routers/test_registry.py b/src/tests/_internal/proxy/gateway/routers/test_registry.py index 20a0e148ce..48c7a41d5f 100644 --- a/src/tests/_internal/proxy/gateway/routers/test_registry.py +++ b/src/tests/_internal/proxy/gateway/routers/test_registry.py @@ -226,6 +226,13 @@ async def test_register_with_model(self, tmp_path: Path, system_mocks: Mocks) -> format_spec=OpenAIChatModelFormat(prefix="/v1"), ) ] + resp = await client.post( + "/api/registry/test-proj/services/test-run/replicas/register", + json=register_replica_payload(), + ) + assert resp.status_code == 200 + conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text() + assert "proxy_buffering off;" in conf async def test_register_with_rate_limits(self, tmp_path: Path, system_mocks: Mocks) -> None: client = make_client(tmp_path) @@ -325,6 +332,7 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None: assert (m1 := re.search(r"server unix:/(.+)/replica.sock; # replica xxx-xxx", conf)) assert (m2 := re.search(r"server unix:/(.+)/replica.sock; # replica yyy-yyy", conf)) assert m1.group(1) != m2.group(1) + assert "proxy_buffering" not in conf assert system_mocks.reload_nginx.call_count == 3 assert system_mocks.open_conn.call_count == 2 From 878efaded9398f49f01795ed5e4bafe2781864e2 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 30 Sep 2026 13:41:13 +0500 Subject: [PATCH 613/640] Apply DSTACK_SERVICE_CLIENT_TIMEOUT to gateway nginx and app (#4327) * Apply DSTACK_SERVICE_CLIENT_TIMEOUT to gateway nginx and app The server now passes the timeout to gateways on service and entrypoint registration. The default is raised from 60 to 300 seconds to match the previous nginx timeout. * Fix docstrings * Clean up tests --- mkdocs/docs/reference/env.md | 2 +- src/dstack/_internal/proxy/gateway/models.py | 2 + .../gateway/resources/nginx/entrypoint.jinja2 | 2 +- .../gateway/resources/nginx/service.jinja2 | 4 +- .../proxy/gateway/routers/registry.py | 2 + .../proxy/gateway/schemas/registry.py | 3 ++ .../_internal/proxy/gateway/services/nginx.py | 2 + .../proxy/gateway/services/registry.py | 6 +++ src/dstack/_internal/proxy/lib/const.py | 2 + src/dstack/_internal/proxy/lib/models.py | 3 ++ .../proxy/lib/services/service_connection.py | 7 +--- .../pipeline_tasks/gateway_replicas.py | 1 + .../server/services/gateways/client.py | 9 ++++- .../_internal/server/services/proxy/repo.py | 6 ++- src/dstack/_internal/server/settings.py | 1 + .../proxy/gateway/routers/test_registry.py | 39 ++++++++++++++++++- .../pipeline_tasks/test_gateway_replicas.py | 5 +++ 17 files changed, 83 insertions(+), 13 deletions(-) diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 4a169fc22f..c721de395c 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -132,7 +132,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_OTEL_METRICS_ENABLED`{ #DSTACK_OTEL_METRICS_ENABLED } – Enables OpenTelemetry metrics if set to any value. Requires the `otel` extra. - `DSTACK_OTEL_METRICS_EXPORTERS`{ #DSTACK_OTEL_METRICS_EXPORTERS } – A comma-separated list of OpenTelemetry metrics exporters: `otlp` (push via OTLP) and/or `prometheus` (expose on the `/metrics` endpoint). Defaults to `otlp`. - `DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE`{ #DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE } – Request body size limit for services running with a gateway, in bytes. Defaults to 64 MiB. -- `DSTACK_SERVICE_CLIENT_TIMEOUT`{ #DSTACK_SERVICE_CLIENT_TIMEOUT } – Timeout in seconds for HTTP requests sent from the in-server proxy and gateways to service replicas. Defaults to 60. +- `DSTACK_SERVICE_CLIENT_TIMEOUT`{ #DSTACK_SERVICE_CLIENT_TIMEOUT } – How long (in seconds) the in-server proxy and gateways wait for a service replica to send data. Defaults to 300. - `DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY`{ #DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY } – Forbids registering new services without a gateway if set to any value. - `DSTACK_FORBID_DSTACK_IN_RUNS`{ #DSTACK_FORBID_DSTACK_IN_RUNS } – Forbids submitting runs with `dstack: true` (dstack server access inside runs) if set to any value. - `DSTACK_SERVER_CODE_UPLOAD_LIMIT`{ #DSTACK_SERVER_CODE_UPLOAD_LIMIT } - The repo size limit when uploading diffs or local repos, in bytes. Set to `0` to disable size limits. Defaults to `2MiB`. diff --git a/src/dstack/_internal/proxy/gateway/models.py b/src/dstack/_internal/proxy/gateway/models.py index 3e0a76c032..80ce0b8abe 100644 --- a/src/dstack/_internal/proxy/gateway/models.py +++ b/src/dstack/_internal/proxy/gateway/models.py @@ -4,6 +4,7 @@ from pydantic import AnyHttpUrl +from dstack._internal.proxy.lib.const import DEFAULT_SERVICE_READ_TIMEOUT from dstack._internal.proxy.lib.models import ImmutableModel @@ -11,6 +12,7 @@ class ModelEntrypoint(ImmutableModel): project_name: str domain: str https: bool + read_timeout: int = DEFAULT_SERVICE_READ_TIMEOUT class ACMESettings(ImmutableModel): diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 index bcb66e1dd4..ce0160f558 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 @@ -4,7 +4,7 @@ server { proxy_pass http://localhost:{{ proxy_port }}/api/models/{{ project_name }}/; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; - proxy_read_timeout 300s; + proxy_read_timeout {{ read_timeout }}s; } listen 80; {% if https %} diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 index 4f2adda389..e3eced5f6d 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 @@ -68,7 +68,7 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; - proxy_read_timeout 300s; + proxy_read_timeout {{ read_timeout }}s; {% else %} return 503; {% endif %} @@ -88,7 +88,7 @@ server { proxy_pass http://{{ domain }}.upstream; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; - proxy_read_timeout 300s; + proxy_read_timeout {{ read_timeout }}s; {% if not proxy_buffering %} proxy_buffering off; {% endif %} diff --git a/src/dstack/_internal/proxy/gateway/routers/registry.py b/src/dstack/_internal/proxy/gateway/routers/registry.py index c5fb7c0d6a..954ae2933a 100644 --- a/src/dstack/_internal/proxy/gateway/routers/registry.py +++ b/src/dstack/_internal/proxy/gateway/routers/registry.py @@ -35,6 +35,7 @@ async def register_service( rate_limits=body.rate_limits, auth=body.auth, client_max_body_size=body.client_max_body_size, + read_timeout=body.read_timeout, model=body.options.openai.model if body.options.openai is not None else None, ssh_private_key=body.ssh_private_key, repo=repo, @@ -139,6 +140,7 @@ async def register_entrypoint( project_name=project_name.lower(), domain=body.domain.lower(), https=body.https, + read_timeout=body.read_timeout, repo=repo, nginx=nginx, ) diff --git a/src/dstack/_internal/proxy/gateway/schemas/registry.py b/src/dstack/_internal/proxy/gateway/schemas/registry.py index 9a800c2ceb..25f801252a 100644 --- a/src/dstack/_internal/proxy/gateway/schemas/registry.py +++ b/src/dstack/_internal/proxy/gateway/schemas/registry.py @@ -3,6 +3,7 @@ from pydantic import BaseModel, Field from dstack._internal.core.models.instances import SSHConnectionParams +from dstack._internal.proxy.lib.const import DEFAULT_SERVICE_READ_TIMEOUT from dstack._internal.proxy.lib.models import RateLimit @@ -47,6 +48,7 @@ class RegisterServiceRequest(BaseModel): ssh_private_key: str rate_limits: tuple[RateLimit, ...] = () has_router_replica: bool = False + read_timeout: int = DEFAULT_SERVICE_READ_TIMEOUT class SetServiceIdRequest(BaseModel): @@ -68,3 +70,4 @@ class RegisterReplicaRequest(BaseModel): class RegisterEntrypointRequest(BaseModel): domain: str https: bool + read_timeout: int = DEFAULT_SERVICE_READ_TIMEOUT diff --git a/src/dstack/_internal/proxy/gateway/services/nginx.py b/src/dstack/_internal/proxy/gateway/services/nginx.py index 1486ea0c22..df71594f5d 100644 --- a/src/dstack/_internal/proxy/gateway/services/nginx.py +++ b/src/dstack/_internal/proxy/gateway/services/nginx.py @@ -69,11 +69,13 @@ class ServiceConfig(SiteConfig): has_router_replica: bool = False cors_enabled: bool = False proxy_buffering: bool = True + read_timeout: int class ModelEntrypointConfig(SiteConfig): type: Literal["entrypoint"] = "entrypoint" project_name: str + read_timeout: int class Nginx: diff --git a/src/dstack/_internal/proxy/gateway/services/registry.py b/src/dstack/_internal/proxy/gateway/services/registry.py index c69b57136a..2ce9ea9ffd 100644 --- a/src/dstack/_internal/proxy/gateway/services/registry.py +++ b/src/dstack/_internal/proxy/gateway/services/registry.py @@ -42,6 +42,7 @@ async def register_service( rate_limits: tuple[models.RateLimit, ...], auth: bool, client_max_body_size: int, + read_timeout: int, model: Optional[schemas.AnyModel], ssh_private_key: str, repo: GatewayProxyRepo, @@ -63,6 +64,7 @@ async def register_service( has_router_replica=has_router_replica, cors_enabled=cors_enabled, proxy_buffering=model is None, + read_timeout=read_timeout, ) async with lock: @@ -252,6 +254,7 @@ async def register_model_entrypoint( project_name: str, domain: str, https: bool, + read_timeout: int, repo: GatewayProxyRepo, nginx: Nginx, ) -> None: @@ -259,6 +262,7 @@ async def register_model_entrypoint( project_name=project_name, domain=domain, https=https, + read_timeout=read_timeout, ) logger.debug("Registering entrypoint %s in project %s", domain, project_name) await apply_entrypoint(entrypoint, repo, nginx) @@ -409,6 +413,7 @@ async def get_nginx_service_config( has_router_replica=service.has_router_replica, cors_enabled=service.cors_enabled, proxy_buffering=service.proxy_buffering, + read_timeout=service.read_timeout, ) @@ -419,6 +424,7 @@ async def apply_entrypoint( domain=entrypoint.domain, https=entrypoint.https, project_name=entrypoint.project_name, + read_timeout=entrypoint.read_timeout, ) acme = (await repo.get_config()).acme_settings await nginx.register(config, acme) diff --git a/src/dstack/_internal/proxy/lib/const.py b/src/dstack/_internal/proxy/lib/const.py index 43ede03ac8..0824a5edb7 100644 --- a/src/dstack/_internal/proxy/lib/const.py +++ b/src/dstack/_internal/proxy/lib/const.py @@ -2,6 +2,8 @@ Shared constants for proxy components (gateway + in-server proxy). """ +DEFAULT_SERVICE_READ_TIMEOUT = 300 + # Inference endpoints exposed by the in-replica HTTP router. Applies to both # SGLang's router and Dynamo's `dynamo.frontend` — they share the # OpenAI-compatible endpoint surface. diff --git a/src/dstack/_internal/proxy/lib/models.py b/src/dstack/_internal/proxy/lib/models.py index 511f27d70c..44d5878bbd 100644 --- a/src/dstack/_internal/proxy/lib/models.py +++ b/src/dstack/_internal/proxy/lib/models.py @@ -7,6 +7,7 @@ from typing_extensions import Annotated from dstack._internal.core.models.instances import SSHConnectionParams +from dstack._internal.proxy.lib.const import DEFAULT_SERVICE_READ_TIMEOUT from dstack._internal.proxy.lib.errors import UnexpectedProxyError @@ -66,6 +67,8 @@ class Service(ImmutableModel): proxy_buffering: bool = True """Only used on gateways. Disabled for services with a model so that streamed responses reach the client as the replica emits them.""" + read_timeout: int = DEFAULT_SERVICE_READ_TIMEOUT + """Seconds to wait for data from a replica between two successive reads.""" @model_validator(mode="before") @classmethod diff --git a/src/dstack/_internal/proxy/lib/services/service_connection.py b/src/dstack/_internal/proxy/lib/services/service_connection.py index 37bdc5083a..c8229ad53a 100644 --- a/src/dstack/_internal/proxy/lib/services/service_connection.py +++ b/src/dstack/_internal/proxy/lib/services/service_connection.py @@ -19,14 +19,11 @@ from dstack._internal.proxy.lib.models import Project, Replica, Service from dstack._internal.proxy.lib.repo import BaseProxyRepo from dstack._internal.utils.common import get_or_error -from dstack._internal.utils.env import environ from dstack._internal.utils.logging import get_logger from dstack._internal.utils.path import FileContent logger = get_logger(__name__) OPEN_TUNNEL_TIMEOUT = 10 -HTTP_TIMEOUT = environ.get_int("DSTACK_SERVICE_CLIENT_TIMEOUT", default=60) -# Same as default Nginx proxy timeout; override via DSTACK_SERVICE_CLIENT_TIMEOUT class ServiceClient(httpx.AsyncClient): @@ -75,7 +72,7 @@ def __init__(self, project: Project, service: Service, replica: Replica) -> None # The hostname in base_url is there for troubleshooting, as it may appear in # logs and in the Host header. The actual destination is the Unix socket. base_url=f"http://{replica.id}-{service.run_name}/", - timeout=HTTP_TIMEOUT, + timeout=service.read_timeout, ) self._is_open = asyncio.locks.Event() @@ -148,7 +145,7 @@ async def get_service_replica_client( return httpx.AsyncClient( base_url="http://127.0.0.1", headers={"Host": service.domain}, - timeout=HTTP_TIMEOUT, + timeout=service.read_timeout, ) # Nginx not available, forward directly to the tunnel replica = random.choice(service.replicas) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index 44fd687cda..7e51f90eff 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -1288,6 +1288,7 @@ async def _register_service( ), auth=run_spec.configuration.auth, client_max_body_size=settings.DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE, + read_timeout=settings.SERVICE_CLIENT_TIMEOUT, options=service_spec.options, rate_limits=run_spec.configuration.rate_limits, ssh_private_key=run_model.project.ssh_private_key, diff --git a/src/dstack/_internal/server/services/gateways/client.py b/src/dstack/_internal/server/services/gateways/client.py index ed6c8d78b9..0c9defecca 100644 --- a/src/dstack/_internal/server/services/gateways/client.py +++ b/src/dstack/_internal/server/services/gateways/client.py @@ -45,6 +45,7 @@ async def register_service( gateway_https: bool, auth: bool, client_max_body_size: int, + read_timeout: int, options: dict, rate_limits: list[RateLimit], ssh_private_key: str, @@ -52,7 +53,7 @@ async def register_service( ): if "openai" in options: entrypoint = f"gateway.{domain.split('.', maxsplit=1)[1]}" - await self.register_openai_entrypoint(project, entrypoint, gateway_https) + await self.register_openai_entrypoint(project, entrypoint, gateway_https, read_timeout) payload = { "id": run_id.hex, @@ -61,6 +62,7 @@ async def register_service( "https": service_https, "auth": auth, "client_max_body_size": client_max_body_size, + "read_timeout": read_timeout, "options": options, "rate_limits": [limit.model_dump() for limit in rate_limits], "ssh_private_key": ssh_private_key, @@ -160,12 +162,15 @@ async def set_service_id(self, project: str, run_name: str, run_id: uuid.UUID) - resp.raise_for_status() self.is_server_ready = True - async def register_openai_entrypoint(self, project: str, domain: str, https: bool): + async def register_openai_entrypoint( + self, project: str, domain: str, https: bool, read_timeout: int + ): resp = await self._client.post( self._url(f"/api/registry/{project}/entrypoints/register"), json={ "domain": domain, "https": https, + "read_timeout": read_timeout, }, ) if resp.status_code == 400: diff --git a/src/dstack/_internal/server/services/proxy/repo.py b/src/dstack/_internal/server/services/proxy/repo.py index d0afd41def..7ea7df43cd 100644 --- a/src/dstack/_internal/server/services/proxy/repo.py +++ b/src/dstack/_internal/server/services/proxy/repo.py @@ -32,7 +32,10 @@ from dstack._internal.server.services.instances import get_instance_remote_connection_info from dstack._internal.server.services.jobs import get_job_spec from dstack._internal.server.services.runs import get_run_spec -from dstack._internal.server.settings import DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE +from dstack._internal.server.settings import ( + DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE, + SERVICE_CLIENT_TIMEOUT, +) from dstack._internal.utils.common import get_or_error _ANY_MODEL_ADAPTER = pydantic.TypeAdapter(AnyModel) @@ -136,6 +139,7 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic strip_prefix=run_spec.configuration.strip_prefix, replicas=tuple(replicas), has_router_replica=has_router_replica, + read_timeout=SERVICE_CLIENT_TIMEOUT, ) async def list_models(self, project_name: str) -> List[ChatModel]: diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index efe6e320fc..c077feb4ea 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -180,6 +180,7 @@ def get_database_url() -> str: DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE = int( os.getenv("DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE", 64 * 1024 * 1024) ) +SERVICE_CLIENT_TIMEOUT = environ.get_int("DSTACK_SERVICE_CLIENT_TIMEOUT", default=300) SERVER_DEFAULT_DOCKER_REGISTRY = os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY") or None SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME = ( diff --git a/src/tests/_internal/proxy/gateway/routers/test_registry.py b/src/tests/_internal/proxy/gateway/routers/test_registry.py index 48c7a41d5f..393b901acc 100644 --- a/src/tests/_internal/proxy/gateway/routers/test_registry.py +++ b/src/tests/_internal/proxy/gateway/routers/test_registry.py @@ -31,8 +31,9 @@ def register_service_payload( client_max_body_size: int = 1024, options: Optional[dict] = None, rate_limits: Optional[list[dict]] = None, + read_timeout: Optional[int] = None, ) -> dict: - return { + payload = { "id": uuid.uuid4().hex, "run_name": run_name, "domain": domain, @@ -43,6 +44,9 @@ def register_service_payload( "rate_limits": rate_limits or [], "ssh_private_key": "private-key", } + if read_timeout is not None: + payload["read_timeout"] = read_timeout + return payload def register_replica_payload(job_id: str = "xxx-xxx") -> dict: @@ -234,6 +238,26 @@ async def test_register_with_model(self, tmp_path: Path, system_mocks: Mocks) -> conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text() assert "proxy_buffering off;" in conf + async def test_register_with_read_timeout(self, tmp_path: Path, system_mocks: Mocks) -> None: + repo = GatewayProxyRepo() + client = make_client(tmp_path, repo=repo) + resp = await client.post( + "/api/registry/test-proj/services/register", + json=register_service_payload(run_name="test-run", read_timeout=900), + ) + assert resp.status_code == 200 + service = await repo.get_service("test-proj", "test-run") + assert service is not None + assert service.read_timeout == 900 + resp = await client.post( + "/api/registry/test-proj/services/test-run/replicas/register", + json=register_replica_payload(), + ) + assert resp.status_code == 200 + conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text() + assert conf.count("proxy_read_timeout 900s;") == 2 + assert "proxy_read_timeout 300s;" not in conf + async def test_register_with_rate_limits(self, tmp_path: Path, system_mocks: Mocks) -> None: client = make_client(tmp_path) resp = await client.post( @@ -599,6 +623,19 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None: assert system_mocks.reload_nginx.call_count == 1 assert system_mocks.run_certbot.call_count == 0 + async def test_register_with_read_timeout(self, tmp_path: Path, system_mocks: Mocks) -> None: + repo = GatewayProxyRepo() + client = make_client(tmp_path, repo=repo) + resp = await client.post( + "/api/registry/test-proj/entrypoints/register", + json={"domain": "gateway.gtw.test", "https": False, "read_timeout": 900}, + ) + assert resp.status_code == 200 + conf = (tmp_path / "sites-enabled" / "443-gateway.gtw.test.conf").read_text() + assert "proxy_read_timeout 900s;" in conf + [entrypoint] = await repo.list_entrypoints() + assert entrypoint.read_timeout == 900 + async def test_register_with_https(self, tmp_path: Path, system_mocks: Mocks) -> None: client = make_client(tmp_path) resp = await client.post( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py index 07f0a00b5f..aaddd0b7eb 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_gateway_replicas.py @@ -20,6 +20,7 @@ from dstack._internal.core.models.instances import InstanceStatus from dstack._internal.core.models.runs import JobStatus, RunStatus, ServiceSpec from dstack._internal.proxy.gateway.schemas.services import ServiceListItem, ServiceListReplicaItem +from dstack._internal.server import settings from dstack._internal.server.background.pipeline_tasks.gateway_replicas import ( GatewayReplicaFetcher, GatewayReplicaPipeline, @@ -817,7 +818,9 @@ async def test_registers_new_service_and_replica( session: AsyncSession, worker: GatewayReplicaWorker, mock_gateway_connection: AsyncMock, + monkeypatch: pytest.MonkeyPatch, ): + monkeypatch.setattr(settings, "SERVICE_CLIENT_TIMEOUT", 900) project = await create_project(session=session) user = await create_user(session=session) repo = await create_repo(session=session, project_id=project.id) @@ -858,6 +861,7 @@ async def test_registers_new_service_and_replica( gateway_https=ANY, auth=ANY, client_max_body_size=ANY, + read_timeout=900, options={}, rate_limits=[], ssh_private_key=project.ssh_private_key, @@ -1432,6 +1436,7 @@ async def test_unregisters_and_reregisters_legacy_service_without_id_and_replica gateway_https=ANY, auth=ANY, client_max_body_size=ANY, + read_timeout=ANY, options={}, rate_limits=[], ssh_private_key=project.ssh_private_key, From 1284826412ce75cfb639392a5fe74668ce67f7af Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:00:23 +0200 Subject: [PATCH 614/640] Add Daytona backend (#4321) * Add Daytona backend * Shorten Daytona volume documentation * Validate Daytona API key permissions Document required permissions and remove redundant Daytona tests. * Pin gpuhunt 0.1.31 * Clarify Daytona volume sharing * Clarify Daytona volume documentation * Clean up Daytona docs, timeouts, and tests --------- Co-authored-by: Andrey Cheptsov --- frontend/src/types/backend.d.ts | 1 + frontend/src/types/volume.d.ts | 4 +- mkdocs/docs/concepts/backends.md | 28 + mkdocs/docs/concepts/volumes.md | 27 +- mkdocs/docs/reference/dstack.yml/volume.md | 8 + mkdocs/docs/reference/server/config.yml.md | 17 + pyproject.toml | 3 +- .../cli/services/configurators/volume.py | 10 +- .../_internal/core/backends/configurators.py | 7 + .../core/backends/daytona/__init__.py | 0 .../core/backends/daytona/api_client.py | 267 ++++++ .../core/backends/daytona/backend.py | 16 + .../core/backends/daytona/compute.py | 449 +++++++++ .../core/backends/daytona/configurator.py | 75 ++ .../_internal/core/backends/daytona/models.py | 42 + src/dstack/_internal/core/backends/models.py | 7 + .../_internal/core/models/backends/base.py | 2 + src/dstack/_internal/core/models/volumes.py | 41 +- .../background/pipeline_tasks/common.py | 2 + .../pipeline_tasks/jobs_submitted.py | 5 +- .../_internal/server/services/instances.py | 18 +- .../server/services/jobs/__init__.py | 5 +- .../_internal/server/services/offers.py | 19 +- .../_internal/server/services/volumes.py | 7 +- .../core/backends/daytona/__init__.py | 0 .../core/backends/daytona/test_api_client.py | 242 +++++ .../core/backends/daytona/test_compute.py | 867 ++++++++++++++++++ .../backends/daytona/test_configurator.py | 96 ++ .../core/backends/daytona/test_models.py | 26 + .../_internal/core/models/test_volumes.py | 53 ++ .../_internal/server/routers/test_backends.py | 1 + .../server/services/jobs/test_jobs.py | 44 + .../server/services/test_instances.py | 70 +- .../_internal/server/services/test_offers.py | 31 +- .../_internal/server/services/test_volumes.py | 28 +- 35 files changed, 2474 insertions(+), 44 deletions(-) create mode 100644 src/dstack/_internal/core/backends/daytona/__init__.py create mode 100644 src/dstack/_internal/core/backends/daytona/api_client.py create mode 100644 src/dstack/_internal/core/backends/daytona/backend.py create mode 100644 src/dstack/_internal/core/backends/daytona/compute.py create mode 100644 src/dstack/_internal/core/backends/daytona/configurator.py create mode 100644 src/dstack/_internal/core/backends/daytona/models.py create mode 100644 src/tests/_internal/core/backends/daytona/__init__.py create mode 100644 src/tests/_internal/core/backends/daytona/test_api_client.py create mode 100644 src/tests/_internal/core/backends/daytona/test_compute.py create mode 100644 src/tests/_internal/core/backends/daytona/test_configurator.py create mode 100644 src/tests/_internal/core/backends/daytona/test_models.py diff --git a/frontend/src/types/backend.d.ts b/frontend/src/types/backend.d.ts index f09bf1b208..d42a26ba17 100644 --- a/frontend/src/types/backend.d.ts +++ b/frontend/src/types/backend.d.ts @@ -4,6 +4,7 @@ declare type TBackendType = | 'crusoe' | 'cudo' | 'datacrunch' + | 'daytona' | 'dstack' | 'gcp' | 'kubernetes' diff --git a/frontend/src/types/volume.d.ts b/frontend/src/types/volume.d.ts index e704e02b07..c72878f3d3 100644 --- a/frontend/src/types/volume.d.ts +++ b/frontend/src/types/volume.d.ts @@ -11,7 +11,7 @@ declare interface IVolumeConfiguration { type: "volume", name?: string, backend: TBackendType, - region: string, + region?: string, size?: number, volume_id?: string } @@ -19,7 +19,7 @@ declare interface IVolumeConfiguration { declare interface IVolumeProvisioningData { backend?: TBackendType, volume_id: string, - size_gb: number, + size_gb: number | null, availability_zone?: string price?:number attachable: boolean diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index 110c1b78a5..fd23b1ad5f 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -1473,6 +1473,34 @@ projects: - **Runs always execute as `root`.** `enroot` is a single-user container runtime: it can only map the invoking host user inside the container to either the same UID/GID or to `root` (`0:0`). `dstack` uses the latter, so runs always execute as `root`, and both the image's default user and the run configuration's `user` property are ignored. The single-user model is a fundamental `enroot` limitation, not specific to the `dstack` integration. - **Private registries via `registry_auth` are not supported.** The `registry_auth` run configuration property is rejected. You can still pull images from a private registry by preconfiguring `enroot` credentials on the cluster's compute nodes — see the [enroot import documentation](https://github.com/NVIDIA/enroot/blob/main/doc/cmd/import.md#description). +### Daytona + +Create an API key in the [Daytona dashboard](https://app.daytona.io/) for the organization +that will own the sandboxes. Then configure the backend: + +
+ +```yaml +projects: + - name: main + backends: + - type: daytona + creds: + type: api_key + api_key: YOUR_DAYTONA_API_KEY +``` + +
+ +??? info "Required permissions" + The API key requires `write:sandboxes`, `delete:sandboxes`, and `read:limits`. + + Additional permissions depend on the features used: + + * Private images: `write:registries` and `delete:registries` + * Creating volumes: `read:volumes`, `write:volumes`, and `delete:volumes` + * Registering existing volumes: `read:volumes` + ### Runpod Log into your [Runpod](https://www.runpod.io/console/) console, click Settings in the sidebar, expand the `API Keys` section, and click diff --git a/mkdocs/docs/concepts/volumes.md b/mkdocs/docs/concepts/volumes.md index 19be81952d..6aa26ead5a 100644 --- a/mkdocs/docs/concepts/volumes.md +++ b/mkdocs/docs/concepts/volumes.md @@ -16,7 +16,7 @@ Useful as a cache for cloud fleets or for persistent storage with SSH fleets. ## Network volumes -> Network volumes are currently supported for the `aws`, `gcp`, `runpod`, and `kubernetes` backends. +> Network volumes are currently supported for the `aws`, `gcp`, `runpod`, `kubernetes`, and `daytona` backends. ### Apply a configuration @@ -42,6 +42,16 @@ size: 100GB If you use this configuration, `dstack` will create a new volume based on the specified options. +??? info "Daytona" + Daytona volumes use [S3-backed FUSE storage](https://www.daytona.io/docs/en/volumes/). + Storage grows with usage, and the same volume can be mounted in any Daytona region. + + ```yaml + type: volume + backend: daytona + name: my-volume + ``` + ??? info "Kubernetes" Set `region` to a kubeconfig context name enabled in the [backend configuration](backends.md#kubernetes). Omit it only for legacy configurations without `contexts`. @@ -86,7 +96,7 @@ Volume my-volume does not exist yet. Create the volume? [y/n]: y Once created, the volume can be attached to dev environments, tasks, and services. -> When creating a new network volume, `dstack` automatically creates an `ext4` filesystem on it. +> When creating a new network volume in AWS or GCP, `dstack` automatically creates an `ext4` filesystem on it. #### Register existing volumes @@ -110,7 +120,7 @@ volume_id: vol1235
-If you register an existing volume, you must ensure the volume already has a filesystem. +If you register an existing volume in AWS or GCP, you must ensure the volume already has a filesystem. ??? info "Kubernetes" @@ -254,7 +264,7 @@ If you've registered an existing volume, it will be de-registered with `dstack` ??? info "Can I use network volumes across backends?" - Since volumes are backed up by cloud network disks, you can only use them within the same cloud. If you need to access + Volumes can only be used with their backend. If you need to access data across different backends, you should either use object storage or replicate the data across multiple volumes. ??? info "Can I use network volumes across regions?" @@ -262,19 +272,22 @@ If you've registered an existing volume, it will be de-registered with `dstack` Typically, network volumes are associated with specific regions, so you can't use them in other regions. Often, volumes are also linked to availability zones, but some providers support volumes that can be used across different availability zones within the same region. - + If you don't want to limit a run to one particular region, you can create different volumes for different regions and specify them for the same mount point as [documented above](#attach-network-volume). + Daytona volumes can be mounted in any Daytona region. + ??? info "Can I attach network volumes to multiple runs or instances?" - You can mount a volume in multiple runs. This feature is currently supported only by the `runpod` backend. + The `runpod` and `daytona` backends support mounting the same volume in multiple runs at the same time. + For `kubernetes`, the storage class and access modes must support sharing, such as `ReadWriteMany`. ## Instance volumes Instance volumes allow mapping any directory on the instance where the run is executed to any path inside the container. This means that the data in instance volumes is persisted only if the run is executed on the same instance. -> Instance volumes are currently supported for all backends except `runpod` and `vastai`, and can also be used with [SSH fleets](fleets.md#ssh-fleets). +> Instance volumes are currently supported for all backends except `daytona`, `runpod`, and `vastai`, and can also be used with [SSH fleets](fleets.md#ssh-fleets). ### Attach a volume diff --git a/mkdocs/docs/reference/dstack.yml/volume.md b/mkdocs/docs/reference/dstack.yml/volume.md index 23eccccd11..58883b4744 100644 --- a/mkdocs/docs/reference/dstack.yml/volume.md +++ b/mkdocs/docs/reference/dstack.yml/volume.md @@ -26,6 +26,14 @@ The `volume` configuration type allows creating, registering, and updating [volu backend: required: true +=== "Daytona" + + #SCHEMA# dstack._internal.core.models.volumes.DaytonaVolumeConfiguration + overrides: + show_root_heading: false + backend: + required: true + === "Kubernetes" Kubernetes backend volumes are mapped to [`PersistentVolumeClaim`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) objects. diff --git a/mkdocs/docs/reference/server/config.yml.md b/mkdocs/docs/reference/server/config.yml.md index e6b5359084..538f703a89 100644 --- a/mkdocs/docs/reference/server/config.yml.md +++ b/mkdocs/docs/reference/server/config.yml.md @@ -92,6 +92,23 @@ to configure [backends](../../concepts/backends.md) and other [server-level sett type: required: true +##### `projects[n].backends[type=daytona]` { #daytona data-toc-label="daytona" } + +#SCHEMA# dstack._internal.core.backends.daytona.models.DaytonaBackendConfigWithCreds + overrides: + show_root_heading: false + type: + required: true + item_id_prefix: daytona- + +###### `projects[n].backends[type=daytona].creds` { #daytona-creds data-toc-label="creds" } + +#SCHEMA# dstack._internal.core.backends.daytona.models.DaytonaAPIKeyCreds + overrides: + show_root_heading: false + type: + required: true + ##### `projects[n].backends[type=gcp]` { #gcp data-toc-label="gcp" } #SCHEMA# dstack._internal.core.backends.gcp.models.GCPBackendConfigWithCreds diff --git a/pyproject.toml b/pyproject.toml index 4c623306c4..5c51736acb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "python-multipart>=0.0.16", "filelock", "psutil", - "gpuhunt==0.1.30", + "gpuhunt==0.1.31", "argcomplete>=3.5.0", "ignore-python>=0.2.0", "apscheduler<4", @@ -110,6 +110,7 @@ include = [ "src/dstack/_internal/server", "src/dstack/_internal/core/services", "src/dstack/_internal/core/backends/aws", + "src/dstack/_internal/core/backends/daytona", "src/dstack/_internal/core/backends/kubernetes", "src/dstack/_internal/core/backends/runpod", "src/dstack/_internal/core/backends/slurm", diff --git a/src/dstack/_internal/cli/services/configurators/volume.py b/src/dstack/_internal/cli/services/configurators/volume.py index 2449de0da7..f8e2dea334 100644 --- a/src/dstack/_internal/cli/services/configurators/volume.py +++ b/src/dstack/_internal/cli/services/configurators/volume.py @@ -17,6 +17,7 @@ AnyVolumeConfiguration, Volume, VolumeConfigurationWithRegion, + VolumeConfigurationWithSize, VolumePlan, VolumeSpec, VolumeStatus, @@ -204,9 +205,6 @@ def th(s: str) -> str: configuration_table.add_row(th("Type"), plan.spec.configuration.type) volume_type = "managed" - size = "-" - if plan.spec.configuration.size is not None: - size = str(plan.spec.configuration.size) if plan.spec.configuration.is_external: volume_type = "external" @@ -214,7 +212,11 @@ def th(s: str) -> str: configuration_table.add_row(th("Backend"), plan.spec.configuration.backend.value) if isinstance(plan.spec.configuration, VolumeConfigurationWithRegion): configuration_table.add_row(th("Region"), plan.spec.configuration.region) - configuration_table.add_row(th("Size"), size) + if isinstance(plan.spec.configuration, VolumeConfigurationWithSize): + size = "-" + if plan.spec.configuration.size is not None: + size = str(plan.spec.configuration.size) + configuration_table.add_row(th("Size"), size) console.print(configuration_table) console.print() diff --git a/src/dstack/_internal/core/backends/configurators.py b/src/dstack/_internal/core/backends/configurators.py index 2fa3267be4..7bad991340 100644 --- a/src/dstack/_internal/core/backends/configurators.py +++ b/src/dstack/_internal/core/backends/configurators.py @@ -53,6 +53,13 @@ except ImportError: pass +try: + from dstack._internal.core.backends.daytona.configurator import DaytonaConfigurator + + _CONFIGURATOR_CLASSES.append(DaytonaConfigurator) +except ImportError: + pass + try: from dstack._internal.core.backends.digitalocean.configurator import ( DigitalOceanConfigurator, diff --git a/src/dstack/_internal/core/backends/daytona/__init__.py b/src/dstack/_internal/core/backends/daytona/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/dstack/_internal/core/backends/daytona/api_client.py b/src/dstack/_internal/core/backends/daytona/api_client.py new file mode 100644 index 0000000000..c1a9981059 --- /dev/null +++ b/src/dstack/_internal/core/backends/daytona/api_client.py @@ -0,0 +1,267 @@ +import time +from typing import Any, Dict, List, Optional +from urllib.parse import quote, urlsplit + +import requests +from gpuhunt.providers.daytona import API_URL + +from dstack._internal.core.errors import BackendError, BackendInvalidCredentialsError + +REQUEST_TIMEOUT_SECONDS = 30 + + +class DaytonaAPIError(BackendError): + def __init__(self, message: str, status_code: Optional[int] = None): + super().__init__(message) + self.status_code = status_code + + +class DaytonaNotFoundError(DaytonaAPIError): + pass + + +class DaytonaAPIClient: + def __init__(self, api_key: str): + self.api_key = api_key + + def get_current_api_key(self) -> Dict[str, Any]: + return _object(self._make_request("GET", "/api-keys/current")) + + def get_organization_usage(self, organization_id: str) -> Dict[str, Any]: + return _object( + self._make_request("GET", f"/organizations/{_path_id(organization_id)}/usage") + ) + + def get_available_sandbox_classes(self, organization_id: str) -> List[Dict[str, Any]]: + return _objects( + self._make_request( + "GET", f"/organizations/{_path_id(organization_id)}/available-sandbox-classes" + ) + ) + + def get_shared_regions(self) -> List[Dict[str, Any]]: + return _objects(self._make_request("GET", "/shared-regions")) + + def create_sandbox(self, payload: Dict[str, Any]) -> Dict[str, Any]: + # Do not retry creates: a lost response does not mean the sandbox was not created. + return _object(self._make_request("POST", "/sandbox", json=payload)) + + def get_sandbox(self, sandbox_id_or_name: str) -> Optional[Dict[str, Any]]: + try: + return _object(self._make_request("GET", f"/sandbox/{_path_id(sandbox_id_or_name)}")) + except DaytonaNotFoundError: + return None + + def delete_sandbox(self, sandbox_id_or_name: str) -> None: + self._delete(f"/sandbox/{_path_id(sandbox_id_or_name)}") + + def create_registry(self, payload: Dict[str, Any]) -> Dict[str, Any]: + return _object(self._make_request("POST", "/docker-registry", json=payload)) + + def get_registries(self) -> List[Dict[str, Any]]: + return _objects(self._make_request("GET", "/docker-registry")) + + def delete_registry(self, registry_id: str) -> None: + self._delete(f"/docker-registry/{_path_id(registry_id)}") + + def create_volume(self, name: str) -> Dict[str, Any]: + return _object(self._make_request("POST", "/volumes", json={"name": name})) + + def get_volume(self, volume_id: str) -> Optional[Dict[str, Any]]: + try: + return _object(self._make_request("GET", f"/volumes/{_path_id(volume_id)}")) + except DaytonaNotFoundError: + return None + + def get_volume_by_name(self, name: str) -> Optional[Dict[str, Any]]: + try: + return _object(self._make_request("GET", f"/volumes/by-name/{_path_id(name)}")) + except DaytonaNotFoundError: + return None + + def delete_volume(self, volume_id: str) -> None: + self._delete(f"/volumes/{_path_id(volume_id)}") + + def get_toolbox_url(self, sandbox_id: str) -> str: + data = _object( + self._make_request("GET", f"/sandbox/{_path_id(sandbox_id)}/toolbox-proxy-url") + ) + proxy_url = data.get("url") + if not isinstance(proxy_url, str): + raise DaytonaAPIError("Daytona returned an invalid toolbox proxy URL") + _validate_toolbox_url(proxy_url) + # The API returns the proxy base; Toolbox routes also need the sandbox ID. + return f"{proxy_url.rstrip('/')}/{_path_id(sandbox_id)}" + + def get_session(self, toolbox_url: str, session_id: str) -> Optional[Dict[str, Any]]: + try: + return _object( + self._make_toolbox_request( + "GET", toolbox_url, f"/process/session/{_path_id(session_id)}" + ) + ) + except DaytonaNotFoundError: + return None + + def create_session(self, toolbox_url: str, session_id: str) -> None: + self._make_toolbox_request( + "POST", toolbox_url, "/process/session", json={"sessionId": session_id} + ) + + def execute_session_command( + self, + toolbox_url: str, + session_id: str, + command: str, + *, + run_async: bool = True, + ) -> Dict[str, Any]: + return _object( + self._make_toolbox_request( + "POST", + toolbox_url, + f"/process/session/{_path_id(session_id)}/exec", + json={"command": command, "runAsync": run_async}, + ) + ) + + def create_ssh_access( + self, sandbox_id_or_name: str, *, expires_in_minutes: int + ) -> Dict[str, Any]: + if expires_in_minutes <= 0: + raise ValueError("SSH access lifetime must be positive") + return _object( + self._make_request( + "POST", + f"/sandbox/{_path_id(sandbox_id_or_name)}/ssh-access", + params={"expiresInMinutes": expires_in_minutes}, + ) + ) + + def _delete(self, path: str) -> None: + # Deletes are idempotent. Retry transient failures, including during rollback + # when the server has not yet persisted the resource's provisioning data. + for attempt in range(3): + try: + self._make_request("DELETE", path) + return + except DaytonaNotFoundError: + return + except DaytonaAPIError as e: + if attempt == 2 or ( + e.status_code is not None + and e.status_code not in (408, 429) + and e.status_code < 500 + ): + raise + time.sleep(attempt + 1) + + def _make_request( + self, + method: str, + path: str, + *, + json: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + ) -> Any: + return self._request(method, API_URL + path, json=json, params=params) + + def _make_toolbox_request( + self, + method: str, + toolbox_url: str, + path: str, + *, + json: Optional[Dict[str, Any]] = None, + ) -> Any: + _validate_toolbox_url(toolbox_url) + return self._request(method, toolbox_url.rstrip("/") + path, json=json) + + def _request( + self, + method: str, + url: str, + *, + json: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + ) -> Any: + try: + response = requests.request( + method=method, + url=url, + headers={"Authorization": f"Bearer {self.api_key}"}, + json=json, + params=params, + timeout=REQUEST_TIMEOUT_SECONDS, + allow_redirects=False, + ) + except requests.Timeout as e: + raise DaytonaAPIError("Daytona API request timed out") from e + except requests.RequestException as e: + raise DaytonaAPIError("Daytona API request failed") from e + if response.status_code == 401: + raise BackendInvalidCredentialsError(fields=[["creds", "api_key"]]) + if not 200 <= response.status_code < 300: + message = _response_message(response) + if self.api_key: + message = message.replace(self.api_key, "[redacted]") + error = DaytonaNotFoundError if response.status_code == 404 else DaytonaAPIError + raise error(message, status_code=response.status_code) + if not response.content: + return None + try: + return response.json() + except ValueError as e: + raise DaytonaAPIError("Daytona API returned a non-JSON response") from e + + +def _object(data: Any) -> Dict[str, Any]: + if not isinstance(data, dict): + raise DaytonaAPIError("Daytona API returned an invalid object response") + return data + + +def _objects(data: Any) -> List[Dict[str, Any]]: + if not isinstance(data, list) or not all(isinstance(item, dict) for item in data): + raise DaytonaAPIError("Daytona API returned an invalid list response") + return data + + +def _path_id(value: str) -> str: + return quote(value, safe="") + + +def _validate_toolbox_url(url: str) -> None: + try: + parsed = urlsplit(url) + valid = ( + parsed.scheme == "https" + and parsed.hostname is not None + and parsed.hostname.endswith((".daytona.io", ".daytona.work")) + and parsed.port in (None, 443) + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + ) + except ValueError: + valid = False + if not valid: + raise DaytonaAPIError("Daytona returned an invalid toolbox proxy URL") + + +def _response_message(response: requests.Response) -> str: + message = f"Daytona API returned HTTP {response.status_code}" + try: + data = response.json() + except ValueError: + data = None + if isinstance(data, dict): + for field in ("message", "error", "detail"): + value = data.get(field) + if isinstance(value, str) and value: + return f"{message}: {value}" + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return f"{message}: {'; '.join(value)}" + # Do not include arbitrary HTML/text responses, which can reflect request credentials. + return message diff --git a/src/dstack/_internal/core/backends/daytona/backend.py b/src/dstack/_internal/core/backends/daytona/backend.py new file mode 100644 index 0000000000..e89594c514 --- /dev/null +++ b/src/dstack/_internal/core/backends/daytona/backend.py @@ -0,0 +1,16 @@ +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.backends.daytona.compute import DaytonaCompute +from dstack._internal.core.backends.daytona.models import DaytonaConfig +from dstack._internal.core.models.backends.base import BackendType + + +class DaytonaBackend(Backend): + TYPE = BackendType.DAYTONA + COMPUTE_CLASS = DaytonaCompute + + def __init__(self, config: DaytonaConfig): + self.config = config + self._compute = DaytonaCompute(self.config) + + def compute(self) -> DaytonaCompute: + return self._compute diff --git a/src/dstack/_internal/core/backends/daytona/compute.py b/src/dstack/_internal/core/backends/daytona/compute.py new file mode 100644 index 0000000000..c30a4a661a --- /dev/null +++ b/src/dstack/_internal/core/backends/daytona/compute.py @@ -0,0 +1,449 @@ +import re +import shlex +import time +from typing import Any, List, Optional + +import gpuhunt +from gpuhunt.providers.daytona import DaytonaProvider + +from dstack._internal.core.backends.base.authorized_keys import build_authorized_keys +from dstack._internal.core.backends.base.compute import ( + Compute, + ComputeWithFilteredOffersCached, + ComputeWithVolumeSupport, + generate_unique_instance_name_for_job, + generate_unique_volume_name, + get_docker_commands, +) +from dstack._internal.core.backends.base.offers import get_catalog_offers +from dstack._internal.core.backends.daytona.api_client import DaytonaAPIClient, DaytonaAPIError +from dstack._internal.core.backends.daytona.models import DaytonaConfig +from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT +from dstack._internal.core.errors import ComputeError, NotYetTerminated, ProvisioningError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.common import CoreModel, RegistryAuth, validate_json_extra_ignore +from dstack._internal.core.models.instances import ( + InstanceAvailability, + InstanceOffer, + InstanceOfferWithAvailability, + InstanceRuntime, + SSHConnectionParams, +) +from dstack._internal.core.models.placement import PlacementGroup +from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run +from dstack._internal.core.models.volumes import ( + DaytonaVolumeConfiguration, + Volume, + VolumeMountPoint, + VolumeProvisioningData, +) +from dstack._internal.utils.common import get_or_error +from dstack._internal.utils.docker import DEFAULT_REGISTRY, is_default_registry, parse_image_name +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +MAX_INSTANCE_NAME_LEN = 63 +RUNNER_SESSION = "dstack-runner" +# A replacement token revokes the previous one for new connections. Request one +# ten-year credential at provisioning and reuse it for the sandbox's lifetime. +SSH_ACCESS_MINUTES = 3650 * 24 * 60 +VOLUME_READY_TIMEOUT_SECONDS = 60 + + +class DaytonaCompute(ComputeWithFilteredOffersCached, ComputeWithVolumeSupport, Compute): + def __init__(self, config: DaytonaConfig): + super().__init__() + self.config = config + self.api_client = DaytonaAPIClient(config.creds.api_key) + self._catalog = gpuhunt.Catalog(balance_resources=False, auto_reload=False) + self._catalog.add_provider(DaytonaProvider(api_key=config.creds.api_key)) + self._organization_id: Optional[str] = None + + def get_offers_by_requirements( + self, requirements: Requirements, full_offers: bool, unallocated_resources: bool + ) -> List[InstanceOfferWithAvailability]: + offers = get_catalog_offers( + backend=BackendType.DAYTONA, + locations=self.config.regions or None, + requirements=requirements, + catalog=self._catalog, + ) + if not offers: + return [] + organization_id = self._organization_id + if organization_id is None: + organization_id = self.api_client.get_current_api_key().get("organizationId") + if not isinstance(organization_id, str) or not organization_id: + raise DaytonaAPIError("Daytona API key has no organization") + self._organization_id = organization_id + usage = self.api_client.get_organization_usage(organization_id) + classes = self.api_client.get_available_sandbox_classes(organization_id) + region_usage = { + row["regionId"]: row + for row in usage["regionUsage"] + if row["sandboxClass"] == "container" + } + region_classes = { + row["regionId"]: row for row in classes if row["sandboxClass"] == "container" + } + return [ + offer.with_availability( + availability=( + InstanceAvailability.AVAILABLE + if _has_quota( + offer, + region_usage.get(offer.region), + region_classes.get(offer.region), + ) + else InstanceAvailability.NO_QUOTA + ), + instance_runtime=InstanceRuntime.RUNNER, + ) + for offer in offers + ] + + def run_job( + self, + run: Run, + job: Job, + instance_offer: InstanceOfferWithAvailability, + project_ssh_public_key: str, + project_ssh_private_key: str, + volumes: List[Volume], + placement_group: Optional[PlacementGroup], + requirements: Requirements, + extra_authorized_keys: list[str], + ) -> JobProvisioningData: + image_name = job.job_spec.image_name + # Whitespace could inject additional Dockerfile instructions. + if not image_name or re.search(r"\s", image_name): + raise ComputeError("Invalid Daytona container image name") + instance_name = generate_unique_instance_name_for_job( + run, job, max_length=MAX_INSTANCE_NAME_LEN + ) + commands = get_docker_commands( + build_authorized_keys(project_ssh_public_key, extra_authorized_keys) + ) + backend_data = DaytonaInstanceBackendData( + startup_command=shlex.join(["sh", "-c", " && ".join(commands)]) + ) + registry: Optional[str] = None + if job.job_spec.registry_auth is not None: + image = parse_image_name(image_name) + registry = image.registry or DEFAULT_REGISTRY + if is_default_registry(registry): + registry = DEFAULT_REGISTRY + # Daytona's Dockerfile builder includes organization credentials only + # for FROM images with a registry prefix. + suffix = f"@{image.digest}" if image.digest else f":{image.tag}" + image_name = f"{registry}/{image.repo}{suffix}" + resources = instance_offer.instance.resources + payload = { + "name": instance_name, + "buildInfo": { + # The sandbox's user field does not override the image USER. + # Daytona caches builds, so updating a mutable tag such as latest may + # still reuse an older image. A digest-pinned image avoids this. + "dockerfileContent": ( + f'FROM {image_name}\nUSER root\nENTRYPOINT []\nCMD ["sleep", "infinity"]\n' + ) + }, + "user": "root", + "target": instance_offer.region, + "cpu": resources.cpus, + "memory": round(resources.memory_mib / 1024), + "disk": round(resources.disk.size_mib / 1024), + "gpu": len(resources.gpus), + "spot": resources.spot, + "public": False, + "autoStopInterval": 0, + "autoPauseInterval": 0, + "autoDeleteInterval": 0, + "ttlMinutes": 0, + } + if resources.gpus: + payload["gpuType"] = [_get_gpu_type(instance_offer)] + if volumes: + mount_points = job.job_spec.volumes + if mount_points is None: + mount_points = run.run_spec.configuration.volumes + paths = {} + for mount in mount_points: + if isinstance(mount, VolumeMountPoint): + names = [mount.name] if isinstance(mount.name, str) else mount.name + paths.update((name, mount.path) for name in names) + payload["volumes"] = [ + {"volumeId": get_or_error(volume.volume_id), "mountPath": paths[volume.name]} + for volume in volumes + ] + if job.job_spec.registry_auth is not None: + # Daytona selects registry credentials at organization scope, without a + # per-sandbox selector. Another record for the same registry can affect this pull. + backend_data.registry_id = self._create_registry( + f"{instance_name}-registry", get_or_error(registry), job.job_spec.registry_auth + ) + instance_id = instance_name + try: + sandbox = self.api_client.create_sandbox(payload) + if isinstance(sandbox.get("id"), str) and sandbox["id"]: + instance_id = sandbox["id"] + except DaytonaAPIError as e: + if e.status_code is not None and e.status_code < 500: + self._cleanup_failed_registry(backend_data.registry_id) + raise + # A timed-out create may have succeeded. Persist its unique name so + # provisioning can find it and termination can clean it up; never retry POST. + logger.warning( + "Daytona sandbox %s creation response was lost; checking by name", instance_name + ) + except Exception: + self._cleanup_failed_registry(backend_data.registry_id) + raise + return JobProvisioningData( + backend=BackendType.DAYTONA, + instance_type=instance_offer.instance, + instance_id=instance_id, + region=instance_offer.region, + price=instance_offer.price, + username="root", + dockerized=False, + backend_data=backend_data.model_dump_json(), + ) + + def _create_registry(self, name: str, url: str, auth: RegistryAuth) -> str: + try: + registry = self.api_client.create_registry( + {"name": name, "url": url, "username": auth.username, "password": auth.password} + ) + return registry["id"] + except DaytonaAPIError as e: + if e.status_code is not None and e.status_code < 500: + raise + # Recover the exact per-job record if its create response was lost. + for registry in self.api_client.get_registries(): + if registry["name"] == name: + return registry["id"] + raise + + def _cleanup_failed_registry(self, registry_id: Optional[str]) -> None: + if registry_id is None: + return + try: + self.api_client.delete_registry(registry_id) + except Exception: + logger.exception( + "Failed to delete Daytona registry %s after provisioning failed. " + "Delete it manually in Daytona.", + registry_id, + ) + + def update_provisioning_data( + self, + provisioning_data: JobProvisioningData, + project_ssh_public_key: str, + project_ssh_private_key: str, + ): + if provisioning_data.hostname is not None and provisioning_data.ssh_port is not None: + return + sandbox = self.api_client.get_sandbox(provisioning_data.instance_id) + if sandbox is None: + # The create response may have been lost before the sandbox became visible. + # The server's provisioning timeout still schedules deletion by name. + return + state = sandbox.get("state") + if state in { + "error", + "build_failed", + "destroyed", + "destroying", + "stopped", + "stopping", + "archived", + "archiving", + "paused", + "pausing", + }: + reason = sandbox.get("errorReason") or state + raise ProvisioningError(f"Daytona sandbox failed to start: {reason}") + if state != "started": + return + backend_data = validate_json_extra_ignore( + DaytonaInstanceBackendData, provisioning_data.backend_data or "{}" + ) + toolbox_url = self.api_client.get_toolbox_url(sandbox["id"]) + self._start_runner(toolbox_url, get_or_error(backend_data.startup_command)) + access = self.api_client.create_ssh_access( + sandbox["id"], expires_in_minutes=SSH_ACCESS_MINUTES + ) + provisioning_data.ssh_proxy = _get_ssh_proxy(access) + provisioning_data.hostname = "localhost" + provisioning_data.ssh_port = DSTACK_RUNNER_SSH_PORT + # The server checks runner readiness through the normal SSH connection. + backend_data.startup_command = None + provisioning_data.backend_data = ( + backend_data.model_dump_json() if backend_data.registry_id is not None else None + ) + + def _start_runner(self, toolbox_url: str, startup_command: str) -> None: + session = self.api_client.get_session(toolbox_url, RUNNER_SESSION) + if session is None: + self.api_client.create_session(toolbox_url, RUNNER_SESSION) + session = {"commands": []} + commands = session["commands"] + if not commands: + self.api_client.execute_session_command( + toolbox_url, RUNNER_SESSION, startup_command, run_async=True + ) + return + for command in commands: + if command.get("exitCode") is not None: + raise ProvisioningError( + "Daytona runner setup exited with code " + f"{command['exitCode']}. Check the dstack-runner session in Daytona." + ) + + def terminate_instance( + self, instance_id: str, region: str, backend_data: Optional[str] = None + ): + sandbox = self.api_client.get_sandbox(instance_id) + if sandbox is not None and sandbox.get("state") != "destroyed": + if sandbox.get("state") != "destroying": + self.api_client.delete_sandbox(instance_id) + raise NotYetTerminated("Waiting for Daytona sandbox deletion") + data = validate_json_extra_ignore(DaytonaInstanceBackendData, backend_data or "{}") + if data.registry_id is not None: + self.api_client.delete_registry(data.registry_id) + + def register_volume(self, volume: Volume) -> VolumeProvisioningData: + assert isinstance(volume.configuration, DaytonaVolumeConfiguration) + volume_id = get_or_error(volume.configuration.volume_id) + self._wait_for_volume(volume_id) + return _volume_provisioning_data(volume_id) + + def create_volume(self, volume: Volume) -> VolumeProvisioningData: + name = generate_unique_volume_name(volume) + try: + data = self.api_client.create_volume(name) + except DaytonaAPIError as e: + if e.status_code is not None and e.status_code < 500: + raise + data = self.api_client.get_volume_by_name(name) + if data is None: + raise + volume_id = data["id"] + try: + self._wait_for_volume(volume_id) + except Exception: + try: + self.api_client.delete_volume(volume_id) + except Exception: + logger.exception( + "Failed to delete Daytona volume %s after provisioning failed. " + "Delete it manually in Daytona.", + volume_id, + ) + raise + return _volume_provisioning_data(volume_id) + + def _wait_for_volume(self, volume_id: str) -> None: + deadline = time.monotonic() + VOLUME_READY_TIMEOUT_SECONDS + while True: + data = self.api_client.get_volume(volume_id) + if data is None: + raise ComputeError(f"Daytona volume {volume_id} not found") + state = data["state"] + if state == "ready": + return + if state not in {"pending_create", "creating"}: + raise ComputeError( + f"Daytona volume {volume_id} is {state}: {data.get('errorReason') or state}" + ) + if time.monotonic() >= deadline: + raise ComputeError(f"Timed out waiting for Daytona volume {volume_id}") + time.sleep(2) + + def delete_volume(self, volume: Volume) -> None: + if volume.volume_id is None: + return + data = self.api_client.get_volume(volume.volume_id) + if data is None or data["state"] in {"pending_delete", "deleting", "deleted"}: + return + self.api_client.delete_volume(volume.volume_id) + + +class DaytonaInstanceBackendData(CoreModel): + startup_command: Optional[str] = None + registry_id: Optional[str] = None + + +def _volume_provisioning_data(volume_id: str) -> VolumeProvisioningData: + return VolumeProvisioningData( + backend=BackendType.DAYTONA, + volume_id=volume_id, + size_gb=None, + price=0, + attachable=False, + detachable=False, + ) + + +def _has_quota( + offer: InstanceOffer, + usage: Optional[dict[str, Any]], + sandbox_class: Optional[dict[str, Any]], +) -> bool: + if usage is None or sandbox_class is None: + return False + resources = offer.instance.resources + sizes = { + "Cpu": resources.cpus, + "Memory": resources.memory_mib / 1024, + "Disk": resources.disk.size_mib / 1024, + } + gpu_count = len(resources.gpus) + if gpu_count: + if not sandbox_class.get("gpuAvailable"): + return False + if not resources.spot and gpu_count > usage["totalGpuQuota"] - usage["currentGpuUsage"]: + return False + for limits in (usage, sandbox_class): + allowed_types = limits.get("allowedGpuTypes") + if allowed_types is not None and _get_gpu_type(offer) not in allowed_types: + return False + # GPU sandboxes use separate quotas: CPU/RAM/disk aggregate totals may be zero. + # Check per-GPU limits, falling back to per-sandbox limits. + for name, size in sizes.items(): + limit = usage.get(f"max{name}PerGpu") + if limit is not None: + limit *= gpu_count + else: + limit = usage.get(f"max{name}PerSandbox") + if limit is not None and size > limit: + return False + return True + for name, size in sizes.items(): + if size > usage[f"total{name}Quota"] - usage[f"current{name}Usage"]: + return False + limit = usage.get(f"max{name}PerSandbox") + if limit is not None and size > limit: + return False + return True + + +def _get_gpu_type(offer: InstanceOffer) -> str: + return offer.backend_data.get("gpu_type") or offer.instance.resources.gpus[0].name + + +def _get_ssh_proxy(access: dict[str, Any]) -> SSHConnectionParams: + # Parse only the endpoint; never execute the provider's command or log its token. + try: + parts = shlex.split(access["sshCommand"]) + target = next(part for part in parts[1:] if "@" in part) + _, hostname = target.rsplit("@", 1) + port = int(parts[parts.index("-p") + 1]) if "-p" in parts else 22 + if parts[0] != "ssh" or not hostname or not access["token"] or not 1 <= port <= 65535: + raise ValueError + except (KeyError, IndexError, StopIteration, TypeError, ValueError) as e: + raise ProvisioningError("Daytona returned invalid SSH connection details") from e + return SSHConnectionParams(hostname=hostname, port=port, username=access["token"]) diff --git a/src/dstack/_internal/core/backends/daytona/configurator.py b/src/dstack/_internal/core/backends/daytona/configurator.py new file mode 100644 index 0000000000..184e51d4ac --- /dev/null +++ b/src/dstack/_internal/core/backends/daytona/configurator.py @@ -0,0 +1,75 @@ +import json + +from gpuhunt.providers.daytona import GPU_REGION + +from dstack._internal.core.backends.base.configurator import BackendRecord, Configurator +from dstack._internal.core.backends.daytona.api_client import DaytonaAPIClient +from dstack._internal.core.backends.daytona.backend import DaytonaBackend +from dstack._internal.core.backends.daytona.models import ( + DaytonaBackendConfig, + DaytonaBackendConfigWithCreds, + DaytonaConfig, + DaytonaCreds, + DaytonaStoredConfig, +) +from dstack._internal.core.errors import BackendInvalidCredentialsError, ServerClientError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore + + +class DaytonaConfigurator(Configurator[DaytonaBackendConfig, DaytonaBackendConfigWithCreds]): + TYPE = BackendType.DAYTONA + BACKEND_CLASS = DaytonaBackend + + def validate_config(self, config: DaytonaBackendConfigWithCreds, default_creds_enabled: bool): + client = DaytonaAPIClient(api_key=config.creds.api_key) + api_key = client.get_current_api_key() + required_permissions = {"write:sandboxes", "delete:sandboxes", "read:limits"} + missing_permissions = sorted(required_permissions - set(api_key["permissions"])) + if missing_permissions: + raise BackendInvalidCredentialsError( + msg=f"Daytona API key is missing permissions: {', '.join(missing_permissions)}", + fields=[["creds", "api_key"]], + ) + if not config.regions: + return + regions = {GPU_REGION, *(region["id"] for region in client.get_shared_regions())} + invalid_regions = sorted(set(config.regions) - regions) + if invalid_regions: + raise ServerClientError( + msg=( + f"Unsupported Daytona regions: {invalid_regions}. " + f"Supported regions: {sorted(regions)}." + ), + fields=[["regions"]], + ) + + def create_backend( + self, project_name: str, config: DaytonaBackendConfigWithCreds + ) -> BackendRecord: + return BackendRecord( + config=DaytonaStoredConfig( + **validate_extra_ignore(DaytonaBackendConfig, config).model_dump() + ).model_dump_json(), + auth=DaytonaCreds.model_validate(config.creds).model_dump_json(), + ) + + def get_backend_config_with_creds( + self, record: BackendRecord + ) -> DaytonaBackendConfigWithCreds: + return validate_extra_ignore(DaytonaBackendConfigWithCreds, self._get_config(record)) + + def get_backend_config_without_creds(self, record: BackendRecord) -> DaytonaBackendConfig: + return validate_extra_ignore(DaytonaBackendConfig, self._get_config(record)) + + def get_backend(self, record: BackendRecord) -> DaytonaBackend: + return DaytonaBackend(config=self._get_config(record)) + + def _get_config(self, record: BackendRecord) -> DaytonaConfig: + return validate_extra_ignore( + DaytonaConfig, + { + **json.loads(record.config), + "creds": validate_json_extra_ignore(DaytonaCreds, record.auth), + }, + ) diff --git a/src/dstack/_internal/core/backends/daytona/models.py b/src/dstack/_internal/core/backends/daytona/models.py new file mode 100644 index 0000000000..3ee230d33d --- /dev/null +++ b/src/dstack/_internal/core/backends/daytona/models.py @@ -0,0 +1,42 @@ +from typing import Annotated, Literal, Optional, Union + +from pydantic import Field + +from dstack._internal.core.models.common import CoreModel + + +class DaytonaAPIKeyCreds(CoreModel): + type: Annotated[Literal["api_key"], Field(description="The type of credentials")] = "api_key" + api_key: Annotated[str, Field(description="The Daytona API key", min_length=1)] + + +AnyDaytonaCreds = DaytonaAPIKeyCreds +DaytonaCreds = AnyDaytonaCreds + + +class DaytonaBackendConfig(CoreModel): + type: Annotated[Literal["daytona"], Field(description="The type of backend")] = "daytona" + regions: Annotated[ + Optional[list[str]], + Field( + description=( + "The list of Daytona regions. GPU sandboxes use `earth`; CPU sandboxes use " + "shared regions such as `us` and `eu`. Omit to use all regions" + ) + ), + ] = None + + +class DaytonaBackendConfigWithCreds(DaytonaBackendConfig): + creds: Annotated[AnyDaytonaCreds, Field(description="The credentials")] + + +AnyDaytonaBackendConfig = Union[DaytonaBackendConfig, DaytonaBackendConfigWithCreds] + + +class DaytonaStoredConfig(DaytonaBackendConfig): + pass + + +class DaytonaConfig(DaytonaStoredConfig): + creds: AnyDaytonaCreds diff --git a/src/dstack/_internal/core/backends/models.py b/src/dstack/_internal/core/backends/models.py index 071d2b1b9f..94d9dcd094 100644 --- a/src/dstack/_internal/core/backends/models.py +++ b/src/dstack/_internal/core/backends/models.py @@ -23,6 +23,10 @@ CudoBackendConfig, CudoBackendConfigWithCreds, ) +from dstack._internal.core.backends.daytona.models import ( + DaytonaBackendConfig, + DaytonaBackendConfigWithCreds, +) from dstack._internal.core.backends.digitalocean_base.models import ( BaseDigitalOceanBackendConfig, BaseDigitalOceanBackendConfigWithCreds, @@ -102,6 +106,7 @@ CloudRiftBackendConfig, CrusoeBackendConfig, CudoBackendConfig, + DaytonaBackendConfig, BaseDigitalOceanBackendConfig, GCPBackendConfig, HotAisleBackendConfig, @@ -131,6 +136,7 @@ CrusoeBackendConfigWithCreds, CudoBackendConfigWithCreds, VerdaBackendConfigWithCreds, + DaytonaBackendConfigWithCreds, BaseDigitalOceanBackendConfigWithCreds, GCPBackendConfigWithCreds, HotAisleBackendConfigWithCreds, @@ -175,6 +181,7 @@ class BackendConfigWithCreds(RootModel[AnyBackendConfigWithCredsTagged]): CrusoeBackendFileConfigWithCreds, CudoBackendConfigWithCreds, VerdaBackendConfigWithCreds, + DaytonaBackendConfigWithCreds, BaseDigitalOceanBackendConfigWithCreds, GCPBackendFileConfigWithCreds, HotAisleBackendFileConfigWithCreds, diff --git a/src/dstack/_internal/core/models/backends/base.py b/src/dstack/_internal/core/models/backends/base.py index b42b7bf575..6ee2931b18 100644 --- a/src/dstack/_internal/core/models/backends/base.py +++ b/src/dstack/_internal/core/models/backends/base.py @@ -11,6 +11,7 @@ class BackendType(str, enum.Enum): CRUSOE (BackendType): Crusoe CUDO (BackendType): Cudo DATACRUNCH (BackendType): DataCrunch (for backward compatibility) + DAYTONA (BackendType): Daytona DIGITALOCEAN (BackendType): DigitalOcean DSTACK (BackendType): dstack Sky GCP (BackendType): Google Cloud Platform @@ -37,6 +38,7 @@ class BackendType(str, enum.Enum): CUDO = "cudo" DATACRUNCH = "datacrunch" """`DATACRUNCH` is kept as a `BackendType` for backward compatibility.""" + DAYTONA = "daytona" DIGITALOCEAN = "digitalocean" DSTACK = "dstack" GCP = "gcp" diff --git a/src/dstack/_internal/core/models/volumes.py b/src/dstack/_internal/core/models/volumes.py index d651e717a4..92b3a650ee 100644 --- a/src/dstack/_internal/core/models/volumes.py +++ b/src/dstack/_internal/core/models/volumes.py @@ -42,10 +42,6 @@ class BaseVolumeConfiguration(CoreModel): > Variable is mutable so its type is invariant """ name: Annotated[Optional[str], Field(description="The volume name")] = None - size: Annotated[ - Optional[Memory], - Field(description="The volume size. Must be specified when creating new volumes"), - ] = None auto_cleanup_duration: Annotated[ OptionalIdleDuration, Field( @@ -81,6 +77,13 @@ def external_volume_id(self) -> Optional[str]: def is_external(self) -> bool: return self.external_volume_id is not None + +class VolumeConfigurationWithSize(BaseVolumeConfiguration): + size: Annotated[ + Optional[Memory], + Field(description="The volume size. Must be specified when creating new volumes"), + ] = None + @property def size_gb(self) -> int: return int(get_or_error(self.size)) @@ -108,7 +111,9 @@ def external_volume_id(self) -> Optional[str]: class AWSVolumeConfiguration( - VolumeConfigurationWithAvailibilityZone, VolumeConfigurationWithVolumeID + VolumeConfigurationWithAvailibilityZone, + VolumeConfigurationWithVolumeID, + VolumeConfigurationWithSize, ): backend: Annotated[Literal[BackendType.AWS], Field(description="The volume backend")] = ( BackendType.AWS @@ -116,14 +121,18 @@ class AWSVolumeConfiguration( class GCPVolumeConfiguration( - VolumeConfigurationWithAvailibilityZone, VolumeConfigurationWithVolumeID + VolumeConfigurationWithAvailibilityZone, + VolumeConfigurationWithVolumeID, + VolumeConfigurationWithSize, ): backend: Annotated[Literal[BackendType.GCP], Field(description="The volume backend")] = ( BackendType.GCP ) -class RunpodVolumeConfiguration(VolumeConfigurationWithRegion, VolumeConfigurationWithVolumeID): +class RunpodVolumeConfiguration( + VolumeConfigurationWithRegion, VolumeConfigurationWithVolumeID, VolumeConfigurationWithSize +): backend: Annotated[Literal[BackendType.RUNPOD], Field(description="The volume backend")] = ( BackendType.RUNPOD ) @@ -131,7 +140,7 @@ class RunpodVolumeConfiguration(VolumeConfigurationWithRegion, VolumeConfigurati """Runpod doesn't have AZs but we accept this field for compatibility with older clients.""" -class KubernetesVolumeConfiguration(VolumeConfigurationWithRegion): +class KubernetesVolumeConfiguration(VolumeConfigurationWithRegion, VolumeConfigurationWithSize): backend: Annotated[ Literal[BackendType.KUBERNETES], Field(description="The volume backend") ] = BackendType.KUBERNETES @@ -173,11 +182,18 @@ def external_volume_id(self) -> Optional[str]: return self.claim_name +class DaytonaVolumeConfiguration(VolumeConfigurationWithVolumeID): + backend: Annotated[Literal[BackendType.DAYTONA], Field(description="The volume backend")] = ( + BackendType.DAYTONA + ) + + AnyVolumeConfiguration = Union[ AWSVolumeConfiguration, GCPVolumeConfiguration, RunpodVolumeConfiguration, KubernetesVolumeConfiguration, + DaytonaVolumeConfiguration, ] @@ -202,7 +218,8 @@ class VolumeSpec(CoreModel): class VolumeProvisioningData(CoreModel): backend: Optional[BackendType] = None volume_id: str - size_gb: int + size_gb: Optional[int] + """The provisioned capacity, or `None` for volumes without a fixed capacity.""" availability_zone: Optional[str] = None price: Optional[float] = None """`price` stores the monthly price.""" @@ -263,6 +280,12 @@ def get_attachment_data_for_instance(self, instance_id: str) -> Optional[VolumeA def get_backend(self) -> BackendType: return self.configuration.backend + def matches_location(self, backend: BackendType, region: str) -> bool: + return self.get_backend() == backend and ( + not isinstance(self.configuration, VolumeConfigurationWithRegion) + or self.configuration.region.lower() == region.lower() + ) + def get_region(self) -> str: """ Returns the volume region or an empty string if the volume (that is, its backend) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/common.py b/src/dstack/_internal/server/background/pipeline_tasks/common.py index 3d0327075c..d6f9564452 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/common.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/common.py @@ -12,6 +12,8 @@ def get_provisioning_timeout(backend_type: BackendType, instance_type_name: str) return timedelta(minutes=30) if backend_type == BackendType.RUNPOD: return timedelta(minutes=20) + if backend_type == BackendType.DAYTONA: + return timedelta(minutes=20) if backend_type == BackendType.SEEWEB: # Seeweb GPU servers install drivers/Docker on first boot, which can exceed 10 minutes. return timedelta(minutes=20) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 4a1796847d..b0f592b0e6 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -2724,10 +2724,7 @@ def _get_offer_mount_point_volume( offer: InstanceOfferWithAvailability, ) -> Volume: for volume in volumes: - if ( - volume.get_backend() != offer.backend - or volume.get_region().lower() != offer.region.lower() - ): + if not volume.matches_location(offer.backend, offer.region): continue return volume raise ServerClientError("Failed to find an eligible volume for the mount point") diff --git a/src/dstack/_internal/server/services/instances.py b/src/dstack/_internal/server/services/instances.py index 2b763c93ad..cdcab7ce34 100644 --- a/src/dstack/_internal/server/services/instances.py +++ b/src/dstack/_internal/server/services/instances.py @@ -43,7 +43,7 @@ TerminationPolicy, ) from dstack._internal.core.models.runs import JobProvisioningData, Requirements -from dstack._internal.core.models.volumes import Volume +from dstack._internal.core.models.volumes import Volume, VolumeConfigurationWithRegion from dstack._internal.core.services.profiles import get_termination from dstack._internal.server import settings as server_settings from dstack._internal.server.models import ( @@ -610,14 +610,17 @@ def filter_instances( backend_types: Optional[list[BackendType]] = profile.backends regions: Optional[list[str]] = profile.regions zones: Optional[list[str]] = profile.availability_zones - # (BackendType, region.lower() | "", availability_zone.lower() | None) - volumes_locations: Optional[set[tuple[BackendType, str, Optional[str]]]] = None + # (BackendType, region.lower() | None, availability_zone.lower() | None). + # A None region matches any region; an empty string remains an exact region. + volumes_locations: Optional[set[tuple[BackendType, Optional[str], Optional[str]]]] = None if volumes: volumes_locations = set() for volume in volumes[0]: volume_backend = volume.get_backend() - volume_region = volume.get_region().lower() + volume_region = None + if isinstance(volume.configuration, VolumeConfigurationWithRegion): + volume_region = volume.get_region().lower() # If the volume has an AZ, it's added twice -- with and without an AZ. # When the instance location is checked against the available volumes locations (see # below) the instance with an AZ matches only the volume with the same AZ, while @@ -677,7 +680,12 @@ def filter_instances( instance_zone = jpd.availability_zone if instance_zone is not None: instance_zone = instance_zone.lower() - if (instance_backend, instance_region, instance_zone) not in volumes_locations: + instance_location = (instance_backend, instance_region, instance_zone) + regionless_location = (instance_backend, None, instance_zone) + if ( + instance_location not in volumes_locations + and regionless_location not in volumes_locations + ): continue filtered_instances.append(instance) return filtered_instances diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index ffd3078c21..0326be9aca 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -597,9 +597,8 @@ def _get_job_mount_point_attached_volume( Returns the volume attached to the job among the list of possible mount point volumes. """ for volume in volumes: - if ( - volume.get_backend() != job_provisioning_data.get_base_backend() - or volume.get_region().lower() != job_provisioning_data.region.lower() + if not volume.matches_location( + job_provisioning_data.get_base_backend(), job_provisioning_data.region ): continue if ( diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index 8193ab0863..7fbe24f5c3 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -22,7 +22,7 @@ from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.runs import JobProvisioningData, Requirements -from dstack._internal.core.models.volumes import Volume +from dstack._internal.core.models.volumes import Volume, VolumeConfigurationWithRegion from dstack._internal.server.models import ProjectModel from dstack._internal.server.services import backends as backends_services @@ -49,11 +49,19 @@ async def get_offers_by_requirements( regions: Optional[list[str]] = profile.regions availability_zones: Optional[list[str]] = profile.availability_zones instance_types: Optional[list[str]] = profile.instance_types - # (BackendType, region.lower() | "") - volumes_locations: Optional[set[tuple[BackendType, str]]] = None + # (BackendType, region.lower() | None). None means any region. + volumes_locations: Optional[set[tuple[BackendType, Optional[str]]]] = None if volumes: - volumes_locations = {(v.get_backend(), v.get_region().lower()) for v in volumes[0]} + volumes_locations = { + ( + v.get_backend(), + v.get_region().lower() + if isinstance(v.configuration, VolumeConfigurationWithRegion) + else None, + ) + for v in volumes[0] + } if multinode: if backend_types is None: @@ -206,7 +214,7 @@ def _filter_offers( availability_zones: Optional[List[str]] = None, instance_types: Optional[List[str]] = None, placement_group: Optional[PlacementGroup] = None, - volumes_locations: Optional[Container[tuple[BackendType, str]]] = None, + volumes_locations: Optional[Container[tuple[BackendType, Optional[str]]]] = None, ) -> Iterator[Tuple[Backend, InstanceOfferWithAvailability]]: """ Yields filtered offers. May return modified offers to match the filters. @@ -246,6 +254,7 @@ def _filter_offers( if ( volumes_locations is not None and (offer.backend, offer.region.lower()) not in volumes_locations + and (offer.backend, None) not in volumes_locations ): continue yield (b, offer) diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index 374fe76a4c..9d648eee46 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -22,6 +22,7 @@ VolumeAttachment, VolumeAttachmentData, VolumeConfiguration, + VolumeConfigurationWithSize, VolumeInstance, VolumeProvisioningData, VolumeSpec, @@ -474,7 +475,11 @@ async def generate_volume_name(session: AsyncSession, project: ProjectModel) -> def _validate_volume_configuration(configuration: AnyVolumeConfiguration): - if configuration.external_volume_id is None and configuration.size is None: + if ( + isinstance(configuration, VolumeConfigurationWithSize) + and configuration.external_volume_id is None + and configuration.size is None + ): raise ServerClientError("Volume must specify either existing identifier or size") backends_services.check_backend_type_available(configuration.backend) if configuration.backend not in BACKENDS_WITH_VOLUMES_SUPPORT: diff --git a/src/tests/_internal/core/backends/daytona/__init__.py b/src/tests/_internal/core/backends/daytona/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tests/_internal/core/backends/daytona/test_api_client.py b/src/tests/_internal/core/backends/daytona/test_api_client.py new file mode 100644 index 0000000000..a3198c6a81 --- /dev/null +++ b/src/tests/_internal/core/backends/daytona/test_api_client.py @@ -0,0 +1,242 @@ +import pytest +import requests + +from dstack._internal.core.backends.daytona.api_client import ( + API_URL, + DaytonaAPIClient, + DaytonaAPIError, +) +from dstack._internal.core.errors import BackendInvalidCredentialsError, NoCapacityError + +TOOLBOX_URL = "https://proxy.app.daytona.io/toolbox/sandbox-id" + + +class TestDaytonaAPIClient: + def test_identity_uses_bearer_auth(self, requests_mock): + requests_mock.get( + f"{API_URL}/api-keys/current", json={"organizationId": "organization-id"} + ) + + identity = DaytonaAPIClient("test-api-key").get_current_api_key() + + assert identity["organizationId"] == "organization-id" + assert requests_mock.last_request.headers["Authorization"] == "Bearer test-api-key" + assert requests_mock.last_request.timeout == 30 + + def test_unauthorized_maps_to_invalid_credentials(self, requests_mock): + requests_mock.get(f"{API_URL}/api-keys/current", status_code=401) + + with pytest.raises(BackendInvalidCredentialsError) as exc: + DaytonaAPIClient("bad-key").get_current_api_key() + + assert exc.value.fields == [["creds", "api_key"]] + + @pytest.mark.parametrize( + ["status", "message"], + [ + (400, "No capacity available"), + (429, "Too many requests"), + (503, "Service unavailable"), + ], + ) + def test_http_errors_preserve_status_and_reason(self, requests_mock, status, message): + requests_mock.post(f"{API_URL}/sandbox", status_code=status, json={"message": message}) + + with pytest.raises(DaytonaAPIError, match=message) as exc: + DaytonaAPIClient("test-api-key").create_sandbox({"name": "test"}) + + assert exc.value.status_code == status + assert not isinstance(exc.value, NoCapacityError) + assert requests_mock.call_count == 1 + + @pytest.mark.parametrize("error", [requests.ReadTimeout, requests.ConnectionError]) + def test_network_errors_preserve_cause_without_retrying(self, requests_mock, error): + cause = error("Connection failed") + requests_mock.post(f"{API_URL}/sandbox", exc=cause) + client = DaytonaAPIClient("test-api-key") + + with pytest.raises(DaytonaAPIError) as exc: + client.create_sandbox({"name": "test"}) + + assert exc.value.__cause__ is cause + assert exc.value.status_code is None + assert requests_mock.call_count == 1 + + def test_error_response_redacts_api_key(self, requests_mock): + requests_mock.get( + f"{API_URL}/api-keys/current", + status_code=403, + json={"message": "Denied for Bearer secret-api-key"}, + ) + + with pytest.raises(DaytonaAPIError, match="Denied for Bearer \\[redacted\\]"): + DaytonaAPIClient("secret-api-key").get_current_api_key() + + def test_non_json_error_omits_response_body(self, requests_mock): + requests_mock.get( + f"{API_URL}/api-keys/current", status_code=502, text="gateway HTML with credentials" + ) + + with pytest.raises(DaytonaAPIError) as exc: + DaytonaAPIClient("test-api-key").get_current_api_key() + + assert str(exc.value) == "Daytona API returned HTTP 502" + + def test_redirect_is_not_followed(self, requests_mock): + requests_mock.get( + f"{API_URL}/api-keys/current", + status_code=302, + headers={"Location": "https://untrusted.example/identity"}, + ) + + with pytest.raises(DaytonaAPIError) as exc: + DaytonaAPIClient("test-api-key").get_current_api_key() + + assert exc.value.status_code == 302 + assert requests_mock.call_count == 1 + + def test_get_sandbox_accepts_id_or_name_and_returns_none_only_for_404(self, requests_mock): + requests_mock.get(f"{API_URL}/sandbox/sandbox-id", json={"id": "sandbox-id"}) + requests_mock.get(f"{API_URL}/sandbox/job-name", json={"id": "sandbox-id"}) + requests_mock.get(f"{API_URL}/sandbox/missing", status_code=404) + requests_mock.get(f"{API_URL}/sandbox/forbidden", status_code=403) + client = DaytonaAPIClient("test-api-key") + + assert client.get_sandbox("sandbox-id") == {"id": "sandbox-id"} + assert client.get_sandbox("job-name") == {"id": "sandbox-id"} + assert client.get_sandbox("missing") is None + with pytest.raises(DaytonaAPIError): + client.get_sandbox("forbidden") + + def test_volume_create_lookup_and_deleted_tombstone(self, requests_mock): + pending = {"id": "volume-id", "state": "pending_create"} + deleted = {"id": "volume-id", "state": "deleted"} + requests_mock.post(f"{API_URL}/volumes", json=pending) + requests_mock.get(f"{API_URL}/volumes/by-name/volume-name", json=pending) + requests_mock.get(f"{API_URL}/volumes/volume-id", json=deleted) + requests_mock.get(f"{API_URL}/volumes/missing", status_code=404) + requests_mock.get(f"{API_URL}/volumes/by-name/missing", status_code=404) + client = DaytonaAPIClient("test-api-key") + + assert client.create_volume("volume-name") == pending + assert requests_mock.last_request.json() == {"name": "volume-name"} + assert client.get_volume_by_name("volume-name") == pending + assert client.get_volume("volume-id") == deleted + assert client.get_volume("missing") is None + assert client.get_volume_by_name("missing") is None + + @pytest.mark.parametrize( + ["method", "path", "status"], + [ + ("delete_sandbox", "/sandbox/resource-id", 200), + ("delete_registry", "/docker-registry/resource-id", 204), + ("delete_volume", "/volumes/resource-id", 404), + ], + ) + def test_delete_endpoints_are_idempotent(self, requests_mock, method, path, status): + requests_mock.delete(API_URL + path, status_code=status) + + getattr(DaytonaAPIClient("test-api-key"), method)("resource-id") + + @pytest.mark.parametrize( + ["method", "path", "failure"], + [ + ("delete_sandbox", "/sandbox/resource-id", {"status_code": 503}), + ("delete_registry", "/docker-registry/resource-id", {"status_code": 429}), + ("delete_volume", "/volumes/resource-id", {"exc": requests.ReadTimeout}), + ], + ) + def test_delete_retries_transient_errors(self, requests_mock, mocker, method, path, failure): + sleep = mocker.patch("dstack._internal.core.backends.daytona.api_client.time.sleep") + requests_mock.delete(API_URL + path, [failure, {"status_code": 404}]) + + getattr(DaytonaAPIClient("test-api-key"), method)("resource-id") + + assert requests_mock.call_count == 2 + sleep.assert_called_once_with(1) + + def test_delete_stops_retrying_after_three_attempts(self, requests_mock, mocker): + sleep = mocker.patch("dstack._internal.core.backends.daytona.api_client.time.sleep") + requests_mock.delete(f"{API_URL}/docker-registry/registry-id", status_code=503) + + with pytest.raises(DaytonaAPIError) as exc: + DaytonaAPIClient("test-api-key").delete_registry("registry-id") + + assert exc.value.status_code == 503 + assert requests_mock.call_count == 3 + assert [call.args[0] for call in sleep.call_args_list] == [1, 2] + + def test_delete_does_not_retry_permission_errors(self, requests_mock, mocker): + sleep = mocker.patch("dstack._internal.core.backends.daytona.api_client.time.sleep") + requests_mock.delete(f"{API_URL}/docker-registry/registry-id", status_code=403) + + with pytest.raises(DaytonaAPIError) as exc: + DaytonaAPIClient("test-api-key").delete_registry("registry-id") + + assert exc.value.status_code == 403 + assert requests_mock.call_count == 1 + sleep.assert_not_called() + + def test_toolbox_url_includes_sandbox_id(self, requests_mock): + requests_mock.get( + f"{API_URL}/sandbox/sandbox-id/toolbox-proxy-url", + json={"url": "https://proxy.app.daytona.io/toolbox/"}, + ) + + assert DaytonaAPIClient("test-api-key").get_toolbox_url("sandbox-id") == TOOLBOX_URL + + @pytest.mark.parametrize( + "url", + [ + "http://proxy.app.daytona.io/toolbox", + "https://daytona.io.attacker.example/toolbox", + "https://attacker.example/toolbox", + "https://user:password@proxy.app.daytona.io/toolbox", + "https://proxy.app.daytona.io/toolbox?token=secret", + "https://proxy.app.daytona.io:8443/toolbox", + ], + ) + def test_toolbox_rejects_untrusted_url_before_sending_credentials(self, requests_mock, url): + with pytest.raises(DaytonaAPIError, match="invalid toolbox proxy URL"): + DaytonaAPIClient("test-api-key").get_session(url, "runner") + + assert requests_mock.call_count == 0 + + def test_session_creation_and_async_command(self, requests_mock): + requests_mock.post(f"{TOOLBOX_URL}/process/session", status_code=201) + requests_mock.post( + f"{TOOLBOX_URL}/process/session/runner/exec", json={"cmdId": "command-id"} + ) + client = DaytonaAPIClient("test-api-key") + + client.create_session(TOOLBOX_URL, "runner") + command = client.execute_session_command(TOOLBOX_URL, "runner", "/tmp/dstack-runner start") + + assert requests_mock.request_history[0].json() == {"sessionId": "runner"} + assert requests_mock.last_request.json() == { + "command": "/tmp/dstack-runner start", + "runAsync": True, + } + assert requests_mock.last_request.headers["Authorization"] == "Bearer test-api-key" + assert command == {"cmdId": "command-id"} + + def test_missing_session(self, requests_mock): + requests_mock.get(f"{TOOLBOX_URL}/process/session/runner", status_code=404) + client = DaytonaAPIClient("test-api-key") + + assert client.get_session(TOOLBOX_URL, "runner") is None + + def test_ssh_access_preserves_explicit_long_lifetime(self, requests_mock): + access = { + "token": "ssh-access-token", + "sshCommand": "ssh -p 2222 ssh-access-token@ssh.app.daytona.io", + "expiresAt": "2027-09-28T00:00:00Z", + } + requests_mock.post(f"{API_URL}/sandbox/sandbox-id/ssh-access", json=access) + + result = DaytonaAPIClient("test-api-key").create_ssh_access( + "sandbox-id", expires_in_minutes=365 * 24 * 60 + ) + + assert result == access + assert requests_mock.last_request.qs == {"expiresinminutes": ["525600"]} diff --git a/src/tests/_internal/core/backends/daytona/test_compute.py b/src/tests/_internal/core/backends/daytona/test_compute.py new file mode 100644 index 0000000000..b4d6566878 --- /dev/null +++ b/src/tests/_internal/core/backends/daytona/test_compute.py @@ -0,0 +1,867 @@ +import json +import shlex +from types import SimpleNamespace +from unittest.mock import patch + +import gpuhunt +import pytest + +from dstack._internal.core.backends.daytona.api_client import ( + API_URL, + DaytonaAPIError, +) +from dstack._internal.core.backends.daytona.compute import DaytonaCompute +from dstack._internal.core.backends.daytona.models import DaytonaConfig, DaytonaCreds +from dstack._internal.core.errors import ComputeError, NotYetTerminated, ProvisioningError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.common import RegistryAuth +from dstack._internal.core.models.instances import ( + InstanceAvailability, + InstanceRuntime, +) +from dstack._internal.core.models.resources import ResourcesSpec +from dstack._internal.core.models.runs import JobProvisioningData, Requirements +from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration, VolumeMountPoint + +COMPUTE_MODULE = "dstack._internal.core.backends.daytona.compute" +PUBLIC_KEY = ( + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINOmx0T+hBRaJ6jCi21ZYe2NW3EZS8e0Mdwl+yZJt+kD project" +) +TOOLBOX_URL = "https://proxy.app.daytona.io/sandbox-id" +STARTUP_COMMANDS = ["install runner", "runner --key 'a key with spaces'"] + + +@pytest.fixture +def compute(): + with patch(f"{COMPUTE_MODULE}.DaytonaAPIClient", autospec=True) as client_class: + compute = DaytonaCompute(DaytonaConfig(creds=DaytonaCreds(api_key="test-api-key"))) + client = client_class.return_value + client.get_current_api_key.return_value = {"organizationId": "org-id"} + client.get_organization_usage.return_value = {"regionUsage": [_cpu_usage(), _gpu_usage()]} + client.get_available_sandbox_classes.return_value = [ + {"regionId": "us", "sandboxClass": "container", "gpuAvailable": False}, + {"regionId": "earth", "sandboxClass": "container", "gpuAvailable": True}, + ] + client.create_sandbox.return_value = {"id": "sandbox-id", "state": "creating"} + client.create_registry.return_value = {"id": "registry-id"} + client.get_registries.return_value = [] + client.create_volume.return_value = {"id": "volume-id", "state": "pending_create"} + client.get_volume.return_value = {"id": "volume-id", "state": "ready"} + client.get_sandbox.return_value = {"id": "sandbox-id", "state": "started"} + client.get_toolbox_url.return_value = TOOLBOX_URL + client.get_session.return_value = { + "sessionId": "dstack-runner", + "commands": [{"id": "command-id", "command": "runner"}], + } + client.create_ssh_access.return_value = { + "token": "ssh-token_123", + "sshCommand": "ssh ssh-token_123@ssh.app.daytona.io -p 2222", + } + yield compute + + +def _cpu_usage(**overrides): + return { + "regionId": "us", + "sandboxClass": "container", + "totalCpuQuota": 100, + "totalMemoryQuota": 200, + "totalDiskQuota": 1000, + "currentCpuUsage": 0, + "currentMemoryUsage": 0, + "currentDiskUsage": 0, + "maxCpuPerSandbox": None, + "maxMemoryPerSandbox": None, + "maxDiskPerSandbox": None, + **overrides, + } + + +def _gpu_usage(**overrides): + return { + "regionId": "earth", + "sandboxClass": "container", + # GPU sandbox CPU/RAM/disk are not charged against these aggregate quotas. + "totalCpuQuota": 0, + "totalMemoryQuota": 0, + "totalDiskQuota": 0, + "currentCpuUsage": 0, + "currentMemoryUsage": 0, + "currentDiskUsage": 0, + "totalGpuQuota": 4, + "currentGpuUsage": 0, + "maxCpuPerGpu": 16, + "maxMemoryPerGpu": 192, + "maxDiskPerGpu": 512, + **overrides, + } + + +def _item(*, gpu=False, spot=False, cpu=4, memory=16, disk=101, region=None): + return gpuhunt.CatalogItem( + provider="daytona", + instance_name="test-shape", + location=region or ("earth" if gpu else "us"), + price=0.512345, + cpu=cpu, + memory=memory, + disk_size=disk, + gpu_count=2 if gpu else 0, + gpu_name="RTXPRO6000" if gpu else None, + gpu_vendor=gpuhunt.AcceleratorVendor.NVIDIA if gpu else None, + gpu_memory=96 if gpu else None, + spot=spot, + provider_data={"gpu_type": "RTX-PRO-6000"} if gpu else {}, + ) + + +def _offers(compute, *items): + with patch.object(compute._catalog, "query", return_value=list(items)): + return compute.get_offers_by_requirements( + Requirements(resources=ResourcesSpec()), + full_offers=False, + unallocated_resources=False, + ) + + +def _run_job(compute, offer=None, instance_name="dstack-test-unique", **overrides): + job_spec = SimpleNamespace( + image_name="pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime", + registry_auth=None, + volumes=[], + ) + for name in ("image_name", "registry_auth", "mount_points"): + if name in overrides: + setattr(job_spec, "volumes" if name == "mount_points" else name, overrides.pop(name)) + kwargs = { + "run": SimpleNamespace(), + "job": SimpleNamespace(job_spec=job_spec), + "instance_offer": offer or _offers(compute, _item())[0], + "project_ssh_public_key": PUBLIC_KEY, + "project_ssh_private_key": "private-key", + "volumes": [], + "placement_group": None, + "requirements": Requirements(resources=ResourcesSpec()), + "extra_authorized_keys": [], + **overrides, + } + with ( + patch( + f"{COMPUTE_MODULE}.generate_unique_instance_name_for_job", + return_value=instance_name, + ), + patch(f"{COMPUTE_MODULE}.get_docker_commands", return_value=list(STARTUP_COMMANDS)), + ): + return compute.run_job(**kwargs) + + +def _update(compute, data): + compute.update_provisioning_data(data, PUBLIC_KEY, "private-key") + + +class TestRunJob: + @pytest.mark.parametrize("gpu,spot", [(False, False), (True, False), (True, True)]) + def test_preserves_priced_resources_and_native_gpu_type(self, compute, gpu, spot): + offer = _offers(compute, _item(gpu=gpu, spot=spot))[0] + + data = _run_job(compute, offer) + + assert offer.availability == InstanceAvailability.AVAILABLE + assert offer.instance_runtime == InstanceRuntime.RUNNER + payload = compute.api_client.create_sandbox.call_args.args[0] + assert payload["name"] == "dstack-test-unique" + assert payload["target"] == ("earth" if gpu else "us") + assert (payload["cpu"], payload["memory"], payload["disk"]) == (4, 16, 101) + assert payload["gpu"] == (2 if gpu else 0) + assert payload["spot"] is spot + if gpu: + assert offer.backend_data == {"gpu_type": "RTX-PRO-6000"} + assert offer.instance.resources.gpus[0].name == "RTXPRO6000" + assert payload["gpuType"] == ["RTX-PRO-6000"] + else: + assert "gpuType" not in payload + assert data.backend == BackendType.DAYTONA + assert data.instance_id == "sandbox-id" + assert data.price == offer.price == 0.512345 + assert data.instance_type.resources.disk.size_mib == 101 * 1024 + assert data.username == "root" + assert data.dockerized is False + assert data.hostname is None + assert data.ssh_port is None + compute.api_client.create_ssh_access.assert_not_called() + compute.api_client.create_registry.assert_not_called() + + def test_creates_root_image_without_user_entrypoint_or_idle_shutdown(self, compute): + data = _run_job(compute) + + payload = compute.api_client.create_sandbox.call_args.args[0] + assert payload["buildInfo"]["dockerfileContent"] == ( + "FROM pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime\n" + 'USER root\nENTRYPOINT []\nCMD ["sleep", "infinity"]\n' + ) + assert payload["user"] == "root" + assert payload["public"] is False + for field in ("autoStopInterval", "autoPauseInterval", "autoDeleteInterval", "ttlMinutes"): + assert payload[field] == 0 + startup = json.loads(data.backend_data)["startup_command"] + assert shlex.split(startup) == [ + "sh", + "-c", + " && ".join(STARTUP_COMMANDS), + ] + + @pytest.mark.parametrize( + "override", + [ + {"image_name": "ubuntu:24.04\nRUN touch /injected"}, + {"image_name": ""}, + ], + ) + def test_rejects_invalid_image_names_before_creating_sandbox(self, compute, override): + with pytest.raises(ComputeError): + _run_job(compute, **override) + + compute.api_client.create_sandbox.assert_not_called() + + @pytest.mark.parametrize("status_code", [None, 500]) + def test_lost_create_response_retains_unique_name_for_recovery_and_cleanup( + self, compute, status_code + ): + compute.api_client.create_sandbox.side_effect = DaytonaAPIError( + "response lost", status_code=status_code + ) + + data = _run_job(compute) + + assert data.instance_id == "dstack-test-unique" + compute.api_client.create_sandbox.assert_called_once() + _update(compute, data) + compute.api_client.get_sandbox.assert_called_with("dstack-test-unique") + compute.api_client.create_ssh_access.assert_called_once_with( + "sandbox-id", expires_in_minutes=3650 * 24 * 60 + ) + with pytest.raises(NotYetTerminated): + compute.terminate_instance(data.instance_id, data.region) + compute.api_client.delete_sandbox.assert_called_once_with("dstack-test-unique") + + @pytest.mark.parametrize("status_code", [400, 429]) + def test_definite_create_failure_is_not_treated_as_success(self, compute, status_code): + error = DaytonaAPIError("creation rejected", status_code=status_code) + compute.api_client.create_sandbox.side_effect = error + + with pytest.raises(DaytonaAPIError) as exc_info: + _run_job(compute) + + assert exc_info.value is error + compute.api_client.create_sandbox.assert_called_once() + + +class TestRegistryAuth: + @pytest.mark.parametrize( + "image,registry,qualified", + [ + ("org/private:v1", "docker.io", "docker.io/org/private:v1"), + ("index.docker.io/org/private:v1", "docker.io", "docker.io/org/private:v1"), + ("ghcr.io/org/image@sha256:abc", "ghcr.io", "ghcr.io/org/image@sha256:abc"), + ( + "registry.example:5000/image:v1", + "registry.example:5000", + "registry.example:5000/image:v1", + ), + ], + ) + def test_creates_job_registry_and_qualifies_private_base_image( + self, compute, image, registry, qualified + ): + auth = RegistryAuth(username="registry-user", password="registry-password") + data = _run_job(compute, image_name=image, registry_auth=auth) + + compute.api_client.create_registry.assert_called_once_with( + { + "name": "dstack-test-unique-registry", + "url": registry, + "username": auth.username, + "password": auth.password, + } + ) + assert compute.api_client.create_sandbox.call_args.args[0]["buildInfo"][ + "dockerfileContent" + ].startswith(f"FROM {qualified}\n") + assert json.loads(data.backend_data)["registry_id"] == "registry-id" + assert auth.password not in data.backend_data + + def test_keeps_separate_ids_for_jobs_with_the_same_registry_credentials(self, compute): + auth = RegistryAuth(username="user", password="password") + compute.api_client.create_registry.side_effect = [ + {"id": "registry-a"}, + {"id": "registry-b"}, + ] + a = _run_job(compute, registry_auth=auth, instance_name="job-a") + b = _run_job(compute, registry_auth=auth, instance_name="job-b") + + assert json.loads(a.backend_data)["registry_id"] == "registry-a" + assert json.loads(b.backend_data)["registry_id"] == "registry-b" + assert [c.args[0]["name"] for c in compute.api_client.create_registry.call_args_list] == [ + "job-a-registry", + "job-b-registry", + ] + compute.api_client.get_sandbox.return_value = None + compute.terminate_instance(a.instance_id, a.region, a.backend_data) + compute.api_client.delete_registry.assert_called_once_with("registry-a") + + def test_recovers_only_own_registry_after_lost_create_response(self, compute): + compute.api_client.create_registry.side_effect = DaytonaAPIError("response lost") + compute.api_client.get_registries.return_value = [ + {"name": "someone-else", "id": "other-id"}, + {"name": "dstack-test-unique-registry", "id": "recovered-id"}, + ] + + data = _run_job(compute, registry_auth=RegistryAuth(username="user", password="password")) + + assert json.loads(data.backend_data)["registry_id"] == "recovered-id" + compute.api_client.create_registry.assert_called_once() + compute.api_client.delete_registry.assert_not_called() + + def test_registry_failure_does_not_create_a_sandbox_or_delete_existing_credentials( + self, compute + ): + compute.api_client.create_registry.side_effect = DaytonaAPIError( + "forbidden", status_code=403 + ) + + with pytest.raises(DaytonaAPIError, match="forbidden"): + _run_job(compute, registry_auth=RegistryAuth(username="user", password="password")) + + compute.api_client.create_sandbox.assert_not_called() + compute.api_client.delete_registry.assert_not_called() + + def test_lost_sandbox_response_keeps_credentials_until_sandbox_is_gone(self, compute): + compute.api_client.create_sandbox.side_effect = DaytonaAPIError("response lost") + data = _run_job(compute, registry_auth=RegistryAuth(username="user", password="password")) + + with pytest.raises(NotYetTerminated): + compute.terminate_instance(data.instance_id, data.region, data.backend_data) + compute.api_client.delete_registry.assert_not_called() + compute.api_client.get_sandbox.return_value = None + compute.terminate_instance(data.instance_id, data.region, data.backend_data) + compute.api_client.delete_registry.assert_called_once_with("registry-id") + + def test_failed_registry_rollback_reports_manual_cleanup(self, compute, caplog): + error = DaytonaAPIError("invalid image", status_code=400) + compute.api_client.create_sandbox.side_effect = error + compute.api_client.delete_registry.side_effect = DaytonaAPIError( + "temporary outage", status_code=503 + ) + + with pytest.raises(DaytonaAPIError, match="invalid image") as exc: + _run_job( + compute, + registry_auth=RegistryAuth(username="user", password="password"), + ) + + assert exc.value is error + compute.api_client.delete_registry.assert_called_once_with("registry-id") + assert "registry-id" in caplog.text + assert "Delete it manually in Daytona" in caplog.text + + def test_registry_cleanup_survives_provisioning_and_server_restart(self, compute): + data = _run_job(compute, registry_auth=RegistryAuth(username="user", password="password")) + _update(compute, data) + data = JobProvisioningData.model_validate_json(data.model_dump_json()) + assert json.loads(data.backend_data) == { + "startup_command": None, + "registry_id": "registry-id", + } + restarted = DaytonaCompute(compute.config) + restarted.api_client = compute.api_client + compute.api_client.get_sandbox.return_value = None + compute.api_client.delete_registry.side_effect = [ + DaytonaAPIError("temporary outage"), + None, + ] + + with pytest.raises(DaytonaAPIError, match="temporary outage"): + restarted.terminate_instance(data.instance_id, data.region, data.backend_data) + restarted.terminate_instance(data.instance_id, data.region, data.backend_data) + + assert compute.api_client.delete_registry.call_count == 2 + + +class TestVolumes: + def test_mounts_selected_alternatives_and_multiple_volumes(self, compute): + volumes = [ + SimpleNamespace(name="data", volume_id="data-id"), + SimpleNamespace(name="cache", volume_id="cache-id"), + ] + _run_job( + compute, + volumes=volumes, + mount_points=[ + VolumeMountPoint(name=["another-provider", "data"], path="/data"), + VolumeMountPoint(name="cache", path="/cache"), + ], + ) + + assert compute.api_client.create_sandbox.call_args.args[0]["volumes"] == [ + {"volumeId": "data-id", "mountPath": "/data"}, + {"volumeId": "cache-id", "mountPath": "/cache"}, + ] + + def test_creates_volume_without_a_size_or_region_and_waits_until_ready(self, compute): + compute.api_client.get_volume.side_effect = [{"state": "creating"}, {"state": "ready"}] + with ( + patch(f"{COMPUTE_MODULE}.generate_unique_volume_name", return_value="unique-volume"), + patch(f"{COMPUTE_MODULE}.time.sleep") as sleep, + ): + data = compute.create_volume(SimpleNamespace()) + + compute.api_client.create_volume.assert_called_once_with("unique-volume") + sleep.assert_called_once_with(2) + assert data.volume_id == "volume-id" + assert data.size_gb is None + assert data.price == 0 + assert data.attachable is False + assert data.detachable is False + + def test_recovers_created_volume_by_name_after_lost_response(self, compute): + compute.api_client.create_volume.side_effect = DaytonaAPIError("response lost") + compute.api_client.get_volume_by_name.return_value = {"id": "recovered-id"} + with patch(f"{COMPUTE_MODULE}.generate_unique_volume_name", return_value="unique-volume"): + data = compute.create_volume(SimpleNamespace()) + + assert data.volume_id == "recovered-id" + compute.api_client.create_volume.assert_called_once() + compute.api_client.get_volume_by_name.assert_called_once_with("unique-volume") + + def test_creation_timeout_is_bounded_and_cleans_up(self, compute): + compute.api_client.get_volume.return_value = {"state": "creating"} + with ( + patch(f"{COMPUTE_MODULE}.generate_unique_volume_name", return_value="unique-volume"), + patch(f"{COMPUTE_MODULE}.time.monotonic", side_effect=[0, 61]), + ): + with pytest.raises(ComputeError, match="Timed out"): + compute.create_volume(SimpleNamespace()) + compute.api_client.delete_volume.assert_called_once_with("volume-id") + + def test_failed_volume_rollback_reports_manual_cleanup(self, compute, mocker, caplog): + mocker.patch(f"{COMPUTE_MODULE}.generate_unique_volume_name", return_value="unique-volume") + compute.api_client.get_volume.return_value = { + "state": "error", + "errorReason": "storage failed", + } + compute.api_client.delete_volume.side_effect = DaytonaAPIError( + "temporary outage", status_code=503 + ) + + with pytest.raises(ComputeError, match="storage failed"): + compute.create_volume(SimpleNamespace()) + + compute.api_client.delete_volume.assert_called_once_with("volume-id") + assert "volume-id" in caplog.text + assert "Delete it manually in Daytona" in caplog.text + + def test_registers_an_existing_volume_without_modifying_it(self, compute): + volume = SimpleNamespace(configuration=DaytonaVolumeConfiguration(volume_id="external-id")) + data = compute.register_volume(volume) + + assert data.volume_id == "external-id" + compute.api_client.get_volume.assert_called_once_with("external-id") + compute.api_client.create_volume.assert_not_called() + compute.api_client.delete_volume.assert_not_called() + + @pytest.mark.parametrize("state", [None, {"state": "deleted"}, {"state": "error"}]) + def test_rejects_unusable_external_volume_without_deleting_it(self, compute, state): + compute.api_client.get_volume.return_value = state + volume = SimpleNamespace(configuration=DaytonaVolumeConfiguration(volume_id="external-id")) + with pytest.raises(ComputeError): + compute.register_volume(volume) + compute.api_client.delete_volume.assert_not_called() + + def test_deletion_accepts_pending_request_without_repeating_delete(self, compute): + compute.api_client.get_volume.side_effect = [ + {"state": "ready"}, + {"state": "pending_delete"}, + {"state": "deleting"}, + {"state": "deleted"}, + None, + ] + volume = SimpleNamespace(volume_id="volume-id") + for _ in range(5): + compute.delete_volume(volume) + + compute.api_client.delete_volume.assert_called_once_with("volume-id") + + +class TestUpdateProvisioningData: + def test_queues_runner_and_returns_connection_data_once(self, compute): + data = _run_job(compute) + startup_command = json.loads(data.backend_data)["startup_command"] + compute.api_client.get_session.return_value = None + + _update(compute, data) + + compute.api_client.create_session.assert_called_once_with(TOOLBOX_URL, "dstack-runner") + compute.api_client.execute_session_command.assert_called_once_with( + TOOLBOX_URL, + "dstack-runner", + startup_command, + run_async=True, + ) + assert data.hostname == "localhost" + assert data.ssh_port == 10022 + assert data.ssh_proxy.hostname == "ssh.app.daytona.io" + assert data.ssh_proxy.port == 2222 + assert data.ssh_proxy.username == "ssh-token_123" + assert data.backend_data is None + # Reconstruct the persisted state, as a new server process would do. + data = JobProvisioningData.model_validate_json(data.model_dump_json()) + restarted = DaytonaCompute(compute.config) + restarted.api_client = compute.api_client + + _update(restarted, data) + _update(restarted, data) + + compute.api_client.get_session.assert_called_once() + compute.api_client.execute_session_command.assert_called_once() + compute.api_client.create_session.assert_called_once() + compute.api_client.create_ssh_access.assert_called_once_with( + "sandbox-id", expires_in_minutes=3650 * 24 * 60 + ) + + def test_executes_an_existing_empty_session_after_interrupted_bootstrap(self, compute): + data = _run_job(compute) + compute.api_client.get_session.return_value = { + "sessionId": "dstack-runner", + "commands": [], + } + + _update(compute, data) + + compute.api_client.create_session.assert_not_called() + compute.api_client.execute_session_command.assert_called_once() + compute.api_client.create_ssh_access.assert_called_once() + assert data.hostname == "localhost" + + def test_recovers_existing_startup_after_server_restart_before_connection_data_saved( + self, compute + ): + data = _run_job(compute) + compute.api_client.get_session.side_effect = [ + None, + {"commands": [{"id": "running", "command": "runner"}]}, + ] + access = compute.api_client.create_ssh_access.return_value + compute.api_client.create_ssh_access.side_effect = [ + DaytonaAPIError("response lost"), + access, + ] + + with pytest.raises(DaytonaAPIError, match="response lost"): + _update(compute, data) + assert data.hostname is None + data = JobProvisioningData.model_validate_json(data.model_dump_json()) + restarted = DaytonaCompute(compute.config) + restarted.api_client = compute.api_client + + _update(restarted, data) + + assert data.hostname == "localhost" + compute.api_client.create_session.assert_called_once() + compute.api_client.execute_session_command.assert_called_once() + + @pytest.mark.parametrize("sandbox", [None, {"id": "sandbox-id", "state": "building_snapshot"}]) + def test_waits_for_pending_or_not_yet_visible_sandbox(self, compute, sandbox): + data = _run_job(compute) + compute.api_client.get_sandbox.return_value = sandbox + + _update(compute, data) + + assert data.hostname is None + compute.api_client.get_toolbox_url.assert_not_called() + compute.api_client.create_ssh_access.assert_not_called() + + @pytest.mark.parametrize("state", ["error", "build_failed", "stopped", "destroying", "paused"]) + def test_reports_terminal_sandbox_failure(self, compute, state): + data = _run_job(compute) + compute.api_client.get_sandbox.return_value = { + "id": "sandbox-id", + "state": state, + "errorReason": "image pull failed", + } + + with pytest.raises(ProvisioningError, match="image pull failed"): + _update(compute, data) + + compute.api_client.create_ssh_access.assert_not_called() + + @pytest.mark.parametrize("exit_code", [0, 1]) + def test_reports_exited_runner_setup_instead_of_executing_it_again(self, compute, exit_code): + data = _run_job(compute) + compute.api_client.get_session.return_value = { + "commands": [{"id": "cmd", "command": "runner", "exitCode": exit_code}] + } + + with pytest.raises(ProvisioningError, match=f"exited with code {exit_code}"): + _update(compute, data) + + compute.api_client.execute_session_command.assert_not_called() + compute.api_client.create_ssh_access.assert_not_called() + + def test_recovers_when_async_exec_response_is_lost(self, compute): + data = _run_job(compute) + compute.api_client.get_session.side_effect = [ + {"commands": []}, + {"commands": [{"id": "running", "command": "runner"}]}, + ] + compute.api_client.execute_session_command.side_effect = DaytonaAPIError("response lost") + + with pytest.raises(DaytonaAPIError, match="response lost"): + _update(compute, data) + _update(compute, data) + + assert data.hostname == "localhost" + compute.api_client.execute_session_command.assert_called_once() + compute.api_client.create_ssh_access.assert_called_once() + + @pytest.mark.parametrize( + "command", + [ + "ssh ssh-token_123@ssh.app.daytona.io", # Default SSH port is supported. + "ssh -p 2222 ssh-token_123@ssh.app.daytona.io", + ], + ) + def test_accepts_native_ssh_command_argument_order(self, compute, command): + data = _run_job(compute) + compute.api_client.create_ssh_access.return_value["sshCommand"] = command + + _update(compute, data) + + assert data.ssh_proxy.port == (2222 if "-p" in command else 22) + + def test_uses_api_token_and_provider_supplied_ssh_gateway(self, compute): + data = _run_job(compute) + compute.api_client.create_ssh_access.return_value = { + "token": "current-token", + "sshCommand": "ssh previous-token@custom-gateway.example -p 2222", + } + + _update(compute, data) + + assert data.ssh_proxy.username == "current-token" + assert data.ssh_proxy.hostname == "custom-gateway.example" + assert data.ssh_proxy.port == 2222 + + @pytest.mark.parametrize( + "command", + [ + "ssh ssh.app.daytona.io", + "ssh ssh-token_123@", + "ssh ssh-token_123@ssh.app.daytona.io -p 0", + "ssh ssh-token_123@ssh.app.daytona.io -p 70000", + "ssh ssh-token_123@ssh.app.daytona.io -p invalid", + "echo ssh-token_123@ssh.app.daytona.io", + ], + ) + def test_rejects_invalid_native_ssh_details(self, compute, command): + data = _run_job(compute) + compute.api_client.create_ssh_access.return_value["sshCommand"] = command + + with pytest.raises(ProvisioningError, match="invalid SSH connection details"): + _update(compute, data) + + assert data.hostname is None + + def test_rejects_missing_ssh_token(self, compute): + data = _run_job(compute) + del compute.api_client.create_ssh_access.return_value["token"] + + with pytest.raises(ProvisioningError, match="invalid SSH connection details"): + _update(compute, data) + + assert data.hostname is None + + +class TestTerminateInstance: + def test_deletion_is_idempotent_while_provider_is_destroying(self, compute): + compute.api_client.get_sandbox.side_effect = [ + {"state": "started"}, + {"state": "destroying"}, + {"state": "destroyed"}, + None, + ] + + for _ in range(2): + with pytest.raises(NotYetTerminated): + compute.terminate_instance("sandbox-id", "us") + compute.terminate_instance("sandbox-id", "us") + compute.terminate_instance("sandbox-id", "us") + + compute.api_client.delete_sandbox.assert_called_once_with("sandbox-id") + + +class TestGetOffersByRequirements: + def test_offer_queries_use_only_api_key_accessible_endpoints(self, requests_mock): + # Organization details require dashboard authentication, even for a valid API key. + # No organization-details mock is registered so using that endpoint fails this test. + paths = [ + "/api-keys/current", + "/organizations/org-id/usage", + "/organizations/org-id/available-sandbox-classes", + ] + requests_mock.get(API_URL + paths[0], json={"organizationId": "org-id"}) + requests_mock.get(API_URL + paths[1], json={"regionUsage": [_cpu_usage(), _gpu_usage()]}) + requests_mock.get( + API_URL + paths[2], + json=[ + {"regionId": "us", "sandboxClass": "container", "gpuAvailable": False}, + {"regionId": "earth", "sandboxClass": "container", "gpuAvailable": True}, + ], + ) + compute = DaytonaCompute(DaytonaConfig(creds=DaytonaCreds(api_key="test-api-key"))) + + offers = _offers(compute, _item(), _item(gpu=True)) + + assert [offer.availability for offer in offers] == [InstanceAvailability.AVAILABLE] * 2 + assert [request.url for request in requests_mock.request_history] == [ + API_URL + path for path in paths + ] + assert all( + request.headers["Authorization"] == "Bearer test-api-key" + for request in requests_mock.request_history + ) + + def test_organization_id_is_cached_but_quotas_are_refreshed(self, compute): + assert _offers(compute, _item())[0].availability == InstanceAvailability.AVAILABLE + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _cpu_usage(currentCpuUsage=100) + ] + + assert _offers(compute, _item())[0].availability == InstanceAvailability.NO_QUOTA + compute.api_client.get_current_api_key.assert_called_once_with() + assert compute.api_client.get_organization_usage.call_count == 2 + + def test_skips_quota_calls_without_catalog_offers(self, compute): + assert _offers(compute) == [] + + compute.api_client.get_current_api_key.assert_not_called() + compute.api_client.get_organization_usage.assert_not_called() + + def test_filters_configured_regions(self, compute): + compute.config.regions = ["earth"] + + offers = _offers(compute, _item(), _item(gpu=True)) + + assert [offer.region for offer in offers] == ["earth"] + + @pytest.mark.parametrize("resource", ["Cpu", "Memory", "Disk"]) + def test_cpu_aggregate_quotas_include_current_usage(self, compute, resource): + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _cpu_usage(**{f"current{resource}Usage": 999}) + ] + + assert _offers(compute, _item())[0].availability == InstanceAvailability.NO_QUOTA + + @pytest.mark.parametrize("resource", ["Cpu", "Memory", "Disk"]) + def test_cpu_region_per_sandbox_limit_is_enforced(self, compute, resource): + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _cpu_usage(**{f"max{resource}PerSandbox": 1}) + ] + + assert _offers(compute, _item())[0].availability == InstanceAvailability.NO_QUOTA + + def test_ignores_other_sandbox_classes_when_checking_usage(self, compute): + compute.api_client.get_organization_usage.return_value["regionUsage"].append( + _cpu_usage(sandboxClass="windows", totalCpuQuota=0) + ) + compute.api_client.get_available_sandbox_classes.return_value.append( + {"regionId": "earth", "sandboxClass": "windows", "gpuAvailable": False} + ) + + assert _offers(compute, _item())[0].availability == InstanceAvailability.AVAILABLE + assert _offers(compute, _item(gpu=True))[0].availability == InstanceAvailability.AVAILABLE + + @pytest.mark.parametrize("missing", ["usage", "sandbox_class"]) + def test_region_requires_both_quota_and_container_class(self, compute, missing): + if missing == "usage": + compute.api_client.get_organization_usage.return_value["regionUsage"] = [] + else: + compute.api_client.get_available_sandbox_classes.return_value = [] + + assert _offers(compute, _item())[0].availability == InstanceAvailability.NO_QUOTA + + @pytest.mark.parametrize("spot", [False, True]) + @pytest.mark.parametrize("total,used", [(4, 3), (0, 0)]) + def test_gpu_quota_uses_remaining_count_only_for_on_demand(self, compute, spot, total, used): + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _gpu_usage(totalGpuQuota=total, currentGpuUsage=used) + ] + + expected = InstanceAvailability.AVAILABLE if spot else InstanceAvailability.NO_QUOTA + assert _offers(compute, _item(gpu=True, spot=spot))[0].availability == expected + + @pytest.mark.parametrize( + "allowed,expected", + [ + (None, InstanceAvailability.AVAILABLE), + ([], InstanceAvailability.NO_QUOTA), + (["RTX-PRO-6000"], InstanceAvailability.AVAILABLE), + (["H100"], InstanceAvailability.NO_QUOTA), + ], + ) + @pytest.mark.parametrize("source", ["usage", "sandbox_class"]) + @pytest.mark.parametrize("spot", [False, True]) + def test_gpu_allowlist_uses_api_gpu_name(self, compute, allowed, expected, source, spot): + if source == "usage": + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _gpu_usage(allowedGpuTypes=allowed) + ] + else: + compute.api_client.get_available_sandbox_classes.return_value[1]["allowedGpuTypes"] = ( + allowed + ) + + assert _offers(compute, _item(gpu=True, spot=spot))[0].availability == expected + + @pytest.mark.parametrize("shape", [{"cpu": 33}, {"memory": 385}, {"disk": 1025}]) + @pytest.mark.parametrize("spot", [False, True]) + def test_gpu_resource_limit_is_multiplied_by_gpu_count(self, compute, shape, spot): + boundary = {name: value - 1 for name, value in shape.items()} + + assert ( + _offers(compute, _item(gpu=True, spot=spot, **boundary))[0].availability + == InstanceAvailability.AVAILABLE + ) + assert ( + _offers(compute, _item(gpu=True, spot=spot, **shape))[0].availability + == InstanceAvailability.NO_QUOTA + ) + + @pytest.mark.parametrize("resource", ["Cpu", "Memory", "Disk"]) + def test_gpu_null_limit_falls_back_to_absolute_region_per_sandbox_limit( + self, compute, resource + ): + usage = _gpu_usage(**{f"max{resource}PerGpu": None}) + sizes = {"Cpu": 4, "Memory": 16, "Disk": 101} + usage[f"max{resource}PerSandbox"] = sizes[resource] - 1 + compute.api_client.get_organization_usage.return_value["regionUsage"] = [usage] + + assert _offers(compute, _item(gpu=True))[0].availability == InstanceAvailability.NO_QUOTA + + def test_gpu_unknown_resource_limits_do_not_impose_guessed_global_defaults(self, compute): + compute.api_client.get_organization_usage.return_value["regionUsage"] = [ + _gpu_usage(maxCpuPerGpu=None, maxMemoryPerGpu=None, maxDiskPerGpu=None) + ] + + assert ( + _offers(compute, _item(gpu=True, cpu=32, memory=384, disk=1024))[0].availability + == InstanceAvailability.AVAILABLE + ) + + @pytest.mark.parametrize("spot", [False, True]) + def test_gpu_class_must_allow_gpu(self, compute, spot): + compute.api_client.get_available_sandbox_classes.return_value[1]["gpuAvailable"] = False + + assert ( + _offers(compute, _item(gpu=True, spot=spot))[0].availability + == InstanceAvailability.NO_QUOTA + ) diff --git a/src/tests/_internal/core/backends/daytona/test_configurator.py b/src/tests/_internal/core/backends/daytona/test_configurator.py new file mode 100644 index 0000000000..8e272dd376 --- /dev/null +++ b/src/tests/_internal/core/backends/daytona/test_configurator.py @@ -0,0 +1,96 @@ +import json +from unittest.mock import patch + +import pytest + +from dstack._internal.core.backends.daytona.api_client import API_URL +from dstack._internal.core.backends.daytona.configurator import DaytonaConfigurator +from dstack._internal.core.backends.daytona.models import ( + DaytonaBackendConfigWithCreds, + DaytonaCreds, +) +from dstack._internal.core.errors import ( + BackendInvalidCredentialsError, + ServerClientError, +) + + +@pytest.fixture +def client(): + with patch( + "dstack._internal.core.backends.daytona.configurator.DaytonaAPIClient", autospec=True + ) as client_class: + client = client_class.return_value + client.get_current_api_key.return_value = { + "organizationId": "test-org", + "permissions": ["write:sandboxes", "delete:sandboxes", "read:limits"], + } + client.get_shared_regions.return_value = [{"id": "us"}, {"id": "eu"}, {"id": "ap"}] + yield client + + +class TestDaytonaConfigurator: + def test_validate_config_accepts_only_required_permissions(self, client): + config = DaytonaBackendConfigWithCreds(creds=DaytonaCreds(api_key="test-key")) + + DaytonaConfigurator().validate_config(config, default_creds_enabled=False) + + client.get_current_api_key.assert_called_once_with() + client.get_shared_regions.assert_not_called() + + @pytest.mark.parametrize( + "permissions,missing", + [ + (["write:sandboxes", "delete:sandboxes"], "read:limits"), + (["write:sandboxes", "read:limits"], "delete:sandboxes"), + (["delete:sandboxes", "read:limits"], "write:sandboxes"), + ], + ) + def test_validate_config_reports_missing_permissions( + self, requests_mock, permissions, missing + ): + requests_mock.get(f"{API_URL}/api-keys/current", json={"permissions": permissions}) + config = DaytonaBackendConfigWithCreds(creds=DaytonaCreds(api_key="test-key")) + + with pytest.raises(BackendInvalidCredentialsError) as exc: + DaytonaConfigurator().validate_config(config, default_creds_enabled=False) + + assert exc.value.msg == f"Daytona API key is missing permissions: {missing}" + assert exc.value.fields == [["creds", "api_key"]] + assert requests_mock.call_count == 1 + + def test_validate_config_accepts_gpu_and_discovered_cpu_regions(self, client): + config = DaytonaBackendConfigWithCreds( + creds=DaytonaCreds(api_key="test-key"), regions=["earth", "eu", "ap"] + ) + + DaytonaConfigurator().validate_config(config, default_creds_enabled=False) + + client.get_current_api_key.assert_called_once_with() + client.get_shared_regions.assert_called_once_with() + + def test_validate_config_rejects_unknown_regions(self, client): + config = DaytonaBackendConfigWithCreds( + creds=DaytonaCreds(api_key="test-key"), regions=["unknown"] + ) + + with pytest.raises(ServerClientError) as exc_info: + DaytonaConfigurator().validate_config(config, default_creds_enabled=False) + + assert exc_info.value.fields == [["regions"]] + assert "unknown" in exc_info.value.msg + + def test_stored_config_keeps_credentials_separate(self): + config = DaytonaBackendConfigWithCreds( + creds=DaytonaCreds(api_key="test-secret"), regions=["earth", "us"] + ) + configurator = DaytonaConfigurator() + + record = configurator.create_backend(project_name="main", config=config) + + assert json.loads(record.config) == {"type": "daytona", "regions": ["earth", "us"]} + assert json.loads(record.auth) == {"type": "api_key", "api_key": "test-secret"} + assert configurator.get_backend_config_with_creds(record) == config + public_config = configurator.get_backend_config_without_creds(record).model_dump() + assert public_config == json.loads(record.config) + assert "test-secret" not in json.dumps(public_config) diff --git a/src/tests/_internal/core/backends/daytona/test_models.py b/src/tests/_internal/core/backends/daytona/test_models.py new file mode 100644 index 0000000000..3b9a823084 --- /dev/null +++ b/src/tests/_internal/core/backends/daytona/test_models.py @@ -0,0 +1,26 @@ +from pydantic import Field, TypeAdapter +from typing_extensions import Annotated + +from dstack._internal.core.backends.daytona.models import DaytonaBackendConfigWithCreds +from dstack._internal.core.backends.models import ( + AnyBackendFileConfigWithCreds, + BackendConfigWithCreds, +) + + +class TestDaytonaBackendConfig: + def test_api_and_server_file_configs_accept_daytona(self): + data = { + "type": "daytona", + "regions": ["earth", "eu"], + "creds": {"type": "api_key", "api_key": "test-key"}, + } + api_config = BackendConfigWithCreds.model_validate(data).root + file_config = TypeAdapter( + Annotated[AnyBackendFileConfigWithCreds, Field(discriminator="type")] + ).validate_python(data) + + assert isinstance(api_config, DaytonaBackendConfigWithCreds) + assert api_config == file_config + assert api_config.regions == ["earth", "eu"] + assert api_config.creds.api_key == "test-key" diff --git a/src/tests/_internal/core/models/test_volumes.py b/src/tests/_internal/core/models/test_volumes.py index 9f95b1bffe..46b692e2aa 100644 --- a/src/tests/_internal/core/models/test_volumes.py +++ b/src/tests/_internal/core/models/test_volumes.py @@ -1,13 +1,66 @@ +import json + import pytest from pydantic import ValidationError from dstack._internal.core.models.volumes import ( + DaytonaVolumeConfiguration, InstanceMountPoint, + VolumeConfiguration, + VolumeConfigurationWithSize, VolumeMountPoint, + VolumeProvisioningData, + VolumeSpec, parse_mount_point, ) +@pytest.mark.parametrize("backend", ["aws", "gcp", "runpod", "kubernetes"]) +class TestSizedVolumeConfiguration: + def test_managed_size_keeps_numeric_wire_format(self, backend): + spec = VolumeSpec.model_validate( + {"configuration": {"backend": backend, "region": "us", "size": "120GB"}} + ) + + assert isinstance(spec.configuration, VolumeConfigurationWithSize) + assert spec.configuration.size_gb == 120 + assert not spec.configuration.is_external + assert json.loads(spec.model_dump_json())["configuration"]["size"] == 120.0 + restored = VolumeSpec.model_validate_json(spec.model_dump_json()) + assert restored == spec + + +class TestDaytonaVolumeConfiguration: + @pytest.mark.parametrize("volume_id", [None, "existing-volume"]) + def test_roundtrip_without_region_or_size(self, volume_id): + spec = VolumeSpec.model_validate( + {"configuration": {"backend": "daytona", "name": "cache", "volume_id": volume_id}} + ) + + assert isinstance(spec.configuration, DaytonaVolumeConfiguration) + assert spec.configuration.is_external == (volume_id is not None) + assert "region" not in spec.configuration.model_dump() + assert "size" not in spec.configuration.model_dump() + assert VolumeSpec.model_validate_json(spec.model_dump_json()) == spec + + @pytest.mark.parametrize(("field", "value"), [("size", 100), ("region", "us")]) + def test_rejects_fixed_size_and_region(self, field, value): + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + VolumeConfiguration.model_validate({"backend": "daytona", field: value}) + + +class TestVolumeProvisioningData: + def test_existing_capacity_remains_an_integer(self): + data = VolumeProvisioningData.model_validate( + {"backend": "aws", "volume_id": "volume-id", "size_gb": 120} + ) + + payload = json.loads(data.model_dump_json()) + assert type(payload["size_gb"]) is int + assert payload["size_gb"] == 120 + assert VolumeProvisioningData.model_validate(payload).size_gb == 120 + + class TestVolumeMountPoint: def test_parse(self): assert VolumeMountPoint.parse("my-vol:/path/./to///dir/") == VolumeMountPoint( diff --git a/src/tests/_internal/server/routers/test_backends.py b/src/tests/_internal/server/routers/test_backends.py index 9aed7be000..824674c364 100644 --- a/src/tests/_internal/server/routers/test_backends.py +++ b/src/tests/_internal/server/routers/test_backends.py @@ -88,6 +88,7 @@ async def test_returns_backend_types(self, client: AsyncClient): "cloudrift", "crusoe", "datacrunch", + "daytona", "digitalocean", "gcp", "hotaisle", diff --git a/src/tests/_internal/server/services/jobs/test_jobs.py b/src/tests/_internal/server/services/jobs/test_jobs.py index 089b5b54bb..96b127e823 100644 --- a/src/tests/_internal/server/services/jobs/test_jobs.py +++ b/src/tests/_internal/server/services/jobs/test_jobs.py @@ -4,17 +4,26 @@ import pytest import dstack._internal.server.settings as server_settings +from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import RegistryAuth from dstack._internal.core.models.configurations import TaskConfiguration from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.repos.local import LocalRunRepoData from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import JobSpec, RunSpec +from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration from dstack._internal.server.services.docker import ImageConfig from dstack._internal.server.services.jobs import ( + _get_job_mount_point_attached_volume, get_job_specs_from_run_spec, job_spec_updatable_in_place, ) +from dstack._internal.server.testing.common import ( + get_job_provisioning_data, + get_volume, + get_volume_configuration, + get_volume_provisioning_data, +) @pytest.mark.parametrize( @@ -175,3 +184,38 @@ def test_does_not_mutate_the_new_spec(self): job_spec_updatable_in_place(old_job_spec, new_job_spec) assert new_job_spec.requirements.resources.cpu.arch is None + + +class TestGetJobMountPointAttachedVolume: + @pytest.mark.parametrize(("region", "gpu_count"), [("us", 0), ("earth", 1)]) + def test_global_volume_matches_cpu_and_gpu_regions(self, region, gpu_count): + volume = get_volume(configuration=DaytonaVolumeConfiguration()) + other_backend_volume = get_volume( + configuration=get_volume_configuration(backend=BackendType.AWS, region=region) + ) + provisioning_data = get_job_provisioning_data( + backend=BackendType.DAYTONA, region=region, gpu_count=gpu_count + ) + + assert ( + _get_job_mount_point_attached_volume([other_backend_volume, volume], provisioning_data) + == volume + ) + + def test_regional_volume_preserves_region_and_zone_matching(self): + volumes = [ + get_volume( + configuration=get_volume_configuration(backend=BackendType.AWS, region=region), + provisioning_data=get_volume_provisioning_data(availability_zone=zone), + ) + for region, zone in [ + ("eu-west-1", "eu-west-1a"), + ("us-east-1", "us-east-1b"), + ("us-east-1", "us-east-1a"), + ] + ] + provisioning_data = get_job_provisioning_data( + backend=BackendType.AWS, region="US-EAST-1", availability_zone="US-EAST-1A" + ) + + assert _get_job_mount_point_attached_volume(volumes, provisioning_data) == volumes[2] diff --git a/src/tests/_internal/server/services/test_instances.py b/src/tests/_internal/server/services/test_instances.py index defd1498d4..6b332a3e73 100644 --- a/src/tests/_internal/server/services/test_instances.py +++ b/src/tests/_internal/server/services/test_instances.py @@ -23,6 +23,7 @@ Profile, ) from dstack._internal.core.models.runs import JobStatus +from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration from dstack._internal.server.models import InstanceModel from dstack._internal.server.schemas.runner import TaskListItem, TaskListResponse, TaskStatus from dstack._internal.server.services.runner.client import ShimClient @@ -90,6 +91,67 @@ async def test_includes_termination_reason_in_event_message_when_switching_direc class TestFilterInstances: + @pytest.mark.parametrize("backend", [BackendType.AWS, BackendType.GCP]) + @pytest.mark.parametrize( + ("instance_zone", "volume_zone", "matches"), + [ + (None, None, True), + (None, "us-1b", True), + ("us-1a", None, False), + ("us-1a", "us-1a", True), + ("US-1A", "us-1a", True), + ("us-1a", "us-1b", False), + ], + ) + def test_preserves_volume_availability_zone_matching( + self, backend, instance_zone, volume_zone, matches + ): + instance = InstanceModel( + unreachable=False, + health=HealthStatus.HEALTHY, + total_blocks=1, + job_provisioning_data=get_job_provisioning_data( + backend=backend, region="US-1", availability_zone=instance_zone + ).model_dump_json(), + ) + volume = get_volume( + configuration=get_volume_configuration(backend=backend, region="us-1"), + provisioning_data=get_volume_provisioning_data( + backend=backend, availability_zone=volume_zone + ), + ) + + result = instances_services.filter_instances( + instances=[instance], profile=Profile(), volumes=[[volume]] + ) + + assert result == ([instance] if matches else []) + + def test_global_volume_matches_cpu_and_gpu_regions(self): + instances = [ + InstanceModel( + unreachable=False, + health=HealthStatus.HEALTHY, + total_blocks=1, + job_provisioning_data=get_job_provisioning_data( + backend=backend, region=region, gpu_count=gpu_count + ).model_dump_json(), + ) + for backend, region, gpu_count in [ + (BackendType.DAYTONA, "us", 0), + (BackendType.DAYTONA, "earth", 1), + (BackendType.AWS, "us", 0), + ] + ] + + result = instances_services.filter_instances( + instances=instances, + profile=Profile(), + volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], + ) + + assert result == instances[:2] + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_returns_all_instances(self, test_db, session: AsyncSession): @@ -257,7 +319,13 @@ async def test_returns_volume_instances_without_region(self, test_db, session: A project=project, backend=BackendType.KUBERNETES, ) - instances = [aws_instance, kubernetes_instance] + other_cluster_instance = await create_instance( + session=session, + project=project, + backend=BackendType.KUBERNETES, + region="other-cluster", + ) + instances = [aws_instance, kubernetes_instance, other_cluster_instance] volume = get_volume( configuration=get_kubernetes_volume_configuration(), provisioning_data=get_volume_provisioning_data( diff --git a/src/tests/_internal/server/services/test_offers.py b/src/tests/_internal/server/services/test_offers.py index 017d6aa813..7ff0b4fe4a 100644 --- a/src/tests/_internal/server/services/test_offers.py +++ b/src/tests/_internal/server/services/test_offers.py @@ -6,6 +6,7 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import Requirements +from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration from dstack._internal.server.services.offers import get_offers_by_requirements from dstack._internal.server.testing.common import ( get_instance_offer_with_availability, @@ -100,6 +101,31 @@ async def test_returns_volume_offers(self): m.assert_awaited_once() assert res == [(runpod_backend_mock, runpod_offer2)] + @pytest.mark.asyncio + async def test_global_volume_matches_cpu_and_gpu_regions(self): + daytona_backend = Mock(TYPE=BackendType.DAYTONA) + cpu_offer = get_instance_offer_with_availability(backend=BackendType.DAYTONA, region="us") + gpu_offer = get_instance_offer_with_availability( + backend=BackendType.DAYTONA, region="earth", gpu_count=1 + ) + daytona_backend.compute.return_value.get_offers.return_value = [cpu_offer, gpu_offer] + aws_backend = Mock(TYPE=BackendType.AWS) + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us") + aws_backend.compute.return_value.get_offers.return_value = [aws_offer] + + with patch( + "dstack._internal.server.services.backends.get_project_backends", + return_value=[daytona_backend, aws_backend], + ): + offers = await get_offers_by_requirements( + project=Mock(), + profile=Profile(), + requirements=Requirements(resources=ResourcesSpec()), + volumes=[[get_volume(configuration=DaytonaVolumeConfiguration())]], + ) + + assert offers == [(daytona_backend, cpu_offer), (daytona_backend, gpu_offer)] + @pytest.mark.asyncio async def test_returns_volume_offers_without_region(self): profile = Profile(name="test") @@ -117,7 +143,10 @@ async def test_returns_volume_offers_without_region(self): availability_zones=None, ) kubernetes_backend_mock.compute.return_value.get_offers.return_value = [ - kubernetes_offer + kubernetes_offer, + get_instance_offer_with_availability( + backend=BackendType.KUBERNETES, region="other-cluster" + ), ] m.return_value = [aws_backend_mock, kubernetes_backend_mock] res = await get_offers_by_requirements( diff --git a/src/tests/_internal/server/services/test_volumes.py b/src/tests/_internal/server/services/test_volumes.py index 82477812ab..b05455ed31 100644 --- a/src/tests/_internal/server/services/test_volumes.py +++ b/src/tests/_internal/server/services/test_volumes.py @@ -5,7 +5,12 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.volumes import AWSVolumeConfiguration, VolumeStatus +from dstack._internal.core.models.volumes import ( + AWSVolumeConfiguration, + DaytonaVolumeConfiguration, + VolumeStatus, + parse_volume_configuration, +) from dstack._internal.server.services.volumes import ( _get_volume_cost, _validate_volume_configuration, @@ -17,6 +22,27 @@ class TestValidateVolumeConfiguration: + def test_daytona_does_not_require_size_or_region(self): + _validate_volume_configuration(DaytonaVolumeConfiguration()) + + @pytest.mark.parametrize( + ("backend", "identifier_field"), + [ + ("aws", "volume_id"), + ("gcp", "volume_id"), + ("runpod", "volume_id"), + ("kubernetes", "claim_name"), + ], + ) + def test_regional_volume_requires_size_only_when_managed(self, backend, identifier_field): + configuration = {"backend": backend, "region": "us"} + with pytest.raises(ServerClientError, match="existing identifier or size"): + _validate_volume_configuration(parse_volume_configuration(configuration)) + + _validate_volume_configuration( + parse_volume_configuration({**configuration, identifier_field: "existing-volume"}) + ) + def test_external_volume_with_auto_cleanup_duration_raises_error(self): """External volumes (with volume_id) should not allow auto_cleanup_duration""" config = AWSVolumeConfiguration( From 4faf0ba1624d24df30f4f9141eefbd68157aef2e Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:31:56 +0000 Subject: [PATCH 615/640] Pin Nebius spot price at provisioning time (#4328) Starting October 8, Nebius is introducing Dynamic Spot pricing for preemptible VMs. The price can change for existing instances during their lifetime. To prevent the price from increasing above the one `dstack` shows, `dstack` now caps the price at its current value at provisioning time using a pricing policy. If the price grows above the cap, the instance is interrupted by Nebius. The actual price may still drop below the one `dstack` shows. Preemptible L40S is still sold at a flat rate and is exempt from pricing policies. Full support for Nebius Dynamic Spot pricing (tracking the current price and showing it in `dstack`, user-specified caps, etc.) may be added later upon request. --- pyproject.toml | 2 +- .../_internal/core/backends/nebius/compute.py | 83 ++++++++++++- .../_internal/core/backends/nebius/models.py | 4 + .../core/backends/nebius/resources.py | 45 +++++++ src/dstack/_internal/core/models/instances.py | 2 +- .../core/backends/nebius/test_compute.py | 111 ++++++++++++++++++ 6 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 src/tests/_internal/core/backends/nebius/test_compute.py diff --git a/pyproject.toml b/pyproject.toml index 5c51736acb..c755837f70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -298,7 +298,7 @@ oci = [ "dstack[server]", ] nebius = [ - "nebius>=0.3.4,<0.4", + "nebius>=0.6.13,<0.7", "dstack[server]", ] fluentbit = [ diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index b3103f8903..4e4e645eab 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -3,6 +3,7 @@ import shlex import time from collections.abc import Iterable +from decimal import ROUND_CEILING, Decimal from functools import cached_property from typing import List, Optional @@ -66,6 +67,7 @@ max=Memory.parse("8192GB"), # max for the NETWORK_SSD disk type ) WAIT_FOR_DISK_TIMEOUT = 20 +WAIT_FOR_PRICING_POLICY_TIMEOUT = 10 WAIT_FOR_INSTANCE_TIMEOUT = 30 WAIT_FOR_INSTANCE_UPDATE_INTERVAL = 2.5 DELETE_INSTANCE_TIMEOUT = 25 @@ -136,7 +138,11 @@ def get_all_offers_with_availability( extra_filter=_supported_instances, ) return [ - offer.with_availability(availability=InstanceAvailability.UNKNOWN) for offer in offers + offer.with_availability( + availability=InstanceAvailability.UNKNOWN, + price=_get_price_adjusted_for_pricing_policy(offer), + ) + for offer in offers ] def get_offers_modifiers( @@ -156,6 +162,9 @@ def create_instance( # instance. instance_name = generate_unique_instance_name(instance_config) platform, preset = instance_offer.instance.name.split() + offer_backend_data = validate_extra_ignore( + NebiusOfferBackendData, instance_offer.backend_data + ) cluster_id = None if placement_group: assert placement_group.provisioning_data is not None @@ -186,6 +195,7 @@ def create_instance( labels=labels, ) create_instance_op = None + create_pricing_policy_op = None try: logger.debug("Blocking until disk %s is created", create_disk_op.resource_id) resources.wait_for_operation(create_disk_op, timeout=WAIT_FOR_DISK_TIMEOUT) @@ -195,6 +205,27 @@ def create_instance( f"Create disk operation failed. Message: {raw_op.status.message}." f" Details: {raw_op.status.details}" ) + if ( + instance_offer.instance.resources.spot + and not offer_backend_data.is_preemptible_flat_rate + ): + create_pricing_policy_op = resources.create_pricing_policy( + sdk=self._sdk, + name=instance_name, + project_id=self._region_to_project_id[instance_offer.region], + platform=platform, + # Prevent the spot price from growing above what dstack shows + max_price=str(_get_pricing_policy_max_price(instance_offer)), + ) + resources.wait_for_operation( + create_pricing_policy_op, timeout=WAIT_FOR_PRICING_POLICY_TIMEOUT + ) + if not create_pricing_policy_op.successful(): + raw_op = create_pricing_policy_op.raw() + raise ProvisioningError( + f"Create pricing policy operation failed. Message: {raw_op.status.message}." + f" Details: {raw_op.status.details}" + ) create_instance_op = resources.create_instance( sdk=self._sdk, name=instance_name, @@ -209,6 +240,11 @@ def create_instance( disk_id=create_disk_op.resource_id, subnet_id=self._get_subnet_id(instance_offer.region), preemptible=instance_offer.instance.resources.spot, + pricing_policy_id=( + create_pricing_policy_op.resource_id + if create_pricing_policy_op is not None + else None + ), labels=labels, ) _wait_for_instance(self._sdk, create_instance_op) @@ -226,6 +262,18 @@ def create_instance( logger.exception( "Could not delete instance %s: %s", create_instance_op.resource_id, e ) + if create_pricing_policy_op is not None: + try: + with resources.ignore_errors([StatusCode.NOT_FOUND]): + resources.delete_pricing_policy( + self._sdk, create_pricing_policy_op.resource_id + ) + except Exception as e: + logger.exception( + "Could not delete pricing policy %s: %s", + create_pricing_policy_op.resource_id, + e, + ) try: with resources.ignore_errors([StatusCode.NOT_FOUND]): resources.delete_disk(self._sdk, create_disk_op.resource_id) @@ -245,7 +293,12 @@ def create_instance( username="ubuntu", dockerized=True, backend_data=NebiusInstanceBackendData( - boot_disk_id=create_disk_op.resource_id + boot_disk_id=create_disk_op.resource_id, + pricing_policy_id=( + create_pricing_policy_op.resource_id + if create_pricing_policy_op is not None + else None + ), ).model_dump_json(), ) @@ -284,6 +337,9 @@ def terminate_instance( ) with resources.ignore_errors([StatusCode.NOT_FOUND]): resources.delete_disk(self._sdk, backend_data_parsed.boot_disk_id) + if backend_data_parsed.pricing_policy_id is not None: + with resources.ignore_errors([StatusCode.NOT_FOUND]): + resources.delete_pricing_policy(self._sdk, backend_data_parsed.pricing_policy_id) def create_placement_group( self, @@ -348,6 +404,7 @@ def is_suitable_placement_group( class NebiusInstanceBackendData(CoreModel): boot_disk_id: str + pricing_policy_id: str | None = None @classmethod def load(cls, raw: Optional[str]) -> "NebiusInstanceBackendData": @@ -406,3 +463,25 @@ def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None: def _supported_instances(offer: InstanceOffer) -> bool: platform, _ = offer.instance.name.split() return platform in SUPPORTED_PLATFORMS + + +def _get_price_adjusted_for_pricing_policy(offer: InstanceOffer) -> float: + if ( + not offer.instance.resources.spot + or validate_extra_ignore( + NebiusOfferBackendData, offer.backend_data + ).is_preemptible_flat_rate + ): + return offer.price + max_price = _get_pricing_policy_max_price(offer) + return float(max_price * _get_pricing_policy_price_units(offer)) + + +def _get_pricing_policy_max_price(offer: InstanceOffer) -> Decimal: + # Pricing policies allow at most 3 decimal places. Round upward. + price = Decimal(str(offer.price)) / _get_pricing_policy_price_units(offer) + return price.quantize(Decimal("0.001"), rounding=ROUND_CEILING) + + +def _get_pricing_policy_price_units(offer: InstanceOffer) -> int: + return len(offer.instance.resources.gpus) or 1 diff --git a/src/dstack/_internal/core/backends/nebius/models.py b/src/dstack/_internal/core/backends/nebius/models.py index 6c5e05dbae..34af0e6fa1 100644 --- a/src/dstack/_internal/core/backends/nebius/models.py +++ b/src/dstack/_internal/core/backends/nebius/models.py @@ -182,6 +182,10 @@ class NebiusConfig(NebiusStoredConfig): class NebiusOfferBackendData(CoreModel): fabrics: set[str] = set() + is_preemptible_flat_rate: bool = False + """ + True if the price of this spot instance does not change and pricing policies are not supported. + """ @field_serializer("fabrics") def _serialize_fabrics(self, value: set[str]) -> list[str]: diff --git a/src/dstack/_internal/core/backends/nebius/resources.py b/src/dstack/_internal/core/backends/nebius/resources.py index 0c56fd9572..4bf79cb14d 100644 --- a/src/dstack/_internal/core/backends/nebius/resources.py +++ b/src/dstack/_internal/core/backends/nebius/resources.py @@ -11,6 +11,16 @@ from nebius.aio.operation import Operation as SDKOperation from nebius.aio.service_error import RequestError, StatusCode from nebius.aio.token.renewable import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS +from nebius.api.nebius.billing.v1 import ( + ComputeInstanceSpec, + ComputeInstanceSpecV1, + CreatePricingPolicyRequest, + DeletePricingPolicyRequest, + MaxPriceV1, + PricingMethod, + PricingPolicyServiceClient, + PricingPolicySpec, +) from nebius.api.nebius.common.v1 import Operation, ResourceMetadata from nebius.api.nebius.compute.v1 import ( AttachedDiskSpec, @@ -37,6 +47,7 @@ PublicIPAddress, ResourcesSpec, SourceImageFamily, + SpotPricingPolicySpec, ) from nebius.api.nebius.iam.v1 import ( Container, @@ -299,6 +310,34 @@ def delete_disk(sdk: SDK, disk_id: str) -> None: ) +def create_pricing_policy( + sdk: SDK, name: str, project_id: str, platform: str, max_price: str +) -> SDKOperation[Operation]: + client = PricingPolicyServiceClient(sdk) + request = CreatePricingPolicyRequest( + metadata=ResourceMetadata(name=name, parent_id=project_id), + spec=PricingPolicySpec( + compute_instance_spec=ComputeInstanceSpec(v1=ComputeInstanceSpecV1(platform=platform)), + pricing=PricingMethod(max_price_v1=MaxPriceV1(max_price=max_price)), + ), + ) + return LOOP.await_( + client.create( + request, per_retry_timeout=REQUEST_TIMEOUT, auth_options=REQUEST_AUTH_OPTIONS + ) + ) + + +def delete_pricing_policy(sdk: SDK, pricing_policy_id: str) -> None: + LOOP.await_( + PricingPolicyServiceClient(sdk).delete( + DeletePricingPolicyRequest(id=pricing_policy_id), + per_retry_timeout=REQUEST_TIMEOUT, + auth_options=REQUEST_AUTH_OPTIONS, + ) + ) + + def create_instance( sdk: SDK, name: str, @@ -310,6 +349,7 @@ def create_instance( disk_id: str, subnet_id: str, preemptible: bool, + pricing_policy_id: Optional[str], labels: Dict[str, str], ) -> SDKOperation[Operation]: client = InstanceServiceClient(sdk) @@ -341,6 +381,11 @@ def create_instance( if preemptible else None, recovery_policy=InstanceRecoveryPolicy.FAIL if preemptible else None, + spot_pricing_policy=( + SpotPricingPolicySpec(id=pricing_policy_id) + if pricing_policy_id is not None + else None + ), ), ) with wrap_capacity_errors(): diff --git a/src/dstack/_internal/core/models/instances.py b/src/dstack/_internal/core/models/instances.py index d4f5900eb7..68e5c11028 100644 --- a/src/dstack/_internal/core/models/instances.py +++ b/src/dstack/_internal/core/models/instances.py @@ -205,7 +205,7 @@ def with_availability(self, **kwargs) -> "InstanceOfferWithAvailability": """Convert to InstanceOfferWithAvailability without re-serializing/re-validating fields. The result shares nested objects with self. This is generally safe because callers discard the original InstanceOffer after conversion.""" - return InstanceOfferWithAvailability.model_construct(**self.__dict__, **kwargs) + return InstanceOfferWithAvailability.model_construct(**{**self.__dict__, **kwargs}) class InstanceOfferWithAvailability(InstanceOffer): diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py new file mode 100644 index 0000000000..e40416b225 --- /dev/null +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -0,0 +1,111 @@ +import pytest + +from dstack._internal.core.backends.nebius import compute as compute_module +from dstack._internal.core.backends.nebius.compute import NebiusCompute +from dstack._internal.core.backends.nebius.models import ( + NebiusConfig, + NebiusServiceAccountCreds, +) +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.instances import ( + Gpu, + InstanceAvailability, + InstanceOffer, + InstanceType, + Resources, +) + + +def make_compute() -> NebiusCompute: + return NebiusCompute( + NebiusConfig( + creds=NebiusServiceAccountCreds( + service_account_id="service-account-id", + public_key_id="public-key-id", + private_key_content="private-key", + ) + ) + ) + + +def make_offer( + price: float, spot: bool, is_preemptible_flat_rate: bool, gpu_count: int = 8 +) -> InstanceOffer: + return InstanceOffer( + backend=BackendType.NEBIUS, + instance=InstanceType( + name=f"gpu-h100-sxm {gpu_count}gpu-128vcpu-1600gb", + resources=Resources( + cpus=128, + memory_mib=1600 * 1024, + gpus=[Gpu(name="H100", memory_mib=80 * 1024)] * gpu_count, + spot=spot, + ), + ), + region="eu-north1", + price=price, + backend_data={ + "fabrics": ["fabric-2"], + "is_preemptible_flat_rate": is_preemptible_flat_rate, + }, + ) + + +class TestGetAllOffersWithAvailability: + @pytest.fixture(autouse=True) + def _mock_region_to_project_id(self, mocker): + mocker.patch.object(NebiusCompute, "_region_to_project_id", {"eu-north1": "project-id"}) + + @pytest.mark.parametrize( + ("price", "gpu_count", "expected_price"), + [ + (1.23, 1, 1.23), + (1.2, 1, 1.2), + (1.2341, 1, 1.235), + (1.2349, 1, 1.235), + (1.2350001, 1, 1.236), + (0.0001, 1, 0.001), + (2.007, 1, 2.007), + (9.84, 8, 9.84), + (1.23, 8, 1.232), + (1.2341, 0, 1.235), + ], + ) + def test_rounds_spot_price_up_to_3_decimal_places_per_gpu( + self, mocker, price, gpu_count, expected_price + ): + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[ + make_offer(price, spot=True, is_preemptible_flat_rate=False, gpu_count=gpu_count) + ], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + assert len(offers) == 1 + assert offers[0].price == expected_price + assert offers[0].availability == InstanceAvailability.UNKNOWN + + def test_keeps_on_demand_price_as_is(self, mocker): + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[make_offer(1.2341, spot=False, is_preemptible_flat_rate=False)], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + assert offers[0].price == 1.2341 + + def test_keeps_flat_rate_spot_price_as_is(self, mocker): + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[make_offer(1.2341, spot=True, is_preemptible_flat_rate=True)], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + assert offers[0].price == 1.2341 From 70c26168448e615100df267baafd413c1edaceec Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:32:38 +0000 Subject: [PATCH 616/640] Add OCI `experimental_instance_types` setting (#4329) Add an OCI backend setting that accepts a list of shape names to allow provisioning in addition to the standard supported instance families. Only works for shapes included in `dstack`'s pricing catalog (`gpuhunt`). This setting can be used to test various shapes that were not yet tested internally at `dstack`. ```yaml experimental_instance_types: [BM.GPU.RTXPRO.8] ``` --- .../_internal/core/backends/oci/compute.py | 16 ++++++++++++---- src/dstack/_internal/core/backends/oci/models.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/dstack/_internal/core/backends/oci/compute.py b/src/dstack/_internal/core/backends/oci/compute.py index ab9be4908b..b4705bb0a9 100644 --- a/src/dstack/_internal/core/backends/oci/compute.py +++ b/src/dstack/_internal/core/backends/oci/compute.py @@ -1,6 +1,6 @@ -from collections.abc import Iterable +from collections.abc import Container, Iterable from concurrent.futures import ThreadPoolExecutor -from functools import cached_property +from functools import cached_property, partial from typing import List, Optional import oci @@ -64,6 +64,10 @@ def __init__(self, config: OCIConfig): super().__init__() self.config = config self.regions = make_region_clients_map(config.regions or [], config.creds) + self._supported_instances = partial( + _supported_instances, + experimental_instance_types=set(self.config.experimental_instance_types or []), + ) @cached_property def shapes_quota(self) -> resources.ShapesQuota: @@ -75,7 +79,7 @@ def get_all_offers_with_availability( offers = get_catalog_offers( backend=BackendType.OCI, locations=self.config.regions, - extra_filter=_supported_instances, + extra_filter=self._supported_instances, ) with ThreadPoolExecutor(max_workers=8) as executor: @@ -206,7 +210,11 @@ def update_provisioning_data( provisioning_data.internal_ip = vnic.private_ip -def _supported_instances(offer: InstanceOffer) -> bool: +def _supported_instances( + offer: InstanceOffer, experimental_instance_types: Container[str] +) -> bool: + if offer.instance.name in experimental_instance_types: + return True if "Flex" in offer.instance.name: return False return any(map(offer.instance.name.startswith, SUPPORTED_SHAPE_FAMILIES)) diff --git a/src/dstack/_internal/core/backends/oci/models.py b/src/dstack/_internal/core/backends/oci/models.py index 20f22b793d..f7d17c862b 100644 --- a/src/dstack/_internal/core/backends/oci/models.py +++ b/src/dstack/_internal/core/backends/oci/models.py @@ -70,6 +70,16 @@ class OCIBackendConfig(CoreModel): ) ), ] = None + experimental_instance_types: Annotated[ + Optional[List[str]], + Field( + description=( + "The list of shape names to allow provisioning in addition to" + " the standard supported shape families. Only works for shapes" + " included in `dstack`'s pricing catalog (`gpuhunt`)" + ) + ), + ] = None class OCIBackendConfigWithCreds(OCIBackendConfig): From c2be409beb6b127d022d3c129ff12cb267b4b6f5 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:26:17 +0200 Subject: [PATCH 617/640] Add Hot Aisle bare metal support (#4325) * Add Hot Aisle bare metal support Co-Authored-By: Claude Opus 5.5 (1M context) * Add a flag to release Hot Aisle bare metal without force Co-Authored-By: Claude Opus 5.5 (1M context) * Start Hot Aisle shim with nohup instead of disown The shim start is now retried until the SSH command succeeds, and `disown` fails in shells other than bash even though the shim starts. Co-Authored-By: Claude Opus 5.5 (1M context) * Connect to Hot Aisle bare metal via ssh_access The bare metal server's ip_address is private (100.64.0.0/10); SSH is exposed on ssh_access. Co-Authored-By: Claude Opus 5.5 (1M context) * Don't fail Hot Aisle docs commands when nothing is in stock Co-Authored-By: Claude Opus 5.5 (1M context) * Tighten Hot Aisle minimum reservation note Co-Authored-By: Claude Opus 5.5 (1M context) * Silence curl progress in Hot Aisle docs commands Co-Authored-By: Claude Opus 5.5 (1M context) * Recommend fixed fleet nodes for prepaid Hot Aisle instances Co-Authored-By: Claude Opus 5.5 (1M context) * Mark Hot Aisle bare metal as experimental Keep the no-force flag enabled; drop it once offers carry the minimum reservation period and instances stay idle until it ends. Co-Authored-By: Claude Opus 5.5 (1M context) * Make Hot Aisle bare metal opt-in via `bare_metal: true` Bare metal offers are suggested only if the backend sets `bare_metal: true`, like RunPod's `community_cloud`. Exclude the field from client requests when unset to keep older servers working. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Andrey Cheptsov Co-authored-by: Claude Opus 5.5 (1M context) --- mkdocs/docs/concepts/backends.md | 27 +- .../_internal/core/backends/base/offers.py | 1 + .../core/backends/hotaisle/api_client.py | 33 ++ .../core/backends/hotaisle/compute.py | 165 ++++++-- .../core/backends/hotaisle/models.py | 17 + .../_internal/core/compatibility/backends.py | 15 + src/dstack/_internal/settings.py | 10 + src/dstack/api/server/_backends.py | 7 +- .../core/backends/hotaisle/__init__.py | 0 .../core/backends/hotaisle/test_api_client.py | 76 ++++ .../core/backends/hotaisle/test_compute.py | 396 ++++++++++++++++++ .../_internal/core/compatibility/__init__.py | 0 .../core/compatibility/test_backends.py | 27 ++ 13 files changed, 717 insertions(+), 57 deletions(-) create mode 100644 src/dstack/_internal/core/compatibility/backends.py create mode 100644 src/tests/_internal/core/backends/hotaisle/__init__.py create mode 100644 src/tests/_internal/core/backends/hotaisle/test_api_client.py create mode 100644 src/tests/_internal/core/backends/hotaisle/test_compute.py create mode 100644 src/tests/_internal/core/compatibility/__init__.py create mode 100644 src/tests/_internal/core/compatibility/test_backends.py diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index fd23b1ad5f..58cff86371 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -976,24 +976,29 @@ projects:
??? info "Required permissions" - The API key must have the following roles assigned: + The API key requires the `owner` role for your user and the `operator` and `user` roles for the team specified in `team_handle`. - * **Owner role for the user** - Required for creating and managing SSH keys - * **Operator role for the team** - Required for managing virtual machines within the team +??? info "Bare metal" + Bare metal servers (`bm-mi300x-8`) are experimental and disabled by default. To enable them, set `bare_metal: true` in the backend settings: -??? info "Pricing" - `dstack` shows the hourly price for Hot Aisle instances. Some instances also require an upfront payment for a minimum reservation period, which is usually a few hours. You will be charged for the full minimum period even if you stop the instance early. - - See the Hot Aisle API for the minimum reservation period for each instance type: - -
+
- ```shell - $ curl -H "Authorization: Token $API_KEY" https://admin.hotaisle.app/api/teams/$TEAM_HANDLE/virtual_machines/available/ | jq ".[] | {gpus: .Specs.gpus, MinimumReservationMinutes}" + ```yaml + projects: + - name: main + backends: + - type: hotaisle + team_handle: hotaisle-team-handle + creds: + type: api_key + api_key: 9c27a4bb7a8e472fae12ab34.3f2e3c1db75b9a0187fd2196c6b3e56d2b912e1c439ba08d89e7b6fcd4ef1d3f + bare_metal: true ```
+ Bare metal servers are prepaid for 8 hours. To use the full period, use a fleet with a fixed number of [`nodes`](../concepts/fleets.md#nodes), or at least set [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) to cover it. Within the period, `dstack` doesn't terminate them but stops tracking them — terminate them manually in Hot Aisle. After the period, `dstack` terminates them automatically. + ### JarvisLabs Log into your [JarvisLabs](https://cloud.jarvislabs.ai/) account and create an API key. diff --git a/src/dstack/_internal/core/backends/base/offers.py b/src/dstack/_internal/core/backends/base/offers.py index 33d745c1ba..43d6cb484d 100644 --- a/src/dstack/_internal/core/backends/base/offers.py +++ b/src/dstack/_internal/core/backends/base/offers.py @@ -30,6 +30,7 @@ "gcp-dws-calendar-mode", "runpod-cpu", "runpod-cluster", + "hotaisle-bm", ] diff --git a/src/dstack/_internal/core/backends/hotaisle/api_client.py b/src/dstack/_internal/core/backends/hotaisle/api_client.py index a3cc355fcd..003953d80b 100644 --- a/src/dstack/_internal/core/backends/hotaisle/api_client.py +++ b/src/dstack/_internal/core/backends/hotaisle/api_client.py @@ -3,6 +3,7 @@ import requests from dstack._internal.core.backends.base.configurator import raise_invalid_credentials_error +from dstack._internal.core.errors import BackendError, NoCapacityError from dstack._internal.utils.logging import get_logger API_URL = "https://admin.hotaisle.app/api" @@ -88,6 +89,38 @@ def terminate_virtual_machine(self, vm_name: str) -> None: return response.raise_for_status() + def reserve_bare_metal_server(self, specs: Dict[str, Any], description: str) -> Dict[str, Any]: + url = f"{API_URL}/teams/{self.team_handle}/bare_metal/" + payload = {"specs": specs, "description": description} + response = self._make_request("POST", url, json=payload) + # 403: the team's bare metal server limit is reached or the API key lacks permissions. + # 404: no available server matches the specs, e.g. another team reserved it. + if response.status_code in [403, 404]: + raise NoCapacityError(response.text) + response.raise_for_status() + return response.json() + + def get_bare_metal_server(self, server_id: str) -> Dict[str, Any]: + url = f"{API_URL}/teams/{self.team_handle}/bare_metal/{server_id}/" + response = self._make_request("GET", url) + response.raise_for_status() + return response.json() + + def release_bare_metal_server(self, server_id: str, force: bool = True) -> None: + url = f"{API_URL}/teams/{self.team_handle}/bare_metal/{server_id}/" + # force releases even if min reservation time not met + params = {"force": "true"} if force else None + response = self._make_request("DELETE", url, params=params) + if response.status_code == 404: + logger.debug("Hot Aisle bare metal server %s not found", server_id) + return + if response.status_code == 400 and not force: + raise BackendError( + f"Hot Aisle refused to release bare metal server {server_id}" + f" before its minimum reservation period ends: {response.text}" + ) + response.raise_for_status() + def _make_request( self, method: str, diff --git a/src/dstack/_internal/core/backends/hotaisle/compute.py b/src/dstack/_internal/core/backends/hotaisle/compute.py index 2fbbb37da4..5e1d798396 100644 --- a/src/dstack/_internal/core/backends/hotaisle/compute.py +++ b/src/dstack/_internal/core/backends/hotaisle/compute.py @@ -1,7 +1,6 @@ import shlex import subprocess import tempfile -from threading import Thread from typing import Any, List, Optional import gpuhunt @@ -18,6 +17,7 @@ from dstack._internal.core.backends.base.offers import get_catalog_offers from dstack._internal.core.backends.hotaisle.api_client import HotAisleAPIClient from dstack._internal.core.backends.hotaisle.models import HotAisleConfig +from dstack._internal.core.errors import ProvisioningError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import ( CoreModel, @@ -32,12 +32,16 @@ ) from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.runs import JobProvisioningData +from dstack._internal.settings import FeatureFlags +from dstack._internal.utils.common import get_or_error from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) SUPPORTED_GPUS = ["MI300X"] +SSH_CONNECT_TIMEOUT_SECONDS = 10 +SSH_LAUNCH_TIMEOUT_SECONDS = 60 class HotAisleCompute( @@ -63,7 +67,9 @@ def get_all_offers_with_availability( backend=BackendType.HOTAISLE, locations=self.config.regions or None, catalog=self.catalog, - extra_filter=_supported_instances, + extra_filter=lambda o: ( + _supported_instances(o) and (self.config.allow_bare_metal or not _is_bare_metal(o)) + ), ) return [ offer.with_availability(availability=InstanceAvailability.AVAILABLE) @@ -81,11 +87,24 @@ def create_instance( offer_backend_data = validate_extra_ignore( HotAisleOfferBackendData, instance_offer.backend_data ) - vm_data = self.api_client.create_virtual_machine(offer_backend_data.vm_specs) + if offer_backend_data.bare_metal_specs is not None: + server_data = self.api_client.reserve_bare_metal_server( + specs=offer_backend_data.bare_metal_specs, + description=instance_config.instance_name, + ) + # The deployment ID identifies this reservation, the name identifies the server. + instance_id = server_data["deployment_id"] + ip_address = server_data["ip_address"] + else: + vm_data = self.api_client.create_virtual_machine( + get_or_error(offer_backend_data.vm_specs) + ) + instance_id = vm_data["name"] + ip_address = vm_data["ip_address"] return JobProvisioningData( backend=instance_offer.backend, instance_type=instance_offer.instance, - instance_id=vm_data["name"], + instance_id=instance_id, hostname=None, internal_ip=None, region=instance_offer.region, @@ -95,7 +114,8 @@ def create_instance( dockerized=True, ssh_proxy=None, backend_data=HotAisleInstanceBackendData( - ip_address=vm_data["ip_address"] + ip_address=ip_address, + bare_metal=offer_backend_data.bare_metal_specs is not None, ).model_dump_json(), ) @@ -105,38 +125,56 @@ def update_provisioning_data( project_ssh_public_key: str, project_ssh_private_key: str, ): - vm_state = self.api_client.get_vm_state(provisioning_data.instance_id) - if vm_state == "running": - if provisioning_data.hostname is None and provisioning_data.backend_data: - backend_data = HotAisleInstanceBackendData.load(provisioning_data.backend_data) - provisioning_data.hostname = backend_data.ip_address - commands = get_shim_commands(arch=provisioning_data.instance_type.resources.cpu_arch) - launch_command = "sudo sh -c " + shlex.quote(" && ".join(commands)) - thread = Thread( - target=_start_runner, - kwargs={ - "hostname": provisioning_data.hostname, - "project_ssh_private_key": project_ssh_private_key, - "launch_command": launch_command, - }, - daemon=True, - ) - thread.start() + backend_data = HotAisleInstanceBackendData.load(provisioning_data.backend_data) + hostname = backend_data.ip_address + port = 22 + if backend_data.bare_metal: + server_data = self.api_client.get_bare_metal_server(provisioning_data.instance_id) + os_install_status = (server_data.get("os_status") or {}).get("os_install_status") + if os_install_status == "failed": + raise ProvisioningError("Hot Aisle bare metal server OS installation failed") + if os_install_status != "installed": + return + # The bare metal server's ip_address is private, SSH is exposed via ssh_access. + ssh_access = server_data.get("ssh_access") or {} + hostname = ssh_access.get("ip_address") or hostname + port = ssh_access.get("port") or port + elif self.api_client.get_vm_state(provisioning_data.instance_id) != "running": + return + # Retried on the next check until the shim starts. + if not _start_runner( + hostname=hostname, + port=port, + project_ssh_private_key=project_ssh_private_key, + arch=provisioning_data.instance_type.resources.cpu_arch, + ): + return + provisioning_data.hostname = hostname + provisioning_data.ssh_port = port def terminate_instance( self, instance_id: str, region: str, backend_data: Optional[str] = None ): + if backend_data is not None and HotAisleInstanceBackendData.load(backend_data).bare_metal: + self.api_client.release_bare_metal_server( + instance_id, force=not FeatureFlags.HOTAISLE_BARE_METAL_NO_FORCE_RELEASE + ) + return vm_name = instance_id self.api_client.terminate_virtual_machine(vm_name) def _start_runner( hostname: str, + port: int, project_ssh_private_key: str, - launch_command: str, -): - _launch_runner( + arch: Optional[str], +) -> bool: + commands = get_shim_commands(arch=arch) + launch_command = "sudo sh -c " + shlex.quote(" && ".join(commands)) + return _launch_runner( hostname=hostname, + port=port, ssh_private_key=project_ssh_private_key, launch_command=launch_command, ) @@ -144,36 +182,68 @@ def _start_runner( def _launch_runner( hostname: str, + port: int, ssh_private_key: str, launch_command: str, -): - daemonized_command = f"{launch_command.rstrip('&')} >/tmp/dstack-shim.log 2>&1 & disown" - _run_ssh_command( +) -> bool: + # nohup instead of disown, which isn't available in all shells and would fail the exit code. + daemonized_command = ( + f"nohup {launch_command.rstrip('&')} >/tmp/dstack-shim.log 2>&1 bool: with tempfile.NamedTemporaryFile("w+", 0o600) as f: f.write(ssh_private_key) f.flush() - subprocess.run( - [ - "ssh", - "-F", - "none", - "-o", - "StrictHostKeyChecking=no", - "-i", - f.name, - f"hotaisle@{hostname}", - command, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + try: + proc = subprocess.run( + [ + "ssh", + "-F", + "none", + "-o", + "BatchMode=yes", + "-o", + f"ConnectTimeout={SSH_CONNECT_TIMEOUT_SECONDS}", + "-o", + "ConnectionAttempts=1", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "LogLevel=ERROR", + "-i", + f.name, + "-p", + str(port), + f"hotaisle@{hostname}", + command, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + timeout=SSH_LAUNCH_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + logger.debug("Timed out running SSH command on Hot Aisle instance %s", hostname) + return False + if proc.returncode != 0: + logger.debug( + "SSH command failed on Hot Aisle instance %s: exit_code=%s stderr=%r", + hostname, + proc.returncode, + proc.stderr[-1000:], ) + return False + return True def _supported_instances(offer: InstanceOffer) -> bool: @@ -182,8 +252,14 @@ def _supported_instances(offer: InstanceOffer) -> bool: ) +def _is_bare_metal(offer: InstanceOffer) -> bool: + offer_backend_data = validate_extra_ignore(HotAisleOfferBackendData, offer.backend_data) + return offer_backend_data.bare_metal_specs is not None + + class HotAisleInstanceBackendData(CoreModel): ip_address: str + bare_metal: bool = False @classmethod def load(cls, raw: Optional[str]) -> "HotAisleInstanceBackendData": @@ -192,4 +268,5 @@ def load(cls, raw: Optional[str]) -> "HotAisleInstanceBackendData": class HotAisleOfferBackendData(CoreModel): - vm_specs: dict[str, Any] + vm_specs: Optional[dict[str, Any]] = None + bare_metal_specs: Optional[dict[str, Any]] = None diff --git a/src/dstack/_internal/core/backends/hotaisle/models.py b/src/dstack/_internal/core/backends/hotaisle/models.py index efee6b4e93..7bc730a98e 100644 --- a/src/dstack/_internal/core/backends/hotaisle/models.py +++ b/src/dstack/_internal/core/backends/hotaisle/models.py @@ -4,6 +4,8 @@ from dstack._internal.core.models.common import CoreModel +HOTAISLE_BARE_METAL_DEFAULT = False + class HotAisleAPIKeyCreds(CoreModel): type: Annotated[Literal["api_key"], Field(description="The type of credentials")] = "api_key" @@ -24,6 +26,15 @@ class HotAisleBackendConfig(CoreModel): Optional[List[str]], Field(description="The list of Hot Aisle regions. Omit to use all regions"), ] = None + bare_metal: Annotated[ + Optional[bool], + Field( + description=( + "Whether bare metal offers can be suggested in addition to VMs (experimental)." + f" Defaults to `{str(HOTAISLE_BARE_METAL_DEFAULT).lower()}`" + ) + ), + ] = None class HotAisleBackendConfigWithCreds(HotAisleBackendConfig): @@ -43,3 +54,9 @@ class HotAisleStoredConfig(HotAisleBackendConfig): class HotAisleConfig(HotAisleStoredConfig): creds: AnyHotAisleCreds + + @property + def allow_bare_metal(self) -> bool: + if self.bare_metal is not None: + return self.bare_metal + return HOTAISLE_BARE_METAL_DEFAULT diff --git a/src/dstack/_internal/core/compatibility/backends.py b/src/dstack/_internal/core/compatibility/backends.py new file mode 100644 index 0000000000..a4bb7f21a5 --- /dev/null +++ b/src/dstack/_internal/core/compatibility/backends.py @@ -0,0 +1,15 @@ +from dstack._internal.core.backends.hotaisle.models import HotAisleBackendConfigWithCreds +from dstack._internal.core.backends.models import AnyBackendConfigWithCreds +from dstack._internal.core.models.common import IncludeExcludeDictType + + +def get_backend_config_excludes(config: AnyBackendConfigWithCreds) -> IncludeExcludeDictType: + """ + Returns `config` exclude mapping to exclude certain fields from the create/update backend + request. Use this method to exclude new fields when they are not set to keep + clients backward-compatibility with older servers. + """ + excludes: IncludeExcludeDictType = {} + if isinstance(config, HotAisleBackendConfigWithCreds) and config.bare_metal is None: + excludes["bare_metal"] = True + return excludes diff --git a/src/dstack/_internal/settings.py b/src/dstack/_internal/settings.py index b04b94b856..1d890f1643 100644 --- a/src/dstack/_internal/settings.py +++ b/src/dstack/_internal/settings.py @@ -50,3 +50,13 @@ class FeatureFlags: """If DSTACK_FF_CLI_PRINT_JOB_CONNECTION_INFO enabled, `dstack apply` command prints server-provided IDE URL(s) and SSH command(s) before job logs (for dev-environments only). """ + + # TODO: Drop once offers carry the minimum reservation period and `dstack` keeps such instances + # idle until it ends. + HOTAISLE_BARE_METAL_NO_FORCE_RELEASE = ( + os.getenv("DSTACK_FF_HOTAISLE_BARE_METAL_NO_FORCE_RELEASE", "1") != "0" + ) + """Enabled unless set to `0`. If enabled, Hot Aisle bare metal servers are deleted without `force`, + so a server still within its minimum reservation period isn't deleted and must be released + manually. This prevents accidentally losing a prepaid server. + """ diff --git a/src/dstack/api/server/_backends.py b/src/dstack/api/server/_backends.py index 37afa04e05..0b180a42cc 100644 --- a/src/dstack/api/server/_backends.py +++ b/src/dstack/api/server/_backends.py @@ -6,6 +6,7 @@ AnyBackendConfigWithCreds, AnyBackendConfigWithCredsTagged, ) +from dstack._internal.core.compatibility.backends import get_backend_config_excludes from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.server.schemas.backends import DeleteBackendsRequest @@ -27,7 +28,8 @@ def create( self, project_name: str, config: AnyBackendConfigWithCreds ) -> AnyBackendConfigWithCreds: resp = self._request( - f"/api/project/{project_name}/backends/create", body=config.model_dump_json() + f"/api/project/{project_name}/backends/create", + body=config.model_dump_json(exclude=get_backend_config_excludes(config)), ) return validate_extra_ignore(AnyBackendConfigWithCredsTagged, resp.json()) @@ -35,7 +37,8 @@ def update( self, project_name: str, config: AnyBackendConfigWithCreds ) -> AnyBackendConfigWithCreds: resp = self._request( - f"/api/project/{project_name}/backends/update", body=config.model_dump_json() + f"/api/project/{project_name}/backends/update", + body=config.model_dump_json(exclude=get_backend_config_excludes(config)), ) return validate_extra_ignore(AnyBackendConfigWithCredsTagged, resp.json()) diff --git a/src/tests/_internal/core/backends/hotaisle/__init__.py b/src/tests/_internal/core/backends/hotaisle/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tests/_internal/core/backends/hotaisle/test_api_client.py b/src/tests/_internal/core/backends/hotaisle/test_api_client.py new file mode 100644 index 0000000000..c69c969d32 --- /dev/null +++ b/src/tests/_internal/core/backends/hotaisle/test_api_client.py @@ -0,0 +1,76 @@ +import pytest +import requests + +from dstack._internal.core.backends.hotaisle.api_client import API_URL, HotAisleAPIClient +from dstack._internal.core.errors import BackendError, NoCapacityError + +BARE_METAL_URL = f"{API_URL}/teams/test-team/bare_metal/" +SERVER_URL = f"{BARE_METAL_URL}deployment-id/" + + +def _client() -> HotAisleAPIClient: + return HotAisleAPIClient(api_key="test-key", team_handle="test-team") + + +class TestReserveBareMetalServer: + def test_posts_specs_and_description(self, requests_mock): + requests_mock.post(BARE_METAL_URL, json={"deployment_id": "deployment-id"}) + + server_data = _client().reserve_bare_metal_server( + specs={"cpu_cores": 104}, description="test-instance" + ) + + assert server_data == {"deployment_id": "deployment-id"} + assert requests_mock.last_request.json() == { + "specs": {"cpu_cores": 104}, + "description": "test-instance", + } + + @pytest.mark.parametrize( + ("status_code", "text"), + [(403, "tenant limit exceeded"), (404, "no available servers")], + ) + def test_raises_no_capacity(self, requests_mock, status_code, text): + requests_mock.post(BARE_METAL_URL, status_code=status_code, text=text) + + with pytest.raises(NoCapacityError, match=text): + _client().reserve_bare_metal_server(specs={}, description="test-instance") + + def test_raises_on_other_errors(self, requests_mock): + requests_mock.post(BARE_METAL_URL, status_code=402) + + with pytest.raises(requests.HTTPError): + _client().reserve_bare_metal_server(specs={}, description="test-instance") + + +class TestReleaseBareMetalServer: + def test_forces_release(self, requests_mock): + requests_mock.delete(SERVER_URL, status_code=204) + + _client().release_bare_metal_server("deployment-id") + + assert requests_mock.last_request.qs == {"force": ["true"]} + + def test_releases_without_force(self, requests_mock): + requests_mock.delete(SERVER_URL, status_code=204) + + _client().release_bare_metal_server("deployment-id", force=False) + + assert requests_mock.last_request.qs == {} + + def test_raises_if_refused_without_force(self, requests_mock): + requests_mock.delete(SERVER_URL, status_code=400, text="minimum usage requirement") + + with pytest.raises(BackendError, match="minimum usage requirement"): + _client().release_bare_metal_server("deployment-id", force=False) + + def test_ignores_missing_server(self, requests_mock): + requests_mock.delete(SERVER_URL, status_code=404) + + _client().release_bare_metal_server("deployment-id") + + def test_raises_on_other_errors(self, requests_mock): + requests_mock.delete(SERVER_URL, status_code=400) + + with pytest.raises(requests.HTTPError): + _client().release_bare_metal_server("deployment-id") diff --git a/src/tests/_internal/core/backends/hotaisle/test_compute.py b/src/tests/_internal/core/backends/hotaisle/test_compute.py new file mode 100644 index 0000000000..562014770a --- /dev/null +++ b/src/tests/_internal/core/backends/hotaisle/test_compute.py @@ -0,0 +1,396 @@ +import subprocess +from typing import Optional +from unittest.mock import MagicMock, call, patch + +import pytest +from gpuhunt.providers.hotaisle import API_URL + +from dstack._internal.core.backends.hotaisle.compute import ( + SSH_CONNECT_TIMEOUT_SECONDS, + SSH_LAUNCH_TIMEOUT_SECONDS, + HotAisleCompute, + HotAisleInstanceBackendData, + _launch_runner, + _run_ssh_command, +) +from dstack._internal.core.backends.hotaisle.models import HotAisleAPIKeyCreds, HotAisleConfig +from dstack._internal.core.errors import ProvisioningError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + InstanceAvailability, + InstanceConfiguration, + InstanceOfferWithAvailability, + InstanceType, + Resources, + SSHKey, +) +from dstack._internal.core.models.runs import JobProvisioningData +from dstack._internal.settings import FeatureFlags + +VM_SPECS = { + "cpu_cores": 13, + "ram_capacity": 224 * 1024**3, + "disk_capacity": 12288 * 1024**3, + "cpus": {"count": 1, "manufacturer": "Intel", "model": "Xeon Platinum 8470"}, + "gpus": [{"count": 1, "manufacturer": "AMD", "model": "MI300X"}], +} + +BARE_METAL_SPECS = { + "cpu_cores": 104, + "ram_capacity": 2048 * 1024**3, + "disk_capacity": 123839994396672, + "cpus": [{"count": 2, "manufacturer": "Intel", "model": "Xeon Platinum 8470", "cores": 52}], + "gpus": [{"count": 8, "manufacturer": "AMD", "model": "MI300X"}], +} + + +def _compute(bare_metal: Optional[bool] = None) -> HotAisleCompute: + return HotAisleCompute( + HotAisleConfig( + team_handle="test-team", + creds=HotAisleAPIKeyCreds(api_key="test-key"), + bare_metal=bare_metal, + ) + ) + + +def _compute_with_mocked_api_client() -> HotAisleCompute: + compute = _compute() + compute.api_client = MagicMock() + return compute + + +def _instance_type(name: str) -> InstanceType: + return InstanceType( + name=name, + resources=Resources( + cpus=13, + memory_mib=224 * 1024, + gpus=[Gpu(name="MI300X", memory_mib=192 * 1024)], + spot=False, + disk=Disk(size_mib=12288 * 1024), + ), + ) + + +def _offer(name: str, backend_data: dict) -> InstanceOfferWithAvailability: + return InstanceOfferWithAvailability( + backend=BackendType.HOTAISLE, + instance=_instance_type(name), + region="us-michigan-1", + price=1.99, + backend_data=backend_data, + availability=InstanceAvailability.AVAILABLE, + ) + + +def _instance_config() -> InstanceConfiguration: + return InstanceConfiguration( + project_name="test-project", + instance_name="test-instance", + user="test-user", + ssh_keys=[SSHKey(public="ssh-rsa AAAA test")], + ) + + +def _provisioning_data( + backend_data: Optional[HotAisleInstanceBackendData], +) -> JobProvisioningData: + return JobProvisioningData( + backend=BackendType.HOTAISLE, + instance_type=_instance_type("vm-mi300x-1"), + instance_id="instance-id", + hostname=None, + internal_ip=None, + region="us-michigan-1", + price=1.99, + username="hotaisle", + ssh_port=22, + dockerized=True, + ssh_proxy=None, + backend_data=backend_data.model_dump_json() if backend_data is not None else None, + ) + + +class TestGetAllOffersWithAvailability: + @pytest.mark.parametrize( + ("bare_metal", "expected_instance_types"), + [ + (None, ["vm-mi300x-1"]), + (False, ["vm-mi300x-1"]), + (True, ["vm-mi300x-1", "bm-mi300x-8"]), + ], + ids=["default", "disabled", "enabled"], + ) + def test_returns_bare_metal_offers_only_if_enabled( + self, requests_mock, bare_metal, expected_instance_types + ): + requests_mock.get( + f"{API_URL}/teams/test-team/virtual_machines/available/", + json=[{"OnDemandPrice": 199, "Specs": VM_SPECS}], + ) + requests_mock.get( + f"{API_URL}/teams/test-team/bare_metal/available/", + json=[{"OnDemandPrice": 2712, "Specs": BARE_METAL_SPECS}], + ) + + offers = _compute(bare_metal=bare_metal).get_all_offers_with_availability( + unallocated_resources=False + ) + + assert [offer.instance.name for offer in offers] == expected_instance_types + + def test_keeps_specs_in_backend_data(self, requests_mock): + requests_mock.get( + f"{API_URL}/teams/test-team/virtual_machines/available/", + json=[{"OnDemandPrice": 199, "Specs": VM_SPECS}], + ) + requests_mock.get( + f"{API_URL}/teams/test-team/bare_metal/available/", + json=[{"OnDemandPrice": 2712, "Specs": BARE_METAL_SPECS}], + ) + + offers = _compute(bare_metal=True).get_all_offers_with_availability( + unallocated_resources=False + ) + + assert [offer.backend_data for offer in offers] == [ + {"vm_specs": VM_SPECS}, + {"bare_metal_specs": BARE_METAL_SPECS}, + ] + + +class TestCreateInstance: + def test_creates_vm(self): + compute = _compute_with_mocked_api_client() + compute.api_client.create_virtual_machine.return_value = { + "name": "vm-name", + "ip_address": "10.0.0.1", + } + + provisioning_data = compute.create_instance( + _offer("vm-mi300x-1", {"vm_specs": VM_SPECS}), _instance_config(), None + ) + + compute.api_client.upload_ssh_key.assert_called_once_with("ssh-rsa AAAA test") + compute.api_client.create_virtual_machine.assert_called_once_with(VM_SPECS) + compute.api_client.reserve_bare_metal_server.assert_not_called() + assert provisioning_data.instance_id == "vm-name" + assert HotAisleInstanceBackendData.load( + provisioning_data.backend_data + ) == HotAisleInstanceBackendData(ip_address="10.0.0.1", bare_metal=False) + + def test_reserves_bare_metal_server(self): + compute = _compute_with_mocked_api_client() + compute.api_client.reserve_bare_metal_server.return_value = { + "deployment_id": "deployment-id", + "name": "server-01", + "ip_address": "10.0.0.2", + } + + provisioning_data = compute.create_instance( + _offer("bm-mi300x-8", {"bare_metal_specs": BARE_METAL_SPECS}), + _instance_config(), + None, + ) + + # The key must be uploaded before reserving, so the server accepts it. + assert compute.api_client.mock_calls == [ + call.upload_ssh_key("ssh-rsa AAAA test"), + call.reserve_bare_metal_server(specs=BARE_METAL_SPECS, description="test-instance"), + ] + assert provisioning_data.instance_id == "deployment-id" + assert provisioning_data.username == "hotaisle" + assert HotAisleInstanceBackendData.load( + provisioning_data.backend_data + ) == HotAisleInstanceBackendData(ip_address="10.0.0.2", bare_metal=True) + + +@patch("dstack._internal.core.backends.hotaisle.compute._run_ssh_command", return_value=True) +class TestUpdateProvisioningData: + def test_starts_shim_on_running_vm(self, ssh_mock): + compute = _compute_with_mocked_api_client() + compute.api_client.get_vm_state.return_value = "running" + provisioning_data = _provisioning_data(HotAisleInstanceBackendData(ip_address="10.0.0.1")) + + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + + compute.api_client.get_vm_state.assert_called_once_with("instance-id") + assert provisioning_data.hostname == "10.0.0.1" + assert provisioning_data.ssh_port == 22 + ssh_mock.assert_called_once() + assert ssh_mock.call_args.kwargs["hostname"] == "10.0.0.1" + + def test_retries_if_shim_fails_to_start(self, ssh_mock): + ssh_mock.return_value = False + compute = _compute_with_mocked_api_client() + compute.api_client.get_vm_state.return_value = "running" + provisioning_data = _provisioning_data(HotAisleInstanceBackendData(ip_address="10.0.0.1")) + + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + + ssh_mock.assert_called_once() + assert provisioning_data.hostname is None + + def test_waits_for_vm_to_run(self, ssh_mock): + compute = _compute_with_mocked_api_client() + compute.api_client.get_vm_state.return_value = "shut off" + provisioning_data = _provisioning_data(HotAisleInstanceBackendData(ip_address="10.0.0.1")) + + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + + assert provisioning_data.hostname is None + ssh_mock.assert_not_called() + + @pytest.mark.parametrize( + ("ssh_access", "hostname", "port"), + [ + ({"ip_address": "203.0.113.10", "port": 2222}, "203.0.113.10", 2222), + (None, "10.0.0.2", 22), + ], + ids=["ssh-access", "no-ssh-access"], + ) + def test_starts_shim_on_installed_bare_metal_server( + self, ssh_mock, ssh_access, hostname, port + ): + compute = _compute_with_mocked_api_client() + compute.api_client.get_bare_metal_server.return_value = { + "ip_address": "10.0.0.2", + "ssh_access": ssh_access, + "os_status": {"os_install_status": "installed"}, + } + provisioning_data = _provisioning_data( + HotAisleInstanceBackendData(ip_address="10.0.0.2", bare_metal=True) + ) + + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + + compute.api_client.get_bare_metal_server.assert_called_once_with("instance-id") + compute.api_client.get_vm_state.assert_not_called() + assert provisioning_data.hostname == hostname + assert provisioning_data.ssh_port == port + ssh_mock.assert_called_once() + assert ssh_mock.call_args.kwargs["hostname"] == hostname + assert ssh_mock.call_args.kwargs["port"] == port + + @pytest.mark.parametrize( + "server_data", + [ + {"os_status": {"os_install_status": "installing_os"}}, + {"os_status": {"os_install_status": "first_boot_tasks"}}, + {"os_status": None}, + {}, + ], + ) + def test_waits_for_bare_metal_os_installation(self, ssh_mock, server_data): + compute = _compute_with_mocked_api_client() + compute.api_client.get_bare_metal_server.return_value = server_data + provisioning_data = _provisioning_data( + HotAisleInstanceBackendData(ip_address="10.0.0.2", bare_metal=True) + ) + + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + + assert provisioning_data.hostname is None + ssh_mock.assert_not_called() + + def test_raises_on_failed_bare_metal_os_installation(self, ssh_mock): + compute = _compute_with_mocked_api_client() + compute.api_client.get_bare_metal_server.return_value = { + "os_status": {"os_install_status": "failed"} + } + provisioning_data = _provisioning_data( + HotAisleInstanceBackendData(ip_address="10.0.0.2", bare_metal=True) + ) + + with pytest.raises(ProvisioningError): + compute.update_provisioning_data(provisioning_data, "public-key", "private-key") + ssh_mock.assert_not_called() + + +class TestTerminateInstance: + def test_terminates_vm(self): + compute = _compute_with_mocked_api_client() + + compute.terminate_instance( + "vm-name", + "us-michigan-1", + HotAisleInstanceBackendData(ip_address="10.0.0.1").model_dump_json(), + ) + + compute.api_client.terminate_virtual_machine.assert_called_once_with("vm-name") + compute.api_client.release_bare_metal_server.assert_not_called() + + def test_terminates_vm_without_backend_data(self): + compute = _compute_with_mocked_api_client() + + compute.terminate_instance("vm-name", "us-michigan-1", None) + + compute.api_client.terminate_virtual_machine.assert_called_once_with("vm-name") + + @pytest.mark.parametrize( + ("no_force_release", "force"), + [(False, True), (True, False)], + ids=["force", "no-force-flag"], + ) + def test_releases_bare_metal_server(self, no_force_release, force): + compute = _compute_with_mocked_api_client() + + with patch.object(FeatureFlags, "HOTAISLE_BARE_METAL_NO_FORCE_RELEASE", no_force_release): + compute.terminate_instance( + "deployment-id", + "us-michigan-1", + HotAisleInstanceBackendData( + ip_address="10.0.0.2", bare_metal=True + ).model_dump_json(), + ) + + compute.api_client.release_bare_metal_server.assert_called_once_with( + "deployment-id", force=force + ) + compute.api_client.terminate_virtual_machine.assert_not_called() + + +class TestLaunchRunner: + @patch("dstack._internal.core.backends.hotaisle.compute._run_ssh_command", return_value=True) + def test_daemonizes_without_disown(self, ssh_mock): + assert _launch_runner("10.0.0.1", 22, "private-key", "sudo sh -c 'dstack-shim'") + + ssh_mock.assert_called_once_with( + hostname="10.0.0.1", + port=22, + ssh_private_key="private-key", + command="nohup sudo sh -c 'dstack-shim' >/tmp/dstack-shim.log 2>&1 dict: + return json.loads(config.model_dump_json(exclude=get_backend_config_excludes(config))) + + +class TestGetBackendConfigExcludes: + def test_excludes_unset_hotaisle_bare_metal(self): + config = HotAisleBackendConfigWithCreds( + team_handle="test-team", creds=HotAisleAPIKeyCreds(api_key="test-key") + ) + + assert "bare_metal" not in _request_body(config) + + def test_keeps_set_hotaisle_bare_metal(self): + config = HotAisleBackendConfigWithCreds( + team_handle="test-team", creds=HotAisleAPIKeyCreds(api_key="test-key"), bare_metal=True + ) + + assert _request_body(config)["bare_metal"] is True From e316e088fd608f5f0441c9942a9f1dd812513a8f Mon Sep 17 00:00:00 2001 From: jvstme <36324149+jvstme@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:22:34 +0000 Subject: [PATCH 618/640] Correctly support NVIDIA B300 in Vast.ai (#4330) - Report `B300 PC` as `B300`. - Make the `gpu: B300` filter work reliably for both `B300 AC` and `B300 PC` variants. Before: ```shell $ dstack offer -b vastai --gpu B300 No matching instance offers available. Possible reasons: https://dstack.ai/docs/guides/troubleshooting/#no-offers ``` After: ```shell $ dstack offer -b vastai --gpu B300 # BACKEND RESOURCES INSTANCE TYPE PRICE 1 vastai (us-california) cpu=x86:256 mem=3095.7GB disk=25751.7GB gpu=B300:270GB:8 (spot) 53587615 $73.5377 2 vastai (us-california) cpu=x86:256 mem=3095.7GB disk=25751.7GB gpu=B300:270GB:8 53587615 $94.871 3 vastai (us-california) cpu=x86:128 mem=773.9GB disk=12875.8GB gpu=B300:270GB:4 (spot) 53587617 $36.7688 4 vastai (us-california) cpu=x86:128 mem=773.9GB disk=12875.8GB gpu=B300:270GB:4 53587617 $47.4355 5 vastai (hr-croatia) cpu=x86:43 mem=32.2GB disk=810.2GB gpu=B300:270GB:1 (spot) 53579010 $5.411 6 vastai (hr-croatia) cpu=x86:43 mem=32.2GB disk=810.2GB gpu=B300:270GB:1 53579010 $10.1485 7 vastai (hr-croatia) cpu=x86:86 mem=129GB disk=1620.5GB gpu=B300:270GB:2 (spot) 53579004 $10.822 8 vastai (hr-croatia) cpu=x86:86 mem=129GB disk=1620.5GB gpu=B300:270GB:2 53579004 $20.297 ``` The fix is in the `gpuhunt` package. This commit bumps the `gpuhunt` version in `dstack`. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c755837f70..bba8433701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "python-multipart>=0.0.16", "filelock", "psutil", - "gpuhunt==0.1.31", + "gpuhunt==0.1.32", "argcomplete>=3.5.0", "ignore-python>=0.2.0", "apscheduler<4", From 90887d0ac79c6b621fda66f16f89e30fb1a0a6f9 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov <54148038+peterschmidt85@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:47:45 +0200 Subject: [PATCH 619/640] Update Azure GRID driver to 580.178.04 (#4332) * Update Azure GRID driver to 580.178.04 NVIDIA's 2026-09-30 security bulletin covers A10 vGPU. Azure has patched the NVadsA10_v5 hosts and now supports only the patched guest drivers: vGPU 19.6 (580.178.04) and vGPU 20.2. The 580 driver also adds CUDA 13.0 support on A10. With the previous 570 driver, CUDA 13 images failed with "driver too old" unless they bundled CUDA forward compatibility libraries, as dstack's base image does. This matches the 580 driver already used in dstack-cuda images. Also allow republishing an existing Azure image as a new gallery version, so dstack servers using dstack-grid-0.14 pick up the new driver without a release: - azure_variant input to build only some Azure image variants - azure_new_gallery_version input to replace the build image and publish the next gallery version; without it, publishing stops if a version exists Co-Authored-By: Claude Opus 5.5 * Stop Azure image publish if gallery versions can't be listed Without this, a failed version listing looked like an empty gallery, so the script tried to publish 0.0.1 again instead of the next version. Also describe azure_new_gallery_version as a rebuild, since packer -force deletes and rebuilds the existing managed image. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Andrey Cheptsov Co-authored-by: Claude Opus 5.5 --- .github/workflows/vm-images.yml | 15 +++++++-- .../install-nvidia-grid-driver-for-azure.sh | 2 +- scripts/publish_azure_image.sh | 32 +++++++++++++++++-- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/.github/workflows/vm-images.yml b/.github/workflows/vm-images.yml index 6251fa2027..f86769834b 100644 --- a/.github/workflows/vm-images.yml +++ b/.github/workflows/vm-images.yml @@ -19,6 +19,15 @@ on: description: "Build Azure images" type: boolean default: true + azure_variant: + description: "Azure image variants to build" + type: choice + options: ["all", "standard", "cuda", "grid"] + default: "all" + azure_new_gallery_version: + description: "Rebuild Azure images that already exist and publish them as a new gallery version" + type: boolean + default: false build_gcp: description: "Build GCP images" type: boolean @@ -72,7 +81,7 @@ jobs: VERSION: ${{ github.run_number }} strategy: matrix: - variant: ["", "-cuda", "-grid"] + variant: ${{ fromJSON('{"all":["","-cuda","-grid"],"standard":[""],"cuda":["-cuda"],"grid":["-grid"]}')[inputs.azure_variant] }} steps: - uses: actions/checkout@v4 - uses: Azure/login@v2 @@ -86,13 +95,13 @@ jobs: chmod +x packer - name: Run packer run: | - ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json + ./packer build ${{ inputs.azure_new_gallery_version && '-force' || '' }} -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json - name: Publish azure image if: ${{ !inputs.staging }} run: | IMAGE_DEFINITION=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} IMAGE_NAME=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} - ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME + ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME ${{ inputs.azure_new_gallery_version }} build-gcp-images: if: inputs.build_gcp diff --git a/scripts/packer/provisioners/install-nvidia-grid-driver-for-azure.sh b/scripts/packer/provisioners/install-nvidia-grid-driver-for-azure.sh index 8a57b2a7cf..1ad4266759 100755 --- a/scripts/packer/provisioners/install-nvidia-grid-driver-for-azure.sh +++ b/scripts/packer/provisioners/install-nvidia-grid-driver-for-azure.sh @@ -8,7 +8,7 @@ sudo apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install build-essential linux-azure -y wget --no-verbose -O NVIDIA-Linux-x86_64-grid.run \ - https://download.microsoft.com/download/2a04ca6a-9eec-40d9-9564-9cdea1ab795f/NVIDIA-Linux-x86_64-570.211.01-grid-azure.run + https://download.microsoft.com/download/4efc1974-7b6f-431f-ab2d-b8d99bb8a512/NVIDIA-Linux-x86_64-580.178.04-grid-azure.run chmod +x NVIDIA-Linux-x86_64-grid.run sudo ./NVIDIA-Linux-x86_64-grid.run --silent --disable-nouveau rm NVIDIA-Linux-x86_64-grid.run diff --git a/scripts/publish_azure_image.sh b/scripts/publish_azure_image.sh index 3139693809..f69291844f 100755 --- a/scripts/publish_azure_image.sh +++ b/scripts/publish_azure_image.sh @@ -1,7 +1,11 @@ #!/bin/bash +set -o pipefail + image_definition=$1 image_name=$2 +# If true, publish a new gallery version when the image definition already has versions +new_gallery_version=${3:-false} subscription_id=$AZURE_SUBSCRIPTION_ID if [ -z "$subscription_id" ]; then @@ -37,17 +41,39 @@ function create_image_definition() { --features DiskControllerTypes=SCSI,NVMe } +function get_latest_image_version() { + az sig image-version list \ + --resource-group $resource_group \ + --gallery-name $gallery_name \ + --gallery-image-definition $image_definition \ + --query "[].name" \ + --output tsv | + sort -V | + tail -1 +} + function create_image_version() { - echo Creating image version... + echo Creating image version $1... az sig image-version create \ --resource-group $resource_group \ --gallery-name $gallery_name \ --gallery-image-definition $image_definition \ - --gallery-image-version "0.0.1" \ + --gallery-image-version "$1" \ --target-regions "australiaeast" "brazilsouth" "canadacentral" "centralindia" "centralus" "eastasia" "eastus" "eastus2" "francecentral" "germanywestcentral" "japaneast" "koreacentral" "northeurope" "norwayeast" "qatarcentral" "southafricanorth" "southcentralus" "southeastasia" "swedencentral" "switzerlandnorth" "uaenorth" "uksouth" "westeurope" "westus2" "westus3" \ --replica-count 1 \ --managed-image "/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.Compute/images/${image_name}" } get_image_definition > /dev/null || create_image_definition -create_image_version + +latest_version=$(get_latest_image_version) || exit 1 +if [ -z "$latest_version" ]; then + image_version=0.0.1 +elif [ "$new_gallery_version" = true ]; then + image_version="${latest_version%.*}.$((${latest_version##*.} + 1))" +else + echo "$image_definition already has gallery version $latest_version." \ + "Enable azure_new_gallery_version to publish a new one." + exit 1 +fi +create_image_version $image_version From 0d578c8087537c0998ac6ca8d85c51312022fa96 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 5 Oct 2026 13:33:57 +0000 Subject: [PATCH 620/640] [Docs] Update vLLM PD example (#4336) --- mkdocs/docs/examples/inference/vllm.md | 33 +++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/mkdocs/docs/examples/inference/vllm.md b/mkdocs/docs/examples/inference/vllm.md index 0bc271c8d2..0fac3d2370 100644 --- a/mkdocs/docs/examples/inference/vllm.md +++ b/mkdocs/docs/examples/inference/vllm.md @@ -144,7 +144,7 @@ groups: - replicas: 1 python: "3.12" commands: - - pip install smg + - pip install smg==1.11.0 - | smg launch \ --pd-disaggregation \ @@ -162,37 +162,47 @@ groups: scaling: metric: rps target: 3 - image: ghcr.io/lightseekorg/smg:1.4.1-vllm-v0.18.0 + image: ghcr.io/smg-project/smg:1.11.0-vllm-v0.27.1 commands: + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP - | python3 -m vllm.entrypoints.grpc_server \ --model "$MODEL_ID" \ --host 0.0.0.0 \ --port 8000 \ + --max-model-len 65536 \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' resources: gpu: H200 + disk: 200GB.. - replicas: 1..8 scaling: metric: rps target: 2 - image: ghcr.io/lightseekorg/smg:1.4.1-vllm-v0.18.0 + image: ghcr.io/smg-project/smg:1.11.0-vllm-v0.27.1 commands: + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP - | python3 -m vllm.entrypoints.grpc_server \ --model "$MODEL_ID" \ --host 0.0.0.0 \ --port 8000 \ + --max-model-len 65536 \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}' resources: gpu: H200 + disk: 200GB.. port: 8000 ```
+Use the same SMG version for the router and the worker images: the worker image bundles the SMG gRPC servicer, which must return the KV transfer parameters that the router relays from prefill to decode. + +`VLLM_NIXL_SIDE_CHANNEL_HOST` sets the address of the NIXL handshake listener, which the prefill worker also passes to the decode worker. It defaults to `localhost`, in which case the decode worker connects to itself instead of the prefill worker. `DSTACK_MASTER_NODE_IP` is the internal IP of the replica's instance. + > To use the [Mooncake Transfer](https://github.com/kvcache-ai/Mooncake) backend, set `"kv_connector": "MooncakeConnector"` in `--kv-transfer-config`. Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics are coming soon. @@ -202,6 +212,23 @@ Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics While the prefill and decode replicas run on GPUs, the router replica requires a CPU instance in the same cluster. + For production, use instances with RDMA (InfiniBand or RoCE). Without RDMA, NIXL transfers the KV cache over TCP, which works but may be slower than recomputing the prompt on the decode worker. + +??? info "Instances without RDMA" + If an instance has both a private and a public network interface and no RDMA, UCX (the default NIXL transport) may advertise the public address, which may be unreachable from other instances. In this case, restrict UCX to the interface that has the instance's internal IP by adding the following command before starting the worker: + + ```yaml + - | + export UCX_NET_DEVICES=$(python3 -c ' + import os, psutil + ip = os.environ["DSTACK_MASTER_NODE_IP"] + ifaces = psutil.net_if_addrs().items() + print(next(name for name, addrs in ifaces if any(a.address == ip for a in addrs))) + ') + ``` + + Don't use this on instances with RDMA, as it prevents UCX from using RDMA devices. + ## What's next? 1. Read about [services](../../concepts/services.md) and [gateways](../../concepts/gateways.md) From 46c9e831cc65010b539a7199493dea06899f8c07 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 7 Oct 2026 13:01:02 +0500 Subject: [PATCH 621/640] Fix pyright with kubernetes 37 (#4347) --- .../_internal/core/backends/kubernetes/resources.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/core/backends/kubernetes/resources.py b/src/dstack/_internal/core/backends/kubernetes/resources.py index 393d2b786a..b049eccc8c 100644 --- a/src/dstack/_internal/core/backends/kubernetes/resources.py +++ b/src/dstack/_internal/core/backends/kubernetes/resources.py @@ -9,10 +9,8 @@ import gpuhunt from gpuhunt import KNOWN_AMD_GPUS, KNOWN_NVIDIA_GPUS, AcceleratorVendor - -# XXX: kubernetes.utils is missing in the stubs package -from kubernetes import utils as _kubernetes_utils # pyright: ignore[reportAttributeAccessIssue] from kubernetes.client import CoreV1Api, V1Node, V1Taint +from kubernetes.utils.quantity import parse_quantity as _parse_quantity from typing_extensions import Self from dstack._internal.core.backends.base.compute import normalize_arch @@ -344,9 +342,7 @@ def build_dockerconfigjson(image_name: str, username: str, password: str) -> str return json.dumps({"auths": {registry: entry}}) -parse_quantity = cast( - Callable[[Union[str, int, float, Decimal]], Decimal], _kubernetes_utils.parse_quantity -) +parse_quantity = cast(Callable[[Union[str, int, float, Decimal]], Decimal], _parse_quantity) def format_memory(memory: Memory) -> str: From 07174d93280a88b18af24094af8c395a8dfdaefc Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 7 Oct 2026 13:01:39 +0500 Subject: [PATCH 622/640] Skip placement groups for AWS Capacity Blocks (#4345) * Skip placement groups for AWS Capacity Blocks AWS rejects launches into Capacity Blocks that specify a placement group. Fixes #4344. * Drop AWS Capacity Block lookup cache and handle missing ReservationType AWS omits ReservationType for regular capacity reservations. * Use fleet reservation when provisioning runs --- .../_internal/core/backends/aws/compute.py | 19 ++++++ .../_internal/core/backends/aws/resources.py | 7 ++- .../_internal/core/backends/base/compute.py | 18 ++++-- .../_internal/core/backends/gcp/compute.py | 6 +- .../instances/cloud_provisioning.py | 8 ++- .../pipeline_tasks/jobs_submitted.py | 8 ++- .../_internal/server/services/placement.py | 14 +++++ .../test_instances/test_cloud_provisioning.py | 61 +++++++++++++++++++ .../pipeline_tasks/test_submitted_jobs.py | 60 ++++++++++++++++++ 9 files changed, 186 insertions(+), 15 deletions(-) diff --git a/src/dstack/_internal/core/backends/aws/compute.py b/src/dstack/_internal/core/backends/aws/compute.py index 9579b31ba6..b5e9af0bdc 100644 --- a/src/dstack/_internal/core/backends/aws/compute.py +++ b/src/dstack/_internal/core/backends/aws/compute.py @@ -528,6 +528,25 @@ def is_suitable_placement_group( return False return placement_group.configuration.region == instance_offer.region + def are_placement_groups_compatible_with_reservation( + self, + instance_offer: InstanceOffer, + reservation: str, + ) -> bool: + # AWS rejects launches into Capacity Blocks that specify a placement group. + # Capacity Block instances are already placed close together in EC2 UltraClusters. + try: + capacity_block = aws_resources.get_reservation( + ec2_client=self.session.client("ec2", region_name=instance_offer.region), + reservation_id=reservation, + is_capacity_block=True, + active_only=False, + ) + except botocore.exceptions.ClientError as e: + logger.warning("Failed to get reservation %s: %s", reservation, e) + return True + return capacity_block is None + def create_gateway_replica( self, configuration: GatewayReplicaConfiguration, diff --git a/src/dstack/_internal/core/backends/aws/resources.py b/src/dstack/_internal/core/backends/aws/resources.py index 5ee3f63191..aa0648fc46 100644 --- a/src/dstack/_internal/core/backends/aws/resources.py +++ b/src/dstack/_internal/core/backends/aws/resources.py @@ -678,8 +678,11 @@ def get_reservation( instance_count: int = 0, instance_types: Optional[List[str]] = None, is_capacity_block: bool = False, + active_only: bool = True, ) -> Optional[Dict[str, Any]]: - filters = [{"Name": "state", "Values": ["active"]}] + filters = [] + if active_only: + filters.append({"Name": "state", "Values": ["active"]}) if instance_types: filters.append({"Name": "instance-type", "Values": instance_types}) try: @@ -707,7 +710,7 @@ def get_reservation( if instance_count > 0 and reservation["AvailableInstanceCount"] < instance_count: return None - if is_capacity_block and reservation["ReservationType"] != "capacity-block": + if is_capacity_block and reservation.get("ReservationType") != "capacity-block": return None return reservation diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 9bf202d3f9..2a952aff7e 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -26,7 +26,6 @@ DSTACK_RUNNER_SSH_PORT, DSTACK_SHIM_HTTP_PORT, ) -from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.compute_groups import ComputeGroup, ComputeGroupProvisioningData from dstack._internal.core.models.gateways import ( GatewayLoadBalancerConfiguration, @@ -415,7 +414,7 @@ def run_job( user=run.user, ssh_keys=[SSHKey(public=project_ssh_public_key.strip())], volumes=volumes, - reservation=job.job_spec.requirements.reservation, + reservation=requirements.reservation, tags=run.run_spec.merged_profile.tags, ) instance_offer = instance_offer.model_copy() @@ -559,13 +558,20 @@ def is_suitable_placement_group( """ pass - def are_placement_groups_compatible_with_reservations(self, backend_type: BackendType) -> bool: + def are_placement_groups_compatible_with_reservation( + self, + instance_offer: InstanceOffer, + reservation: str, + ) -> bool: """ - Whether placement groups can be used for instances provisioned in reservations. + Whether a placement group can be used for an instance provisioned in the reservation. + + May perform API calls. Arguments: - backend_type: matches the backend type of this compute, unless this compute is a proxy - for other backends (dstack Sky) + instance_offer: the offer to provision. Its backend matches the backend type of this + compute, unless this compute is a proxy for other backends (dstack Sky) + reservation: the reservation to provision the instance in """ return True diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index 8ab25584f6..662a5455c3 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -594,7 +594,11 @@ def is_suitable_placement_group( ) -> bool: return placement_group.configuration.region == instance_offer.region - def are_placement_groups_compatible_with_reservations(self, backend_type: BackendType) -> bool: + def are_placement_groups_compatible_with_reservation( + self, + instance_offer: InstanceOffer, + reservation: str, + ) -> bool: # Cannot use our own placement policies when provisioning in a reservation. # Instead, we use the placement policy defined in reservation settings. return False diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py index cb87d0294a..bd869b8325 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py @@ -47,6 +47,7 @@ from dstack._internal.server.services.logging import fmt from dstack._internal.server.services.offers import get_instance_offer_with_restricted_az from dstack._internal.server.services.placement import ( + can_use_placement_groups, get_fleet_placement_group_models, placement_group_model_to_placement_group, placement_group_model_to_placement_group_optional, @@ -140,9 +141,10 @@ async def create_cloud_instance(instance_model: InstanceModel) -> ProcessResult: and cluster_context.is_current_instance_master and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT and isinstance(compute, ComputeWithPlacementGroupSupport) - and ( - compute.are_placement_groups_compatible_with_reservations(instance_offer.backend) - or instance_configuration.reservation is None + and await can_use_placement_groups( + compute=compute, + instance_offer=instance_offer, + reservation=instance_configuration.reservation, ) ): ( diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index b0f592b0e6..1ab7875d04 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -122,6 +122,7 @@ ) from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.placement import ( + can_use_placement_groups, find_or_create_suitable_placement_group, get_placement_group_model_for_job, placement_group_model_to_placement_group_optional, @@ -2457,9 +2458,10 @@ async def _provision_new_capacity( and is_cloud_cluster(fleet_model) and offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT and isinstance(compute, ComputeWithPlacementGroupSupport) - and ( - compute.are_placement_groups_compatible_with_reservations(offer.backend) - or job.job_spec.requirements.reservation is None + and await can_use_placement_groups( + compute=compute, + instance_offer=offer, + reservation=requirements.reservation, ) ): placement_group_model = await find_or_create_suitable_placement_group( diff --git a/src/dstack/_internal/server/services/placement.py b/src/dstack/_internal/server/services/placement.py index 6544442fff..9d169c3299 100644 --- a/src/dstack/_internal/server/services/placement.py +++ b/src/dstack/_internal/server/services/placement.py @@ -121,6 +121,20 @@ def get_placement_group_model_for_job( return placement_group_model +async def can_use_placement_groups( + compute: ComputeWithPlacementGroupSupport, + instance_offer: InstanceOffer, + reservation: Optional[str], +) -> bool: + if reservation is None: + return True + return await run_async( + compute.are_placement_groups_compatible_with_reservation, + instance_offer, + reservation, + ) + + async def find_or_create_suitable_placement_group( fleet_model: FleetModel, placement_groups: list[PlacementGroupModel], diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py index c559931ac8..45f29c8976 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py @@ -31,6 +31,7 @@ create_project, get_fleet_configuration, get_fleet_spec, + get_instance_configuration, get_instance_offer_with_availability, get_job_provisioning_data, get_placement_group_provisioning_data, @@ -749,6 +750,66 @@ async def test_create_placement_group_if_placement_cluster( assert backend_mock.compute.return_value.create_placement_group.call_count == 0 assert len(placement_groups) == 0 + @pytest.mark.parametrize("compatible", [True, False]) + async def test_creates_placement_group_in_reservation_only_if_compatible( + self, + test_db, + session: AsyncSession, + worker: InstanceWorker, + compatible: bool, + ) -> None: + project = await create_project(session=session) + fleet = await create_fleet( + session, + project, + spec=get_fleet_spec( + conf=get_fleet_configuration( + placement=InstanceGroupPlacement.CLUSTER, + nodes=FleetNodesSpec(min=1, target=1, max=1), + ) + ), + ) + instance_configuration = get_instance_configuration() + instance_configuration.reservation = "test-reservation" + instance = await create_instance( + session=session, + project=project, + fleet=fleet, + status=InstanceStatus.PENDING, + offer=None, + job_provisioning_data=None, + instance_configuration=instance_configuration, + ) + await _set_current_master_instance(session, fleet, instance) + offer = get_instance_offer_with_availability() + backend_mock = Mock() + backend_mock.TYPE = BackendType.AWS + compute_mock = Mock(spec=ComputeMockSpec) + backend_mock.compute.return_value = compute_mock + compute_mock.get_offers.return_value = [offer] + compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible + compute_mock.create_instance.return_value = get_job_provisioning_data() + compute_mock.create_placement_group.return_value = get_placement_group_provisioning_data() + with patch("dstack._internal.server.services.backends.get_project_backends") as m: + m.return_value = [backend_mock] + await process_instance(session, worker, instance) + + await session.refresh(instance) + assert instance.status == InstanceStatus.PROVISIONING + compute_mock.are_placement_groups_compatible_with_reservation.assert_called_once_with( + offer, "test-reservation" + ) + placement_groups = (await session.execute(select(PlacementGroupModel))).scalars().all() + created_placement_group = compute_mock.create_instance.call_args[0][2] + if compatible: + assert compute_mock.create_placement_group.call_count == 1 + assert len(placement_groups) == 1 + assert isinstance(created_placement_group, PlacementGroup) + else: + assert compute_mock.create_placement_group.call_count == 0 + assert len(placement_groups) == 0 + assert created_placement_group is None + @pytest.mark.parametrize("can_reuse", [True, False]) async def test_reuses_placement_group_between_offers_if_the_group_is_suitable( self, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py index e511241d7f..536df52cc1 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py @@ -803,6 +803,66 @@ async def test_creates_placement_group_for_cluster_fleet( placement_group = (await session.execute(select(PlacementGroupModel))).scalar() assert placement_group is not None + @pytest.mark.parametrize("compatible", [True, False]) + async def test_creates_placement_group_in_reservation_only_if_compatible( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker, compatible: bool + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.placement = InstanceGroupPlacement.CLUSTER + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=None) + fleet = await create_fleet(session=session, project=project, spec=fleet_spec) + run_spec = get_run_spec(run_name="test-run", repo_id=repo.name) + run_spec.configuration.reservation = "test-reservation" + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + fleet=fleet, + run_name="test-run", + run_spec=run_spec, + ) + job = await create_job(session=session, run=run, instance_assigned=True) + offer = get_instance_offer_with_availability(backend=BackendType.AWS) + + with patch("dstack._internal.server.services.backends.get_project_backends") as m: + backend_mock = Mock() + compute_mock = Mock(spec=ComputeMockSpec) + backend_mock.TYPE = BackendType.AWS + backend_mock.compute.return_value = compute_mock + m.return_value = [backend_mock] + compute_mock.get_offers.return_value = [offer] + compute_mock.are_placement_groups_compatible_with_reservation.return_value = compatible + compute_mock.run_job.return_value = get_job_provisioning_data( + backend=BackendType.AWS, + ) + compute_mock.create_placement_group.return_value = ( + get_placement_group_provisioning_data() + ) + + await _process_job(session=session, worker=worker, job_model=job) + + await session.refresh(job) + assert job.status == JobStatus.PROVISIONING + compute_mock.are_placement_groups_compatible_with_reservation.assert_called_once() + assert ( + compute_mock.are_placement_groups_compatible_with_reservation.call_args[0][1] + == "test-reservation" + ) + compute_mock.run_job.assert_called_once() + placement_group = (await session.execute(select(PlacementGroupModel))).scalar() + if compatible: + compute_mock.create_placement_group.assert_called_once() + assert isinstance(compute_mock.run_job.call_args[0][6], PlacementGroup) + assert placement_group is not None + else: + compute_mock.create_placement_group.assert_not_called() + assert compute_mock.run_job.call_args[0][6] is None + assert placement_group is None + async def test_marks_unused_existing_placement_groups_for_cleanup( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): From c2850b2afc6556f324bae0d14996158ad993092a Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 7 Oct 2026 13:56:32 +0500 Subject: [PATCH 623/640] Warn dstack skill about heredocs split across commands items (#4348) --- skills/dstack/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/skills/dstack/SKILL.md b/skills/dstack/SKILL.md index a16851fe79..eb722e8941 100644 --- a/skills/dstack/SKILL.md +++ b/skills/dstack/SKILL.md @@ -172,6 +172,7 @@ Unless you use **Multi-node tasks** (see `### 2. Tasks`) or **Multi-replica serv - Prefer giving configurations a `name` property for easier management - When configurations need credentials (API keys, tokens), list only env var names in the `env` section (e.g., `- HF_TOKEN`), not values. Recommend storing actual values in a `.envrc` file alongside the configuration, applied via `source .envrc && dstack apply`. - `python` and `image` are mutually exclusive in run configurations. If `image` is set, do not set `python`. +- `commands` items are joined into one shell line with ` && `, so keep multi-line shell such as heredocs inside a single item (a `|` block). A heredoc that ends an item swallows every item after it, and the run exits 0 without running them. The same applies to a trailing `# comment` in an item, which comments out the rest of the line. ### `files` and `repos` intent policy From 9ebdd3f36a54aac041c8e700f12ccafed7cac2f7 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 8 Oct 2026 11:51:26 +0500 Subject: [PATCH 624/640] Bound SSHTunnel control commands with timeouts and BatchMode (#4354) * Bound SSHTunnel control commands with timeouts and BatchMode * Drop ConnectTimeout from SSHTunnel control commands * Drop -n from SSHTunnel exec command and document control batch mode --- .../_internal/core/services/ssh/tunnel.py | 116 +++++++++++++----- .../server/services/gateways/__init__.py | 4 +- .../core/services/ssh/test_tunnel.py | 114 ++++++++++++++--- 3 files changed, 188 insertions(+), 46 deletions(-) diff --git a/src/dstack/_internal/core/services/ssh/tunnel.py b/src/dstack/_internal/core/services/ssh/tunnel.py index f4d6a17f70..9dff46c6e0 100644 --- a/src/dstack/_internal/core/services/ssh/tunnel.py +++ b/src/dstack/_internal/core/services/ssh/tunnel.py @@ -88,6 +88,9 @@ def __init__( configured `destination` with `ProxyJump` in the `ssh_config_path` config, the proxy jump connection will ignore this option -- in that case, you should replace `ProxyJump` with explicit `ProxyCommand=ssh [...] -o BatchMode=yes` in your config. + Control commands (`check`, `close`, `exec`) always run in batch mode, since they + only talk to the local master and must not prompt if ssh falls back to a direct + connection. """ self.destination = destination self.forwarded_sockets = list(forwarded_sockets) @@ -173,13 +176,13 @@ def open_command(self) -> List[str]: return command def close_command(self) -> List[str]: - return [self.ssh_exec_path, "-S", self.control_sock_path, "-O", "exit", self.destination] + return [*self._control_command_prefix(), "-O", "exit", self.destination] def check_command(self) -> List[str]: - return [self.ssh_exec_path, "-S", self.control_sock_path, "-O", "check", self.destination] + return [*self._control_command_prefix(), "-O", "check", self.destination] def exec_command(self) -> List[str]: - return [self.ssh_exec_path, "-S", self.control_sock_path, self.destination] + return [*self._control_command_prefix(), self.destination] def open(self) -> None: # We cannot use `stderr=subprocess.PIPE` here since the forked process (daemon) does not @@ -225,9 +228,17 @@ def close(self) -> None: "Control socket does not exist, it seems that ssh process has already exited" ) return - proc = subprocess.run( - self.close_command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) + try: + proc = subprocess.run( + self.close_command(), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=SSH_TIMEOUT, + ) + except subprocess.TimeoutExpired: + logger.error("Failed to close SSH tunnel in %d seconds", SSH_TIMEOUT) + return if proc.returncode: logger.error( "Failed to close SSH tunnel, exit status: %d, output: %s", @@ -241,37 +252,52 @@ async def aclose(self) -> None: "Control socket does not exist, it seems that ssh process has already exited" ) return - proc = await asyncio.create_subprocess_exec( - *self.close_command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - await proc.wait() - if proc.returncode: + try: + returncode, stdout, stderr = await _arun(self.close_command(), SSH_TIMEOUT) + except asyncio.TimeoutError: + logger.error("Failed to close SSH tunnel in %d seconds", SSH_TIMEOUT) + return + if returncode: logger.error( "Failed to close SSH tunnel, exit status: %d, output: %s", - proc.returncode, - proc.stdout, + returncode, + stdout + stderr, ) def check(self) -> bool: - proc = subprocess.run( - self.check_command(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + try: + proc = subprocess.run( + self.check_command(), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=SSH_TIMEOUT, + ) + except subprocess.TimeoutExpired: + logger.debug("SSH tunnel check did not complete in %d seconds", SSH_TIMEOUT) + return False return proc.returncode == 0 async def acheck(self) -> bool: - proc = await asyncio.create_subprocess_exec( - *self.check_command(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - await proc.wait() - ok = proc.returncode == 0 - return ok + try: + returncode, _, _ = await _arun(self.check_command(), SSH_TIMEOUT) + except asyncio.TimeoutError: + logger.debug("SSH tunnel check did not complete in %d seconds", SSH_TIMEOUT) + return False + return returncode == 0 - async def aexec(self, command: str) -> str: - proc = await asyncio.create_subprocess_exec( - *self.exec_command(), command, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - if proc.returncode != 0: + async def aexec(self, command: str, timeout: float = SSH_TIMEOUT) -> str: + """ + Runs `command` on the remote host over the open tunnel. + + :param timeout: Seconds to wait for `command` to complete before killing it + and raising `SSHError`. + """ + try: + returncode, stdout, stderr = await _arun([*self.exec_command(), command], timeout) + except asyncio.TimeoutError as e: + raise SSHError(f"Command did not complete in {timeout} seconds") from e + if returncode != 0: raise SSHError(stderr.decode()) return stdout.decode() @@ -289,6 +315,21 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): await self.aclose() + def _control_command_prefix(self) -> List[str]: + # If the master does not complete the initial exchange (or, for exec, the control socket + # is missing), OpenSSH falls back to connecting to `destination` directly, even for `-O` + # commands. Ignore the user's ssh config and disable prompts so that such a connection + # fails instead of waiting for input on the terminal. + return [ + self.ssh_exec_path, + "-F", + "none", + "-o", + "BatchMode=yes", + "-S", + self.control_sock_path, + ] + def _get_proxy_command(self) -> Optional[str]: proxy_command: Optional[str] = None for params, identity_path in self.ssh_proxies: @@ -366,6 +407,25 @@ def _get_identity_path(self, identity: FilePathOrContent, tmp_filename: str) -> return identity_path +async def _arun(command: List[str], timeout: float) -> tuple[int, bytes, bytes]: + """ + Runs `command` with stdin redirected from /dev/null and returns its exit status, stdout, + and stderr. Kills the process and raises `asyncio.TimeoutError` if it does not exit in + `timeout` seconds. + """ + proc = await asyncio.create_subprocess_exec( + *command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise + assert proc.returncode is not None + return proc.returncode, stdout, stderr + + def ports_to_forwarded_sockets( ports: Dict[int, int], bind_local: str = "localhost" ) -> List[SocketPair]: diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 100a955a37..793654ded1 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -832,13 +832,15 @@ async def _update_gateway_replica( + " " + shlex.quote(target_version or "") ) - stdout = await connection.tunnel.aexec(command) + stdout = await connection.tunnel.aexec(command, timeout=_GATEWAY_UPDATE_TIMEOUT) if "Update successfully completed" in stdout: logger.info("Gateway replica %s updated", connection.ip_address) return True return False +_GATEWAY_UPDATE_TIMEOUT = 600 + # Blue/green: install the new build into the currently inactive venv and flip to it _GATEWAY_UPDATE_SCRIPT = """\ set -e diff --git a/src/tests/_internal/core/services/ssh/test_tunnel.py b/src/tests/_internal/core/services/ssh/test_tunnel.py index b97c86a1e6..da61536bd5 100644 --- a/src/tests/_internal/core/services/ssh/test_tunnel.py +++ b/src/tests/_internal/core/services/ssh/test_tunnel.py @@ -1,7 +1,12 @@ +import asyncio +import subprocess from pathlib import Path +from typing import NoReturn, Optional +from unittest.mock import Mock import pytest +from dstack._internal.core.errors import SSHError from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.core.services.ssh.client import SSHClientInfo from dstack._internal.core.services.ssh.tunnel import ( @@ -221,29 +226,104 @@ def test_open_command_with_forwarding(self) -> None: def test_check_command(self, sample_tunnel_with_all_params: SSHTunnel) -> None: command = sample_tunnel_with_all_params.check_command() - assert command == [ - "/usr/bin/ssh", - "-S", - "/tmp/control.sock", - "-O", - "check", - "ubuntu@my-server", - ] + assert " ".join(command) == ( + "/usr/bin/ssh -F none -o BatchMode=yes -S /tmp/control.sock -O check ubuntu@my-server" + ) def test_close_command(self, sample_tunnel_with_all_params: SSHTunnel) -> None: command = sample_tunnel_with_all_params.close_command() - assert command == [ - "/usr/bin/ssh", - "-S", - "/tmp/control.sock", - "-O", - "exit", - "ubuntu@my-server", - ] + assert " ".join(command) == ( + "/usr/bin/ssh -F none -o BatchMode=yes -S /tmp/control.sock -O exit ubuntu@my-server" + ) def test_exec_command(self, sample_tunnel_with_all_params: SSHTunnel) -> None: command = sample_tunnel_with_all_params.exec_command() - assert command == ["/usr/bin/ssh", "-S", "/tmp/control.sock", "ubuntu@my-server"] + assert " ".join(command) == ( + "/usr/bin/ssh -F none -o BatchMode=yes -S /tmp/control.sock ubuntu@my-server" + ) + + +class TestSSHTunnelControlTimeouts: + @pytest.fixture + def tunnel(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> SSHTunnel: + monkeypatch.setattr( + "dstack._internal.core.services.ssh.client._ssh_client_info", + SSHClientInfo.from_raw_version("OpenSSH_9.7p1", Path("/usr/bin/ssh")), + ) + control_sock_path = tmp_path / "control.sock" + control_sock_path.touch() + return SSHTunnel( + destination="ubuntu@my-server", + identity=FilePath("/home/user/.ssh/id_rsa"), + control_sock_path=control_sock_path, + ) + + @pytest.fixture + def hanging_process(self, monkeypatch: pytest.MonkeyPatch) -> "_HangingProcess": + process = _HangingProcess() + + async def create_subprocess_exec(*args, **kwargs): + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess_exec) + monkeypatch.setattr("dstack._internal.core.services.ssh.tunnel.SSH_TIMEOUT", 0) + return process + + @pytest.fixture + def timing_out_run(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + run = Mock(side_effect=subprocess.TimeoutExpired(cmd="ssh", timeout=0)) + monkeypatch.setattr(subprocess, "run", run) + return run + + def test_check_returns_false_on_timeout(self, tunnel: SSHTunnel, timing_out_run: Mock) -> None: + assert tunnel.check() is False + assert timing_out_run.call_args.kwargs["stdin"] == subprocess.DEVNULL + + def test_close_does_not_raise_on_timeout( + self, tunnel: SSHTunnel, timing_out_run: Mock + ) -> None: + tunnel.close() + assert timing_out_run.call_args.kwargs["stdin"] == subprocess.DEVNULL + + @pytest.mark.asyncio + async def test_acheck_kills_process_and_returns_false_on_timeout( + self, tunnel: SSHTunnel, hanging_process: "_HangingProcess" + ) -> None: + assert await tunnel.acheck() is False + assert hanging_process.killed + + @pytest.mark.asyncio + async def test_aclose_kills_process_on_timeout( + self, tunnel: SSHTunnel, hanging_process: "_HangingProcess" + ) -> None: + await tunnel.aclose() + assert hanging_process.killed + + @pytest.mark.asyncio + async def test_aexec_kills_process_and_raises_on_timeout( + self, tunnel: SSHTunnel, hanging_process: "_HangingProcess" + ) -> None: + with pytest.raises(SSHError, match="did not complete"): + await tunnel.aexec("true", timeout=0) + assert hanging_process.killed + + +class _HangingProcess: + def __init__(self) -> None: + self.returncode: Optional[int] = None + self.killed = False + + async def communicate(self) -> NoReturn: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + def kill(self) -> None: + self.killed = True + self.returncode = -9 + + async def wait(self) -> int: + assert self.returncode is not None + return self.returncode def test_ports_to_forwarded_sockets() -> None: From 5e56fd6880dcc657e870983446aa0ea1dcbb5a9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Jan 2026 17:41:21 +0000 Subject: [PATCH 625/640] Fix stgn Dockerfile: copy examples dir for uv.lock dependency (cherry picked from commit 49e871539069521db09b83845dd0da201ab5e8b1) --- docker/server/stgn/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/server/stgn/Dockerfile b/docker/server/stgn/Dockerfile index b2c4f96370..d14f944170 100644 --- a/docker/server/stgn/Dockerfile +++ b/docker/server/stgn/Dockerfile @@ -22,6 +22,7 @@ ENV PATH="/root/.local/bin/:$PATH" COPY pyproject.toml uv.lock README.md ./ COPY src src +COPY examples examples RUN uv sync --extra all COPY docker/server/entrypoint.sh entrypoint.sh From ac8f29edaf6c0b4197bb1ac37638ea2ffcb79d1e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Jan 2026 17:49:09 +0000 Subject: [PATCH 626/640] Fix stgn Dockerfile: add .venv/bin to PATH for dstack command (cherry picked from commit c1408049cb8044e72cd8727c4304ea08a0dda8bf) --- docker/server/stgn/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/server/stgn/Dockerfile b/docker/server/stgn/Dockerfile index d14f944170..03c163dc15 100644 --- a/docker/server/stgn/Dockerfile +++ b/docker/server/stgn/Dockerfile @@ -25,6 +25,9 @@ COPY src src COPY examples examples RUN uv sync --extra all +# Add virtualenv to PATH so entrypoint can find dstack +ENV PATH="/dstack-server/.venv/bin:$PATH" + COPY docker/server/entrypoint.sh entrypoint.sh RUN chmod 777 entrypoint.sh From 5ddb26ac9ad659a7f7b8fd5533bc0876dfeb856e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Jan 2026 18:15:16 +0000 Subject: [PATCH 627/640] Add frontend build stage to stgn Dockerfile for UI support (cherry picked from commit 3a5fd6ebd693f1bfc7e18f9422d5e4a3d7c77dc1) --- docker/server/stgn/Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/server/stgn/Dockerfile b/docker/server/stgn/Dockerfile index 03c163dc15..cfbf0a05ee 100644 --- a/docker/server/stgn/Dockerfile +++ b/docker/server/stgn/Dockerfile @@ -1,3 +1,13 @@ +# Stage 1: Build frontend +FROM node:20-slim AS frontend-builder + +WORKDIR /frontend +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +# Stage 2: Python server FROM python:3.11-slim ENV PYTHONUNBUFFERED=1 @@ -23,6 +33,10 @@ ENV PATH="/root/.local/bin/:$PATH" COPY pyproject.toml uv.lock README.md ./ COPY src src COPY examples examples + +# Copy built frontend to statics directory +COPY --from=frontend-builder /frontend/build src/dstack/_internal/server/statics + RUN uv sync --extra all # Add virtualenv to PATH so entrypoint can find dstack From 1cde2f5090edbfdc99e803d51043e74dce7a099a Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Fri, 16 Jan 2026 18:43:21 +0100 Subject: [PATCH 628/640] chore: add depot.json (depot build project id) Adapted from fork commit f3cb27cef (fix(plugin): remove example plugin). The pyproject.toml change (comment out the example plugin) is dropped: upstream now keeps dstack-plugin-server in its docs group, and docker/server/stgn/Dockerfile copies examples/ (fork commit 49e871539), so the editable plugin path resolves. Keeping pyproject.toml identical to upstream keeps uv.lock consistent for a reproducible image build. (cherry picked from commit f3cb27cef, adapted) Co-Authored-By: Claude Opus 5.5 (1M context) --- depot.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 depot.json diff --git a/depot.json b/depot.json new file mode 100644 index 0000000000..8a3924b61d --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{"id":"kxhkdgdcdn"} From ec5de7bea2ae9d536b2bf80d89eb3f4642e5ecdd Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:45:48 +0200 Subject: [PATCH 629/640] verda: port the fork's SFS volumes, host SSH keys and debug key onto upstream Ports the net change of these fork commits to verda/compute.py onto the upstream file (per-instance startup scripts and SSH keys #3718, instance volumes #3758, Pydantic v2 #4077) with one 3-way merge, because a per-commit cherry-pick hit the same conflicts eight times: 792baf1b6 Add SFS (Shared File System) volume support for Verda/DataCrunch 530297b15 Fix shim to handle NFS volumes without block device (verda part) f7142376d Fix _get_volume_by_id to handle SDK objects, not just dicts 9661390f3 Use direct HTTP request for volume API instead of SDK 5e2714152 Add attach/detach volume methods for SFS volumes d5e7a6115 Update Verda SFS volume support with actual API attach/detach 00987a4e9 Include user SSH key in addition to project SSH key e6212a780 Restore verbose logging and debug SSH key support Adaptations to upstream: - VerdaCompute keeps upstream ComputeWithInstanceVolumesSupport and adds ComputeWithVolumeSupport (register, attach, detach of SFS volumes). - run_job takes the upstream signature (requirements, extra_authorized_keys). The host keys are the project key plus extra_authorized_keys (upstream's source of the user key, was run_spec.ssh_key_pub). reservation comes from requirements. instance_offer.model_copy() (Pydantic v2). - create_instance keeps the upstream per-instance key and script lifecycle (cleanup on failure). The debug key from DSTACK_DEBUG_SSH_PUBLIC_KEY is a per-instance key too, so the cleanup removes it. The SFS NFS mount commands run before the shim commands, as in the fork. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/verda/compute.py | 506 +++++++++++++++++- 1 file changed, 503 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/core/backends/verda/compute.py b/src/dstack/_internal/core/backends/verda/compute.py index c8b71b4058..eb52a163a6 100644 --- a/src/dstack/_internal/core/backends/verda/compute.py +++ b/src/dstack/_internal/core/backends/verda/compute.py @@ -1,3 +1,5 @@ +import json +import os from collections.abc import Iterable from typing import Dict, List, Optional @@ -11,7 +13,9 @@ ComputeWithCreateInstanceSupport, ComputeWithInstanceVolumesSupport, ComputeWithPrivilegedSupport, + ComputeWithVolumeSupport, generate_unique_instance_name, + get_job_instance_name, get_shim_commands, ) from dstack._internal.core.backends.base.offers import ( @@ -22,6 +26,7 @@ from dstack._internal.core.backends.verda.models import VerdaConfig from dstack._internal.core.errors import ( BackendError, + ComputeError, NoCapacityError, NotYetTerminated, ProvisioningError, @@ -33,13 +38,21 @@ InstanceConfiguration, InstanceOffer, InstanceOfferWithAvailability, + SSHKey, ) from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.resources import Memory, Range -from dstack._internal.core.models.runs import JobProvisioningData, Requirements +from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run +from dstack._internal.core.models.volumes import ( + Volume, + VolumeAttachmentData, + VolumeMountPoint, + VolumeProvisioningData, +) +from dstack._internal.utils.common import get_or_error from dstack._internal.utils.logging import get_logger -logger = get_logger("verda.compute") +logger = get_logger(__name__) MAX_INSTANCE_NAME_LEN = 60 @@ -53,6 +66,7 @@ class VerdaCompute( ComputeWithCreateInstanceSupport, ComputeWithPrivilegedSupport, ComputeWithInstanceVolumesSupport, + ComputeWithVolumeSupport, Compute, ): def __init__(self, config: VerdaConfig, backend_type: BackendType): @@ -102,11 +116,84 @@ def _get_offers_with_availability( return availability_offers + def run_job( + self, + run: Run, + job: Job, + instance_offer: InstanceOfferWithAvailability, + project_ssh_public_key: str, + project_ssh_private_key: str, + volumes: List[Volume], + placement_group: Optional[PlacementGroup], + requirements: Requirements, + extra_authorized_keys: list[str], + ) -> JobProvisioningData: + """ + Override run_job to handle SFS volume mounting. + SFS volumes are mounted via NFS in the startup script. + """ + # Fork: authorize the caller's keys (typically the user key) on the Verda host too, + # next to the project key, for host-level debug access. Upstream passes them as + # `extra_authorized_keys` (VM backends otherwise send them to the shim only). + ssh_keys = [SSHKey(public=project_ssh_public_key.strip())] + for key in extra_authorized_keys: + if key and key.strip(): + ssh_keys.append(SSHKey(public=key.strip())) + + instance_config = InstanceConfiguration( + project_name=run.project_name, + instance_name=get_job_instance_name(run, job), + user=run.user, + ssh_keys=ssh_keys, + volumes=volumes, + reservation=requirements.reservation, + tags=run.run_spec.merged_profile.tags, + ) + + # Build volume mount mapping: volume_id -> mount_path + logger.info( + "run_job called with %d volumes: %s", + len(volumes), + [(v.name, v.volume_id, v.provisioning_data.backend_data if v.provisioning_data else None) for v in volumes] + ) + + volume_mounts: Dict[str, str] = {} + if run.run_spec.configuration.volumes: + for mount_point in run.run_spec.configuration.volumes: + if isinstance(mount_point, VolumeMountPoint): + # Find the matching volume + names = ( + mount_point.name + if isinstance(mount_point.name, list) + else [mount_point.name] + ) + for vol in volumes: + if vol.name in names and vol.volume_id: + volume_mounts[vol.volume_id] = mount_point.path + logger.info( + "Mapped volume %s (id=%s) to mount path %s", + vol.name, vol.volume_id, mount_point.path + ) + break + + logger.info("Final volume_mounts: %s", volume_mounts) + + instance_offer = instance_offer.model_copy() + self._restrict_instance_offer_az_to_volumes_az(instance_offer, volumes) + + return self.create_instance( + instance_offer, + instance_config, + placement_group=placement_group, + volume_mounts=volume_mounts, + ) + def create_instance( self, instance_offer: InstanceOfferWithAvailability, instance_config: InstanceConfiguration, placement_group: Optional[PlacementGroup], + volume_mounts: Optional[Dict[str, str]] = None, ) -> JobProvisioningData: instance_name = generate_unique_instance_name( instance_config, max_length=MAX_INSTANCE_NAME_LEN @@ -124,8 +211,35 @@ def create_instance( ) ) - commands = get_shim_commands() + # Fork: an optional debug SSH key from the server env (DSTACK_DEBUG_SSH_PUBLIC_KEY). + # Created per instance like the other keys, so the upstream cleanup removes it. + debug_ssh_key = os.environ.get("DSTACK_DEBUG_SSH_PUBLIC_KEY") + if debug_ssh_key: + logger.info("Adding debug SSH key from DSTACK_DEBUG_SSH_PUBLIC_KEY") + ssh_ids.append( + _create_ssh_key( + client=self.client, + name=f"{instance_name}-debug.key", + public_key=debug_ssh_key.strip(), + ) + ) + + # Fork: mount Verda SFS volumes over NFS before the shim starts. + sfs_mount_commands = _get_sfs_mount_commands( + volumes=instance_config.volumes or [], + volume_mounts=volume_mounts or {}, + instance_region=instance_offer.region, + ) + logger.info( + "SFS mount commands for instance %s: %s", instance_name, sfs_mount_commands + ) + commands = sfs_mount_commands + get_shim_commands() startup_script = " ".join([" && ".join(commands)]) + logger.info( + "Full startup script for instance %s: %s", + instance_name, + startup_script[:500] + "..." if len(startup_script) > 500 else startup_script, + ) script_name = f"{instance_name}.sh" startup_script_id = _create_startup_script( client=self.client, @@ -236,6 +350,392 @@ def update_provisioning_data( if instance.status == "running": provisioning_data.hostname = instance.ip + def register_volume(self, volume: Volume) -> VolumeProvisioningData: + """ + Register an existing SFS (Shared File System) volume from Verda. + + SFS volumes are NFS-based shared filesystems that can be mounted to instances. + The volume must already exist in Verda and be of type *_Shared (e.g. NVMe_Shared). + """ + volume_id = get_or_error(volume.configuration.volume_id) + volume_data = _get_volume_by_id(self.config, volume_id) + + if volume_data is None: + raise ComputeError(f"Volume {volume_id} not found in Verda") + + volume_type = volume_data.get("type", "") + if "Shared" not in volume_type: + raise ComputeError( + f"Volume {volume_id} is not an SFS volume (type: {volume_type}). " + "Only shared filesystem volumes (HDD_Shared, NVMe_Shared) are supported." + ) + + location = volume_data.get("location", "") + if volume.configuration.region and location.upper() != volume.configuration.region.upper(): + raise ComputeError( + f"Volume {volume_id} is in region {location}, " + f"but configuration specifies region {volume.configuration.region}" + ) + + pseudo_path = volume_data.get("pseudo_path") + if not pseudo_path: + raise ComputeError(f"Volume {volume_id} has no pseudo_path for NFS mounting") + + size_gb = volume_data.get("size", 0) + + # Store SFS-specific data for NFS mounting + backend_data = json.dumps({ + "pseudo_path": pseudo_path, + "location_code": location, + "volume_type": volume_type, + }) + + return VolumeProvisioningData( + backend=self.backend_type, + volume_id=volume_id, + size_gb=size_gb, + availability_zone=location, + # SFS volumes don't have dynamic pricing; use monthly price / 730 hours as estimate + price=volume_data.get("monthly_price"), + # SFS volumes must be attached via Verda API to authorize instance IP access + attachable=True, + detachable=True, + backend_data=backend_data, + ) + + def create_volume(self, volume: Volume) -> VolumeProvisioningData: + """ + Creating SFS volumes via dstack is not supported. + Create volumes directly in the Verda dashboard and use volume_id to register them. + """ + raise ComputeError( + "Creating SFS volumes via dstack is not supported for Verda/DataCrunch. " + "Please create the volume in the Verda dashboard and register it using volume_id." + ) + + def delete_volume(self, volume: Volume): + """ + Deleting SFS volumes via dstack is not supported. + Delete volumes directly in the Verda dashboard. + """ + raise ComputeError( + "Deleting SFS volumes via dstack is not supported for Verda/DataCrunch. " + "Please delete the volume in the Verda dashboard." + ) + + def attach_volume( + self, volume: Volume, provisioning_data: JobProvisioningData + ) -> VolumeAttachmentData: + """ + Attach an SFS volume to an instance via Verda API. + + This authorizes the instance IP to access the NFS share. + The actual NFS mount is done via the startup script. + + API: PUT /v1/volumes with body {"action": "attach", "id": "", "instance_ids": [""]} + """ + import requests + + volume_id = volume.volume_id + instance_id = provisioning_data.instance_id + + logger.info( + "Attaching SFS volume %s to instance %s", + volume_id, instance_id + ) + + try: + token = _get_verda_access_token( + self.config.creds.client_id, + self.config.creds.client_secret, + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + payload = { + "action": "attach", + "id": volume_id, + "instance_ids": [instance_id], + } + + logger.info( + "Calling Verda volume API: PUT /v1/volumes with payload=%s", + payload + ) + + response = requests.put( + "https://api.datacrunch.io/v1/volumes", + headers=headers, + json=payload, + timeout=60, + ) + + logger.info( + "Verda volume API response: status=%d body=%s", + response.status_code, response.text[:500] if response.text else "(empty)" + ) + + # 202 Accepted is a valid response - means the request was accepted for async processing + if response.status_code not in (200, 201, 202, 204): + error_msg = response.text + try: + error_data = response.json() + error_msg = error_data.get("message", error_data) + except Exception: + pass + raise ComputeError( + f"Failed to attach volume {volume_id} to instance {instance_id}: " + f"HTTP {response.status_code} - {error_msg}" + ) + + logger.info( + "Successfully attached SFS volume %s to instance %s", + volume_id, instance_id + ) + + # SFS volumes don't have a device_name - they're mounted via NFS + return VolumeAttachmentData(device_name=None) + + except requests.exceptions.RequestException as e: + raise ComputeError(f"Failed to attach volume {volume_id}: {e}") + + def detach_volume( + self, volume: Volume, provisioning_data: JobProvisioningData, force: bool = False + ): + """ + Detach an SFS volume from an instance via Verda API. + + This revokes the instance IP's authorization to access the NFS share. + + API: PUT /v1/volumes with body {"action": "detach", "id": "", "instance_ids": [""]} + """ + import requests + + volume_id = volume.volume_id + instance_id = provisioning_data.instance_id + + logger.info( + "Detaching SFS volume %s from instance %s (force=%s)", + volume_id, instance_id, force + ) + + try: + token = _get_verda_access_token( + self.config.creds.client_id, + self.config.creds.client_secret, + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + payload = { + "action": "detach", + "id": volume_id, + "instance_ids": [instance_id], + } + + response = requests.put( + "https://api.datacrunch.io/v1/volumes", + headers=headers, + json=payload, + timeout=60, + ) + + # 202 Accepted is a valid response - means the request was accepted for async processing + if response.status_code not in (200, 201, 202, 204): + error_msg = response.text + try: + error_data = response.json() + error_msg = error_data.get("message", error_data) + except Exception: + pass + # Don't fail on detach errors unless not forcing + if not force: + raise ComputeError( + f"Failed to detach volume {volume_id} from instance {instance_id}: " + f"HTTP {response.status_code} - {error_msg}" + ) + else: + logger.warning( + "Failed to detach volume %s from instance %s (force=True, ignoring): %s", + volume_id, instance_id, error_msg + ) + return + + logger.info( + "Successfully detached SFS volume %s from instance %s", + volume_id, instance_id + ) + + except requests.exceptions.RequestException as e: + if not force: + raise ComputeError(f"Failed to detach volume {volume_id}: {e}") + else: + logger.warning( + "Failed to detach volume %s (force=True, ignoring): %s", + volume_id, e + ) + + +def _get_verda_access_token(client_id: str, client_secret: str) -> str: + """ + Get an OAuth2 access token from the Verda/DataCrunch API using client credentials. + """ + import requests + + token_url = "https://api.datacrunch.io/v1/oauth2/token" + data = { + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + } + + response = requests.post(token_url, data=data, timeout=30) + response.raise_for_status() + token_data = response.json() + return token_data["access_token"] + + +def _get_volume_by_id(config: VerdaConfig, volume_id: str) -> Optional[Dict]: + """ + Fetch volume details from Verda API via direct HTTP request. + The SDK's volume methods may not return all fields we need, so we use the REST API directly. + """ + import requests + + try: + # Get access token using OAuth2 client credentials + token = _get_verda_access_token( + config.creds.client_id, + config.creds.client_secret, + ) + headers = {"Authorization": f"Bearer {token}"} + + response = requests.get( + f"https://api.datacrunch.io/v1/volumes/{volume_id}", + headers=headers, + timeout=30, + ) + + if response.status_code == 404: + return None + + response.raise_for_status() + data = response.json() + logger.debug(f"Volume API response for {volume_id}: {data}") + return data + + except requests.exceptions.HTTPError as e: + if e.response is not None and e.response.status_code == 404: + return None + raise ComputeError(f"Failed to fetch volume {volume_id}: {e}") + except Exception as e: + logger.warning(f"Failed to fetch volume {volume_id}: {e}") + raise ComputeError(f"Failed to fetch volume {volume_id}: {e}") + + +def _get_sfs_mount_commands( + volumes: List[Volume], + volume_mounts: Dict[str, str], + instance_region: str, +) -> List[str]: + """ + Generate NFS mount commands for SFS volumes. + + SFS (Shared File System) volumes in Verda are NFS-based. + Mount command format: mount -t nfs -o nconnect=16 nfs..datacrunch.io: + + The volume is mounted at /mnt/disks/dstack-volumes/ on the host. + The shim then bind-mounts this to the user-specified container path. + + Args: + volumes: List of Volume objects with provisioning data + volume_mounts: Mapping of volume_id -> mount_path (used to filter which volumes to mount) + instance_region: The region/datacenter of the instance (e.g., "FIN-01") + + Returns: + List of shell commands to mount SFS volumes + """ + commands = [] + + logger.info( + "_get_sfs_mount_commands: processing %d volumes, volume_mounts=%s", + len(volumes), volume_mounts + ) + + # Check if we have any volumes to mount - if so, ensure nfs-common is installed + has_volumes_to_mount = any( + volume.provisioning_data and volume.provisioning_data.backend_data + and volume.volume_id and volume.volume_id in volume_mounts + for volume in volumes + ) + if has_volumes_to_mount: + # Install nfs-common if not already installed (needed for NFS mounts) + # Also unmask rpcbind.socket which may be masked on some images + commands.append( + "(which mount.nfs > /dev/null 2>&1 || " + "(systemctl unmask rpcbind.socket 2>/dev/null; " + "apt-get update -qq && apt-get install -y -qq nfs-common)) || " + "echo 'WARNING: Failed to install nfs-common'" + ) + + for volume in volumes: + if not volume.provisioning_data or not volume.provisioning_data.backend_data: + logger.info( + "Skipping volume %s: no provisioning_data or backend_data", + volume.name + ) + continue + + volume_id = volume.volume_id + if not volume_id or volume_id not in volume_mounts: + logger.info( + "Skipping volume %s (id=%s): not in volume_mounts", + volume.name, volume_id + ) + continue + + try: + backend_data = json.loads(volume.provisioning_data.backend_data) + except (json.JSONDecodeError, TypeError): + logger.warning(f"Failed to parse backend_data for volume {volume_id}") + continue + + pseudo_path = backend_data.get("pseudo_path") + location_code = backend_data.get("location_code", instance_region) + + if not pseudo_path: + logger.warning(f"Volume {volume_id} has no pseudo_path, skipping mount") + continue + + # Datacenter code for NFS server (lowercase, e.g., "fin-01") + dc = location_code.lower() + + # Mount at the path the shim expects: /mnt/disks/dstack-volumes/ + # The shim will bind-mount this to the user-specified container path + host_mount_path = f"/mnt/disks/dstack-volumes/{volume.name}" + + # Generate mount commands with error logging but non-blocking + # Use a subshell so failures don't stop the entire startup script + mount_cmd = ( + f"(mkdir -p {host_mount_path} && " + f"mount -t nfs -o nconnect=16 nfs.{dc}.datacrunch.io:{pseudo_path} {host_mount_path} && " + f"echo 'SFS volume {volume.name} mounted successfully') || " + f"echo 'WARNING: Failed to mount SFS volume {volume.name}'" + ) + commands.append(mount_cmd) + + logger.info( + "SFS mount for volume %s: nfs.%s.datacrunch.io:%s -> %s", + volume.name, dc, pseudo_path, host_mount_path + ) + + logger.info("_get_sfs_mount_commands: returning %d commands: %s", len(commands), commands) + return commands + def _get_vm_image_id(instance_offer: InstanceOfferWithAvailability) -> str: # https://api.datacrunch.io/v1/docs#tag/images/GET/v1/images From a00c1e6a7ef47b03af1be9d8abafced451536607 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:47:13 +0200 Subject: [PATCH 630/640] shim: port the fork's NFS mount for Verda SFS volumes onto upstream Ports the net shim and server change of these fork commits: 530297b15 Fix shim to handle NFS volumes without block device (shim part) 526ee157f Add NFS mount support in shim for reused fleet instances Adaptations to upstream: - The volume code moved from runner/internal/shim/docker.go to runner/internal/shim/volumes.go upstream. The NFS branch goes at the top of formatAndMountVolume there. docker.go stays as upstream has it. - VolumeInfo (shim) and ShimVolumeInfo (server) gain nfs_host and nfs_pseudo. The server fills them from the Verda SFS volume backend_data (pseudo_path, location_code) in _volume_to_shim_volume_info. Checked: GOOS=linux go vet ./internal/shim/... clean, the shim builds for linux/amd64 and linux/arm64 with CGO_ENABLED=0. Co-Authored-By: Claude Opus 5.5 (1M context) --- runner/internal/shim/models.go | 3 ++ runner/internal/shim/volumes.go | 34 +++++++++++++++++++ src/dstack/_internal/server/schemas/runner.py | 3 ++ .../server/services/runner/client.py | 20 +++++++++++ 4 files changed, 60 insertions(+) diff --git a/runner/internal/shim/models.go b/runner/internal/shim/models.go index 6166c7c7ff..398539e152 100644 --- a/runner/internal/shim/models.go +++ b/runner/internal/shim/models.go @@ -71,6 +71,9 @@ type VolumeInfo struct { VolumeId string `json:"volume_id"` InitFs bool `json:"init_fs"` DeviceName string `json:"device_name"` + // NFS mount details for SFS/network volumes + NfsHost string `json:"nfs_host,omitempty"` // e.g., "nfs.fin-03.datacrunch.io" + NfsPseudo string `json:"nfs_pseudo,omitempty"` // e.g., "/path/to/share" } type PortMapping struct { diff --git a/runner/internal/shim/volumes.go b/runner/internal/shim/volumes.go index f6d6096b0b..86675f8f7f 100644 --- a/runner/internal/shim/volumes.go +++ b/runner/internal/shim/volumes.go @@ -59,6 +59,40 @@ func unmountVolumes(ctx context.Context, taskConfig TaskConfig) error { } func formatAndMountVolume(ctx context.Context, volume VolumeInfo) error { + mountPoint := getVolumeMountPoint(volume.Name) + + // Fork (Verda SFS): a network volume has no device name. Mount it over NFS when the + // server sends the NFS details, or keep an existing mount (the instance startup + // script can mount it first). Runs before the block-device path below. + if volume.DeviceName == "" { + log.Info(ctx, "Volume has no device name (NFS/network volume)", + "volume", volume.Name, "mountpoint", mountPoint, "nfs_host", volume.NfsHost, "nfs_pseudo", volume.NfsPseudo) + + cmd := exec.CommandContext(ctx, "mountpoint", "-q", mountPoint) + if err := cmd.Run(); err == nil { + log.Info(ctx, "NFS volume already mounted", "volume", volume.Name, "mountpoint", mountPoint) + return nil + } + + if volume.NfsHost != "" && volume.NfsPseudo != "" { + log.Info(ctx, "Mounting NFS volume", "volume", volume.Name, "host", volume.NfsHost, "pseudo", volume.NfsPseudo) + if err := os.MkdirAll(mountPoint, 0o755); err != nil { + return fmt.Errorf("create mount point %s: %w", mountPoint, err) + } + nfsSource := volume.NfsHost + ":" + volume.NfsPseudo + cmd = exec.CommandContext(ctx, "mount", "-t", "nfs", "-o", "nconnect=16", nfsSource, mountPoint) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("mount NFS volume %s from %s to %s: %s: %w", + volume.Name, nfsSource, mountPoint, string(output), err) + } + log.Info(ctx, "NFS volume mounted successfully", "volume", volume.Name, "mountpoint", mountPoint) + return nil + } + + return fmt.Errorf("NFS volume %s not mounted at %s and no NFS mount details provided", + volume.Name, mountPoint) + } + backend, err := backends.GetBackend(volume.Backend) if err != nil { return fmt.Errorf("get backend: %w", err) diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index a9aa404921..59b835f8e4 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -195,6 +195,9 @@ class ShimVolumeInfo(CoreModel): volume_id: str init_fs: bool device_name: Optional[str] = None + # NFS mount details for SFS/network volumes + nfs_host: Optional[str] = None # e.g., "nfs.fin-03.datacrunch.io" + nfs_pseudo: Optional[str] = None # e.g., "/path/to/share" class PortMapping(CoreModel): diff --git a/src/dstack/_internal/server/services/runner/client.py b/src/dstack/_internal/server/services/runner/client.py index 3407c7d8e3..2deee02e1b 100644 --- a/src/dstack/_internal/server/services/runner/client.py +++ b/src/dstack/_internal/server/services/runner/client.py @@ -822,16 +822,36 @@ def instance_info_response_to_gpu_driver( def _volume_to_shim_volume_info(volume: Volume, instance_id: str) -> ShimVolumeInfo: + import json as json_module + device_name = None attachment_data = volume.get_attachment_data_for_instance(instance_id) if attachment_data is not None: device_name = attachment_data.device_name + + # Extract NFS mount details for SFS/network volumes + nfs_host = None + nfs_pseudo = None + if volume.provisioning_data and volume.provisioning_data.backend_data: + try: + backend_data = json_module.loads(volume.provisioning_data.backend_data) + pseudo_path = backend_data.get("pseudo_path") + location_code = backend_data.get("location_code") + if pseudo_path and location_code: + # Build NFS host from location code (e.g., "FIN-03" -> "nfs.fin-03.datacrunch.io") + nfs_host = f"nfs.{location_code.lower()}.datacrunch.io" + nfs_pseudo = pseudo_path + except (json_module.JSONDecodeError, TypeError): + pass + return ShimVolumeInfo( backend=volume.configuration.backend.value, name=volume.name, volume_id=get_or_error(volume.volume_id), init_fs=not volume.external, device_name=device_name, + nfs_host=nfs_host, + nfs_pseudo=nfs_pseudo, ) From a31d10239b5a1cef6f353fc0f94f3f9527bc262a Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:48:53 +0200 Subject: [PATCH 631/640] server: reuse an existing volume attachment on a reused instance Ports fork commit db1d670ed (Fix UNIQUE constraint error when reattaching volume to same instance) to the upstream pipeline. The old process_submitted_jobs.py task is gone upstream. The attach moved to pipeline_tasks/jobs_submitted.py, which still inserts a second VolumeAttachmentModel row when a reused fleet instance already holds the volume, and the job fails with: UNIQUE constraint failed: volumes_attachments.volume_id, volumes_attachments.instance_id Now _process_volume_attachments takes the instance id on the existing-instance path. If the volume (loaded with its attachments) is already attached to that instance, it skips the backend attach call and returns a payload marked already_attached. The apply step inserts no row for it. The volume still counts in job_runtime_data.volume_names. Test first: test_reuses_existing_volume_attachment_on_existing_instance failed with the IntegrityError above (RED), passes now (GREEN). The other volume tests in test_submitted_jobs.py stay green. (cherry picked from commit db1d670ed, adapted) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../pipeline_tasks/jobs_submitted.py | 28 +++++++ .../pipeline_tasks/test_submitted_jobs.py | 78 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 1ab7875d04..9da1c4939b 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -413,6 +413,9 @@ class _VolumeAttachmentPayload: volume_id: uuid.UUID attachment_data: str volume_name: str + # Fork (db1d670ed): True when the instance already holds this attachment (a reused + # fleet instance). No backend attach call and no new attachment row then. + already_attached: bool = False @dataclass @@ -1387,6 +1390,7 @@ async def _process_existing_instance_provisioning( job_model=context.job_model, prepared_job_volumes=prepared_job_volumes, job_provisioning_data=get_or_error(get_instance_provisioning_data(instance_model)), + instance_id=instance_model.id, ) return _ExistingInstanceProvisioning( volume_attachment_result=volume_attachment_result, @@ -1860,6 +1864,7 @@ async def _process_volume_attachments( job_model: JobModel, prepared_job_volumes: _PreparedJobVolumes, job_provisioning_data: JobProvisioningData, + instance_id: Optional[uuid.UUID] = None, ) -> _VolumeAttachmentResult: if len(prepared_job_volumes.volume_model_ids) == 0: return _VolumeAttachmentResult(attachments=[], locked_volume_ids=[]) @@ -1910,6 +1915,27 @@ async def _process_volume_attachments( continue if volume.provisioning_data is None or not volume.provisioning_data.attachable: continue + existing_attachment = None + if instance_id is not None: + existing_attachment = next( + (a for a in volume_model.attachments if a.instance_id == instance_id), + None, + ) + if existing_attachment is not None: + logger.info( + "%s: volume %s is already attached to the instance, skipping attach", + fmt(job_model), + volume.name, + ) + attachments.append( + _VolumeAttachmentPayload( + volume_id=volume_model.id, + attachment_data=existing_attachment.attachment_data or "", + volume_name=volume.name, + already_attached=True, + ) + ) + break attachment_data = await run_async( compute.attach_volume, volume=volume, @@ -2097,6 +2123,8 @@ async def _apply_volume_attachment_result( .values(last_job_processed_at=now) ) for attachment in volume_attachment_result.attachments: + if attachment.already_attached: + continue session.add( VolumeAttachmentModel( volume_id=attachment.volume_id, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py index 536df52cc1..ae8ad30c99 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py @@ -64,6 +64,7 @@ VolumeAttachmentModel, ) from dstack._internal.server.services.docker import ImageConfig +from dstack._internal.server.services.jobs import get_job_runtime_data from dstack._internal.server.services.jobs.configurators.base import JobConfigurator from dstack._internal.server.testing.common import ( ComputeMockSpec, @@ -2475,6 +2476,83 @@ async def test_attaches_volume_on_existing_instance( assert volume.lock_expires_at is None backend_mock.compute.return_value.attach_volume.assert_called_once() + async def test_reuses_existing_volume_attachment_on_existing_instance( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + # Fork (db1d670ed): a reused fleet instance can still hold the attachment row of + # the volume from an earlier job. A second row breaks the UNIQUE constraint on + # (volume_id, instance_id), so the job must reuse the row and skip the attach call. + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + volume = await create_volume( + session=session, + project=project, + user=user, + status=VolumeStatus.ACTIVE, + volume_provisioning_data=get_volume_provisioning_data(), + backend=BackendType.AWS, + region="us-east-1", + ) + fleet = await create_fleet(session=session, project=project) + instance = await create_instance( + session=session, + project=project, + fleet=fleet, + status=InstanceStatus.BUSY, + busy_blocks=1, + backend=BackendType.AWS, + region="us-east-1", + ) + session.add( + VolumeAttachmentModel( + volume_id=volume.id, + instance=instance, + attachment_data=VolumeAttachmentData().model_dump_json(), + ) + ) + await session.commit() + run_spec = get_run_spec(repo_id=repo.name) + run_spec.configuration.volumes = [VolumeMountPoint(name=volume.name, path="/volume")] + run = await create_run( + session=session, project=project, repo=repo, user=user, run_spec=run_spec + ) + job = await create_job( + session=session, + run=run, + instance=instance, + instance_assigned=True, + ) + + with patch("dstack._internal.server.services.backends.get_project_backend_by_type") as m: + backend_mock = Mock() + m.return_value = backend_mock + backend_mock.TYPE = BackendType.AWS + backend_mock.compute.return_value = Mock(spec=ComputeMockSpec) + backend_mock.compute.return_value.attach_volume.return_value = VolumeAttachmentData() + + await _process_job(session=session, worker=worker, job_model=job) + + res = await session.execute( + select(JobModel) + .where(JobModel.id == job.id) + .options( + joinedload(JobModel.instance) + .joinedload(InstanceModel.volume_attachments) + .joinedload(VolumeAttachmentModel.volume) + ) + .execution_options(populate_existing=True) + ) + job = res.unique().scalar_one() + assert job.status == JobStatus.PROVISIONING + assert job.instance is not None + assert len(job.instance.volume_attachments) == 1 + assert job.instance.volume_attachments[0].volume_id == volume.id + backend_mock.compute.return_value.attach_volume.assert_not_called() + job_runtime_data = get_job_runtime_data(job) + assert job_runtime_data is not None + assert job_runtime_data.volume_names == [volume.name] + async def test_terminates_job_when_volume_is_locked_for_processing( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): From 8910240548720a256233bf40b41bde9dc9ad4c00 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:49:37 +0200 Subject: [PATCH 632/640] nebius: accept the gpu-rtx6000-a platform (RTX PRO 6000, uk-south2) Upstream supports the Nebius RTX PRO 6000 natively now (gpuhunt 0.1.32 maps gpu-rtx6000 to RTXPRO6000, and SUPPORTED_PLATFORMS has gpu-rtx6000). That supersedes the fork's live-offer injection and gpuhunt patch (99092f11b, 66fc66faf), which are not carried over. The deployed fork image also served the uk-south2 platform name gpu-rtx6000-a, which upstream does not list. gpuhunt maps it to RTXPRO6000 too, so one list entry is enough. Test first: TestSupportedInstances failed for gpu-rtx6000-a (RED), passes now (GREEN). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/nebius/compute.py | 2 ++ .../core/backends/nebius/test_compute.py | 23 ++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index 4e4e645eab..444f829db4 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -90,6 +90,8 @@ "gpu-l40s-a", "gpu-l40s-d", "gpu-rtx6000", + # Fork: the RTX PRO 6000 platform name in uk-south2. + "gpu-rtx6000-a", "cpu-d3", "cpu-e2", ] diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py index e40416b225..55772f1ed7 100644 --- a/src/tests/_internal/core/backends/nebius/test_compute.py +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -1,7 +1,9 @@ +from types import SimpleNamespace + import pytest from dstack._internal.core.backends.nebius import compute as compute_module -from dstack._internal.core.backends.nebius.compute import NebiusCompute +from dstack._internal.core.backends.nebius.compute import NebiusCompute, _supported_instances from dstack._internal.core.backends.nebius.models import ( NebiusConfig, NebiusServiceAccountCreds, @@ -109,3 +111,22 @@ def test_keeps_flat_rate_spot_price_as_is(self, mocker): offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) assert offers[0].price == 1.2341 + + +class TestSupportedInstances: + # Fork: the RTX PRO 6000 runs as `gpu-rtx6000` (us-central1) and `gpu-rtx6000-a` + # (uk-south2). The deployed fork served both. Upstream lists only `gpu-rtx6000`. + @pytest.mark.parametrize( + "instance_name", + [ + "gpu-rtx6000 1gpu-24vcpu-218gb", + "gpu-rtx6000-a 1gpu-24vcpu-218gb", + ], + ) + def test_accepts_both_rtx_pro_6000_platforms(self, instance_name): + offer = SimpleNamespace(instance=SimpleNamespace(name=instance_name)) + assert _supported_instances(offer) + + def test_rejects_an_unknown_platform(self): + offer = SimpleNamespace(instance=SimpleNamespace(name="gpu-unknown 1gpu-8vcpu-32gb")) + assert not _supported_instances(offer) From 58a4f14feb3876c7e8756834b7cfbe2a73b0fd00 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:50:58 +0200 Subject: [PATCH 633/640] test(nebius): pin the spot create request shape (preemptible + pricing policy) Dry check for Nebius Dynamic Spot pricing (2026-10-08): builds the CreateInstanceRequest through resources.create_instance with the InstanceServiceClient mocked (no network). A spot create carries the recovery policy and the pricing_model oneof set to spot_pricing_policy with the given policy id. An on-demand create carries no pricing model. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../core/backends/nebius/test_resources.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/tests/_internal/core/backends/nebius/test_resources.py b/src/tests/_internal/core/backends/nebius/test_resources.py index 2764e867c0..63830da036 100644 --- a/src/tests/_internal/core/backends/nebius/test_resources.py +++ b/src/tests/_internal/core/backends/nebius/test_resources.py @@ -14,3 +14,46 @@ def test_make_sdk_sets_user_agent_prefix(mocker): resources.make_sdk(creds) assert sdk_cls.call_args.kwargs["user_agent_prefix"] == "dstack/1.2.3" + + +def _capture_create_request(mocker): + client = mocker.patch.object(resources, "InstanceServiceClient") + mocker.patch.object(resources.LOOP, "await_", side_effect=lambda coro: coro) + return client.return_value.create + + +def _create(preemptible, pricing_policy_id): + return resources.create_instance( + sdk=None, + name="test-instance", + project_id="project-test", + user_data="", + platform="gpu-rtx6000", + preset="1gpu-24vcpu-218gb", + cluster_id=None, + disk_id="disk-test", + subnet_id="subnet-test", + preemptible=preemptible, + pricing_policy_id=pricing_policy_id, + labels={}, + ) + + +def test_create_spot_instance_sets_preemptible_and_pricing_policy(mocker): + # Nebius Dynamic Spot pricing (2026-10-08): a preemptible create needs a pricing + # model. dstack pins the spot price with a per-instance pricing policy (#4328). + create = _capture_create_request(mocker) + _create(preemptible=True, pricing_policy_id="pricingpolicy-test") + request = create.call_args.args[0] + assert request.spec.pricing_model is not None + assert request.spec.pricing_model.field == "spot_pricing_policy" + assert request.spec.spot_pricing_policy.id == "pricingpolicy-test" + assert request.spec.recovery_policy is not None + + +def test_create_on_demand_instance_sets_no_preemptible_spec(mocker): + create = _capture_create_request(mocker) + _create(preemptible=False, pricing_policy_id=None) + request = create.call_args.args[0] + # An unset sub-message reads as an empty message in the SDK, so check the oneof. + assert request.spec.pricing_model is None From 9eb058f8e3dc3857f79a2be9b73db07790b2a718 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:52:59 +0200 Subject: [PATCH 634/640] test: cover the fork's Verda SFS, host key, debug key and shim NFS features Tests for the fork features carried by the rebase: - _get_sfs_mount_commands: mounts a mapped SFS volume over NFS, skips an unmapped volume and a volume without backend data. - register_volume: rejects a non-shared volume, registers a shared one with its NFS backend data. - create_instance: the DSTACK_DEBUG_SSH_PUBLIC_KEY key is a per-instance key stored in the backend data (so the upstream cleanup removes it), no extra key when the env is unset, and the SFS mounts run before the shim. - run_job: the host gets the project key and the extra authorized keys. - _volume_to_shim_volume_info: nfs_host and nfs_pseudo from the SFS backend data, none for a block volume. Also reads instance_config.volumes with getattr, because upstream tests (and some callers) pass a config without volumes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/verda/compute.py | 3 +- .../core/backends/verda/test_fork_sfs.py | 214 ++++++++++++++++++ .../server/services/runner/test_client_nfs.py | 34 +++ 3 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 src/tests/_internal/core/backends/verda/test_fork_sfs.py create mode 100644 src/tests/_internal/server/services/runner/test_client_nfs.py diff --git a/src/dstack/_internal/core/backends/verda/compute.py b/src/dstack/_internal/core/backends/verda/compute.py index eb52a163a6..15e5087d22 100644 --- a/src/dstack/_internal/core/backends/verda/compute.py +++ b/src/dstack/_internal/core/backends/verda/compute.py @@ -226,7 +226,8 @@ def create_instance( # Fork: mount Verda SFS volumes over NFS before the shim starts. sfs_mount_commands = _get_sfs_mount_commands( - volumes=instance_config.volumes or [], + # getattr: some callers (and upstream tests) pass a config without volumes. + volumes=getattr(instance_config, "volumes", None) or [], volume_mounts=volume_mounts or {}, instance_region=instance_offer.region, ) diff --git a/src/tests/_internal/core/backends/verda/test_fork_sfs.py b/src/tests/_internal/core/backends/verda/test_fork_sfs.py new file mode 100644 index 0000000000..5affc7d9df --- /dev/null +++ b/src/tests/_internal/core/backends/verda/test_fork_sfs.py @@ -0,0 +1,214 @@ +"""Tests for the fork features of the Verda backend. + +The fork adds Verda SFS (shared file system) volumes mounted over NFS, the +caller's keys on the host, and an optional debug SSH key from the server env. +""" + +import json +import os +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from dstack._internal.core.backends.verda.compute import ( + VerdaCompute, + VerdaInstanceBackendData, + _get_sfs_mount_commands, +) +from dstack._internal.core.errors import ComputeError + + +def _sfs_volume(name="data", volume_id="vol-1", location="FIN-03", pseudo="/share-1"): + return SimpleNamespace( + name=name, + volume_id=volume_id, + provisioning_data=SimpleNamespace( + backend_data=json.dumps( + {"pseudo_path": pseudo, "location_code": location, "volume_type": "NVMe_Shared"} + ) + ), + ) + + +class TestGetSfsMountCommands: + def test_mounts_a_mapped_volume_over_nfs(self): + commands = _get_sfs_mount_commands( + volumes=[_sfs_volume()], + volume_mounts={"vol-1": "/data"}, + instance_region="FIN-03", + ) + mount = [c for c in commands if "mount -t nfs" in c] + assert len(mount) == 1 + assert "nfs.fin-03.datacrunch.io:/share-1 /mnt/disks/dstack-volumes/data" in mount[0] + + def test_skips_a_volume_that_is_not_mounted_by_the_run(self): + commands = _get_sfs_mount_commands( + volumes=[_sfs_volume()], volume_mounts={}, instance_region="FIN-03" + ) + assert commands == [] + + def test_skips_a_volume_without_backend_data(self): + volume = SimpleNamespace( + name="data", volume_id="vol-1", provisioning_data=SimpleNamespace(backend_data=None) + ) + commands = _get_sfs_mount_commands( + volumes=[volume], volume_mounts={"vol-1": "/data"}, instance_region="FIN-03" + ) + assert commands == [] + + +class TestRegisterVolume: + def _volume(self): + return SimpleNamespace( + configuration=SimpleNamespace(volume_id="vol-1", region="FIN-03"), + ) + + def test_rejects_a_volume_that_is_not_shared(self): + compute = VerdaCompute.__new__(VerdaCompute) + compute.config = MagicMock() + with patch( + "dstack._internal.core.backends.verda.compute._get_volume_by_id", + return_value={"type": "NVMe", "location": "FIN-03", "pseudo_path": "/x"}, + ): + with pytest.raises(ComputeError): + compute.register_volume(self._volume()) + + def test_registers_a_shared_volume_with_nfs_backend_data(self): + compute = VerdaCompute.__new__(VerdaCompute) + compute.config = MagicMock() + compute.backend_type = "verda" + with patch( + "dstack._internal.core.backends.verda.compute._get_volume_by_id", + return_value={ + "type": "NVMe_Shared", + "location": "FIN-03", + "pseudo_path": "/share-1", + "size": 100, + }, + ): + data = compute.register_volume(self._volume()) + assert data.attachable is True + assert json.loads(data.backend_data)["pseudo_path"] == "/share-1" + + +class TestCreateInstanceForkFeatures: + def _run(self, env, volumes=None, volume_mounts=None): + compute = VerdaCompute.__new__(VerdaCompute) + compute.client = MagicMock() + instance_offer = SimpleNamespace( + backend="verda", + instance=SimpleNamespace( + name="CPU.4V.16G", + resources=SimpleNamespace(disk=SimpleNamespace(size_mib=102400), gpus=[], spot=False), + ), + region="FIN-03", + price=0.0279, + ) + instance_config = SimpleNamespace( + instance_name="verda-one-node-0", + get_public_keys=lambda: ["ssh-rsa project"], + volumes=volumes or [], + ) + created_keys = [] + + def create_key(client, name, public_key): + created_keys.append((name, public_key)) + return f"key-{len(created_keys)}" + + captured = {} + + def create_script(client, name, script): + captured["script"] = script + return "startup-script-id" + + with ( + patch.dict(os.environ, env, clear=False), + patch( + "dstack._internal.core.backends.verda.compute.generate_unique_instance_name", + return_value="verda-one-node-0", + ), + patch( + "dstack._internal.core.backends.verda.compute.get_shim_commands", + return_value=["echo shim"], + ), + patch( + "dstack._internal.core.backends.verda.compute._create_ssh_key", + side_effect=create_key, + ), + patch( + "dstack._internal.core.backends.verda.compute._create_startup_script", + side_effect=lambda client, name, script: create_script(client, name, script), + ), + patch( + "dstack._internal.core.backends.verda.compute._deploy_instance", + return_value=SimpleNamespace(id="provider-instance-id", location="FIN-03"), + ), + patch( + "dstack._internal.core.backends.verda.compute.JobProvisioningData", + side_effect=lambda **kwargs: SimpleNamespace(**kwargs), + ), + ): + jpd = compute.create_instance( + instance_offer, instance_config, None, volume_mounts=volume_mounts + ) + return jpd, created_keys, captured["script"] + + def test_adds_the_debug_key_as_a_per_instance_key(self): + env = {"DSTACK_DEBUG_SSH_PUBLIC_KEY": "ssh-ed25519 debug"} + jpd, keys, _ = self._run(env) + assert ("verda-one-node-0-debug.key", "ssh-ed25519 debug") in keys + backend_data = VerdaInstanceBackendData.load(jpd.backend_data) + assert len(backend_data.ssh_key_ids) == 2 + + def test_no_debug_key_when_the_env_is_unset(self): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DSTACK_DEBUG_SSH_PUBLIC_KEY", None) + _, keys, _ = self._run({}) + assert [name for name, _ in keys] == ["verda-one-node-0-0.key"] + + def test_mounts_sfs_volumes_before_the_shim_starts(self): + os.environ.pop("DSTACK_DEBUG_SSH_PUBLIC_KEY", None) + _, _, script = self._run( + {}, volumes=[_sfs_volume()], volume_mounts={"vol-1": "/data"} + ) + assert script.index("mount -t nfs") < script.index("echo shim") + + +class TestRunJob: + def test_authorizes_the_project_key_and_the_extra_keys_on_the_host(self): + compute = VerdaCompute.__new__(VerdaCompute) + captured = {} + + def fake_create_instance(instance_offer, instance_config, placement_group, volume_mounts=None): + captured["config"] = instance_config + captured["volume_mounts"] = volume_mounts + return "jpd" + + compute.create_instance = fake_create_instance + compute._restrict_instance_offer_az_to_volumes_az = lambda offer, volumes: None + run = MagicMock() + run.project_name = "main" + run.user = "admin" + run.run_spec.configuration.volumes = [] + run.run_spec.merged_profile.tags = None + requirements = SimpleNamespace(reservation=None) + offer = MagicMock() + with patch( + "dstack._internal.core.backends.verda.compute.get_job_instance_name", + return_value="job-0", + ): + result = compute.run_job( + run, + MagicMock(), + offer, + "ssh-rsa project", + "private", + [], + None, + requirements, + ["ssh-ed25519 user"], + ) + assert result == "jpd" + keys = [k.public for k in captured["config"].ssh_keys] + assert keys == ["ssh-rsa project", "ssh-ed25519 user"] diff --git a/src/tests/_internal/server/services/runner/test_client_nfs.py b/src/tests/_internal/server/services/runner/test_client_nfs.py new file mode 100644 index 0000000000..de9946f269 --- /dev/null +++ b/src/tests/_internal/server/services/runner/test_client_nfs.py @@ -0,0 +1,34 @@ +"""Fork: the server sends the NFS details of a Verda SFS volume to the shim.""" + +import json +from types import SimpleNamespace + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.server.services.runner.client import _volume_to_shim_volume_info + + +def _volume(backend_data): + return SimpleNamespace( + name="data", + volume_id="vol-1", + external=True, + configuration=SimpleNamespace(backend=BackendType.VERDA), + provisioning_data=SimpleNamespace(backend_data=backend_data), + get_attachment_data_for_instance=lambda instance_id: None, + ) + + +def test_fills_nfs_details_from_sfs_backend_data(): + info = _volume_to_shim_volume_info( + _volume(json.dumps({"pseudo_path": "/share-1", "location_code": "FIN-03"})), + instance_id="instance-1", + ) + assert info.nfs_host == "nfs.fin-03.datacrunch.io" + assert info.nfs_pseudo == "/share-1" + assert info.device_name is None + + +def test_no_nfs_details_for_a_block_volume(): + info = _volume_to_shim_volume_info(_volume(None), instance_id="instance-1") + assert info.nfs_host is None + assert info.nfs_pseudo is None From 933b852571b909355fa1c6bb056488862bc3b0e9 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 10:59:35 +0200 Subject: [PATCH 635/640] server: send the shim NFS volume fields only when set ShimVolumeInfo.nfs_host and nfs_pseudo (fork, Verda SFS) serialized as null for every volume, which changed the shim submit body for all backends and broke TestShimClientV1::test_submit and TestShimClientV2::test_submit_task. They are excluded when None now, so a non-SFS volume body is identical to upstream. The Go side (VolumeInfo) already uses omitempty. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/dstack/_internal/server/schemas/runner.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index 59b835f8e4..d40a74d809 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from pydantic import field_validator +from pydantic import Field, field_validator from dstack._internal.core.models.common import CoreModel, NetworkMode from dstack._internal.core.models.repos.remote import RemoteRepoCreds @@ -195,9 +195,11 @@ class ShimVolumeInfo(CoreModel): volume_id: str init_fs: bool device_name: Optional[str] = None - # NFS mount details for SFS/network volumes - nfs_host: Optional[str] = None # e.g., "nfs.fin-03.datacrunch.io" - nfs_pseudo: Optional[str] = None # e.g., "/path/to/share" + # Fork: NFS mount details of a Verda SFS volume, for example + # "nfs.fin-03.datacrunch.io" and "/path/to/share". Sent only when set, so the body + # of every other volume stays as upstream sends it. + nfs_host: Optional[str] = Field(default=None, exclude_if=lambda v: v is None) + nfs_pseudo: Optional[str] = Field(default=None, exclude_if=lambda v: v is None) class PortMapping(CoreModel): From 7bdc44cf6b391635bb8b5216cbbdd87871c65699 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 11:01:13 +0200 Subject: [PATCH 636/640] style: ruff format the ported Verda code and its tests Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/verda/compute.py | 78 ++++++++++--------- .../core/backends/verda/test_fork_sfs.py | 12 +-- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/src/dstack/_internal/core/backends/verda/compute.py b/src/dstack/_internal/core/backends/verda/compute.py index 15e5087d22..20de92a838 100644 --- a/src/dstack/_internal/core/backends/verda/compute.py +++ b/src/dstack/_internal/core/backends/verda/compute.py @@ -154,7 +154,14 @@ def run_job( logger.info( "run_job called with %d volumes: %s", len(volumes), - [(v.name, v.volume_id, v.provisioning_data.backend_data if v.provisioning_data else None) for v in volumes] + [ + ( + v.name, + v.volume_id, + v.provisioning_data.backend_data if v.provisioning_data else None, + ) + for v in volumes + ], ) volume_mounts: Dict[str, str] = {} @@ -172,7 +179,9 @@ def run_job( volume_mounts[vol.volume_id] = mount_point.path logger.info( "Mapped volume %s (id=%s) to mount path %s", - vol.name, vol.volume_id, mount_point.path + vol.name, + vol.volume_id, + mount_point.path, ) break @@ -385,11 +394,13 @@ def register_volume(self, volume: Volume) -> VolumeProvisioningData: size_gb = volume_data.get("size", 0) # Store SFS-specific data for NFS mounting - backend_data = json.dumps({ - "pseudo_path": pseudo_path, - "location_code": location, - "volume_type": volume_type, - }) + backend_data = json.dumps( + { + "pseudo_path": pseudo_path, + "location_code": location, + "volume_type": volume_type, + } + ) return VolumeProvisioningData( backend=self.backend_type, @@ -440,10 +451,7 @@ def attach_volume( volume_id = volume.volume_id instance_id = provisioning_data.instance_id - logger.info( - "Attaching SFS volume %s to instance %s", - volume_id, instance_id - ) + logger.info("Attaching SFS volume %s to instance %s", volume_id, instance_id) try: token = _get_verda_access_token( @@ -461,10 +469,7 @@ def attach_volume( "instance_ids": [instance_id], } - logger.info( - "Calling Verda volume API: PUT /v1/volumes with payload=%s", - payload - ) + logger.info("Calling Verda volume API: PUT /v1/volumes with payload=%s", payload) response = requests.put( "https://api.datacrunch.io/v1/volumes", @@ -475,7 +480,8 @@ def attach_volume( logger.info( "Verda volume API response: status=%d body=%s", - response.status_code, response.text[:500] if response.text else "(empty)" + response.status_code, + response.text[:500] if response.text else "(empty)", ) # 202 Accepted is a valid response - means the request was accepted for async processing @@ -492,8 +498,7 @@ def attach_volume( ) logger.info( - "Successfully attached SFS volume %s to instance %s", - volume_id, instance_id + "Successfully attached SFS volume %s to instance %s", volume_id, instance_id ) # SFS volumes don't have a device_name - they're mounted via NFS @@ -518,8 +523,7 @@ def detach_volume( instance_id = provisioning_data.instance_id logger.info( - "Detaching SFS volume %s from instance %s (force=%s)", - volume_id, instance_id, force + "Detaching SFS volume %s from instance %s (force=%s)", volume_id, instance_id, force ) try: @@ -562,13 +566,14 @@ def detach_volume( else: logger.warning( "Failed to detach volume %s from instance %s (force=True, ignoring): %s", - volume_id, instance_id, error_msg + volume_id, + instance_id, + error_msg, ) return logger.info( - "Successfully detached SFS volume %s from instance %s", - volume_id, instance_id + "Successfully detached SFS volume %s from instance %s", volume_id, instance_id ) except requests.exceptions.RequestException as e: @@ -576,8 +581,7 @@ def detach_volume( raise ComputeError(f"Failed to detach volume {volume_id}: {e}") else: logger.warning( - "Failed to detach volume %s (force=True, ignoring): %s", - volume_id, e + "Failed to detach volume %s (force=True, ignoring): %s", volume_id, e ) @@ -664,13 +668,16 @@ def _get_sfs_mount_commands( logger.info( "_get_sfs_mount_commands: processing %d volumes, volume_mounts=%s", - len(volumes), volume_mounts + len(volumes), + volume_mounts, ) # Check if we have any volumes to mount - if so, ensure nfs-common is installed has_volumes_to_mount = any( - volume.provisioning_data and volume.provisioning_data.backend_data - and volume.volume_id and volume.volume_id in volume_mounts + volume.provisioning_data + and volume.provisioning_data.backend_data + and volume.volume_id + and volume.volume_id in volume_mounts for volume in volumes ) if has_volumes_to_mount: @@ -685,18 +692,12 @@ def _get_sfs_mount_commands( for volume in volumes: if not volume.provisioning_data or not volume.provisioning_data.backend_data: - logger.info( - "Skipping volume %s: no provisioning_data or backend_data", - volume.name - ) + logger.info("Skipping volume %s: no provisioning_data or backend_data", volume.name) continue volume_id = volume.volume_id if not volume_id or volume_id not in volume_mounts: - logger.info( - "Skipping volume %s (id=%s): not in volume_mounts", - volume.name, volume_id - ) + logger.info("Skipping volume %s (id=%s): not in volume_mounts", volume.name, volume_id) continue try: @@ -731,7 +732,10 @@ def _get_sfs_mount_commands( logger.info( "SFS mount for volume %s: nfs.%s.datacrunch.io:%s -> %s", - volume.name, dc, pseudo_path, host_mount_path + volume.name, + dc, + pseudo_path, + host_mount_path, ) logger.info("_get_sfs_mount_commands: returning %d commands: %s", len(commands), commands) diff --git a/src/tests/_internal/core/backends/verda/test_fork_sfs.py b/src/tests/_internal/core/backends/verda/test_fork_sfs.py index 5affc7d9df..f17af11d40 100644 --- a/src/tests/_internal/core/backends/verda/test_fork_sfs.py +++ b/src/tests/_internal/core/backends/verda/test_fork_sfs.py @@ -100,7 +100,9 @@ def _run(self, env, volumes=None, volume_mounts=None): backend="verda", instance=SimpleNamespace( name="CPU.4V.16G", - resources=SimpleNamespace(disk=SimpleNamespace(size_mib=102400), gpus=[], spot=False), + resources=SimpleNamespace( + disk=SimpleNamespace(size_mib=102400), gpus=[], spot=False + ), ), region="FIN-03", price=0.0279, @@ -169,9 +171,7 @@ def test_no_debug_key_when_the_env_is_unset(self): def test_mounts_sfs_volumes_before_the_shim_starts(self): os.environ.pop("DSTACK_DEBUG_SSH_PUBLIC_KEY", None) - _, _, script = self._run( - {}, volumes=[_sfs_volume()], volume_mounts={"vol-1": "/data"} - ) + _, _, script = self._run({}, volumes=[_sfs_volume()], volume_mounts={"vol-1": "/data"}) assert script.index("mount -t nfs") < script.index("echo shim") @@ -180,7 +180,9 @@ def test_authorizes_the_project_key_and_the_extra_keys_on_the_host(self): compute = VerdaCompute.__new__(VerdaCompute) captured = {} - def fake_create_instance(instance_offer, instance_config, placement_group, volume_mounts=None): + def fake_create_instance( + instance_offer, instance_config, placement_group, volume_mounts=None + ): captured["config"] = instance_config captured["volume_mounts"] = volume_mounts return "jpd" From cdaa2eca8c5c2ce60a4942aa213c06c1ad308849 Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 11:26:00 +0200 Subject: [PATCH 637/640] volumes: add VerdaVolumeConfiguration to the backend-tagged volume union Review BLOCKER: upstream turned the volume configuration into a union tagged by backend (Pydantic v2, d10edfdc4) with members for aws, gcp, runpod, kubernetes and daytona only. A Verda SFS volume (backend verda or datacrunch) then fails to parse in the CLI and the server, and the volume rows that the fork stored in the server DB fail to load. VerdaVolumeConfiguration takes region, volume_id and size like the Runpod member, accepts backend verda and datacrunch, and accepts (and drops) the availability_zone field that dstack 0.20.3 stored in each row. register_volume asserts the config type. Test first: test_verda_volume_configuration.py failed at import (RED), passes now (GREEN), including a row in the 0.20.3 shape. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/verda/compute.py | 2 + src/dstack/_internal/core/models/volumes.py | 15 +++++ .../core/backends/verda/test_fork_sfs.py | 3 +- .../models/test_verda_volume_configuration.py | 59 +++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 src/tests/_internal/core/models/test_verda_volume_configuration.py diff --git a/src/dstack/_internal/core/backends/verda/compute.py b/src/dstack/_internal/core/backends/verda/compute.py index 20de92a838..d6f5d2df4f 100644 --- a/src/dstack/_internal/core/backends/verda/compute.py +++ b/src/dstack/_internal/core/backends/verda/compute.py @@ -44,6 +44,7 @@ from dstack._internal.core.models.resources import Memory, Range from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run from dstack._internal.core.models.volumes import ( + VerdaVolumeConfiguration, Volume, VolumeAttachmentData, VolumeMountPoint, @@ -367,6 +368,7 @@ def register_volume(self, volume: Volume) -> VolumeProvisioningData: SFS volumes are NFS-based shared filesystems that can be mounted to instances. The volume must already exist in Verda and be of type *_Shared (e.g. NVMe_Shared). """ + assert isinstance(volume.configuration, VerdaVolumeConfiguration) volume_id = get_or_error(volume.configuration.volume_id) volume_data = _get_volume_by_id(self.config, volume_id) diff --git a/src/dstack/_internal/core/models/volumes.py b/src/dstack/_internal/core/models/volumes.py index 92b3a650ee..b243a82583 100644 --- a/src/dstack/_internal/core/models/volumes.py +++ b/src/dstack/_internal/core/models/volumes.py @@ -140,6 +140,20 @@ class RunpodVolumeConfiguration( """Runpod doesn't have AZs but we accept this field for compatibility with older clients.""" +class VerdaVolumeConfiguration( + VolumeConfigurationWithRegion, VolumeConfigurationWithVolumeID, VolumeConfigurationWithSize +): + """Fork: an existing Verda (DataCrunch) SFS volume, registered by `volume_id` and mounted + over NFS. dstack does not create or delete SFS volumes.""" + + backend: Annotated[ + Literal[BackendType.VERDA, BackendType.DATACRUNCH], + Field(description="The volume backend"), + ] = BackendType.VERDA + availability_zone: Annotated[Optional[str], Field(exclude=True)] = None + """Verda has no AZs. The field is accepted for the rows that dstack 0.20.3 stored.""" + + class KubernetesVolumeConfiguration(VolumeConfigurationWithRegion, VolumeConfigurationWithSize): backend: Annotated[ Literal[BackendType.KUBERNETES], Field(description="The volume backend") @@ -192,6 +206,7 @@ class DaytonaVolumeConfiguration(VolumeConfigurationWithVolumeID): AWSVolumeConfiguration, GCPVolumeConfiguration, RunpodVolumeConfiguration, + VerdaVolumeConfiguration, KubernetesVolumeConfiguration, DaytonaVolumeConfiguration, ] diff --git a/src/tests/_internal/core/backends/verda/test_fork_sfs.py b/src/tests/_internal/core/backends/verda/test_fork_sfs.py index f17af11d40..2709a54725 100644 --- a/src/tests/_internal/core/backends/verda/test_fork_sfs.py +++ b/src/tests/_internal/core/backends/verda/test_fork_sfs.py @@ -17,6 +17,7 @@ _get_sfs_mount_commands, ) from dstack._internal.core.errors import ComputeError +from dstack._internal.core.models.volumes import VerdaVolumeConfiguration def _sfs_volume(name="data", volume_id="vol-1", location="FIN-03", pseudo="/share-1"): @@ -61,7 +62,7 @@ def test_skips_a_volume_without_backend_data(self): class TestRegisterVolume: def _volume(self): return SimpleNamespace( - configuration=SimpleNamespace(volume_id="vol-1", region="FIN-03"), + configuration=VerdaVolumeConfiguration(volume_id="vol-1", region="FIN-03"), ) def test_rejects_a_volume_that_is_not_shared(self): diff --git a/src/tests/_internal/core/models/test_verda_volume_configuration.py b/src/tests/_internal/core/models/test_verda_volume_configuration.py new file mode 100644 index 0000000000..0fcb52a4f9 --- /dev/null +++ b/src/tests/_internal/core/models/test_verda_volume_configuration.py @@ -0,0 +1,59 @@ +"""Fork: Verda SFS volumes need a member in the backend-tagged volume config union.""" + +import pytest + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.volumes import ( + VerdaVolumeConfiguration, + parse_volume_configuration, +) +from dstack._internal.server.services.volumes import _validate_volume_configuration + + +@pytest.mark.parametrize("backend", ["verda", "datacrunch"]) +def test_parses_a_verda_sfs_volume(backend): + configuration = parse_volume_configuration( + { + "type": "volume", + "name": "greencorporafrdata", + "backend": backend, + "region": "FIN-03", + "volume_id": "4a7b-sfs", + } + ) + assert isinstance(configuration, VerdaVolumeConfiguration) + assert configuration.backend == BackendType(backend) + assert configuration.region == "FIN-03" + assert configuration.volume_id == "4a7b-sfs" + assert configuration.external_volume_id == "4a7b-sfs" + + +def test_loads_a_row_written_by_the_old_fork_with_availability_zone(): + # The old flat VolumeConfiguration (dstack 0.20.3) stored availability_zone: null. + configuration = parse_volume_configuration( + { + "type": "volume", + "name": "greencorporafrdata", + "backend": "verda", + "region": "FIN-03", + "availability_zone": None, + "volume_id": "4a7b-sfs", + "size": None, + "auto_cleanup_duration": None, + "tags": None, + } + ) + assert isinstance(configuration, VerdaVolumeConfiguration) + assert "availability_zone" not in configuration.model_dump() + + +def test_an_external_verda_volume_needs_no_size(monkeypatch): + monkeypatch.setattr( + "dstack._internal.server.services.volumes.backends_services.check_backend_type_available", + lambda backend: None, + ) + _validate_volume_configuration( + parse_volume_configuration( + {"backend": "verda", "region": "FIN-03", "volume_id": "4a7b-sfs"} + ) + ) From f9c49610a170be66e71d3bfb5a9790cb82f81a4e Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 11:26:00 +0200 Subject: [PATCH 638/640] docker(stgn): copy skills/ before uv sync Review BLOCKER: upstream pyproject force-includes skills/*/SKILL.md in the wheel, so 'uv sync --extra all' fails in the stgn image build without them. Checked by building the wheel from the Dockerfile's copy set: without skills/ the build fails (rc 2), with it the build passes. Co-Authored-By: Claude Opus 5.5 (1M context) --- docker/server/stgn/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/server/stgn/Dockerfile b/docker/server/stgn/Dockerfile index cfbf0a05ee..4fce817787 100644 --- a/docker/server/stgn/Dockerfile +++ b/docker/server/stgn/Dockerfile @@ -33,6 +33,8 @@ ENV PATH="/root/.local/bin/:$PATH" COPY pyproject.toml uv.lock README.md ./ COPY src src COPY examples examples +# pyproject force-includes skills/*/SKILL.md into the wheel, so uv sync needs them. +COPY skills skills # Copy built frontend to statics directory COPY --from=frontend-builder /frontend/build src/dstack/_internal/server/statics From aa99109fc15ad1ccc83bb675b954c0bb1e18602b Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 12:43:54 +0200 Subject: [PATCH 639/640] nebius: cap spot VMs at the on-demand price of the same type and region Operator decision 2026-10-08: the Nebius pricing policy caps a spot VM at the on-demand price of the same instance type in the same region (for example 1.79 USD/h for gpu-rtx6000 in us-central1), not at the current spot price. Nebius then preempts the VM only when the spot price grows above what the same VM costs on demand. The offer shows the cap, as upstream does, so a run's max_price still filters the offer before any create. Spot offers carry the matching on-demand price in their backend data. Without an on-demand row for that type and region, the cap stays the spot price (upstream behavior). Flat-rate spot offers keep no pricing policy. The cap and its source are logged at create time. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/nebius/compute.py | 46 +++++++++- .../_internal/core/backends/nebius/models.py | 6 ++ .../core/backends/nebius/test_compute.py | 84 +++++++++++++++++++ 3 files changed, 133 insertions(+), 3 deletions(-) diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index 444f829db4..ebf5a52934 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -139,6 +139,7 @@ def get_all_offers_with_availability( locations=list(self._region_to_project_id), extra_filter=_supported_instances, ) + offers = _with_on_demand_price(offers) return [ offer.with_availability( availability=InstanceAvailability.UNKNOWN, @@ -211,13 +212,25 @@ def create_instance( instance_offer.instance.resources.spot and not offer_backend_data.is_preemptible_flat_rate ): + max_price = _get_pricing_policy_max_price(instance_offer) + logger.info( + "Nebius spot cap for %s (%s, %s): %s per unit, source: %s", + instance_name, + instance_offer.instance.name, + instance_offer.region, + max_price, + "on-demand price" + if offer_backend_data.on_demand_price is not None + else "spot price (no on-demand offer)", + ) create_pricing_policy_op = resources.create_pricing_policy( sdk=self._sdk, name=instance_name, project_id=self._region_to_project_id[instance_offer.region], platform=platform, - # Prevent the spot price from growing above what dstack shows - max_price=str(_get_pricing_policy_max_price(instance_offer)), + # Prevent the spot price from growing above what dstack shows, which is + # the on-demand price of the same type when the catalog has it (fork). + max_price=str(max_price), ) resources.wait_for_operation( create_pricing_policy_op, timeout=WAIT_FOR_PRICING_POLICY_TIMEOUT @@ -479,9 +492,36 @@ def _get_price_adjusted_for_pricing_policy(offer: InstanceOffer) -> float: return float(max_price * _get_pricing_policy_price_units(offer)) +def _with_on_demand_price(offers: List[InstanceOffer]) -> List[InstanceOffer]: + """Fork (operator decision 2026-10-08): store on each spot offer the on-demand price of + the same instance type in the same region, so the pricing policy caps the spot price at + the on-demand price, not at the current spot price. Nebius then preempts the VM only when + the spot price grows above what the same VM costs on demand.""" + on_demand_prices = { + (offer.instance.name, offer.region): offer.price + for offer in offers + if not offer.instance.resources.spot + } + result = [] + for offer in offers: + on_demand_price = on_demand_prices.get((offer.instance.name, offer.region)) + if offer.instance.resources.spot and on_demand_price is not None: + offer = offer.model_copy( + update={"backend_data": {**offer.backend_data, "on_demand_price": on_demand_price}} + ) + result.append(offer) + return result + + def _get_pricing_policy_max_price(offer: InstanceOffer) -> Decimal: + # Fork: the cap is the on-demand price of the same type and region when the catalog has + # it (`_with_on_demand_price`), else the offer price (upstream behavior). + on_demand_price = validate_extra_ignore( + NebiusOfferBackendData, offer.backend_data + ).on_demand_price + base_price = on_demand_price if on_demand_price is not None else offer.price # Pricing policies allow at most 3 decimal places. Round upward. - price = Decimal(str(offer.price)) / _get_pricing_policy_price_units(offer) + price = Decimal(str(base_price)) / _get_pricing_policy_price_units(offer) return price.quantize(Decimal("0.001"), rounding=ROUND_CEILING) diff --git a/src/dstack/_internal/core/backends/nebius/models.py b/src/dstack/_internal/core/backends/nebius/models.py index 34af0e6fa1..0b18c88bb0 100644 --- a/src/dstack/_internal/core/backends/nebius/models.py +++ b/src/dstack/_internal/core/backends/nebius/models.py @@ -186,6 +186,12 @@ class NebiusOfferBackendData(CoreModel): """ True if the price of this spot instance does not change and pricing policies are not supported. """ + on_demand_price: Optional[float] = None + """ + Fork: the on-demand price of the same instance type in the same region, set on spot + offers only. The pricing policy caps the spot price at it. `None` when the catalog + has no on-demand row for that type and region: the cap is then the spot price. + """ @field_serializer("fabrics") def _serialize_fabrics(self, value: set[str]) -> list[str]: diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py index 55772f1ed7..06d5e11c02 100644 --- a/src/tests/_internal/core/backends/nebius/test_compute.py +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -113,6 +113,90 @@ def test_keeps_flat_rate_spot_price_as_is(self, mocker): assert offers[0].price == 1.2341 +def make_rtx_offer(price: float, spot: bool, region: str = "us-central1") -> InstanceOffer: + return InstanceOffer( + backend=BackendType.NEBIUS, + instance=InstanceType( + name="gpu-rtx6000 1gpu-24vcpu-218gb", + resources=Resources( + cpus=24, + memory_mib=218 * 1024, + gpus=[Gpu(name="RTXPRO6000", memory_mib=96 * 1024)], + spot=spot, + ), + ), + region=region, + price=price, + backend_data={"fabrics": [], "is_preemptible_flat_rate": False}, + ) + + +class TestSpotCapAtOnDemandPrice: + # Fork (operator decision 2026-10-08): the pricing policy caps a spot VM at the + # on-demand price of the same instance type and region, not at the current spot price. + @pytest.fixture(autouse=True) + def _mock_region_to_project_id(self, mocker): + mocker.patch.object( + NebiusCompute, + "_region_to_project_id", + {"us-central1": "project-a", "uk-south2": "project-b"}, + ) + + def test_spot_offer_is_capped_at_the_on_demand_price_of_the_same_type(self, mocker): + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[make_rtx_offer(0.95, spot=True), make_rtx_offer(1.79, spot=False)], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + spot = [o for o in offers if o.instance.resources.spot][0] + on_demand = [o for o in offers if not o.instance.resources.spot][0] + assert spot.price == 1.79 + assert spot.backend_data["on_demand_price"] == 1.79 + assert on_demand.price == 1.79 + + def test_falls_back_to_the_spot_price_without_an_on_demand_offer(self, mocker): + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[ + make_rtx_offer(0.95, spot=True, region="us-central1"), + make_rtx_offer(1.79, spot=False, region="uk-south2"), + ], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + spot = [o for o in offers if o.instance.resources.spot][0] + assert spot.price == 0.95 + assert spot.backend_data.get("on_demand_price") is None + + def test_flat_rate_spot_price_is_not_capped(self, mocker): + flat = make_rtx_offer(0.95, spot=True) + flat.backend_data["is_preemptible_flat_rate"] = True + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[flat, make_rtx_offer(1.79, spot=False)], + ) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + spot = [o for o in offers if o.instance.resources.spot][0] + assert spot.price == 0.95 + + def test_create_time_cap_is_the_on_demand_price_per_gpu(self): + offer = make_offer(9.84, spot=True, is_preemptible_flat_rate=False, gpu_count=8) + offer.backend_data["on_demand_price"] = 23.6 + assert str(compute_module._get_pricing_policy_max_price(offer)) == "2.950" + + def test_create_time_cap_falls_back_to_the_offer_price(self): + offer = make_rtx_offer(0.95, spot=True) + assert str(compute_module._get_pricing_policy_max_price(offer)) == "0.950" + + class TestSupportedInstances: # Fork: the RTX PRO 6000 runs as `gpu-rtx6000` (us-central1) and `gpu-rtx6000-a` # (uk-south2). The deployed fork served both. Upstream lists only `gpu-rtx6000`. From 62eeb7e304a4a1ca8fbd87c9090d74862fdc5bfe Mon Sep 17 00:00:00 2001 From: Tom HENEAULT Date: Thu, 8 Oct 2026 12:57:19 +0200 Subject: [PATCH 640/640] nebius: cap spot just below on-demand, keep offers sorted, retry out-of-range Review fixes for aa99109fc (review workflow wf_88a5d0da-f11, 3 lanes, a skeptic per finding): - BLOCKER: Nebius accepts a pricing policy bid only up to 0.01 USD per GPU-hour below the on-demand price (OUT_OF_RANGE above it). The cap is now the on-demand price per GPU minus 0.010, rounded down. It is never below the spot price cap. - MAJOR: the shown spot price is the cap, so the catalog order no longer held. get_all_offers_with_availability now returns the offers sorted by price (stable, so spot stays ahead of on-demand on a tie). The server merges the per-backend lists and expects each one sorted. - MINOR: a cap that Nebius refuses as out of range (for example after an on-demand price change that the offline catalog does not show yet) is retried once with the spot price cap, the upstream behavior. Spot offers keep their catalog spot price in backend data for that. - Operator fallback: a spot offer with no on-demand price takes the run's max_price as its cap (offer modifier). Its shown price does not change. Known effect, accepted: dstack shows and records the cap as the price of a Nebius spot VM, so run costs overstate the billed spot price. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/backends/nebius/compute.py | 199 +++++++++++++----- .../_internal/core/backends/nebius/models.py | 15 +- .../core/backends/nebius/test_compute.py | 141 ++++++++++++- 3 files changed, 294 insertions(+), 61 deletions(-) diff --git a/src/dstack/_internal/core/backends/nebius/compute.py b/src/dstack/_internal/core/backends/nebius/compute.py index ebf5a52934..109daceeee 100644 --- a/src/dstack/_internal/core/backends/nebius/compute.py +++ b/src/dstack/_internal/core/backends/nebius/compute.py @@ -3,7 +3,7 @@ import shlex import time from collections.abc import Iterable -from decimal import ROUND_CEILING, Decimal +from decimal import ROUND_CEILING, ROUND_FLOOR, Decimal from functools import cached_property from typing import List, Optional @@ -140,18 +140,27 @@ def get_all_offers_with_availability( extra_filter=_supported_instances, ) offers = _with_on_demand_price(offers) - return [ - offer.with_availability( - availability=InstanceAvailability.UNKNOWN, - price=_get_price_adjusted_for_pricing_policy(offer), - ) - for offer in offers - ] + # Fork: the shown spot price becomes the cap, so the catalog order (by spot price) + # no longer holds. The server merges the per-backend lists and expects each one + # sorted by price. The sort is stable, so spot stays ahead of on-demand on a tie. + return sorted( + ( + offer.with_availability( + availability=InstanceAvailability.UNKNOWN, + price=_get_price_adjusted_for_pricing_policy(offer), + ) + for offer in offers + ), + key=lambda offer: offer.price, + ) def get_offers_modifiers( self, requirements: Requirements, full_offers: bool ) -> Iterable[OfferModifier]: - return [get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements)] + return [ + get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements), + _get_run_max_price_modifier(requirements), + ] def create_instance( self, @@ -212,35 +221,15 @@ def create_instance( instance_offer.instance.resources.spot and not offer_backend_data.is_preemptible_flat_rate ): - max_price = _get_pricing_policy_max_price(instance_offer) - logger.info( - "Nebius spot cap for %s (%s, %s): %s per unit, source: %s", - instance_name, - instance_offer.instance.name, - instance_offer.region, - max_price, - "on-demand price" - if offer_backend_data.on_demand_price is not None - else "spot price (no on-demand offer)", - ) - create_pricing_policy_op = resources.create_pricing_policy( + # Prevent the spot price from growing above the cap. Fork: the cap sits just + # below the on-demand price of the same type when the catalog has it. + create_pricing_policy_op = _create_pricing_policy_with_fallback( sdk=self._sdk, name=instance_name, project_id=self._region_to_project_id[instance_offer.region], platform=platform, - # Prevent the spot price from growing above what dstack shows, which is - # the on-demand price of the same type when the catalog has it (fork). - max_price=str(max_price), - ) - resources.wait_for_operation( - create_pricing_policy_op, timeout=WAIT_FOR_PRICING_POLICY_TIMEOUT + offer=instance_offer, ) - if not create_pricing_policy_op.successful(): - raw_op = create_pricing_policy_op.raw() - raise ProvisioningError( - f"Create pricing policy operation failed. Message: {raw_op.status.message}." - f" Details: {raw_op.status.details}" - ) create_instance_op = resources.create_instance( sdk=self._sdk, name=instance_name, @@ -488,15 +477,20 @@ def _get_price_adjusted_for_pricing_policy(offer: InstanceOffer) -> float: ).is_preemptible_flat_rate ): return offer.price - max_price = _get_pricing_policy_max_price(offer) - return float(max_price * _get_pricing_policy_price_units(offer)) + cap, _ = _get_pricing_policy_cap(offer) + return float(cap * _get_pricing_policy_price_units(offer)) + + +# Nebius accepts a pricing policy bid up to this much (USD per GPU-hour) below the current +# on-demand price, and answers OUT_OF_RANGE above it. +NEBIUS_MAX_BID_BELOW_ON_DEMAND = Decimal("0.010") def _with_on_demand_price(offers: List[InstanceOffer]) -> List[InstanceOffer]: - """Fork (operator decision 2026-10-08): store on each spot offer the on-demand price of - the same instance type in the same region, so the pricing policy caps the spot price at - the on-demand price, not at the current spot price. Nebius then preempts the VM only when - the spot price grows above what the same VM costs on demand.""" + """Fork (operator decision 2026-10-08): store on each spot offer its catalog spot price + and the on-demand price of the same instance type in the same region. The pricing policy + then caps the spot price just below the on-demand price, not at the current spot price, + so Nebius does not preempt the VM for price while spot stays below on-demand.""" on_demand_prices = { (offer.instance.name, offer.region): offer.price for offer in offers @@ -504,26 +498,125 @@ def _with_on_demand_price(offers: List[InstanceOffer]) -> List[InstanceOffer]: } result = [] for offer in offers: - on_demand_price = on_demand_prices.get((offer.instance.name, offer.region)) - if offer.instance.resources.spot and on_demand_price is not None: - offer = offer.model_copy( - update={"backend_data": {**offer.backend_data, "on_demand_price": on_demand_price}} - ) + if offer.instance.resources.spot: + extra = {"spot_price": offer.price} + on_demand_price = on_demand_prices.get((offer.instance.name, offer.region)) + if on_demand_price is not None: + extra["on_demand_price"] = on_demand_price + offer = offer.model_copy(update={"backend_data": {**offer.backend_data, **extra}}) result.append(offer) return result -def _get_pricing_policy_max_price(offer: InstanceOffer) -> Decimal: - # Fork: the cap is the on-demand price of the same type and region when the catalog has - # it (`_with_on_demand_price`), else the offer price (upstream behavior). - on_demand_price = validate_extra_ignore( - NebiusOfferBackendData, offer.backend_data - ).on_demand_price - base_price = on_demand_price if on_demand_price is not None else offer.price - # Pricing policies allow at most 3 decimal places. Round upward. - price = Decimal(str(base_price)) / _get_pricing_policy_price_units(offer) +def _get_run_max_price_modifier(requirements: Requirements) -> OfferModifier: + """Fork: on a spot offer that has no on-demand price, record the run's `max_price` so + the pricing policy caps the spot price at it. The shown price does not change, so the + offer order and the `max_price` filter stay as they are.""" + + def modifier(offer: InstanceOfferWithAvailability) -> Optional[InstanceOfferWithAvailability]: + if not offer.instance.resources.spot or requirements.max_price is None: + return offer + data = validate_extra_ignore(NebiusOfferBackendData, offer.backend_data) + if data.is_preemptible_flat_rate or data.on_demand_price is not None: + return offer + return offer.model_copy( + update={ + "backend_data": {**offer.backend_data, "run_max_price": requirements.max_price} + } + ) + + return modifier + + +def _get_spot_price_cap(offer: InstanceOffer) -> Decimal: + # Upstream behavior: the cap is the catalog spot price. Pricing policies allow at most + # 3 decimal places. Round upward. + spot_price = validate_extra_ignore(NebiusOfferBackendData, offer.backend_data).spot_price + price = Decimal(str(spot_price if spot_price is not None else offer.price)) + price = price / _get_pricing_policy_price_units(offer) return price.quantize(Decimal("0.001"), rounding=ROUND_CEILING) +def _get_pricing_policy_cap(offer: InstanceOffer) -> tuple[Decimal, str]: + """Return the per-GPU pricing policy cap of a spot offer and its source. Fork order: just + below the on-demand price of the same type and region, else the run's `max_price`, else + the spot price (upstream). A cap is never below the spot price cap.""" + data = validate_extra_ignore(NebiusOfferBackendData, offer.backend_data) + units = _get_pricing_policy_price_units(offer) + spot_cap = _get_spot_price_cap(offer) + if data.on_demand_price is not None: + cap = ( + Decimal(str(data.on_demand_price)) / units - NEBIUS_MAX_BID_BELOW_ON_DEMAND + ).quantize(Decimal("0.001"), rounding=ROUND_FLOOR) + if cap >= spot_cap: + return cap, "on-demand price" + if data.run_max_price is not None: + cap = (Decimal(str(data.run_max_price)) / units).quantize( + Decimal("0.001"), rounding=ROUND_FLOOR + ) + if cap >= spot_cap: + return cap, "run max_price" + return spot_cap, "spot price" + + +def _is_out_of_range(code: object) -> bool: + if code == StatusCode.OUT_OF_RANGE: + return True + value = getattr(code, "value", code) + if isinstance(value, tuple): + value = value[0] + return value == StatusCode.OUT_OF_RANGE.value[0] + + +def _create_pricing_policy_with_fallback( + sdk: SDK, name: str, project_id: str, platform: str, offer: InstanceOffer +) -> SDKOperation[Operation]: + """Create the pricing policy of a spot VM at its cap. If Nebius refuses the cap as out of + range (for example after an on-demand price change that the catalog does not show yet), + retry once with the spot price cap, the upstream behavior.""" + cap, source = _get_pricing_policy_cap(offer) + attempts = [(name, cap, source)] + spot_cap = _get_spot_price_cap(offer) + if cap != spot_cap: + attempts.append((f"{name}-spot", spot_cap, "spot price, after OUT_OF_RANGE")) + for index, (policy_name, max_price, cap_source) in enumerate(attempts): + is_last = index == len(attempts) - 1 + logger.info( + "Nebius spot cap for %s (%s, %s): %s per GPU-hour, source: %s", + name, + offer.instance.name, + offer.region, + max_price, + cap_source, + ) + try: + op = resources.create_pricing_policy( + sdk=sdk, + name=policy_name, + project_id=project_id, + platform=platform, + max_price=str(max_price), + ) + except RequestError as e: + if not is_last and _is_out_of_range(e.status.code): + logger.warning("Nebius refused the cap %s as out of range: %s", max_price, e) + continue + raise + resources.wait_for_operation(op, timeout=WAIT_FOR_PRICING_POLICY_TIMEOUT) + if op.successful(): + return op + raw_op = op.raw() + if not is_last and _is_out_of_range(raw_op.status.code): + logger.warning( + "Nebius refused the cap %s as out of range: %s", max_price, raw_op.status.message + ) + continue + raise ProvisioningError( + f"Create pricing policy operation failed. Message: {raw_op.status.message}." + f" Details: {raw_op.status.details}" + ) + raise ProvisioningError("Create pricing policy: no cap left to try") + + def _get_pricing_policy_price_units(offer: InstanceOffer) -> int: return len(offer.instance.resources.gpus) or 1 diff --git a/src/dstack/_internal/core/backends/nebius/models.py b/src/dstack/_internal/core/backends/nebius/models.py index 0b18c88bb0..b719c50445 100644 --- a/src/dstack/_internal/core/backends/nebius/models.py +++ b/src/dstack/_internal/core/backends/nebius/models.py @@ -189,8 +189,19 @@ class NebiusOfferBackendData(CoreModel): on_demand_price: Optional[float] = None """ Fork: the on-demand price of the same instance type in the same region, set on spot - offers only. The pricing policy caps the spot price at it. `None` when the catalog - has no on-demand row for that type and region: the cap is then the spot price. + offers only. The pricing policy caps the spot price just below it (Nebius accepts a + bid up to 0.01 USD per GPU-hour below the on-demand price). `None` when the catalog + has no on-demand row for that type and region. + """ + spot_price: Optional[float] = None + """ + Fork: the catalog spot price of a spot offer, kept because the shown offer price + becomes the cap. Used for the fallback cap when Nebius refuses the on-demand cap. + """ + run_max_price: Optional[float] = None + """ + Fork: the run's `max_price`, set by an offer modifier on a spot offer that has no + on-demand price. The pricing policy then caps the spot price at it. """ @field_serializer("fabrics") diff --git a/src/tests/_internal/core/backends/nebius/test_compute.py b/src/tests/_internal/core/backends/nebius/test_compute.py index 06d5e11c02..4322f2d735 100644 --- a/src/tests/_internal/core/backends/nebius/test_compute.py +++ b/src/tests/_internal/core/backends/nebius/test_compute.py @@ -134,6 +134,8 @@ def make_rtx_offer(price: float, spot: bool, region: str = "us-central1") -> Ins class TestSpotCapAtOnDemandPrice: # Fork (operator decision 2026-10-08): the pricing policy caps a spot VM at the # on-demand price of the same instance type and region, not at the current spot price. + # Nebius accepts a bid only up to 0.01 USD per GPU-hour below the on-demand price + # (OUT_OF_RANGE above it), so the cap is the on-demand price minus 0.01, rounded down. @pytest.fixture(autouse=True) def _mock_region_to_project_id(self, mocker): mocker.patch.object( @@ -142,7 +144,7 @@ def _mock_region_to_project_id(self, mocker): {"us-central1": "project-a", "uk-south2": "project-b"}, ) - def test_spot_offer_is_capped_at_the_on_demand_price_of_the_same_type(self, mocker): + def test_spot_offer_is_capped_just_below_the_on_demand_price_of_the_same_type(self, mocker): mocker.patch.object( compute_module, "get_catalog_offers", @@ -153,10 +155,32 @@ def test_spot_offer_is_capped_at_the_on_demand_price_of_the_same_type(self, mock spot = [o for o in offers if o.instance.resources.spot][0] on_demand = [o for o in offers if not o.instance.resources.spot][0] - assert spot.price == 1.79 + assert spot.price == 1.78 assert spot.backend_data["on_demand_price"] == 1.79 + assert spot.backend_data["spot_price"] == 0.95 assert on_demand.price == 1.79 + def test_offers_stay_sorted_by_the_shown_price(self, mocker): + h100 = make_offer(2.15, spot=True, is_preemptible_flat_rate=False, gpu_count=1) + h100_od = make_offer(4.5, spot=False, is_preemptible_flat_rate=False, gpu_count=1) + b200 = make_offer(3.95, spot=True, is_preemptible_flat_rate=False, gpu_count=1) + b200.instance.name = "gpu-b200-sxm 1gpu-20vcpu-224gb" + b200_od = make_offer(8.5, spot=False, is_preemptible_flat_rate=False, gpu_count=1) + b200_od.instance.name = "gpu-b200-sxm 1gpu-20vcpu-224gb" + mocker.patch.object( + compute_module, + "get_catalog_offers", + return_value=[h100, b200, h100_od, b200_od], + ) + mocker.patch.object(NebiusCompute, "_region_to_project_id", {"eu-north1": "project-id"}) + + offers = make_compute().get_all_offers_with_availability(unallocated_resources=False) + + prices = [o.price for o in offers] + assert prices == sorted(prices) + # At an equal-or-lower price the spot offer stays ahead of the on-demand one. + assert offers[0].instance.resources.spot + def test_falls_back_to_the_spot_price_without_an_on_demand_offer(self, mocker): mocker.patch.object( compute_module, @@ -187,14 +211,119 @@ def test_flat_rate_spot_price_is_not_capped(self, mocker): spot = [o for o in offers if o.instance.resources.spot][0] assert spot.price == 0.95 - def test_create_time_cap_is_the_on_demand_price_per_gpu(self): + def test_create_time_cap_is_the_on_demand_price_per_gpu_minus_one_cent(self): offer = make_offer(9.84, spot=True, is_preemptible_flat_rate=False, gpu_count=8) offer.backend_data["on_demand_price"] = 23.6 - assert str(compute_module._get_pricing_policy_max_price(offer)) == "2.950" + offer.backend_data["spot_price"] = 9.84 + cap, source = compute_module._get_pricing_policy_cap(offer) + assert str(cap) == "2.940" + assert source == "on-demand price" + + def test_create_time_cap_rounds_the_on_demand_price_down(self): + offer = make_rtx_offer(0.95, spot=True) + offer.backend_data["on_demand_price"] = 1.7899 + offer.backend_data["spot_price"] = 0.95 + cap, _ = compute_module._get_pricing_policy_cap(offer) + assert str(cap) == "1.779" + + def test_create_time_cap_uses_the_run_max_price_without_an_on_demand_offer(self): + offer = make_rtx_offer(0.95, spot=True) + offer.backend_data["spot_price"] = 0.95 + offer.backend_data["run_max_price"] = 1.5 + cap, source = compute_module._get_pricing_policy_cap(offer) + assert str(cap) == "1.500" + assert source == "run max_price" + + def test_create_time_cap_falls_back_to_the_spot_price(self): + offer = make_rtx_offer(0.95, spot=True) + cap, source = compute_module._get_pricing_policy_cap(offer) + assert str(cap) == "0.950" + assert source == "spot price" + + def test_spot_cap_uses_the_raw_spot_price_not_the_shown_cap(self): + offer = make_rtx_offer(1.78, spot=True) + offer.backend_data["on_demand_price"] = 1.79 + offer.backend_data["spot_price"] = 0.95 + assert str(compute_module._get_spot_price_cap(offer)) == "0.950" + + +class TestRunMaxPriceModifier: + def make_requirements(self, max_price): + from dstack._internal.core.models.resources import ResourcesSpec + from dstack._internal.core.models.runs import Requirements + + return Requirements(resources=ResourcesSpec(), max_price=max_price, spot=True) - def test_create_time_cap_falls_back_to_the_offer_price(self): + def test_sets_the_run_max_price_on_a_spot_offer_without_an_on_demand_price(self): + offer = make_rtx_offer(0.95, spot=True).with_availability( + availability=InstanceAvailability.UNKNOWN + ) + offer.backend_data["spot_price"] = 0.95 + modified = compute_module._get_run_max_price_modifier(self.make_requirements(1.5))(offer) + assert modified.backend_data["run_max_price"] == 1.5 + assert modified.price == 0.95 + assert "run_max_price" not in offer.backend_data + + def test_leaves_an_offer_with_an_on_demand_price_alone(self): + offer = make_rtx_offer(1.78, spot=True).with_availability( + availability=InstanceAvailability.UNKNOWN + ) + offer.backend_data["on_demand_price"] = 1.79 + modified = compute_module._get_run_max_price_modifier(self.make_requirements(1.9))(offer) + assert "run_max_price" not in modified.backend_data + + def test_leaves_the_offer_alone_without_a_run_max_price(self): + offer = make_rtx_offer(0.95, spot=True).with_availability( + availability=InstanceAvailability.UNKNOWN + ) + modified = compute_module._get_run_max_price_modifier(self.make_requirements(None))(offer) + assert "run_max_price" not in modified.backend_data + + +class TestCreatePricingPolicyOutOfRangeRetry: + def test_retries_once_with_the_spot_cap_on_out_of_range(self, mocker): + from nebius.aio.service_error import RequestError, StatusCode + + calls = [] + + def fake_create(sdk, name, project_id, platform, max_price): + calls.append(max_price) + if len(calls) == 1: + err = RequestError.__new__(RequestError) + err.status = SimpleNamespace(code=StatusCode.OUT_OF_RANGE) + raise err + return SimpleNamespace(resource_id="pp-1", successful=lambda: True) + + mocker.patch.object(compute_module.resources, "create_pricing_policy", fake_create) + mocker.patch.object( + compute_module.resources, "wait_for_operation", lambda op, timeout: None + ) + offer = make_rtx_offer(1.78, spot=True) + offer.backend_data["on_demand_price"] = 1.79 + offer.backend_data["spot_price"] = 0.95 + + op = compute_module._create_pricing_policy_with_fallback( + sdk=None, name="n", project_id="p", platform="gpu-rtx6000", offer=offer + ) + + assert calls == ["1.780", "0.950"] + assert op.resource_id == "pp-1" + + def test_does_not_retry_when_the_cap_already_is_the_spot_cap(self, mocker): + from nebius.aio.service_error import RequestError, StatusCode + + def fake_create(sdk, name, project_id, platform, max_price): + err = RequestError.__new__(RequestError) + err.status = SimpleNamespace(code=StatusCode.OUT_OF_RANGE) + raise err + + mocker.patch.object(compute_module.resources, "create_pricing_policy", fake_create) offer = make_rtx_offer(0.95, spot=True) - assert str(compute_module._get_pricing_policy_max_price(offer)) == "0.950" + + with pytest.raises(RequestError): + compute_module._create_pricing_policy_with_fallback( + sdk=None, name="n", project_id="p", platform="gpu-rtx6000", offer=offer + ) class TestSupportedInstances: