Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions docs/v6/reference/runtime.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ RuntimeConfig(resources=RuntimeResources(tpu=RuntimeTPU(type="v5", topology="2x2
| `compose` | `ComposeProject(document, root, service_access)`. Mutually exclusive with `image`. |
| `resources` | Placement requests: CPU, memory, disk, acceptable GPU types and count, OS, or TPU slice. |
| `limits` | `RuntimeLimits(startup_timeout_s, run_timeout_s)`. |
| `data` | `RuntimeData(files, mount_path, mode)`: platform data files provided to the environment. HUD-hosted EC2 rollouts only. |

Support differs per runtime. Providers reject unsupported requirements except `storage_mb`, which
is best effort: `DockerRuntime` admits against available disk and `DaytonaRuntime` provisions
Expand All @@ -96,6 +97,39 @@ than `rollout_timeout`; an agent timeout must also be less than the actor enviro
SDK adds no overall deadline; configured phase limits and limits imposed by the selected runtime
still apply.

### Data files

`RuntimeData` names platform data files, the ones uploaded on the HUD Data page, by id, and where
the environment sees them. The hosted rollout box mounts them from the platform's storage and binds
each file into the environment container, read-only, so the environment reads them as ordinary
files and never sees other files in the team's store. Bytes are fetched only when read; nothing is
copied ahead of time.

```python
from hud.eval import RuntimeConfig, RuntimeData, RuntimeDataFile

RuntimeConfig(
data=RuntimeData(
files=[RuntimeDataFile(file_id="e9032002-01bb-4511-af4f-d18477f75dda", path="case_room/resume.pdf")],
mount_path="/data",
),
)
```

| Field | Description |
|-------|-------------|
| `files` | One or more `RuntimeDataFile(file_id, path)`. `path` is relative to `mount_path` and defaults to the stored filename; it may not contain `..`, and two entries may not share a path or an id. |
| `mount_path` | Absolute directory inside the container, default `/data`. |
| `mode` | `readonly` or `overlay`. Files are read-only today; `overlay` is reserved for folder-backed data. |

The same `file_id` values appear in task arguments typed with `DataFileArg` and `DataFilesArg`; the
difference is who fetches the bytes. A task argument hands the environment an id to download
itself; `RuntimeData` has the platform mount the file before the environment starts. Declare it on
the deployed environment (`hud deploy`), on synced tasks, or on a single `Task.runtime_config`; a
later layer replaces the block whole. Submission fails with a 400 naming the id when a file does
not exist, is not `ready`, or is not readable by the submitter, and when the hosted environment does
not run on EC2. Local, Docker, Modal, and Daytona runtimes do not support `data`.

When the SDK owns the environment process, its output is part of the local eval:
`SubprocessRuntime`, Docker image and Compose runtimes, Modal image and Compose sandboxes,
and Daytona sandboxes stream stdout and stderr directly to the terminal without
Expand Down
4 changes: 4 additions & 0 deletions hud/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
Run,
Runtime,
RuntimeConfig,
RuntimeData,
RuntimeDataFile,
RuntimeGPU,
RuntimeLimits,
RuntimeResources,
Expand Down Expand Up @@ -48,6 +50,8 @@
"Run",
"Runtime",
"RuntimeConfig",
"RuntimeData",
"RuntimeDataFile",
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
Expand Down
4 changes: 4 additions & 0 deletions hud/eval/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
Provider,
Runtime,
RuntimeConfig,
RuntimeData,
RuntimeDataFile,
RuntimeGPU,
RuntimeLimits,
RuntimeResources,
Expand All @@ -70,6 +72,8 @@
"Run",
"Runtime",
"RuntimeConfig",
"RuntimeData",
"RuntimeDataFile",
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
Expand Down
4 changes: 4 additions & 0 deletions hud/eval/runtime/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
Provider,
Runtime,
RuntimeConfig,
RuntimeData,
RuntimeDataFile,
RuntimeGPU,
RuntimeLimits,
RuntimeResources,
Expand All @@ -29,6 +31,8 @@
"Provider",
"Runtime",
"RuntimeConfig",
"RuntimeData",
"RuntimeDataFile",
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
Expand Down
41 changes: 38 additions & 3 deletions hud/eval/runtime/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import json
from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Protocol, Self, runtime_checkable
from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, runtime_checkable

from pydantic import (
BaseModel,
Expand Down Expand Up @@ -80,6 +80,40 @@ class RuntimeLimits(BaseModel):
run_timeout_s: int | None = Field(default=None, gt=0)


class RuntimeDataFile(BaseModel):
"""A platform data file to provide to the environment.

Same wire shape as ``hud.environment.DataFileRef`` (``file_id`` plus an
optional destination ``path``), kept separate so the runtime module does
not import the environment package.
"""

model_config = ConfigDict(extra="forbid")

file_id: str = Field(min_length=1, description="HUD data-file id")
path: str | None = Field(
default=None,
description="Path under mount_path; defaults to the filename",
)


class RuntimeData(BaseModel):
"""Platform data files the hosted rollout box provides to the environment.

The files appear under ``mount_path`` read-only beneath a writable overlay
(``mode: overlay``) or read-only alone (``mode: readonly``). The platform
resolves ids to storage when it provisions the box; environments never
name buckets. Size the writable layer with ``resources.storage_mb``. EC2
rollouts only.
"""

model_config = ConfigDict(extra="forbid")

files: list[RuntimeDataFile] = Field(min_length=1)
mount_path: str = "/data"
mode: Literal["overlay", "readonly"] = "overlay"


class RuntimeConfig(BaseModel):
"""Typed task-environment launch requirements.

Expand All @@ -96,11 +130,12 @@ class RuntimeConfig(BaseModel):
compose: ComposeProject | None = None
resources: RuntimeResources | None = None
limits: RuntimeLimits | None = None
data: RuntimeData | None = None

@field_serializer("resources", "limits", when_used="json")
@field_serializer("resources", "limits", "data", when_used="json")
def _serialize_options(
self,
value: RuntimeResources | RuntimeLimits | None,
value: RuntimeResources | RuntimeLimits | RuntimeData | None,
info: SerializationInfo,
) -> dict[str, Any] | None:
if value is None:
Expand Down
72 changes: 72 additions & 0 deletions hud/eval/tests/test_runtime_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""``RuntimeConfig.data``: platform data files declared by id for hosted rollouts.

The block is declarative: which data files, where the environment sees them.
It validates like the rest of ``RuntimeConfig`` (unknown keys rejected), is
omitted from JSON when unset, and is replaced whole by an override rather
than merged, so a per-task block never inherits a build's file list.
"""

from __future__ import annotations

import pytest
from pydantic import ValidationError

from hud.eval import RuntimeConfig, RuntimeData, RuntimeDataFile

_FILE_ID = "e9032002-01bb-4511-af4f-d18477f75dda"


def test_data_block_defaults() -> None:
"""Only file ids are required; the mount path and mode have defaults."""
config = RuntimeConfig(data=RuntimeData(files=[RuntimeDataFile(file_id=_FILE_ID)]))

assert config.data is not None
assert config.data.mount_path == "/data"
assert config.data.mode == "overlay"
assert config.data.files[0].path is None


def test_data_block_serializes_only_what_was_set() -> None:
"""JSON carries the block only when set, and only the fields that were set."""
without = RuntimeConfig(image="my-env")
with_data = RuntimeConfig.model_validate(
{"data": {"files": [{"file_id": _FILE_ID, "path": "case_room/resume.pdf"}]}},
)

assert "data" not in without.model_dump(mode="json", exclude_unset=True)
assert with_data.model_dump(mode="json", exclude_unset=True) == {
"data": {"files": [{"file_id": _FILE_ID, "path": "case_room/resume.pdf"}]},
}


@pytest.mark.parametrize(
"payload",
[
{"data": {"files": []}},
{"data": {"files": [{"file_id": ""}]}},
{"data": {"files": [{"file_id": _FILE_ID}], "mode": "rw"}},
{"data": {"files": [{"file_id": _FILE_ID}], "bucket": "some-bucket"}},
{"data": {"files": [{"file_id": _FILE_ID, "size_gb": 1}]}},
],
)
def test_data_block_rejects_malformed_payloads(payload: dict[str, object]) -> None:
"""Empty file lists, blank ids, unknown modes and unknown keys fail validation."""
with pytest.raises(ValidationError):
RuntimeConfig.model_validate(payload)


def test_override_replaces_the_data_block_whole() -> None:
"""A per-task block replaces the base block's file list; it never merges into it."""
base = RuntimeConfig(
image="my-env",
data=RuntimeData(files=[RuntimeDataFile(file_id=_FILE_ID)], mount_path="/mnt/base"),
)
override = RuntimeConfig(data=RuntimeData(files=[RuntimeDataFile(file_id="other-id")]))

merged = base.with_overrides(override)

assert merged.image == "my-env"
assert merged.data is not None
assert [f.file_id for f in merged.data.files] == ["other-id"]
assert merged.data.mount_path == "/data"
assert base.with_overrides(RuntimeConfig(image="new")).data == base.data
2 changes: 2 additions & 0 deletions hud/tests/test_init_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ def test_all_exports(self):
"Run",
"Runtime",
"RuntimeConfig",
"RuntimeData",
"RuntimeDataFile",
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
Expand Down